muninn 7.0.2__tar.gz → 7.2.0__tar.gz

{muninn-7.0.2 → muninn-7.2.0}/CHANGES

@@ -1,3 +1,14 @@
+7.2.0 2025-10-29
+~~~~~~~~~~~~~~~~
+
+* Added support for defining s3fs mappings in the muninn archive config
+  for mapping 'remote_url' s3 urls (mostly useful when storage=none).
+
+7.1.0 2025-10-20
+~~~~~~~~~~~~~~~~
+
+* Added --update-properties option to muninn-pull.
+
 7.0.2 2025-10-10
 ~~~~~~~~~~~~~~~~
 

{muninn-7.0.2 → muninn-7.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: muninn
-Version: 7.0.2
+Version: 7.2.0
 Summary: Data product catalogue and archive system
 Home-page: https://github.com/stcorp/muninn
 Author: S[&]T

{muninn-7.0.2 → muninn-7.2.0}/muninn/__init__.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function
 
-__version__ = "7.0.2"
+__version__ = "7.2.0"
 __copyright__ = "Copyright (C) 2014-2025 S[&]T, The Netherlands."
 
 __all__ = ["Error", "InternalError", "Struct", "Archive", "open", "config_path"]

{muninn-7.0.2 → muninn-7.2.0}/muninn/archive.py

@@ -245,6 +245,15 @@ class Archive(object):
 
         _register_extension(self, remote, 'muninn.remote', 'remote_backend')
 
+        # s3fs mappings
+        self.s3fs_mappings = {}
+        if configuration is not None and 's3fs' in configuration:
+            for mapping in configuration.get('s3fs').get('mappings').split(' '):
+                prefix, path = mapping.split('=')
+                if not prefix.startswith('s3://'):
+                    raise Error("s3fs mapping '%s' does not start with an s3 url" % mapping)
+                self.s3fs_mappings[prefix] = path
+
     def __enter__(self):
         return self
 
@@ -638,8 +647,14 @@ class Archive(object):
         if self._storage is None:
             remote_url = product.core.remote_url
 
+            # apply s3fs mapping if applicable
+            for prefix in self.s3fs_mappings:
+                if remote_url.startswith(prefix):
+                    remote_url = "file://" + self.s3fs_mappings[prefix] + remote_url[len(prefix):]
+                    break
+
             if remote_url.startswith('file://'):
-                product_path = product.core.remote_url[7:]
+                product_path = remote_url[7:]
                 if os.path.isdir(product_path):
                     paths = [os.path.join(product_path, basename) for basename in os.listdir(product_path)]
                 else:
@@ -1315,12 +1330,16 @@ class Archive(object):
             product_path = self._product_path(product)
             return os.path.join(self._storage.global_prefix, product_path)
         if remote_url is not None:
+            # apply s3fs mapping if applicable
+            for prefix in self.s3fs_mappings:
+                if remote_url.startswith(prefix):
+                    return self.s3fs_mappings[prefix] + remote_url[len(prefix):]
             if remote_url.startswith('file://'):
                 return remote_url[7:]
             return remote_url
         return None
 
-    def pull(self, where="", parameters={}, verify_hash=False, verify_hash_download=False):
+    def pull(self, where="", parameters={}, update_properties=False, verify_hash=False, verify_hash_download=False):
         """Pull one or more remote products into the archive.
 
         Products should have a valid remote_url core metadata field and they should not yet exist in the local
@@ -1329,6 +1348,9 @@ class Archive(object):
         Arguments:
         where         --  Search expression or one or more product uuid(s) or properties.
         parameters    --  Parameters referenced in the search expression (if any).
+        update_properties  --  If set to True then, after the pull, metadata and tags will be extracted from the
+                               product using the analyze() function of the product type plugin, and the catalog entry
+                               for the product will be updated with this metadata and tags.
         verify_hash   --  If set to True then, after the pull, the product in the archive will be matched against
                           the hash from the metadata (only if the metadata contained a hash).
         verify_hash_download  --  If set to True then, before the product is stored in the archive, the pulled
@@ -1365,10 +1387,20 @@ class Archive(object):
                 if len(paths) > 1 and not use_enclosing_directory:
                     raise Error("cannot pull multi-file product without enclosing directory")
 
+                # extract properties/tags if an update is requested
+                if update_properties:
+                    properties, tags = self._analyze_paths(plugin, paths)
+                else:
+                    properties = Struct({'core': Struct()})
+                    tags = []
+                core = properties.core
+
                 # reactivate and update archive_date, size
                 product_path = self._product_path(product)
                 size = self._storage.size(product_path)
-                metadata = {'active': True, 'archive_date': self._database.server_time_utc(), 'size': size}
+                core.active = True
+                core.archive_date = self._database.server_time_utc()
+                core.size = size
                 if 'hash' not in product.core or product.core.hash is None:
                     hash_type = self._plugin_hash_type(plugin)
                     if hash_type is not None:
@@ -1376,8 +1408,12 @@ class Archive(object):
                             hash = util.product_hash(paths, hash_type=hash_type)
                         except EnvironmentError as _error:
                             raise Error("cannot determine product hash [%s]" % (_error,))
-                        metadata['hash'] = hash
-                self.update_properties(Struct({'core': metadata}), product.core.uuid)
+                        # update hash
+                        core.hash = hash
+                self.update_properties(properties, product.core.uuid, create_namespaces=update_properties)
+
+                if tags:
+                    self._database.tag(product.core.uuid, tags)
 
                 # verify product hash.
                 if verify_hash and 'hash' in product.core:

{muninn-7.0.2 → muninn-7.2.0}/muninn/remote.py

@@ -289,6 +289,15 @@ class FTPBackend(RemoteBackend):
 
 class S3Backend(RemoteBackend):
     def pull(self, archive, product, target_dir):
+        # apply s3fs mapping if applicable
+        for prefix in archive.s3fs_mappings:
+            if product.core.remote_url.startswith(prefix):
+                source_path = archive.s3fs_mappings[prefix] + product.core.remote_url[len(prefix):]
+                if not os.path.exists(source_path):
+                    raise DownloadError("s3fs path %s (for %s) does not exist" % (source_path, product.core.remote_url))
+                target_path = os.path.join(target_dir, os.path.basename(source_path))
+                util.copy_path(source_path, target_path)
+                return self.auto_extract(target_path, product)
         credentials = get_credentials(archive, product.core.remote_url)
         paths = download_s3(product.core.remote_url, target_dir, credentials=credentials)
         if len(paths) == 1:

{muninn-7.0.2 → muninn-7.2.0}/muninn/tools/pull.py

@@ -15,12 +15,14 @@ class PullProcessor(Processor):
     def __init__(self, args):
         super(PullProcessor, self).__init__(args)
         self.args = args
+        self.update_properties = True if args.update_properties else False
         self.verify_hash = True if args.verify_hash else False
         self.verify_hash_download = True if args.verify_hash_download else False
 
     def perform_operation(self, archive, product):
         archive.pull(
             product.core.uuid,
+            update_properties=self.update_properties,
             verify_hash=self.verify_hash,
             verify_hash_download=self.verify_hash_download
         )
@@ -44,6 +46,8 @@ def pull(args):
 
 def main():
     parser = create_parser(description="Pull remote files into the archive.", parallel=True)
+    parser.add_argument("--update-properties", action="store_true",
+                        help="update properties in catalog as if the product was ingested using muninn-ingest")
     parser.add_argument("--verify-hash", action="store_true",
                         help="verify the hash of the product after it has been put in the archive")
     parser.add_argument("--verify-hash-download", action="store_true",

{muninn-7.0.2 → muninn-7.2.0}/muninn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: muninn
-Version: 7.0.2
+Version: 7.2.0
 Summary: Data product catalogue and archive system
 Home-page: https://github.com/stcorp/muninn
 Author: S[&]T

{muninn-7.0.2 → muninn-7.2.0}/setup.py

@@ -15,7 +15,7 @@ requirements = []
 
 setup(
     name="muninn",
-    version="7.0.2",
+    version="7.2.0",
     description="Data product catalogue and archive system",
     url="https://github.com/stcorp/muninn",
     author="S[&]T",