muninn 6.4__tar.gz → 6.4.2__tar.gz

{muninn-6.4 → muninn-6.4.2}/CHANGES

@@ -1,3 +1,14 @@
+6.4.2 2025-02-24
+~~~~~~~~~~~~~~~~
+
+* Fixed issue with loading spatialite for versions of sqlite that were
+  build to disallow DQS (double-quoted string literals).
+
+6.4.1 2024-08-14
+~~~~~~~~~~~~~~~~
+
+* Fixed issue where only 'dict' values could be stored as a JSON property.
+
 6.4 2024-03-14
 ~~~~~~~~~~~~~~
 

{muninn-6.4 → muninn-6.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: muninn
-Version: 6.4
+Version: 6.4.2
 Summary: Data product catalogue and archive system
 Home-page: https://github.com/stcorp/muninn
 Author: S[&]T
@@ -17,6 +17,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering

{muninn-6.4 → muninn-6.4.2}/muninn/__init__.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function
 
-__version__ = "6.4"
+__version__ = "6.4.2"
 __copyright__ = "Copyright (C) 2014-2024 S[&]T, The Netherlands."
 
 __all__ = ["Error", "InternalError", "Struct", "Archive", "open", "config_path"]

{muninn-6.4 → muninn-6.4.2}/muninn/_compat.py

@@ -5,6 +5,7 @@ Inspired by six, future, and jinja2
 '''
 import sys
 import operator
+import datetime
 
 
 PY3 = sys.version_info[0] == 3
@@ -33,6 +34,9 @@ if PY3:
     def decode(s):
         return s.decode('utf-8')
 
+    def utcnow():
+        return datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
+
     itervalues = operator.methodcaller("values")
 
     imap = map
@@ -55,12 +59,15 @@ else:
     def path_utf8(path):
         return path.decode(sys.getfilesystemencoding()).encode('utf-8')
 
-    def decode(s):
+    def encode(s):
         return s
 
-    def encode(s):
+    def decode(s):
         return s
 
+    def utcnow():
+        return datetime.datetime.utcnow()
+
     dictkeys = operator.methodcaller("keys")
     dictvalues = operator.methodcaller("values")
     itervalues = operator.methodcaller("itervalues")

{muninn-6.4 → muninn-6.4.2}/muninn/archive.py

@@ -413,7 +413,7 @@ class Archive(object):
         if len(set(basenames)) < len(basenames):
             raise Error("basename of each part should be unique for multi-part products")
 
-        return [path.replace('\\', '/') for path in paths]
+        return sorted([path.replace('\\', '/') for path in paths])
 
     def _extract_hash_type(self, hash_value):
         prefix, middle, _ = hash_value.partition(':')

{muninn-6.4 → muninn-6.4.2}/muninn/database/postgresql.py

@@ -481,8 +481,9 @@ class PostgresqlBackend(DatabaseBackend):
             fields.append("uuid")
             parameters.append(uuid)
 
-        # TODO generic conversion? more overlap between sqlite.py and postgresql.py?
-        parameters = [json.dumps(p) if isinstance(p, dict) else p for p in parameters]
+        schema = self._namespace_schema(name)
+        parameters = [json.dumps(p) if f != "uuid" and issubclass(schema[f], JSON) else p for f, p in
+                      zip(fields, parameters)]
 
         # Build and execute INSERT query.
         query = "INSERT INTO %s (%s) VALUES (%s)" % (self._table_name(name), ", ".join(fields),
@@ -775,11 +776,12 @@ class PostgresqlBackend(DatabaseBackend):
             del fields[uuid_index]
             del parameters[uuid_index]
 
+        schema = self._namespace_schema(name)
+        parameters = [json.dumps(p) if issubclass(schema[f], JSON) else p for f, p in zip(fields, parameters)]
+
         # Append the uuid (value) at the end of the list of parameters (will be used in the WHERE clause).
         parameters.append(uuid)
 
-        parameters = [json.dumps(p) if isinstance(p, dict) else p for p in parameters]
-
         # Build and execute UPDATE query.
         set_clause = ", ".join(["%s = %s" % (field, self._placeholder()) for field in fields])
         query = "UPDATE %s SET %s WHERE uuid = %s" % (self._table_name(name), set_clause, self._placeholder())

{muninn-6.4 → muninn-6.4.2}/muninn/database/sqlite.py

@@ -13,7 +13,7 @@ import uuid
 
 from .base import DatabaseBackend
 
-from muninn._compat import dictkeys, dictvalues
+from muninn._compat import decode, dictkeys, dictvalues, utcnow
 
 # Select a version of dbapi2 that's available.
 # Since we need the spatialite module, we need a version of sqlite3 that supports extensions.
@@ -96,6 +96,24 @@ def _cast_geometry(blob):
     return blobgeometry.decode_blob_geometry(blob)
 
 
+def _adapt_datetime(val):
+    return val.isoformat(" ")
+
+
+def _cast_datetime(val):
+    datepart, timepart = decode(val).split(" ")
+    year, month, day = map(int, datepart.split("-"))
+    timepart_full = timepart.split(".")
+    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+    if len(timepart_full) == 2:
+        microseconds = int('{:0<6.6}'.format(timepart_full[1]))
+    else:
+        microseconds = 0
+
+    val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds)
+    return val
+
+
 class SQLiteConnection(object):
     """Wrapper for a sqlite database connection that defers (re)connection until an attempt is made to use the
     connection.
@@ -150,7 +168,7 @@ class SQLiteConnection(object):
         # load the spatialite extension
         try:
             self._connection.enable_load_extension(True)
-            self._connection.execute("select load_extension(\"%s\");" % (self._mod_spatialite))
+            self._connection.execute("select load_extension('%s');" % (self._mod_spatialite))
         except Exception as e:
             raise Error("loading mod_spatialite extension failed (mod_spatialite_path='%s'): %s" %
                         (self._mod_spatialite, str(e)))
@@ -222,6 +240,9 @@ class SQLiteBackend(object):
         dbapi2.register_adapter(geometry.MultiLineString, _adapt_geometry)
         dbapi2.register_adapter(geometry.MultiPolygon, _adapt_geometry)
 
+        dbapi2.register_converter("TIMESTAMP", _cast_datetime)
+        dbapi2.register_adapter(datetime.datetime, _adapt_datetime)
+
         self._connection_string = connection_string
         self._connection = SQLiteConnection(connection_string, mod_spatialite_path, self)
 
@@ -438,7 +459,9 @@ class SQLiteBackend(object):
             fields.append("uuid")
             parameters.append(uuid)
 
-        parameters = [json.dumps(p) if isinstance(p, dict) else p for p in parameters]
+        schema = self._namespace_schema(name)
+        parameters = [json.dumps(p) if f != "uuid" and issubclass(schema[f], JSON) else p for f, p in
+                      zip(fields, parameters)]
 
         # Build and execute INSERT query.
         query = "INSERT INTO %s (%s) VALUES (%s)" % (self._table_name(name), ", ".join(fields),
@@ -733,11 +756,12 @@ class SQLiteBackend(object):
             del fields[uuid_index]
             del parameters[uuid_index]
 
+        schema = self._namespace_schema(name)
+        parameters = [json.dumps(p) if issubclass(schema[f], JSON) else p for f, p in zip(fields, parameters)]
+
         # Append the uuid (value) at the end of the list of parameters (will be used in the WHERE clause).
         parameters.append(uuid)
 
-        parameters = [json.dumps(p) if isinstance(p, dict) else p for p in parameters]
-
         # Build and execute UPDATE query.
         set_clause = ", ".join(["%s = %s" % (field, self._placeholder()) for field in fields])
         query = "UPDATE %s SET %s WHERE uuid = %s" % (self._table_name(name), set_clause, self._placeholder())
@@ -875,7 +899,7 @@ class SQLiteBackend(object):
 
     @translate_sqlite_errors
     def server_time_utc(self):
-        return datetime.datetime.utcnow()
+        return utcnow()
 
     @translate_sqlite_errors
     def source_products(self, uuid):

{muninn-6.4 → muninn-6.4.2}/muninn/language.py

@@ -613,7 +613,7 @@ def _literal_type(literal):
         else:
             return type
 
-    raise Error("unable to determine type of literal value: %r" % literal)
+    raise Error("unable to determine type of literal value: %r" % (literal,))
 
 
 class SemanticAnalysis(Visitor):
@@ -788,7 +788,7 @@ class Identifier(object):
             self.muninn_type = Long
 
         elif not re.match(r'[\w]+\.[\w.]+', canonical_identifier):
-            raise Error("cannot resolve identifier: %r" % canonical_identifier)
+            raise Error("cannot resolve identifier: %r" % (canonical_identifier,))
 
         else:
             segments = canonical_identifier.split('.')
@@ -808,7 +808,7 @@ class Identifier(object):
                 self.namespace, self.identifier, self.subscript = segments
 
             else:
-                raise Error("cannot resolve identifier: %r" % canonical_identifier)
+                raise Error("cannot resolve identifier: %r" % (canonical_identifier,))
 
             # check if namespace is valid
             if self.namespace not in namespace_schemas:

{muninn-6.4 → muninn-6.4.2}/muninn/schema.py

@@ -5,9 +5,10 @@
 from __future__ import absolute_import, division, print_function
 
 import datetime
+import json
+import uuid
 from muninn._compat import long
 from muninn._compat import string_types as basestring
-import uuid
 
 from muninn import geometry
 
@@ -96,7 +97,9 @@ class Timestamp(Type):
 class JSON(Type):
     @classmethod
     def validate(cls, value):
-        if type(value) is not dict:
+        try:
+            json.dumps(value)
+        except Exception:
             raise ValueError("invalid value %r for type %r" % (value, cls.name()))
 
 

{muninn-6.4 → muninn-6.4.2}/muninn/tools/ingest.py

@@ -31,7 +31,7 @@ class CheckProductListAction(argparse.Action):
 
 def expand_stem(stem):
     try:
-        return glob.glob(stem + "*")
+        return sorted(glob.glob(stem + "*"))
     except EnvironmentError as error:
         raise Error("unable to expand stem \"%s\" [%s]" % (stem, error))
 

{muninn-6.4 → muninn-6.4.2}/muninn/tools/update.py

@@ -75,11 +75,8 @@ def update(args):
 
     processor = UpdateProcessor(args)
     with muninn.open(args.archive) as archive:
-        if args.action in ['ingest', 'pull']:
-            # we only need the uuid and the product_name
-            products = archive.search(expression, property_names=['uuid', 'product_name'])
-        else:
-            products = archive.search(expression, namespaces=archive.namespaces())
+        # we only need the uuid and the product_name
+        products = archive.search(expression, property_names=['uuid', 'product_name'])
         if args.parallel:
             if args.processes is not None:
                 pool = multiprocessing.Pool(args.processes)

{muninn-6.4 → muninn-6.4.2}/muninn.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: muninn
-Version: 6.4
+Version: 6.4.2
 Summary: Data product catalogue and archive system
 Home-page: https://github.com/stcorp/muninn
 Author: S[&]T
@@ -17,6 +17,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Scientific/Engineering

{muninn-6.4 → muninn-6.4.2}/setup.py

@@ -15,7 +15,7 @@ requirements = []
 
 setup(
     name="muninn",
-    version="6.4",
+    version="6.4.2",
     description="Data product catalogue and archive system",
     url="https://github.com/stcorp/muninn",
     author="S[&]T",
@@ -51,6 +51,8 @@ setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Topic :: Scientific/Engineering",

{muninn-6.4 → muninn-6.4.2}/test/test.py

@@ -26,7 +26,11 @@ import botocore
 logging.getLogger("boto3").setLevel(logging.CRITICAL)
 
 warnings.filterwarnings('ignore', 'distutils version classes', DeprecationWarning, 'swiftclient.client')
-import swiftclient
+try:
+    import swiftclient
+except ImportError:
+    # allow to run tests where the swift backend is not enabled
+    pass
 logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
 
 sys.path.insert(0, '..')
@@ -1233,6 +1237,13 @@ class TestArchive:
             assert myjson['bla'] == 14
             assert myjson['mylist'] == [1, 2, 3]
 
+        # also test that we can store other json objects than 'dict'
+        props = muninn.Struct({'mynamespace': {'myjson': [1, 2, 3]}})
+        archive.update_properties(props, product.core.uuid)
+        props = archive.retrieve_properties(uuid=product.core.uuid, namespaces=['mynamespace'])
+        myjson = props.mynamespace.myjson
+        assert myjson == [1, 2, 3]
+
     def test_verify_hash(self, archive):
         product = self._ingest_file(archive)