deriva 1.7.1__py3-none-any.whl → 1.7.4__py3-none-any.whl

tests/deriva/core/mmo/base.py

@@ -0,0 +1,300 @@
+"""Base class for MMO test cases.
+"""
+import logging
+import os
+import unittest
+
+from deriva.core import DerivaServer, ErmrestCatalog, get_credential
+from deriva.core.ermrest_model import Schema, Table, Column, Key, ForeignKey, tag, builtin_types
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv('DERIVA_PY_TEST_LOGLEVEL', default=logging.WARNING))
+ermrest_hostname = os.getenv('DERIVA_PY_TEST_HOSTNAME')
+ermrest_catalog_id = os.getenv('DERIVA_PY_TEST_CATALOG')
+catalog = None
+
+# baseline annotation doc for `dept` table
+dept_annotations = {
+    tag.visible_columns: {
+        "compact": [
+            ["dept_schema", "dept_RID_key"],
+            ["dept_schema", "dept_dept_no_key"],
+            "name"
+        ],
+        "detailed": [
+            "RID",
+            "RCT",
+            {
+                "source": "RMT",
+                "markdown_name": "Last Modified Time"
+            },
+            "dept_no",
+            "name",
+            {
+                "source": "street_address",
+                "markdown_name": "Number and Street Name"
+            },
+            "postal_code",
+            {
+                "sourcekey": "head_count",
+                "markdown_name": "Head Count"
+            },
+            {
+                "display": {
+                    "wait_for": [
+                        "personnel"
+                    ],
+                    "template_engine": "handlebars",
+                    "markdown_pattern": "{{#each personnel}}{{{this.values.name}}}{{#unless @last}}, {{/unless}}{{/each}}."
+                },
+                "markdown_name": "Personnel"
+            }
+        ]
+    },
+    tag.visible_foreign_keys: {
+        "*": [
+            [
+                "person_schema",
+                "person_dept_fkey"
+            ]
+        ]
+    },
+    tag.source_definitions: {
+        "columns": [
+            "dept_no",
+            "name",
+            "RID",
+            "country"
+        ],
+        "sources": {
+            "personnel": {
+                "source": [
+                    {
+                        "inbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "name"
+                ]
+            },
+            "head_count": {
+                "source": [
+                    {
+                        "inbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "RID"
+                ],
+                "entity": False,
+                "aggregate": "cnt_d"
+            }
+        }
+    }
+}
+
+# baseline annotation doc for `person` table
+person_annotations = {
+    tag.visible_columns: {
+        "compact": [
+            ["person_schema", "person_RID_key"],
+            "name"
+        ],
+        "detailed": [
+            "RID",
+            "name",
+            ["person_schema", "person_dept_fkey"],
+            {
+                "markdown_name": "Department Name",
+                "source": [
+                    {
+                        "outbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "name"
+                ],
+                "entity": False
+            },
+            {
+                "sourcekey": "dept_size",
+                "markdown_name": "Department Size"
+            },
+            {
+                "sourcekey": "dept_city",
+                "markdown_name": "City"
+            },
+            {
+                "markdown_name": "State or Province",
+                "source": [
+                    {
+                        "outbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "state"
+                ],
+                "entity": False
+            },
+        ]
+    },
+    tag.source_definitions: {
+        "columns": [
+            "RID",
+            "name",
+            "dept"
+        ],
+        "fkeys": [
+            ["person_schema", "person_dept_fkey"]
+        ],
+        "sources": {
+            "dept_size": {
+                "source": [
+                    {
+                        "outbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    {
+                        "inbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "RID"
+                ],
+                "entity": False,
+                "aggregate": "cnt_d"
+            },
+            "dept_city":             {
+                "markdown_name": "City",
+                "source": [
+                    {
+                        "outbound": [
+                            "person_schema",
+                            "person_dept_fkey"
+                        ]
+                    },
+                    "city"
+                ],
+                "entity": False
+            }
+        },
+        "search-box": {
+            "or": [
+                {
+                    "source": "last_name"
+                }
+            ]
+        }
+    }
+}
+
+
+@unittest.skipUnless(ermrest_hostname, 'ERMrest hostname not defined.')
+class BaseMMOTestCase (unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        BaseMMOTestCase.setUpCatalog()
+
+    @classmethod
+    def setUpCatalog(cls):
+        global catalog
+
+        # create catalog
+        server = DerivaServer('https', ermrest_hostname, credentials=get_credential(ermrest_hostname))
+        if ermrest_catalog_id:
+            logger.debug(f'Connecting to {ermrest_hostname}/ermrest/catalog/{ermrest_catalog_id}')
+            catalog = server.connect_ermrest(ermrest_catalog_id)
+        else:
+            catalog = server.create_ermrest_catalog()
+            logger.debug(f'Created {ermrest_hostname}/ermrest/catalog/{catalog.catalog_id}')
+
+        # get the  model
+        model = catalog.getCatalogModel()
+
+        # drop all schemas (except 'public')
+        for sname in [sname for sname in model.schemas if sname != 'public']:
+            model.schemas[sname].drop(cascade=True)
+
+        # recreate schemas
+        for sname in ["dept_schema", "person_schema"]:
+            model.create_schema(Schema.define(sname))
+
+        # create `dept` table
+        model.schemas["dept_schema"].create_table(
+            Table.define(
+                'dept',
+                column_defs=[
+                    Column.define('dept_no', builtin_types.int8),
+                    Column.define('name', builtin_types.text),
+                    Column.define('street_address', builtin_types.text),
+                    Column.define('city', builtin_types.text),
+                    Column.define('state', builtin_types.text),
+                    Column.define('country', builtin_types.text),
+                    Column.define('postal_code', builtin_types.int8)
+                ],
+                key_defs=[
+                    Key.define(['dept_no'])
+                ],
+                annotations=dept_annotations
+            )
+        )
+
+        # create `person` table
+        model.schemas["person_schema"].create_table(
+            Table.define(
+                'person',
+                column_defs=[
+                    Column.define('name', builtin_types.text),
+                    Column.define('dept', builtin_types.int8),
+                    Column.define('last_name', builtin_types.text)
+                ],
+                fkey_defs=[
+                    ForeignKey.define(['dept'], "dept_schema", 'dept', ['dept_no'])
+                ],
+                annotations=person_annotations
+            )
+        )
+
+        # populate for good measure (though not necessary for current set of tests)
+        pbuilder = catalog.getPathBuilder()
+
+        pbuilder.dept_schema.dept.insert([
+            {'dept_no': 1, 'name': 'Dept A', 'street_address': '123 Main St', 'city': 'Anywhere', 'state': 'CA', 'country': 'US', 'postal_code': 98765},
+            {'dept_no': 2, 'name': 'Dept B', 'street_address': '777 Oak Ave', 'city': 'Somewhere', 'state': 'NY', 'country': 'US', 'postal_code': 12345}
+        ])
+
+        pbuilder.person_schema.person.insert([
+            {'name': 'John', 'dept': 1},
+            {'name': 'Helena', 'dept': 1},
+            {'name': 'Ben', 'dept': 1},
+            {'name': 'Sonia', 'dept': 2},
+            {'name': 'Rafael', 'dept': 2},
+        ])
+
+    @classmethod
+    def tearDownClass(cls):
+        BaseMMOTestCase.tearDownCatalog()
+
+    @classmethod
+    def tearDownCatalog(cls):
+        global catalog
+        if not ermrest_catalog_id and isinstance(catalog, ErmrestCatalog) and int(catalog.catalog_id) > 1000:
+            # note: the '... > 1000' clause is intended to safeguard against accidental deletion of production catalogs in the usual (lower) range
+            catalog.delete_ermrest_catalog(really=True)
+        catalog = None
+
+    def setUp(self):
+        # reset annotations to baseline
+        assert isinstance(catalog, ErmrestCatalog)
+        self.model = catalog.getCatalogModel()
+
+    def tearDown(self):
+        pass

tests/deriva/core/mmo/test_mmo_drop.py

@@ -0,0 +1,252 @@
+"""Unit tests for MMO+DDL Drop operations.
+"""
+import os
+import logging
+from deriva.core import mmo
+from deriva.core.ermrest_model import UpdateMappings
+from tests.deriva.core.mmo.base import BaseMMOTestCase
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv('DERIVA_PY_TEST_LOGLEVEL', default=logging.WARNING))
+
+
+class TestMMOxDDLDrop (BaseMMOTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        """Don't bother setting up catalog for the suite.
+        """
+        pass
+
+    def setUp(self):
+        """Setup catalog for each unit test.
+        """
+        TestMMOxDDLDrop.setUpCatalog()
+        super().setUp()
+
+    def test_drop_key(self):
+        kname = ["dept_schema", "dept_dept_no_key"]
+
+        def test():
+            matches = mmo.find(self.model, kname)
+            return len(matches)
+
+        self.assertTrue(test())
+        t = self.model.schemas[kname[0]].tables["dept"]
+        key = t.keys[(t.schema, kname[1])]
+        key.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test())
+
+    def test_drop_key_deferred(self):
+        kname = ["dept_schema", "dept_dept_no_key"]
+
+        def test(model):
+            matches = mmo.find(model, kname)
+            return len(matches)
+
+        self.assertTrue(test(self.model))
+        t = self.model.schemas[kname[0]].tables["dept"]
+        key = t.keys[(t.schema, kname[1])]
+        key.drop(cascade=True, update_mappings=UpdateMappings.deferred)
+        self.assertTrue(test(self.model.catalog.getCatalogModel()))
+        self.model.apply()
+        self.assertFalse(test(self.model.catalog.getCatalogModel()))
+
+    def test_drop_key_cascade(self):
+        kname = ["dept_schema", "dept_dept_no_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        self.assertTrue(test(kname))
+        self.assertTrue(test(fkname))
+        t = self.model.schemas[kname[0]].tables["dept"]
+        key = t.keys[(t.schema, kname[1])]
+        key.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test(kname))
+        self.assertFalse(test(fkname))
+
+    def test_drop_fkey(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test():
+            matches = mmo.find(self.model, fkname)
+            return len(matches)
+
+        self.assertTrue(test())
+        self.model.fkey(fkname).drop(update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test())
+
+    def test_drop_fkey_deferred(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(model):
+            matches = mmo.find(model, fkname)
+            return len(matches)
+
+        self.assertTrue(test(self.model))
+        self.model.fkey(fkname).drop(update_mappings=UpdateMappings.deferred)
+        self.assertTrue(test(self.model.catalog.getCatalogModel()))
+        self.model.apply()
+        self.assertFalse(test(self.model.catalog.getCatalogModel()))
+
+    def test_drop_col(self):
+        cname = ["person_schema", "person", "last_name"]
+
+        def test():
+            matches = mmo.find(self.model, cname)
+            return len(matches)
+
+        self.assertTrue(test())
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        col = t.columns[cname[2]]
+        col.drop(update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test())
+
+    def test_drop_col_deferred(self):
+        cname = ["person_schema", "person", "last_name"]
+
+        def test(model):
+            matches = mmo.find(model, cname)
+            return len(matches)
+
+        self.assertTrue(test(self.model))
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        col = t.columns[cname[2]]
+        col.drop(update_mappings=UpdateMappings.deferred)
+        self.assertTrue(test(self.model.catalog.getCatalogModel()))
+        self.model.apply()
+        self.assertFalse(test(self.model.catalog.getCatalogModel()))
+
+    def test_drop_col_cascade_key(self):
+        cname = ["dept_schema", "dept", "dept_no"]
+        kname = ["dept_schema", "dept_dept_no_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subTest {name}'):
+                self.assertTrue(test(name))
+
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        col = t.columns[cname[2]]
+        col.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+        
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subTest {name}'):
+                self.assertFalse(test(name))
+
+    def test_drop_col_cascade_fkey(self):
+        cname = ["person_schema", "person", "dept"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        self.assertTrue(test(cname))
+        self.assertTrue(test(fkname))
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        col = t.columns[cname[2]]
+        col.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test(cname))
+        self.assertFalse(test(fkname))
+
+    def test_drop_table_cascade(self):
+        cname = ["dept_schema", "dept", "dept_no"]
+        kname = ["dept_schema", "dept_dept_no_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subTest {name}'):
+                self.assertTrue(test(name))
+
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        t.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subTest {name}'):
+                self.assertFalse(test(name))
+
+    def test_drop_table_cascade_deferred(self):
+        cname = ["person_schema", "person", "name"]
+        kname = ["person_schema", "person_RID_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(model, name):
+            matches = mmo.find(model, name)
+            return len(matches)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subtest {name} precondition'):
+                self.assertTrue(test(self.model, name))
+
+        t = self.model.schemas[cname[0]].tables[cname[1]]
+        t.drop(cascade=True, update_mappings=UpdateMappings.deferred)
+
+        # fkname should linger on in the dept table's annotations until the update is applied to the model
+        self.assertTrue(test(self.model.catalog.getCatalogModel(), fkname))
+        self.model.apply()
+        self.assertFalse(test(self.model.catalog.getCatalogModel(), fkname))
+
+    def test_drop_table(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        self.assertTrue(test(fkname))
+        t = self.model.schemas["person_schema"].tables["person"]
+        t.drop(update_mappings=UpdateMappings.immediate)
+        self.assertFalse(test(fkname))
+
+    def test_drop_schema_cascade(self):
+        cname = ["dept_schema", "dept", "dept_no"]
+        kname = ["dept_schema", "dept_dept_no_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(name):
+            matches = mmo.find(self.model, name)
+            return len(matches)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subtest {name} precondition'):
+                self.assertTrue(test(name))
+
+        s = self.model.schemas[cname[0]]
+        s.drop(cascade=True, update_mappings=UpdateMappings.immediate)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subtest {name} postcondition'):
+                self.assertFalse(test(name))
+
+    def test_drop_schema_cascade_deferred(self):
+        cname = ["person_schema", "person", "name"]
+        kname = ["person_schema", "person_RID_key"]
+        fkname = ["person_schema", "person_dept_fkey"]
+
+        def test(model, name):
+            matches = mmo.find(model, name)
+            return len(matches)
+
+        for name in [cname, kname, fkname]:
+            with self.subTest(f'subtest {name} precondition'):
+                self.assertTrue(test(self.model, name))
+
+        s = self.model.schemas[cname[0]]
+        s.drop(cascade=True, update_mappings=UpdateMappings.deferred)
+
+        # fkname should linger on in the dept table's annotations until the update is applied to the model
+        self.assertTrue(test(self.model.catalog.getCatalogModel(), fkname))
+        self.model.apply()
+        self.assertFalse(test(self.model.catalog.getCatalogModel(), fkname))

tests/deriva/core/mmo/test_mmo_find.py

@@ -0,0 +1,90 @@
+"""Unit tests for MMO find operation.
+"""
+import logging
+import os
+
+from deriva.core import mmo
+from deriva.core.ermrest_model import tag
+from tests.deriva.core.mmo.base import BaseMMOTestCase
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv('DERIVA_PY_TEST_LOGLEVEL', default=logging.WARNING))
+
+
+def first_element_match(l1, l2):
+    return isinstance(l1, list) and isinstance(l2, list) and \
+        len(l1) == len(l2) and len(l1) > 1 and \
+        l1[0] == l2[0]
+
+
+class TestMMOFind (BaseMMOTestCase):
+
+    def test_find_key_in_vizcols(self):
+        matches = mmo.find(self.model, ["dept_schema", "dept_RID_key"])
+        self.assertEqual(len(matches), 1)
+
+    def test_find_key_in_vizcols_by_schema_only(self):
+        matches = mmo.find(self.model, ["dept_schema", None])
+        self.assertGreater(len(matches), 0)
+
+    def test_find_col_in_vizcols(self):
+        matches = mmo.find(self.model, ["dept_schema", "dept", "RCT"])
+        self.assertEqual(len(matches), 1)
+
+    def test_find_col_in_vizcols_pseudocol_simple(self):
+        matches = mmo.find(self.model, ["dept_schema", "dept", "RMT"])
+        self.assertEqual(len(matches), 1)
+
+    def test_find_col_in_vizcols_pseudocol(self):
+        matches = mmo.find(self.model, ["dept_schema", "dept", "name"])
+        self.assertTrue(any([m.anchor.name == 'person' and m.tag == tag.visible_columns and isinstance(m.mapping, dict) for m in matches]))
+
+    def test_find_col_in_sourcedefs_columns(self):
+        matches = mmo.find(self.model, ["person_schema", "person", "dept"])
+        self.assertTrue(any([m.anchor.name == 'person' and m.tag == tag.source_definitions and m.mapping == 'dept' for m in matches]))
+
+    def test_find_col_in_sourcedefs_sources(self):
+        matches = mmo.find(self.model, ["person_schema", "person", "RID"])
+        self.assertTrue(any([m.tag == tag.source_definitions and m.mapping == 'dept_size' for m in matches]))
+
+    def test_find_fkey_in_vizfkeys(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.visible_foreign_keys and m.mapping == fkname for m in matches]))
+
+    def test_find_fkey_in_vizfkeys_by_schema_only(self):
+        fkname = ["person_schema", None]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.visible_foreign_keys and first_element_match(m.mapping, fkname) for m in matches]))
+
+    def test_find_fkey_in_vizcols(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.visible_columns and m.mapping == fkname for m in matches]))
+
+    def test_find_fkey_in_vizcols_by_schema_only(self):
+        fkname = ["person_schema", None]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.visible_columns and first_element_match(m.mapping, fkname) for m in matches]))
+
+    def test_find_fkey_in_sourcedefs_sources(self):
+        matches = mmo.find(self.model, ["person_schema", "person_dept_fkey"])
+        self.assertTrue(any([m.tag == tag.source_definitions and m.mapping == 'personnel' for m in matches]))
+
+    def test_find_fkey_in_sourcedefs_sources_by_schema_only(self):
+        matches = mmo.find(self.model, ["person_schema", None])
+        self.assertTrue(any([m.tag == tag.source_definitions and m.mapping == 'personnel' for m in matches]))
+
+    def test_find_fkey_in_sourcedefs_fkeys(self):
+        fkname = ["person_schema", "person_dept_fkey"]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.source_definitions and m.mapping == fkname for m in matches]))
+
+    def test_find_fkey_in_sourcedefs_fkeys_by_schema_only(self):
+        fkname = ["person_schema", None]
+        matches = mmo.find(self.model, fkname)
+        self.assertTrue(any([m.tag == tag.source_definitions and m.context == 'fkeys' and first_element_match(m.mapping, fkname) for m in matches]))
+
+    def test_find_col_in_search_box(self):
+        matches = mmo.find(self.model, ["person_schema", "person", "last_name"])
+        self.assertTrue(len(matches) == 1)