datajoint 0.14.2__py3-none-any.whl → 0.14.4__py3-none-any.whl

datajoint/s3.py

@@ -2,12 +2,14 @@
 AWS S3 operations
 """
 
+import logging
+import uuid
 from io import BytesIO
+from pathlib import Path
+
 import minio  # https://docs.minio.io/docs/python-client-api-reference
 import urllib3
-import uuid
-import logging
-from pathlib import Path
+
 from . import errors
 
 logger = logging.getLogger(__name__.split(".")[0])
@@ -27,7 +29,7 @@ class Folder:
         *,
         secure=False,
         proxy_server=None,
-        **_
+        **_,
     ):
         # from https://docs.min.io/docs/python-client-api-reference
         self.client = minio.Minio(

datajoint/schemas.py

@@ -1,20 +1,20 @@
-import warnings
-import logging
+import collections
 import inspect
-import re
 import itertools
-import collections
+import logging
+import re
+import types
+import warnings
+
 from .connection import conn
-from .diagram import Diagram, _get_tier
-from .settings import config
-from .errors import DataJointError, AccessError
-from .jobs import JobTable
+from .errors import AccessError, DataJointError
 from .external import ExternalMapping
 from .heading import Heading
-from .utils import user_choice, to_camel_case
-from .user_tables import Part, Computed, Imported, Manual, Lookup
-from .table import lookup_class_name, Log, FreeTable
-import types
+from .jobs import JobTable
+from .settings import config
+from .table import FreeTable, Log, lookup_class_name
+from .user_tables import Computed, Imported, Lookup, Manual, Part, _get_tier
+from .utils import to_camel_case, user_choice
 
 logger = logging.getLogger(__name__.split(".")[0])
 
@@ -413,6 +413,7 @@ class Schema:
 
         :return: a string containing the body of a complete Python module defining this schema.
         """
+        self.connection.dependencies.load()
         self._assert_exists()
         module_count = itertools.count()
         # add virtual modules for referenced modules with names vmod0, vmod1, ...
@@ -451,10 +452,8 @@ class Schema:
                 ).replace("\n", "\n    " + indent),
             )
 
-        diagram = Diagram(self)
-        body = "\n\n".join(
-            make_class_definition(table) for table in diagram.topological_sort()
-        )
+        tables = self.connection.dependencies.topo_sort()
+        body = "\n\n".join(make_class_definition(table) for table in tables)
         python_code = "\n\n".join(
             (
                 '"""This module was auto-generated by datajoint from an existing schema"""',
@@ -480,11 +479,12 @@ class Schema:
 
         :return: A list of table names from the database schema.
         """
+        self.connection.dependencies.load()
         return [
             t
             for d, t in (
-                full_t.replace("`", "").split(".")
-                for full_t in Diagram(self).topological_sort()
+                table_name.replace("`", "").split(".")
+                for table_name in self.connection.dependencies.topo_sort()
             )
             if d == self.database
         ]
@@ -533,7 +533,6 @@ class VirtualModule(types.ModuleType):
 
 def list_schemas(connection=None):
     """
-
     :param connection: a dj.Connection object
     :return: list of all accessible schemas on the server
     """

datajoint/settings.py

@@ -1,14 +1,15 @@
 """
-Settings for DataJoint.
+Settings for DataJoint
 """
 
-from contextlib import contextmanager
+import collections
 import json
+import logging
 import os
 import pprint
-import logging
-import collections
+from contextlib import contextmanager
 from enum import Enum
+
 from .errors import DataJointError
 
 LOCALCONFIG = "dj_local_conf.json"
@@ -47,7 +48,9 @@ default = dict(
         "display.show_tuple_count": True,
         "database.use_tls": None,
         "enable_python_native_blobs": True,  # python-native/dj0 encoding support
-        "filepath_checksum_size_limit": None,  # file size limit for when to disable checksums
+        "add_hidden_timestamp": False,
+        # file size limit for when to disable checksums
+        "filepath_checksum_size_limit": None,
     }
 )
 
@@ -116,6 +119,7 @@ class Config(collections.abc.MutableMapping):
         if filename is None:
             filename = LOCALCONFIG
         with open(filename, "r") as fid:
+            logger.info(f"DataJoint is configured from {os.path.abspath(filename)}")
             self._conf.update(json.load(fid))
 
     def save_local(self, verbose=False):
@@ -235,7 +239,8 @@ class Config(collections.abc.MutableMapping):
 
         def __init__(self, *args, **kwargs):
             self._conf = dict(default)
-            self._conf.update(dict(*args, **kwargs))  # use the free update to set keys
+            # use the free update to set keys
+            self._conf.update(dict(*args, **kwargs))
 
         def __getitem__(self, key):
             return self._conf[key]
@@ -246,6 +251,13 @@ class Config(collections.abc.MutableMapping):
                 self._conf[key] = value
             else:
                 raise DataJointError("Validator for {0:s} did not pass".format(key))
+            valid_logging_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
+            if key == "loglevel":
+                if value not in valid_logging_levels:
+                    raise ValueError(
+                        f"'{value}' is not a valid logging value {tuple(valid_logging_levels)}"
+                    )
+                logger.setLevel(value)
 
 
 # Load configuration from file
@@ -254,11 +266,9 @@ config_files = (
     os.path.expanduser(n) for n in (LOCALCONFIG, os.path.join("~", GLOBALCONFIG))
 )
 try:
-    config_file = next(n for n in config_files if os.path.exists(n))
+    config.load(next(n for n in config_files if os.path.exists(n)))
 except StopIteration:
-    pass
-else:
-    config.load(config_file)
+    logger.info("No config file was found.")
 
 # override login credentials with environment variables
 mapping = {
@@ -270,6 +280,7 @@ mapping = {
             "database.password",
             "external.aws_access_key_id",
             "external.aws_secret_access_key",
+            "loglevel",
         ),
         map(
             os.getenv,
@@ -279,11 +290,14 @@ mapping = {
                 "DJ_PASS",
                 "DJ_AWS_ACCESS_KEY_ID",
                 "DJ_AWS_SECRET_ACCESS_KEY",
+                "DJ_LOG_LEVEL",
             ),
         ),
     )
     if v is not None
 }
-config.update(mapping)
+if mapping:
+    logger.info(f"Overloaded settings {tuple(mapping)} from environment variables.")
+    config.update(mapping)
 
 logger.setLevel(log_levels[config["loglevel"]])

datajoint/table.py

@@ -1,30 +1,32 @@
 import collections
-import itertools
+import csv
 import inspect
-import platform
-import numpy as np
-import pandas
+import itertools
+import json
 import logging
-import uuid
-import csv
+import platform
 import re
-import json
+import uuid
 from pathlib import Path
-from .settings import config
-from .declare import declare, alter
-from .condition import make_condition
-from .expression import QueryExpression
+from typing import Union
+
+import numpy as np
+import pandas
+
 from . import blob
-from .utils import user_choice, get_master, is_camel_case
-from .heading import Heading
+from .condition import make_condition
+from .declare import alter, declare
 from .errors import (
-    DuplicateError,
     AccessError,
     DataJointError,
-    UnknownAttributeError,
+    DuplicateError,
     IntegrityError,
+    UnknownAttributeError,
 )
-from typing import Union
+from .expression import QueryExpression
+from .heading import Heading
+from .settings import config
+from .utils import get_master, is_camel_case, user_choice
 from .version import __version__ as version
 
 logger = logging.getLogger(__name__.split(".")[0])
@@ -196,7 +198,6 @@ class Table(QueryExpression):
 
     def children(self, primary=None, as_objects=False, foreign_key_info=False):
         """
-
         :param primary: if None, then all children are returned. If True, then only foreign keys composed of
             primary key attributes are considered.  If False, return foreign keys including at least one
             secondary attribute.
@@ -218,7 +219,6 @@ class Table(QueryExpression):
 
     def descendants(self, as_objects=False):
         """
-
         :param as_objects: False - a list of table names; True - a list of table objects.
         :return: list of tables descendants in topological order.
         """
@@ -230,7 +230,6 @@ class Table(QueryExpression):
 
     def ancestors(self, as_objects=False):
         """
-
         :param as_objects: False - a list of table names; True - a list of table objects.
         :return: list of tables ancestors in topological order.
         """
@@ -246,6 +245,7 @@ class Table(QueryExpression):
 
         :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects.
         """
+        self.connection.dependencies.load(force=False)
         nodes = [
             node
             for node in self.connection.dependencies.nodes
@@ -427,7 +427,8 @@ class Table(QueryExpression):
             self.connection.query(query)
             return
 
-        field_list = []  # collects the field list from first row (passed by reference)
+        # collects the field list from first row (passed by reference)
+        field_list = []
         rows = list(
             self.__make_row_to_insert(row, field_list, ignore_extra_fields)
             for row in rows
@@ -520,7 +521,8 @@ class Table(QueryExpression):
                     delete_count = table.delete_quick(get_count=True)
                 except IntegrityError as error:
                     match = foreign_key_error_regexp.match(error.args[0]).groupdict()
-                    if "`.`" not in match["child"]:  # if schema name missing, use table
+                    # if schema name missing, use table
+                    if "`.`" not in match["child"]:
                         match["child"] = "{}.{}".format(
                             table.full_table_name.split(".")[0], match["child"]
                         )
@@ -644,6 +646,8 @@ class Table(QueryExpression):
                 logger.warn("Nothing to delete.")
             if transaction:
                 self.connection.cancel_transaction()
+        elif not transaction:
+            logger.info("Delete completed")
         else:
             if not safemode or user_choice("Commit deletes?", default="no") == "yes":
                 if transaction:
@@ -962,7 +966,8 @@ def lookup_class_name(name, context, depth=3):
     while nodes:
         node = nodes.pop(0)
         for member_name, member in node["context"].items():
-            if not member_name.startswith("_"):  # skip IPython's implicit variables
+            # skip IPython's implicit variables
+            if not member_name.startswith("_"):
                 if inspect.isclass(member) and issubclass(member, Table):
                     if member.full_table_name == name:  # found it!
                         return ".".join([node["context_name"], member_name]).lstrip(".")

datajoint/user_tables.py

@@ -2,10 +2,12 @@
 Hosts the table tiers, user tables should be derived from.
 """
 
-from .table import Table
+import re
+
 from .autopopulate import AutoPopulate
-from .utils import from_camel_case, ClassProperty
 from .errors import DataJointError
+from .table import Table
+from .utils import ClassProperty, from_camel_case
 
 _base_regexp = r"[a-z][a-z0-9]*(_[a-z][a-z0-9]*)*"
 
@@ -242,3 +244,29 @@ class Part(UserTable):
     def alter(self, prompt=True, context=None):
         # without context, use declaration context which maps master keyword to master table
         super().alter(prompt=prompt, context=context or self.declaration_context)
+
+
+user_table_classes = (Manual, Lookup, Computed, Imported, Part)
+
+
+class _AliasNode:
+    """
+    special class to indicate aliased foreign keys
+    """
+
+    pass
+
+
+def _get_tier(table_name):
+    """given the table name, return the user table class."""
+    if not table_name.startswith("`"):
+        return _AliasNode
+    else:
+        try:
+            return next(
+                tier
+                for tier in user_table_classes
+                if re.fullmatch(tier.tier_regexp, table_name.split("`")[-2])
+            )
+        except StopIteration:
+            return None

datajoint/utils.py

@@ -1,8 +1,9 @@
 """General-purpose utilities"""
 
 import re
-from pathlib import Path
 import shutil
+from pathlib import Path
+
 from .errors import DataJointError
 
 

datajoint/version.py

@@ -1,3 +1,6 @@
-__version__ = "0.14.2"
+# version bump auto managed by Github Actions:
+# label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit)
+# manually set this version will be eventually overwritten by the above actions
+__version__ = "0.14.4"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters