datajoint 0.14.1__py3-none-any.whl → 0.14.3__py3-none-any.whl

datajoint/__init__.py

@@ -1,5 +1,5 @@
 """
-DataJoint for Python is a framework for building data piplines using MySQL databases
+DataJoint for Python is a framework for building data pipelines using MySQL databases
 to represent pipeline structure and bulk storage systems for large objects.
 DataJoint is built on the foundation of the relational data model and prescribes a
 consistent method for organizing, populating, and querying data.
@@ -37,6 +37,7 @@ __all__ = [
     "Part",
     "Not",
     "AndList",
+    "Top",
     "U",
     "Diagram",
     "Di",
@@ -51,6 +52,7 @@ __all__ = [
     "key",
     "key_hash",
     "logger",
+    "cli",
 ]
 
 from .logging import logger
@@ -61,7 +63,7 @@ from .schemas import Schema
 from .schemas import VirtualModule, list_schemas
 from .table import Table, FreeTable
 from .user_tables import Manual, Lookup, Imported, Computed, Part
-from .expression import Not, AndList, U
+from .expression import Not, AndList, U, Top
 from .diagram import Diagram
 from .admin import set_password, kill
 from .blob import MatCell, MatStruct
@@ -70,6 +72,7 @@ from .hash import key_hash
 from .attribute_adapter import AttributeAdapter
 from . import errors
 from .errors import DataJointError
+from .cli import cli
 
 ERD = Di = Diagram  # Aliases for Diagram
 schema = Schema  # Aliases for Schema

datajoint/admin.py

@@ -1,5 +1,6 @@
 import pymysql
 from getpass import getpass
+from packaging import version
 from .connection import conn
 from .settings import config
 from .utils import user_choice
@@ -14,9 +15,16 @@ def set_password(new_password=None, connection=None, update_config=None):
         new_password = getpass("New password: ")
         confirm_password = getpass("Confirm password: ")
         if new_password != confirm_password:
-            logger.warn("Failed to confirm the password! Aborting password change.")
+            logger.warning("Failed to confirm the password! Aborting password change.")
             return
-    connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
+
+    if version.parse(
+        connection.query("select @@version;").fetchone()[0]
+    ) >= version.parse("5.7"):
+        # SET PASSWORD is deprecated as of MySQL 5.7 and removed in 8+
+        connection.query("ALTER USER user() IDENTIFIED BY '%s';" % new_password)
+    else:
+        connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
     logger.info("Password updated.")
 
     if update_config or (

datajoint/autopopulate.py

@@ -1,4 +1,5 @@
 """This module defines class dj.AutoPopulate"""
+
 import logging
 import datetime
 import traceback
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__.split(".")[0])
 
 def _initialize_populate(table, jobs, populate_kwargs):
     """
-    Initialize the process for mulitprocessing.
+    Initialize the process for multiprocessing.
     Saves the unpickled copy of the table to the current process and reconnects.
     """
     process = mp.current_process()
@@ -118,7 +119,7 @@ class AutoPopulate:
 
     def _jobs_to_do(self, restrictions):
         """
-        :return: the query yeilding the keys to be computed (derived from self.key_source)
+        :return: the query yielding the keys to be computed (derived from self.key_source)
         """
         if self.restriction:
             raise DataJointError(
@@ -152,6 +153,7 @@ class AutoPopulate:
     def populate(
         self,
         *restrictions,
+        keys=None,
         suppress_errors=False,
         return_exception_objects=False,
         reserve_jobs=False,
@@ -168,6 +170,8 @@ class AutoPopulate:
 
         :param restrictions: a list of restrictions each restrict
             (table.key_source - target.proj())
+        :param keys: The list of keys (dicts) to send to self.make().
+            If None (default), then use self.key_source to query they keys.
         :param suppress_errors: if True, do not terminate execution.
         :param return_exception_objects: return error objects instead of just error messages
         :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion
@@ -180,6 +184,9 @@ class AutoPopulate:
             to be passed down to each ``make()`` call. Computation arguments should be
             specified within the pipeline e.g. using a `dj.Lookup` table.
         :type make_kwargs: dict, optional
+        :return: a dict with two keys
+            "success_count": the count of successful ``make()`` calls in this ``populate()`` call
+            "error_list": the error list that is filled if `suppress_errors` is True
         """
         if self.connection.in_transaction:
             raise DataJointError("Populate cannot be called during a transaction.")
@@ -202,14 +209,17 @@ class AutoPopulate:
 
             old_handler = signal.signal(signal.SIGTERM, handler)
 
-        keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
+        if keys is None:
+            keys = (self._jobs_to_do(restrictions) - self.target).fetch(
+                "KEY", limit=limit
+            )
 
-        # exclude "error" or "ignore" jobs
+        # exclude "error", "ignore" or "reserved" jobs
         if reserve_jobs:
             exclude_key_hashes = (
                 jobs
                 & {"table_name": self.target.table_name}
-                & 'status in ("error", "ignore")'
+                & 'status in ("error", "ignore", "reserved")'
             ).fetch("key_hash")
             keys = [key for key in keys if key_hash(key) not in exclude_key_hashes]
 
@@ -222,49 +232,62 @@ class AutoPopulate:
 
         keys = keys[:max_calls]
         nkeys = len(keys)
-        if not nkeys:
-            return
-
-        processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
 
         error_list = []
-        populate_kwargs = dict(
-            suppress_errors=suppress_errors,
-            return_exception_objects=return_exception_objects,
-            make_kwargs=make_kwargs,
-        )
+        success_list = []
 
-        if processes == 1:
-            for key in (
-                tqdm(keys, desc=self.__class__.__name__) if display_progress else keys
-            ):
-                error = self._populate1(key, jobs, **populate_kwargs)
-                if error is not None:
-                    error_list.append(error)
-        else:
-            # spawn multiple processes
-            self.connection.close()  # disconnect parent process from MySQL server
-            del self.connection._conn.ctx  # SSLContext is not pickleable
-            with mp.Pool(
-                processes, _initialize_populate, (self, jobs, populate_kwargs)
-            ) as pool, (
-                tqdm(desc="Processes: ", total=nkeys)
-                if display_progress
-                else contextlib.nullcontext()
-            ) as progress_bar:
-                for error in pool.imap(_call_populate1, keys, chunksize=1):
-                    if error is not None:
-                        error_list.append(error)
-                    if display_progress:
-                        progress_bar.update()
-            self.connection.connect()  # reconnect parent process to MySQL server
+        if nkeys:
+            processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
+
+            populate_kwargs = dict(
+                suppress_errors=suppress_errors,
+                return_exception_objects=return_exception_objects,
+                make_kwargs=make_kwargs,
+            )
+
+            if processes == 1:
+                for key in (
+                    tqdm(keys, desc=self.__class__.__name__)
+                    if display_progress
+                    else keys
+                ):
+                    status = self._populate1(key, jobs, **populate_kwargs)
+                    if status is True:
+                        success_list.append(1)
+                    elif isinstance(status, tuple):
+                        error_list.append(status)
+                    else:
+                        assert status is False
+            else:
+                # spawn multiple processes
+                self.connection.close()  # disconnect parent process from MySQL server
+                del self.connection._conn.ctx  # SSLContext is not pickleable
+                with mp.Pool(
+                    processes, _initialize_populate, (self, jobs, populate_kwargs)
+                ) as pool, (
+                    tqdm(desc="Processes: ", total=nkeys)
+                    if display_progress
+                    else contextlib.nullcontext()
+                ) as progress_bar:
+                    for status in pool.imap(_call_populate1, keys, chunksize=1):
+                        if status is True:
+                            success_list.append(1)
+                        elif isinstance(status, tuple):
+                            error_list.append(status)
+                        else:
+                            assert status is False
+                        if display_progress:
+                            progress_bar.update()
+                self.connection.connect()  # reconnect parent process to MySQL server
 
         # restore original signal handler:
         if reserve_jobs:
             signal.signal(signal.SIGTERM, old_handler)
 
-        if suppress_errors:
-            return error_list
+        return {
+            "success_count": sum(success_list),
+            "error_list": error_list,
+        }
 
     def _populate1(
         self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None
@@ -275,55 +298,61 @@ class AutoPopulate:
         :param key: dict specifying job to populate
         :param suppress_errors: bool if errors should be suppressed and returned
         :param return_exception_objects: if True, errors must be returned as objects
-        :return: (key, error) when suppress_errors=True, otherwise None
+        :return: (key, error) when suppress_errors=True,
+            True if successfully invoke one `make()` call, otherwise False
         """
+        # use the legacy `_make_tuples` callback.
         make = self._make_tuples if hasattr(self, "_make_tuples") else self.make
 
-        if jobs is None or jobs.reserve(self.target.table_name, self._job_key(key)):
-            self.connection.start_transaction()
-            if key in self.target:  # already populated
+        if jobs is not None and not jobs.reserve(
+            self.target.table_name, self._job_key(key)
+        ):
+            return False
+
+        self.connection.start_transaction()
+        if key in self.target:  # already populated
+            self.connection.cancel_transaction()
+            if jobs is not None:
+                jobs.complete(self.target.table_name, self._job_key(key))
+            return False
+
+        logger.debug(f"Making {key} -> {self.target.full_table_name}")
+        self.__class__._allow_insert = True
+        try:
+            make(dict(key), **(make_kwargs or {}))
+        except (KeyboardInterrupt, SystemExit, Exception) as error:
+            try:
                 self.connection.cancel_transaction()
-                if jobs is not None:
-                    jobs.complete(self.target.table_name, self._job_key(key))
+            except LostConnectionError:
+                pass
+            error_message = "{exception}{msg}".format(
+                exception=error.__class__.__name__,
+                msg=": " + str(error) if str(error) else "",
+            )
+            logger.debug(
+                f"Error making {key} -> {self.target.full_table_name} - {error_message}"
+            )
+            if jobs is not None:
+                # show error name and error message (if any)
+                jobs.error(
+                    self.target.table_name,
+                    self._job_key(key),
+                    error_message=error_message,
+                    error_stack=traceback.format_exc(),
+                )
+            if not suppress_errors or isinstance(error, SystemExit):
+                raise
             else:
-                logger.debug(f"Making {key} -> {self.target.full_table_name}")
-                self.__class__._allow_insert = True
-                try:
-                    make(dict(key), **(make_kwargs or {}))
-                except (KeyboardInterrupt, SystemExit, Exception) as error:
-                    try:
-                        self.connection.cancel_transaction()
-                    except LostConnectionError:
-                        pass
-                    error_message = "{exception}{msg}".format(
-                        exception=error.__class__.__name__,
-                        msg=": " + str(error) if str(error) else "",
-                    )
-                    logger.debug(
-                        f"Error making {key} -> {self.target.full_table_name} - {error_message}"
-                    )
-                    if jobs is not None:
-                        # show error name and error message (if any)
-                        jobs.error(
-                            self.target.table_name,
-                            self._job_key(key),
-                            error_message=error_message,
-                            error_stack=traceback.format_exc(),
-                        )
-                    if not suppress_errors or isinstance(error, SystemExit):
-                        raise
-                    else:
-                        logger.error(error)
-                        return key, error if return_exception_objects else error_message
-                else:
-                    self.connection.commit_transaction()
-                    logger.debug(
-                        f"Success making {key} -> {self.target.full_table_name}"
-                    )
-                    if jobs is not None:
-                        jobs.complete(self.target.table_name, self._job_key(key))
-                finally:
-                    self.__class__._allow_insert = False
+                logger.error(error)
+                return key, error if return_exception_objects else error_message
+        else:
+            self.connection.commit_transaction()
+            logger.debug(f"Success making {key} -> {self.target.full_table_name}")
+            if jobs is not None:
+                jobs.complete(self.target.table_name, self._job_key(key))
+            return True
+        finally:
+            self.__class__._allow_insert = False
 
     def progress(self, *restrictions, display=False):
         """

datajoint/blob.py

@@ -322,9 +322,11 @@ class Blob:
             + "\0".join(array.dtype.names).encode()  # number of fields
             + b"\0"
             + b"".join(  # field names
-                self.pack_recarray(array[f])
-                if array[f].dtype.fields
-                else self.pack_array(array[f])
+                (
+                    self.pack_recarray(array[f])
+                    if array[f].dtype.fields
+                    else self.pack_array(array[f])
+                )
                 for f in array.dtype.names
             )
         )
@@ -449,7 +451,7 @@ class Blob:
         )
 
     def read_struct(self):
-        """deserialize matlab stuct"""
+        """deserialize matlab struct"""
         n_dims = self.read_value()
         shape = self.read_value(count=n_dims)
         n_elem = np.prod(shape, dtype=int)

datajoint/cli.py

@@ -0,0 +1,77 @@
+import argparse
+from code import interact
+from collections import ChainMap
+import datajoint as dj
+
+
+def cli(args: list = None):
+    """
+    Console interface for DataJoint Python
+
+    :param args: List of arguments to be passed in, defaults to reading stdin
+    :type args: list, optional
+    """
+    parser = argparse.ArgumentParser(
+        prog="datajoint",
+        description="DataJoint console interface.",
+        conflict_handler="resolve",
+    )
+    parser.add_argument(
+        "-V", "--version", action="version", version=f"{dj.__name__} {dj.__version__}"
+    )
+    parser.add_argument(
+        "-u",
+        "--user",
+        type=str,
+        default=dj.config["database.user"],
+        required=False,
+        help="Datajoint username",
+    )
+    parser.add_argument(
+        "-p",
+        "--password",
+        type=str,
+        default=dj.config["database.password"],
+        required=False,
+        help="Datajoint password",
+    )
+    parser.add_argument(
+        "-h",
+        "--host",
+        type=str,
+        default=dj.config["database.host"],
+        required=False,
+        help="Datajoint host",
+    )
+    parser.add_argument(
+        "-s",
+        "--schemas",
+        nargs="+",
+        type=str,
+        required=False,
+        help="A list of virtual module mappings in `db:schema ...` format",
+    )
+    kwargs = vars(parser.parse_args(args))
+    mods = {}
+    if kwargs["user"]:
+        dj.config["database.user"] = kwargs["user"]
+    if kwargs["password"]:
+        dj.config["database.password"] = kwargs["password"]
+    if kwargs["host"]:
+        dj.config["database.host"] = kwargs["host"]
+    if kwargs["schemas"]:
+        for vm in kwargs["schemas"]:
+            d, m = vm.split(":")
+            mods[m] = dj.create_virtual_module(m, d)
+
+    banner = "dj repl\n"
+    if mods:
+        modstr = "\n".join("  - {}".format(m) for m in mods)
+        banner += "\nschema modules:\n\n" + modstr + "\n"
+    interact(banner, local=dict(ChainMap(mods, locals(), globals())))
+
+    raise SystemExit
+
+
+if __name__ == "__main__":
+    cli()

datajoint/condition.py

@@ -10,6 +10,8 @@ import numpy
 import pandas
 import json
 from .errors import DataJointError
+from typing import Union, List
+from dataclasses import dataclass
 
 JSON_PATTERN = re.compile(
     r"^(?P<attr>\w+)(\.(?P<path>[\w.*\[\]]+))?(:(?P<type>[\w(,\s)]+))?$"
@@ -61,6 +63,35 @@ class AndList(list):
             super().append(restriction)
 
 
+@dataclass
+class Top:
+    """
+    A restriction to the top entities of a query.
+    In SQL, this corresponds to ORDER BY ... LIMIT ... OFFSET
+    """
+
+    limit: Union[int, None] = 1
+    order_by: Union[str, List[str]] = "KEY"
+    offset: int = 0
+
+    def __post_init__(self):
+        self.order_by = self.order_by or ["KEY"]
+        self.offset = self.offset or 0
+
+        if self.limit is not None and not isinstance(self.limit, int):
+            raise TypeError("Top limit must be an integer")
+        if not isinstance(self.order_by, (str, collections.abc.Sequence)) or not all(
+            isinstance(r, str) for r in self.order_by
+        ):
+            raise TypeError("Top order_by attributes must all be strings")
+        if not isinstance(self.offset, int):
+            raise TypeError("The offset argument must be an integer")
+        if self.offset and self.limit is None:
+            self.limit = 999999999999  # arbitrary large number to allow query
+        if isinstance(self.order_by, str):
+            self.order_by = [self.order_by]
+
+
 class Not:
     """invert restriction"""
 

datajoint/connection.py

@@ -2,6 +2,7 @@
 This module contains the Connection class that manages the connection to the database, and
 the ``conn`` function that provides access to a persistent connection in datajoint.
 """
+
 import warnings
 from contextlib import contextmanager
 import pymysql as client
@@ -79,6 +80,8 @@ def translate_query_error(client_error, query):
     # Integrity errors
     if err == 1062:
         return errors.DuplicateError(*args)
+    if err == 1217:  # MySQL 8 error code
+        return errors.IntegrityError(*args)
     if err == 1451:
         return errors.IntegrityError(*args)
     if err == 1452:
@@ -113,7 +116,7 @@ def conn(
     :param init_fun: initialization function
     :param reset: whether the connection should be reset or not
     :param use_tls: TLS encryption option. Valid options are: True (required), False
-        (required no TLS), None (TLS prefered, default), dict (Manually specify values per
+        (required no TLS), None (TLS preferred, default), dict (Manually specify values per
         https://dev.mysql.com/doc/refman/5.7/en/connection-options.html#encrypted-connection-options).
     """
     if not hasattr(conn, "connection") or reset:

datajoint/declare.py

@@ -2,12 +2,15 @@
 This module hosts functions to convert DataJoint table definitions into mysql table definitions, and to
 declare the corresponding mysql tables.
 """
+
 import re
 import pyparsing as pp
 import logging
+from hashlib import sha1
 from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH
 from .attribute_adapter import get_adapter
 from .condition import translate_attribute
+from .settings import config
 
 UUID_DATA_TYPE = "binary(16)"
 MAX_TABLE_NAME_LENGTH = 64
@@ -309,6 +312,19 @@ def declare(full_table_name, definition, context):
         external_stores,
     ) = prepare_declare(definition, context)
 
+    if config.get("add_hidden_timestamp", False):
+        metadata_attr_sql = [
+            "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP"
+        ]
+        attribute_sql.extend(
+            attr.format(
+                full_table_name=sha1(
+                    full_table_name.replace("`", "").encode("utf-8")
+                ).hexdigest()
+            )
+            for attr in metadata_attr_sql
+        )
+
     if not primary_key:
         raise DataJointError("Table must have a primary key")
 
@@ -382,9 +398,7 @@ def _make_attribute_alter(new, old, primary_key):
                         command=(
                             "ADD"
                             if (old_name or new_name) not in old_names
-                            else "MODIFY"
-                            if not old_name
-                            else "CHANGE `%s`" % old_name
+                            else "MODIFY" if not old_name else "CHANGE `%s`" % old_name
                         ),
                         new_def=new_def,
                         after="" if after is None else "AFTER `%s`" % after,
@@ -443,9 +457,11 @@ def compile_index(line, index_sql):
             return f"`{attr}`"
         return f"({attr})"
 
-    match = re.match(
-        r"(?P<unique>unique\s+)?index\s*\(\s*(?P<args>.*)\)", line, re.I
-    ).groupdict()
+    match = re.match(r"(?P<unique>unique\s+)?index\s*\(\s*(?P<args>.*)\)", line, re.I)
+    if match is None:
+        raise DataJointError(f'Table definition syntax error in line "{line}"')
+    match = match.groupdict()
+
     attr_list = re.findall(r"(?:[^,(]|\([^)]*\))+", match["args"])
     index_sql.append(
         "{unique}index ({attrs})".format(