datajoint 0.14.0__tar.gz → 0.14.2__tar.gz

{datajoint-0.14.0/datajoint.egg-info → datajoint-0.14.2}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: datajoint
-Version: 0.14.0
+Version: 0.14.2
 Summary: A relational data pipeline framework.
 Home-page: https://datajoint.com
 Author: DataJoint Contributors
 Author-email: support@datajoint.com
 License: GNU LGPL
 Keywords: database,data pipelines,scientific computing,automated research workflows
-Requires-Python: ~=3.7
+Requires-Python: ~=3.8
 License-File: LICENSE.txt
 
 A relational data framework for scientific data pipelines with MySQL backend.

datajoint-0.14.2/README.md

@@ -0,0 +1,50 @@
+[![DOI](https://zenodo.org/badge/16774/datajoint/datajoint-python.svg)](https://zenodo.org/badge/latestdoi/16774/datajoint/datajoint-python)
+[![Coverage Status](https://coveralls.io/repos/datajoint/datajoint-python/badge.svg?branch=master&service=github)](https://coveralls.io/github/datajoint/datajoint-python?branch=master)
+[![PyPI version](https://badge.fury.io/py/datajoint.svg)](http://badge.fury.io/py/datajoint)
+[![Slack](https://img.shields.io/badge/slack-chat-green.svg)](https://datajoint.slack.com/)
+
+# Welcome to DataJoint for Python!
+
+DataJoint for Python is a framework for scientific workflow management based on 
+relational principles. DataJoint is built on the foundation of the relational data 
+model and prescribes a consistent method for organizing, populating, computing, and 
+querying data.
+
+DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at 
+Baylor College of Medicine for the distributed processing and management of large 
+volumes of data streaming from regular experiments. Starting in 2011, DataJoint has 
+been available as an open-source project adopted by other labs and improved through 
+contributions from several developers.
+Presently, the primary developer of DataJoint open-source software is the company 
+DataJoint (https://datajoint.com).
+
+## Data Pipeline Example
+
+![pipeline](https://raw.githubusercontent.com/datajoint/datajoint-python/master/images/pipeline.png)
+
+[Yatsenko et al., bioRxiv 2021](https://doi.org/10.1101/2021.03.30.437358)
+
+## Getting Started
+
+- Install with Conda
+
+     ```bash
+     conda install -c conda-forge datajoint
+     ```
+
+- Install with pip
+
+     ```bash
+     pip install datajoint
+     ```
+
+- [Documentation & Tutorials](https://datajoint.com/docs/core/datajoint-python/)
+
+- [Interactive Tutorials](https://github.com/datajoint/datajoint-tutorials) on GitHub Codespaces
+
+- [DataJoint Elements](https://datajoint.com/docs/elements/) - Catalog of example pipelines for neuroscience experiments
+
+- Contribute
+  - [Development Environment](https://datajoint.com/docs/core/datajoint-python/latest/develop/)
+
+  - [Guidelines](https://datajoint.com/docs/about/contribute/)

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/__init__.py

@@ -1,5 +1,5 @@
 """
-DataJoint for Python is a framework for building data piplines using MySQL databases
+DataJoint for Python is a framework for building data pipelines using MySQL databases
 to represent pipeline structure and bulk storage systems for large objects.
 DataJoint is built on the foundation of the relational data model and prescribes a
 consistent method for organizing, populating, and querying data.

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/admin.py

@@ -1,5 +1,6 @@
 import pymysql
 from getpass import getpass
+from packaging import version
 from .connection import conn
 from .settings import config
 from .utils import user_choice
@@ -8,17 +9,22 @@ import logging
 logger = logging.getLogger(__name__.split(".")[0])
 
 
-def set_password(
-    new_password=None, connection=None, update_config=None
-):  # pragma: no cover
+def set_password(new_password=None, connection=None, update_config=None):
     connection = conn() if connection is None else connection
     if new_password is None:
         new_password = getpass("New password: ")
         confirm_password = getpass("Confirm password: ")
         if new_password != confirm_password:
-            logger.warn("Failed to confirm the password! Aborting password change.")
+            logger.warning("Failed to confirm the password! Aborting password change.")
             return
-    connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
+
+    if version.parse(
+        connection.query("select @@version;").fetchone()[0]
+    ) >= version.parse("5.7"):
+        # SET PASSWORD is deprecated as of MySQL 5.7 and removed in 8+
+        connection.query("ALTER USER user() IDENTIFIED BY '%s';" % new_password)
+    else:
+        connection.query("SET PASSWORD = PASSWORD('%s')" % new_password)
     logger.info("Password updated.")
 
     if update_config or (
@@ -28,7 +34,7 @@ def set_password(
         config.save_local(verbose=True)
 
 
-def kill(restriction=None, connection=None, order_by=None):  # pragma: no cover
+def kill(restriction=None, connection=None, order_by=None):
     """
     view and kill database connections.
 

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/autopopulate.py

@@ -1,4 +1,5 @@
 """This module defines class dj.AutoPopulate"""
+
 import logging
 import datetime
 import traceback
@@ -118,7 +119,7 @@ class AutoPopulate:
 
     def _jobs_to_do(self, restrictions):
         """
-        :return: the query yeilding the keys to be computed (derived from self.key_source)
+        :return: the query yielding the keys to be computed (derived from self.key_source)
         """
         if self.restriction:
             raise DataJointError(
@@ -180,6 +181,9 @@ class AutoPopulate:
             to be passed down to each ``make()`` call. Computation arguments should be
             specified within the pipeline e.g. using a `dj.Lookup` table.
         :type make_kwargs: dict, optional
+        :return: a dict with two keys
+            "success_count": the count of successful ``make()`` calls in this ``populate()`` call
+            "error_list": the error list that is filled if `suppress_errors` is True
         """
         if self.connection.in_transaction:
             raise DataJointError("Populate cannot be called during a transaction.")
@@ -204,12 +208,12 @@ class AutoPopulate:
 
         keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit)
 
-        # exclude "error" or "ignore" jobs
+        # exclude "error", "ignore" or "reserved" jobs
         if reserve_jobs:
             exclude_key_hashes = (
                 jobs
                 & {"table_name": self.target.table_name}
-                & 'status in ("error", "ignore")'
+                & 'status in ("error", "ignore", "reserved")'
             ).fetch("key_hash")
             keys = [key for key in keys if key_hash(key) not in exclude_key_hashes]
 
@@ -222,49 +226,62 @@ class AutoPopulate:
 
         keys = keys[:max_calls]
         nkeys = len(keys)
-        if not nkeys:
-            return
-
-        processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
 
         error_list = []
-        populate_kwargs = dict(
-            suppress_errors=suppress_errors,
-            return_exception_objects=return_exception_objects,
-            make_kwargs=make_kwargs,
-        )
+        success_list = []
 
-        if processes == 1:
-            for key in (
-                tqdm(keys, desc=self.__class__.__name__) if display_progress else keys
-            ):
-                error = self._populate1(key, jobs, **populate_kwargs)
-                if error is not None:
-                    error_list.append(error)
-        else:
-            # spawn multiple processes
-            self.connection.close()  # disconnect parent process from MySQL server
-            del self.connection._conn.ctx  # SSLContext is not pickleable
-            with mp.Pool(
-                processes, _initialize_populate, (self, jobs, populate_kwargs)
-            ) as pool, (
-                tqdm(desc="Processes: ", total=nkeys)
-                if display_progress
-                else contextlib.nullcontext()
-            ) as progress_bar:
-                for error in pool.imap(_call_populate1, keys, chunksize=1):
-                    if error is not None:
-                        error_list.append(error)
-                    if display_progress:
-                        progress_bar.update()
-            self.connection.connect()  # reconnect parent process to MySQL server
+        if nkeys:
+            processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
+
+            populate_kwargs = dict(
+                suppress_errors=suppress_errors,
+                return_exception_objects=return_exception_objects,
+                make_kwargs=make_kwargs,
+            )
+
+            if processes == 1:
+                for key in (
+                    tqdm(keys, desc=self.__class__.__name__)
+                    if display_progress
+                    else keys
+                ):
+                    status = self._populate1(key, jobs, **populate_kwargs)
+                    if status is True:
+                        success_list.append(1)
+                    elif isinstance(status, tuple):
+                        error_list.append(status)
+                    else:
+                        assert status is False
+            else:
+                # spawn multiple processes
+                self.connection.close()  # disconnect parent process from MySQL server
+                del self.connection._conn.ctx  # SSLContext is not pickleable
+                with mp.Pool(
+                    processes, _initialize_populate, (self, jobs, populate_kwargs)
+                ) as pool, (
+                    tqdm(desc="Processes: ", total=nkeys)
+                    if display_progress
+                    else contextlib.nullcontext()
+                ) as progress_bar:
+                    for status in pool.imap(_call_populate1, keys, chunksize=1):
+                        if status is True:
+                            success_list.append(1)
+                        elif isinstance(status, tuple):
+                            error_list.append(status)
+                        else:
+                            assert status is False
+                        if display_progress:
+                            progress_bar.update()
+                self.connection.connect()  # reconnect parent process to MySQL server
 
         # restore original signal handler:
         if reserve_jobs:
             signal.signal(signal.SIGTERM, old_handler)
 
-        if suppress_errors:
-            return error_list
+        return {
+            "success_count": sum(success_list),
+            "error_list": error_list,
+        }
 
     def _populate1(
         self, key, jobs, suppress_errors, return_exception_objects, make_kwargs=None
@@ -275,55 +292,60 @@ class AutoPopulate:
         :param key: dict specifying job to populate
         :param suppress_errors: bool if errors should be suppressed and returned
         :param return_exception_objects: if True, errors must be returned as objects
-        :return: (key, error) when suppress_errors=True, otherwise None
+        :return: (key, error) when suppress_errors=True,
+            True if successfully invoke one `make()` call, otherwise False
         """
         make = self._make_tuples if hasattr(self, "_make_tuples") else self.make
 
-        if jobs is None or jobs.reserve(self.target.table_name, self._job_key(key)):
-            self.connection.start_transaction()
-            if key in self.target:  # already populated
+        if jobs is not None and not jobs.reserve(
+            self.target.table_name, self._job_key(key)
+        ):
+            return False
+
+        self.connection.start_transaction()
+        if key in self.target:  # already populated
+            self.connection.cancel_transaction()
+            if jobs is not None:
+                jobs.complete(self.target.table_name, self._job_key(key))
+            return False
+
+        logger.debug(f"Making {key} -> {self.target.full_table_name}")
+        self.__class__._allow_insert = True
+        try:
+            make(dict(key), **(make_kwargs or {}))
+        except (KeyboardInterrupt, SystemExit, Exception) as error:
+            try:
                 self.connection.cancel_transaction()
-                if jobs is not None:
-                    jobs.complete(self.target.table_name, self._job_key(key))
+            except LostConnectionError:
+                pass
+            error_message = "{exception}{msg}".format(
+                exception=error.__class__.__name__,
+                msg=": " + str(error) if str(error) else "",
+            )
+            logger.debug(
+                f"Error making {key} -> {self.target.full_table_name} - {error_message}"
+            )
+            if jobs is not None:
+                # show error name and error message (if any)
+                jobs.error(
+                    self.target.table_name,
+                    self._job_key(key),
+                    error_message=error_message,
+                    error_stack=traceback.format_exc(),
+                )
+            if not suppress_errors or isinstance(error, SystemExit):
+                raise
             else:
-                logger.debug(f"Making {key} -> {self.target.full_table_name}")
-                self.__class__._allow_insert = True
-                try:
-                    make(dict(key), **(make_kwargs or {}))
-                except (KeyboardInterrupt, SystemExit, Exception) as error:
-                    try:
-                        self.connection.cancel_transaction()
-                    except LostConnectionError:
-                        pass
-                    error_message = "{exception}{msg}".format(
-                        exception=error.__class__.__name__,
-                        msg=": " + str(error) if str(error) else "",
-                    )
-                    logger.debug(
-                        f"Error making {key} -> {self.target.full_table_name} - {error_message}"
-                    )
-                    if jobs is not None:
-                        # show error name and error message (if any)
-                        jobs.error(
-                            self.target.table_name,
-                            self._job_key(key),
-                            error_message=error_message,
-                            error_stack=traceback.format_exc(),
-                        )
-                    if not suppress_errors or isinstance(error, SystemExit):
-                        raise
-                    else:
-                        logger.error(error)
-                        return key, error if return_exception_objects else error_message
-                else:
-                    self.connection.commit_transaction()
-                    logger.debug(
-                        f"Success making {key} -> {self.target.full_table_name}"
-                    )
-                    if jobs is not None:
-                        jobs.complete(self.target.table_name, self._job_key(key))
-                finally:
-                    self.__class__._allow_insert = False
+                logger.error(error)
+                return key, error if return_exception_objects else error_message
+        else:
+            self.connection.commit_transaction()
+            logger.debug(f"Success making {key} -> {self.target.full_table_name}")
+            if jobs is not None:
+                jobs.complete(self.target.table_name, self._job_key(key))
+            return True
+        finally:
+            self.__class__._allow_insert = False
 
     def progress(self, *restrictions, display=False):
         """

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/blob.py

@@ -322,9 +322,11 @@ class Blob:
             + "\0".join(array.dtype.names).encode()  # number of fields
             + b"\0"
             + b"".join(  # field names
-                self.pack_recarray(array[f])
-                if array[f].dtype.fields
-                else self.pack_array(array[f])
+                (
+                    self.pack_recarray(array[f])
+                    if array[f].dtype.fields
+                    else self.pack_array(array[f])
+                )
                 for f in array.dtype.names
             )
         )
@@ -449,7 +451,7 @@ class Blob:
         )
 
     def read_struct(self):
-        """deserialize matlab stuct"""
+        """deserialize matlab struct"""
         n_dims = self.read_value()
         shape = self.read_value(count=n_dims)
         n_elem = np.prod(shape, dtype=int)

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/connection.py

@@ -2,6 +2,7 @@
 This module contains the Connection class that manages the connection to the database, and
 the ``conn`` function that provides access to a persistent connection in datajoint.
 """
+
 import warnings
 from contextlib import contextmanager
 import pymysql as client
@@ -79,6 +80,8 @@ def translate_query_error(client_error, query):
     # Integrity errors
     if err == 1062:
         return errors.DuplicateError(*args)
+    if err == 1217:  # MySQL 8 error code
+        return errors.IntegrityError(*args)
     if err == 1451:
         return errors.IntegrityError(*args)
     if err == 1452:
@@ -113,16 +116,16 @@ def conn(
     :param init_fun: initialization function
     :param reset: whether the connection should be reset or not
     :param use_tls: TLS encryption option. Valid options are: True (required), False
-        (required no TLS), None (TLS prefered, default), dict (Manually specify values per
+        (required no TLS), None (TLS preferred, default), dict (Manually specify values per
         https://dev.mysql.com/doc/refman/5.7/en/connection-options.html#encrypted-connection-options).
     """
     if not hasattr(conn, "connection") or reset:
         host = host if host is not None else config["database.host"]
         user = user if user is not None else config["database.user"]
         password = password if password is not None else config["database.password"]
-        if user is None:  # pragma: no cover
+        if user is None:
             user = input("Please enter DataJoint username: ")
-        if password is None:  # pragma: no cover
+        if password is None:
             password = getpass(prompt="Please enter DataJoint password: ")
         init_fun = (
             init_fun if init_fun is not None else config["connection.init_function"]

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/declare.py

@@ -2,6 +2,7 @@
 This module hosts functions to convert DataJoint table definitions into mysql table definitions, and to
 declare the corresponding mysql tables.
 """
+
 import re
 import pyparsing as pp
 import logging
@@ -382,9 +383,7 @@ def _make_attribute_alter(new, old, primary_key):
                         command=(
                             "ADD"
                             if (old_name or new_name) not in old_names
-                            else "MODIFY"
-                            if not old_name
-                            else "CHANGE `%s`" % old_name
+                            else "MODIFY" if not old_name else "CHANGE `%s`" % old_name
                         ),
                         new_def=new_def,
                         after="" if after is None else "AFTER `%s`" % after,

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/dependencies.py

@@ -127,7 +127,7 @@ class Dependencies(nx.DiGraph):
                 self.add_edge(fk["referenced_table"], alias_node, **props)
                 self.add_edge(alias_node, fk["referencing_table"], **props)
 
-        if not nx.is_directed_acyclic_graph(self):  # pragma: no cover
+        if not nx.is_directed_acyclic_graph(self):
             raise DataJointError("DataJoint can only work with acyclic dependencies")
         self._loaded = True
 

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/diagram.py

@@ -385,11 +385,15 @@ else:
                     assert issubclass(cls, Table)
                     description = cls().describe(context=self.context).split("\n")
                     description = (
-                        "-" * 30
-                        if q.startswith("---")
-                        else q.replace("->", "→")
-                        if "->" in q
-                        else q.split(":")[0]
+                        (
+                            "-" * 30
+                            if q.startswith("---")
+                            else (
+                                q.replace("->", "→")
+                                if "->" in q
+                                else q.split(":")[0]
+                            )
+                        )
                         for q in description
                         if not q.startswith("#")
                     )

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/expression.py

@@ -100,9 +100,11 @@ class QueryExpression:
 
     def from_clause(self):
         support = (
-            "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count)
-            if isinstance(src, QueryExpression)
-            else src
+            (
+                "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count)
+                if isinstance(src, QueryExpression)
+                else src
+            )
             for src in self.support
         )
         clause = next(support)
@@ -704,14 +706,16 @@ class Aggregation(QueryExpression):
             fields=fields,
             from_=self.from_clause(),
             where=self.where_clause(),
-            group_by=""
-            if not self.primary_key
-            else (
-                " GROUP BY `%s`" % "`,`".join(self._grouping_attributes)
-                + (
-                    ""
-                    if not self.restriction
-                    else " HAVING (%s)" % ")AND(".join(self.restriction)
+            group_by=(
+                ""
+                if not self.primary_key
+                else (
+                    " GROUP BY `%s`" % "`,`".join(self._grouping_attributes)
+                    + (
+                        ""
+                        if not self.restriction
+                        else " HAVING (%s)" % ")AND(".join(self.restriction)
+                    )
                 )
             ),
         )
@@ -773,12 +777,16 @@ class Union(QueryExpression):
             # no secondary attributes: use UNION DISTINCT
             fields = arg1.primary_key
             return "SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}`".format(
-                sql1=arg1.make_sql()
-                if isinstance(arg1, Union)
-                else arg1.make_sql(fields),
-                sql2=arg2.make_sql()
-                if isinstance(arg2, Union)
-                else arg2.make_sql(fields),
+                sql1=(
+                    arg1.make_sql()
+                    if isinstance(arg1, Union)
+                    else arg1.make_sql(fields)
+                ),
+                sql2=(
+                    arg2.make_sql()
+                    if isinstance(arg2, Union)
+                    else arg2.make_sql(fields)
+                ),
                 alias=next(self.__count),
             )
         # with secondary attributes, use union of left join with antijoin
@@ -839,7 +847,7 @@ class U:
     >>> dj.U().aggr(expr, n='count(*)')
 
     The following expressions both yield one element containing the number `n` of distinct values of attribute `attr` in
-    query expressio `expr`.
+    query expression `expr`.
 
     >>> dj.U().aggr(expr, n='count(distinct attr)')
     >>> dj.U().aggr(dj.U('attr').aggr(expr), 'n=count(*)')

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/fetch.py

@@ -244,13 +244,15 @@ class Fetch:
                 ]
             else:
                 return_values = [
-                    list(
-                        (to_dicts if as_dict else lambda x: x)(
-                            ret[self._expression.primary_key]
+                    (
+                        list(
+                            (to_dicts if as_dict else lambda x: x)(
+                                ret[self._expression.primary_key]
+                            )
                         )
+                        if is_key(attribute)
+                        else ret[attribute]
                     )
-                    if is_key(attribute)
-                    else ret[attribute]
                     for attribute in attrs
                 ]
                 ret = return_values[0] if len(attrs) == 1 else return_values
@@ -272,12 +274,14 @@ class Fetch:
                     else np.dtype(
                         [
                             (
-                                name,
-                                type(value),
-                            )  # use the first element to determine blob type
-                            if heading[name].is_blob
-                            and isinstance(value, numbers.Number)
-                            else (name, heading.as_dtype[name])
+                                (
+                                    name,
+                                    type(value),
+                                )  # use the first element to determine blob type
+                                if heading[name].is_blob
+                                and isinstance(value, numbers.Number)
+                                else (name, heading.as_dtype[name])
+                            )
                             for value, name in zip(ret[0], heading.as_dtype.names)
                         ]
                     )
@@ -353,9 +357,11 @@ class Fetch1:
                     "fetch1 should only return one tuple. %d tuples found" % len(result)
                 )
             return_values = tuple(
-                next(to_dicts(result[self._expression.primary_key]))
-                if is_key(attribute)
-                else result[attribute][0]
+                (
+                    next(to_dicts(result[self._expression.primary_key]))
+                    if is_key(attribute)
+                    else result[attribute][0]
+                )
                 for attribute in attrs
             )
             ret = return_values[0] if len(attrs) == 1 else return_values

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/heading.py

@@ -193,10 +193,12 @@ class Heading:
         represent heading as the SQL SELECT clause.
         """
         return ",".join(
-            "`%s`" % name
-            if self.attributes[name].attribute_expression is None
-            else self.attributes[name].attribute_expression
-            + (" as `%s`" % name if include_aliases else "")
+            (
+                "`%s`" % name
+                if self.attributes[name].attribute_expression is None
+                else self.attributes[name].attribute_expression
+                + (" as `%s`" % name if include_aliases else "")
+            )
             for name in fields
         )
 
@@ -371,9 +373,11 @@ class Heading:
                     is_blob=category in ("INTERNAL_BLOB", "EXTERNAL_BLOB"),
                     uuid=category == "UUID",
                     is_external=category in EXTERNAL_TYPES,
-                    store=attr["type"].split("@")[1]
-                    if category in EXTERNAL_TYPES
-                    else None,
+                    store=(
+                        attr["type"].split("@")[1]
+                        if category in EXTERNAL_TYPES
+                        else None
+                    ),
                 )
 
             if attr["in_key"] and any(

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/preview.py

@@ -68,9 +68,11 @@ def repr_html(query_expression):
         }
         .Table tr:nth-child(odd){
             background: #ffffff;
+            color: #000000;
         }
         .Table tr:nth-child(even){
             background: #f3f1ff;
+            color: #000000;
         }
         /* Tooltip container */
         .djtooltip {
@@ -124,9 +126,9 @@ def repr_html(query_expression):
             head_template.format(
                 column=c,
                 comment=heading.attributes[c].comment,
-                primary="primary"
-                if c in query_expression.primary_key
-                else "nonprimary",
+                primary=(
+                    "primary" if c in query_expression.primary_key else "nonprimary"
+                ),
             )
             for c in heading.names
         ),
@@ -143,7 +145,9 @@ def repr_html(query_expression):
                 for tup in tuples
             ]
         ),
-        count=("<p>Total: %d</p>" % len(rel))
-        if config["display.show_tuple_count"]
-        else "",
+        count=(
+            ("<p>Total: %d</p>" % len(rel))
+            if config["display.show_tuple_count"]
+            else ""
+        ),
     )

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/s3.py

@@ -1,6 +1,7 @@
 """
 AWS S3 operations
 """
+
 from io import BytesIO
 import minio  # https://docs.minio.io/docs/python-client-api-reference
 import urllib3
@@ -68,7 +69,9 @@ class Folder:
     def get(self, name):
         logger.debug("get: {}:{}".format(self.bucket, name))
         try:
-            return self.client.get_object(self.bucket, str(name)).data
+            with self.client.get_object(self.bucket, str(name)) as result:
+                data = [d for d in result.stream()]
+            return b"".join(data)
         except minio.error.S3Error as e:
             if e.code == "NoSuchKey":
                 raise errors.MissingExternalFile("Missing s3 key %s" % name)

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/schemas.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__.split(".")[0])
 
 def ordered_dir(class_):
     """
-    List (most) attributes of the class including inherited ones, similar to `dir` build-in function,
+    List (most) attributes of the class including inherited ones, similar to `dir` built-in function,
     but respects order of attribute declaration as much as possible.
 
     :param class_: class to list members for

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/settings.py

@@ -1,6 +1,7 @@
 """
 Settings for DataJoint.
 """
+
 from contextlib import contextmanager
 import json
 import os

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/table.py

@@ -15,7 +15,7 @@ from .declare import declare, alter
 from .condition import make_condition
 from .expression import QueryExpression
 from . import blob
-from .utils import user_choice, get_master
+from .utils import user_choice, get_master, is_camel_case
 from .heading import Heading
 from .errors import (
     DuplicateError,
@@ -75,6 +75,10 @@ class Table(QueryExpression):
     def table_name(self):
         return self._table_name
 
+    @property
+    def class_name(self):
+        return self.__class__.__name__
+
     @property
     def definition(self):
         raise NotImplementedError(
@@ -93,6 +97,14 @@ class Table(QueryExpression):
                 "Cannot declare new tables inside a transaction, "
                 "e.g. from inside a populate/make call"
             )
+        # Enforce strict CamelCase #1150
+        if not is_camel_case(self.class_name):
+            raise DataJointError(
+                "Table class name `{name}` is invalid. Please use CamelCase. ".format(
+                    name=self.class_name
+                )
+                + "Classes defining tables should be formatted in strict CamelCase."
+            )
         sql, external_stores = declare(self.full_table_name, self.definition, context)
         sql = sql.format(database=self.database)
         try:
@@ -230,7 +242,7 @@ class Table(QueryExpression):
 
     def parts(self, as_objects=False):
         """
-        return part tables either as entries in a dict with foreign key informaiton or a list of objects
+        return part tables either as entries in a dict with foreign key information or a list of objects
 
         :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects.
         """
@@ -474,6 +486,7 @@ class Table(QueryExpression):
         transaction: bool = True,
         safemode: Union[bool, None] = None,
         force_parts: bool = False,
+        force_masters: bool = False,
     ) -> int:
         """
         Deletes the contents of the table and its dependent tables, recursively.
@@ -485,6 +498,8 @@ class Table(QueryExpression):
             safemode: If `True`, prohibit nested transactions and prompt to confirm. Default
                 is `dj.config['safemode']`.
             force_parts: Delete from parts even when not deleting from their masters.
+            force_masters: If `True`, include part/master pairs in the cascade.
+                Default is `False`.
 
         Returns:
             Number of deleted rows (excluding those from dependent tables).
@@ -495,6 +510,7 @@ class Table(QueryExpression):
             DataJointError: Deleting a part table before its master.
         """
         deleted = set()
+        visited_masters = set()
 
         def cascade(table):
             """service function to perform cascading deletes recursively."""
@@ -547,13 +563,34 @@ class Table(QueryExpression):
                         and match["fk_attrs"] == match["pk_attrs"]
                     ):
                         child._restriction = table._restriction
+                        child._restriction_attributes = table.restriction_attributes
                     elif match["fk_attrs"] != match["pk_attrs"]:
                         child &= table.proj(
                             **dict(zip(match["fk_attrs"], match["pk_attrs"]))
                         )
                     else:
                         child &= table.proj()
-                    cascade(child)
+
+                    master_name = get_master(child.full_table_name)
+                    if (
+                        force_masters
+                        and master_name
+                        and master_name != table.full_table_name
+                        and master_name not in visited_masters
+                    ):
+                        master = FreeTable(table.connection, master_name)
+                        master._restriction_attributes = set()
+                        master._restriction = [
+                            make_condition(  # &= may cause in target tables in subquery
+                                master,
+                                (master.proj() & child.proj()).fetch(),
+                                master._restriction_attributes,
+                            )
+                        ]
+                        visited_masters.add(master_name)
+                        cascade(master)
+                    else:
+                        cascade(child)
                 else:
                     deleted.add(table.full_table_name)
                     logger.info(
@@ -758,9 +795,11 @@ class Table(QueryExpression):
             if do_include:
                 attributes_declared.add(attr.name)
                 definition += "%-20s : %-28s %s\n" % (
-                    attr.name
-                    if attr.default is None
-                    else "%s=%s" % (attr.name, attr.default),
+                    (
+                        attr.name
+                        if attr.default is None
+                        else "%s=%s" % (attr.name, attr.default)
+                    ),
                     "%s%s"
                     % (attr.type, " auto_increment" if attr.autoincrement else ""),
                     "# " + attr.comment if attr.comment else "",

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/user_tables.py

@@ -238,3 +238,7 @@ class Part(UserTable):
             raise DataJointError(
                 "Cannot drop a Part directly.  Delete from master instead"
             )
+
+    def alter(self, prompt=True, context=None):
+        # without context, use declaration context which maps master keyword to master table
+        super().alter(prompt=prompt, context=context or self.declaration_context)

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/utils.py

@@ -53,6 +53,19 @@ def get_master(full_table_name: str) -> str:
     return match["master"] + "`" if match else ""
 
 
+def is_camel_case(s):
+    """
+    Check if a string is in CamelCase notation.
+
+    :param s: string to check
+    :returns: True if the string is in CamelCase notation, False otherwise
+    Example:
+    >>> is_camel_case("TableName")  # returns True
+    >>> is_camel_case("table_name")  # returns False
+    """
+    return bool(re.match(r"^[A-Z][A-Za-z0-9]*$", s))
+
+
 def to_camel_case(s):
     """
     Convert names with under score (_) separation into camel case names.
@@ -82,7 +95,7 @@ def from_camel_case(s):
     def convert(match):
         return ("_" if match.groups()[0] else "") + match.group(0).lower()
 
-    if not re.match(r"[A-Z][a-zA-Z0-9]*", s):
+    if not is_camel_case(s):
         raise DataJointError(
             "ClassName must be alphanumeric in CamelCase, begin with a capital letter"
         )

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.14.0"
+__version__ = "0.14.2"
 
 assert len(__version__) <= 10  # The log table limits version to the 10 characters

{datajoint-0.14.0 → datajoint-0.14.2/datajoint.egg-info}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: datajoint
-Version: 0.14.0
+Version: 0.14.2
 Summary: A relational data pipeline framework.
 Home-page: https://datajoint.com
 Author: DataJoint Contributors
 Author-email: support@datajoint.com
 License: GNU LGPL
 Keywords: database,data pipelines,scientific computing,automated research workflows
-Requires-Python: ~=3.7
+Requires-Python: ~=3.8
 License-File: LICENSE.txt
 
 A relational data framework for scientific data pipelines with MySQL backend.

{datajoint-0.14.0 → datajoint-0.14.2}/datajoint.egg-info/requires.txt

@@ -4,7 +4,7 @@ pyparsing
 ipython
 pandas
 tqdm
-networkx<=2.6.3
+networkx
 pydot
 minio>=7.0.0
 matplotlib

{datajoint-0.14.0 → datajoint-0.14.2}/requirements.txt

@@ -4,7 +4,7 @@ pyparsing
 ipython
 pandas
 tqdm
-networkx<=2.6.3  # until py3.8 is our minimum version
+networkx
 pydot
 minio>=7.0.0
 matplotlib

{datajoint-0.14.0 → datajoint-0.14.2}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 from os import path
 import sys
 
-min_py_version = (3, 7)
+min_py_version = (3, 8)
 
 if sys.version_info < min_py_version:
     sys.exit(

datajoint-0.14.0/README.md

@@ -1,33 +0,0 @@
-[![DOI](https://zenodo.org/badge/16774/datajoint/datajoint-python.svg)](https://zenodo.org/badge/latestdoi/16774/datajoint/datajoint-python)
-[![Build Status](https://travis-ci.org/datajoint/datajoint-python.svg?branch=master)](https://travis-ci.org/datajoint/datajoint-python)
-[![Coverage Status](https://coveralls.io/repos/datajoint/datajoint-python/badge.svg?branch=master&service=github)](https://coveralls.io/github/datajoint/datajoint-python?branch=master)
-[![PyPI version](https://badge.fury.io/py/datajoint.svg)](http://badge.fury.io/py/datajoint)
-[![Requirements Status](https://requires.io/github/datajoint/datajoint-python/requirements.svg?branch=master)](https://requires.io/github/datajoint/datajoint-python/requirements/?branch=master)
-[![Slack](https://img.shields.io/badge/slack-chat-green.svg)](https://datajoint.slack.com/)
-
-# Welcome to DataJoint for Python!
-
-DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
-
-DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at Baylor College of Medicine for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
-Presently, the primary developer of DataJoint open-source software is the company DataJoint (https://datajoint.com).
-
-- [Getting Started](https://datajoint.com/docs/core/datajoint-python/latest/getting-started/)
-- [DataJoint Elements](https://datajoint.com/docs/elements/) - Catalog of example pipelines
-- [DataJoint CodeBook](https://codebook.datajoint.io) - Interactive online tutorials
-- Contribute
-
-  - [Development Environment](https://datajoint.com/docs/core/datajoint-python/latest/develop/)
-  - [Guidelines](https://datajoint.com/docs/community/contribute/)
-
-- Legacy Resources (To be replaced by above)
-  - [Documentation](https://docs.datajoint.org)
-  - [Tutorials](https://tutorials.datajoint.org)
-
-## Citation
-
-- If your work uses DataJoint for Python, please cite the following Research Resource Identifier (RRID) and manuscript.
-
-- DataJoint ([RRID:SCR_014543](https://scicrunch.org/resolver/SCR_014543)) - DataJoint for Python (version `<Enter version number>`)
-
-- Yatsenko D, Reimer J, Ecker AS, Walker EY, Sinz F, Berens P, Hoenselaar A, Cotton RJ, Siapas AS, Tolias AS. DataJoint: managing big scientific data using MATLAB or Python. bioRxiv. 2015 Jan 1:031658. doi: https://doi.org/10.1101/031658