vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl

vortex/layout/monitor.py

@@ -26,24 +26,32 @@ __all__ = []
 
 
 #: Class for possible states of a :class:`InputMonitorEntry` object
-EntryStateTuple = namedtuple('EntryStateTuple',
-                             ['ufo', 'expected', 'available', 'failed'])
+EntryStateTuple = namedtuple(
+    "EntryStateTuple", ["ufo", "expected", "available", "failed"]
+)
 
 #: Predefined :class:`InputMonitorEntry` state values
-EntrySt = EntryStateTuple(ufo='ufo', expected='expected', available='available',
-                          failed='failed')
+EntrySt = EntryStateTuple(
+    ufo="ufo", expected="expected", available="available", failed="failed"
+)
 
 #: Class for possible states of a :class:`_Gang` object
-GangStateTuple = namedtuple('GangStateTuple',
-                            ['ufo', 'collectable', 'pcollectable', 'failed'])
+GangStateTuple = namedtuple(
+    "GangStateTuple", ["ufo", "collectable", "pcollectable", "failed"]
+)
 
 #: Predefined :class:`_Gang` state values
-GangSt = GangStateTuple(ufo='undecided', collectable='collectable',
-                        pcollectable='collectable_partial', failed='failed')
+GangSt = GangStateTuple(
+    ufo="undecided",
+    collectable="collectable",
+    pcollectable="collectable_partial",
+    failed="failed",
+)
 
 
 class LayoutMonitorError(Exception):
     """The default exception for this module."""
+
     pass
 
 
@@ -74,8 +82,9 @@ class _StateFull:
             previous = self._state
             self._state = newstate
             self._state_changed(previous, self._state)
-            self._obsboard.notify_upd(self, dict(state=self._state,
-                                                 previous_state=previous))
+            self._obsboard.notify_upd(
+                self, dict(state=self._state, previous_state=previous)
+            )
 
     state = property(_get_state, _set_state, doc="The entry's state.")
 
@@ -84,7 +93,7 @@ class _StateFullMembersList:
     """Defines an abstract interface: a class with members."""
 
     _mstates = EntrySt  # The name of possible member's states
-    _mcontainer = set   # The container class for the members
+    _mcontainer = set  # The container class for the members
 
     def __init__(self):
         """Initialise the members list."""
@@ -116,7 +125,6 @@ class _StateFullMembersList:
 
 
 class InputMonitorEntry(_StateFull):
-
     def __init__(self, section):
         """An entry manipulated by a :class:`BasicInputMonitor` object.
 
@@ -150,9 +158,14 @@ class _MonitorSilencer(ParallelSilencer):
 
     def export_result(self, key, ts, prevstate, state):
         """Returns the recorded data, plus state related informations."""
-        return dict(report=super().export_result(),
-                    name="Input #{!s}".format(key), key=key,
-                    prevstate=prevstate, state=state, timestamp=ts)
+        return dict(
+            report=super().export_result(),
+            name="Input #{!s}".format(key),
+            key=key,
+            prevstate=prevstate,
+            state=state,
+            timestamp=ts,
+        )
 
 
 class ManualInputMonitor(_StateFullMembersList):
@@ -164,8 +177,14 @@ class ManualInputMonitor(_StateFullMembersList):
 
     _mcontainer = OrderedDict
 
-    def __init__(self, context, targets, caching_freq=20, crawling_threshold=100,
-                 mute=False):
+    def __init__(
+        self,
+        context,
+        targets,
+        caching_freq=20,
+        crawling_threshold=100,
+        mute=False,
+    ):
         """
         If the list of inputs is too long (see the *crawling_threshold*
         option), not all of the inputs will be checked at once: The first
@@ -211,22 +230,24 @@ class ManualInputMonitor(_StateFullMembersList):
         has_term = 0
         map_term = defaultdict(int)
         for e in toclassify:
-            if hasattr(e.section.rh.resource, 'term'):
+            if hasattr(e.section.rh.resource, "term"):
                 has_term += 1
                 map_term[e.section.rh.resource.term.fmthm] += 1
         if toclassify and has_term == len(toclassify):
             toclassify.sort(key=lambda e: e.section.rh.resource.term)
             # Use a crawling threshold that is large enough to span a little bit
             # more than one term.
-            self._crawling_threshold = max(self._crawling_threshold,
-                                           int(max(map_term.values()) * 1.25))
+            self._crawling_threshold = max(
+                self._crawling_threshold, int(max(map_term.values()) * 1.25)
+            )
 
         # Create key/value pairs
         toclassify = [(i, e) for i, e in enumerate(toclassify)]
 
         # Classify the input depending on there stage
-        self._map_stages = dict(expected=EntrySt.expected,
-                                get=EntrySt.available)
+        self._map_stages = dict(
+            expected=EntrySt.expected, get=EntrySt.available
+        )
         while toclassify:
             e = toclassify.pop(0)
             self._append_entry(self._find_state(e[1], onfails=EntrySt.ufo), e)
@@ -234,9 +255,9 @@ class ManualInputMonitor(_StateFullMembersList):
     def start(self):
         """Start the background updater task."""
         self._mpjob = multiprocessing.Process(
-            name='BackgroundUpdater',
+            name="BackgroundUpdater",
             target=self._background_updater_job,
-            args=()
+            args=(),
         )
         self._mpjob.start()
 
@@ -249,18 +270,20 @@ class ManualInputMonitor(_StateFullMembersList):
             t0 = date.now()
             self._mpjob.join(5)
             waiting = date.now() - t0
-            logger.info('Waiting for the background process to stop took %f seconds',
-                        waiting.total_seconds())
+            logger.info(
+                "Waiting for the background process to stop took %f seconds",
+                waiting.total_seconds(),
+            )
             # Be less nice if needed...
             if self._mpjob.is_alive():
-                logger.warning('Force termination of the background process')
+                logger.warning("Force termination of the background process")
                 self._mpjob.terminate()
                 time.sleep(1)  # Allow some time for the process to terminate
             # Wrap up
             rc = not self._mperror.is_set()
-            logger.info('Server still alive ? %s', str(self._mpjob.is_alive()))
+            logger.info("Server still alive ? %s", str(self._mpjob.is_alive()))
             if not rc:
-                raise LayoutMonitorError('The background process ended badly.')
+                raise LayoutMonitorError("The background process ended badly.")
 
     def __enter__(self):
         self.start()
@@ -288,11 +311,11 @@ class ManualInputMonitor(_StateFullMembersList):
 
     def _key_update(self, res):
         """Process a result dictionary of the _background_updater method."""
-        e = self._members[res['prevstate']].pop(res['key'], None)
+        e = self._members[res["prevstate"]].pop(res["key"], None)
         # The entry might be missing if someone mess with the _memebers dicitonary
         if e is not None:
-            self._append_entry(res['state'], (res['key'], e))
-            self._inactive_since = res['timestamp']
+            self._append_entry(res["state"], (res["key"], e))
+            self._inactive_since = res["timestamp"]
 
     def _background_updater(self):
         """This method loops on itself regularly to update the entry's state."""
@@ -302,14 +325,17 @@ class ManualInputMonitor(_StateFullMembersList):
         kangaroo_idx = 0
 
         # Stop if we are asked to or if there is nothing more to do
-        while (not self._mpquit.is_set() and
-               not (len(self._members[EntrySt.expected]) == 0 and
-                    len(self._members[EntrySt.ufo]) == 0)):
-
+        while not self._mpquit.is_set() and not (
+            len(self._members[EntrySt.expected]) == 0
+            and len(self._members[EntrySt.ufo]) == 0
+        ):
             # Tweak the caching_frequency
-            if (len(self._members[EntrySt.ufo]) and
-                    len(self._members[EntrySt.expected]) <= self._crawling_threshold and
-                    not len(self._members[EntrySt.available])):
+            if (
+                len(self._members[EntrySt.ufo])
+                and len(self._members[EntrySt.expected])
+                <= self._crawling_threshold
+                and not len(self._members[EntrySt.available])
+            ):
                 # If UFO are still there and not much resources are expected,
                 # decrease the caching time
                 eff_caching_freq = max(3, self._caching_freq / 5)
@@ -319,37 +345,63 @@ class ManualInputMonitor(_StateFullMembersList):
             curtime = time.time()
             # Crawl into the monitored input if sensible
             if curtime > last_refresh + eff_caching_freq:
-
                 last_refresh = curtime
                 result_stack = list()
 
                 # Crawl into the ufo list
                 # Always process the first self._crawling_threshold elements
-                for k, e in islice(self._members[EntrySt.ufo].items(),
-                                   self._crawling_threshold):
+                for k, e in islice(
+                    self._members[EntrySt.ufo].items(),
+                    self._crawling_threshold,
+                ):
                     if self._mpquit.is_set():  # Are we ordered to stop ?
                         break
-                    with _MonitorSilencer(self._ctx, 'inputmonitor_updater') as psi:
-                        logger.info("First get on local file: %s",
-                                    e.section.rh.container.localpath())
-                        e.section.get(incache=True, fatal=False)  # Do not crash at this stage
-                        res = psi.export_result(k, curtime, e.state, self._find_state(e))
+                    with _MonitorSilencer(
+                        self._ctx, "inputmonitor_updater"
+                    ) as psi:
+                        logger.info(
+                            "First get on local file: %s",
+                            e.section.rh.container.localpath(),
+                        )
+                        e.section.get(
+                            incache=True, fatal=False
+                        )  # Do not crash at this stage
+                        res = psi.export_result(
+                            k, curtime, e.state, self._find_state(e)
+                        )
                     self._mpqueue.put_nowait(res)
                     result_stack.append(res)
 
                 # What are the expected elements we will look for ?
                 # 1. The first self._crawling_threshold elements
-                exp_compress = [1, ] * min(self._crawling_threshold,
-                                           len(self._members[EntrySt.expected]))
+                exp_compress = [
+                    1,
+                ] * min(
+                    self._crawling_threshold,
+                    len(self._members[EntrySt.expected]),
+                )
                 # 2. An additional set of self._crawling_threshold rotating elements
-                for i in range(max(0, len(self._members[EntrySt.expected]) - self._crawling_threshold)):
+                for i in range(
+                    max(
+                        0,
+                        len(self._members[EntrySt.expected])
+                        - self._crawling_threshold,
+                    )
+                ):
                     kdiff = i - kangaroo_idx
-                    exp_compress.append(1 if kdiff >= 0 and kdiff < self._crawling_threshold else 0)
+                    exp_compress.append(
+                        1
+                        if kdiff >= 0 and kdiff < self._crawling_threshold
+                        else 0
+                    )
 
                 # Crawl into the chosen items of the expected list
                 (visited, found, kangaroo_incr) = (0, 0, 0)
-                for i, (k, e) in enumerate(compress(self._members[EntrySt.expected].items(),
-                                                    exp_compress)):
+                for i, (k, e) in enumerate(
+                    compress(
+                        self._members[EntrySt.expected].items(), exp_compress
+                    )
+                ):
                     if self._mpquit.is_set():  # Are we ordered to stop ?
                         break
 
@@ -360,31 +412,49 @@ class ManualInputMonitor(_StateFullMembersList):
                         # If a lot of resources were already found, avoid harassment
                         break
 
-                    logger.debug("Checking local file: %s (kangaroo=%s)",
-                                 e.section.rh.container.localpath(), kangaroo)
+                    logger.debug(
+                        "Checking local file: %s (kangaroo=%s)",
+                        e.section.rh.container.localpath(),
+                        kangaroo,
+                    )
                     e.check_done()
                     # Is the promise file still there or not ?
                     if e.section.rh.is_grabable():
                         visited += 1
-                        with _MonitorSilencer(self._ctx, 'inputmonitor_updater') as psi:
+                        with _MonitorSilencer(
+                            self._ctx, "inputmonitor_updater"
+                        ) as psi:
                             if e.section.rh.is_grabable(check_exists=True):
-                                logger.info("The local resource %s becomes available",
-                                            e.section.rh.container.localpath())
+                                logger.info(
+                                    "The local resource %s becomes available",
+                                    e.section.rh.container.localpath(),
+                                )
                                 # This will crash in case of an error, but this should
                                 # not happen since we checked the resource just above
                                 e.section.get(incache=True)
                                 found += 1
-                                res = psi.export_result(k, curtime, e.state, self._find_state(e))
+                                res = psi.export_result(
+                                    k, curtime, e.state, self._find_state(e)
+                                )
                             else:
-                                logger.warning("The local resource %s has failed",
-                                               e.section.rh.container.localpath())
-                                res = psi.export_result(k, curtime, e.state, EntrySt.failed)
+                                logger.warning(
+                                    "The local resource %s has failed",
+                                    e.section.rh.container.localpath(),
+                                )
+                                res = psi.export_result(
+                                    k, curtime, e.state, EntrySt.failed
+                                )
                         self._mpqueue.put_nowait(res)
                         result_stack.append(res)
 
                 # Update the kangaroo index
                 kangaroo_idx = kangaroo_idx + kangaroo_incr - visited
-                if kangaroo_idx > len(self._members[EntrySt.expected]) - self._crawling_threshold - 1:
+                if (
+                    kangaroo_idx
+                    > len(self._members[EntrySt.expected])
+                    - self._crawling_threshold
+                    - 1
+                ):
                     kangaroo_idx = 0
 
                 # Effectively update the internal _members dictionary
@@ -401,9 +471,9 @@ class ManualInputMonitor(_StateFullMembersList):
             self._background_updater()
         except Exception:
             (exc_type, exc_value, exc_traceback) = sys.exc_info()
-            print('Exception type: {!s}'.format(exc_type))
-            print('Exception info: {!s}'.format(exc_value))
-            print('Traceback:')
+            print("Exception type: {!s}".format(exc_type))
+            print("Exception info: {!s}".format(exc_value))
+            print("Traceback:")
             print("\n".join(traceback.format_tb(exc_traceback)))
             # Alert the main process of the error
             self._mperror.set()
@@ -415,7 +485,7 @@ class ManualInputMonitor(_StateFullMembersList):
         # That's bad...
         if self._mperror.is_set():
             self.stop()
-            raise LayoutMonitorError('The background process ended badly.')
+            raise LayoutMonitorError("The background process ended badly.")
         # Process all the available update messages
         while True:
             try:
@@ -425,8 +495,9 @@ class ManualInputMonitor(_StateFullMembersList):
             if prp is None:
                 prp = ParallelResultParser(self._ctx)
             if not self._mute:
-                self._ctx.system.highlight("The InputMonitor got news for: {!s}"
-                                           .format(r['name']))
+                self._ctx.system.highlight(
+                    "The InputMonitor got news for: {!s}".format(r["name"])
+                )
             prp(r)
             print()
             self._key_update(r)
@@ -435,8 +506,10 @@ class ManualInputMonitor(_StateFullMembersList):
     def all_done(self):
         """Are there any ufo or expected sections left ?"""
         self._refresh()
-        return (len(self._members[EntrySt.expected]) == 0 and
-                len(self._members[EntrySt.ufo]) == 0)
+        return (
+            len(self._members[EntrySt.expected]) == 0
+            and len(self._members[EntrySt.ufo]) == 0
+        )
 
     @property
     def inactive_time(self):
@@ -479,9 +552,13 @@ class ManualInputMonitor(_StateFullMembersList):
         time_now = time.time()
         if time_now - self._last_healthcheck > interval:
             self._last_healthcheck = time_now
-            logger.info("Still waiting (ufo=%d, expected=%d, available=%d, failed=%d)...",
-                        len(self._members[EntrySt.ufo]), len(self._members[EntrySt.expected]),
-                        len(self._members[EntrySt.available]), len(self._members[EntrySt.failed]))
+            logger.info(
+                "Still waiting (ufo=%d, expected=%d, available=%d, failed=%d)...",
+                len(self._members[EntrySt.ufo]),
+                len(self._members[EntrySt.expected]),
+                len(self._members[EntrySt.available]),
+                len(self._members[EntrySt.failed]),
+            )
 
     def is_timedout(self, timeout, exception=None):
         """Check if a timeout occurred.
@@ -493,9 +570,15 @@ class ManualInputMonitor(_StateFullMembersList):
         self._refresh()
         if (timeout > 0) and (self.inactive_time > timeout):
             logger.error("The waiting loop timed out (%d seconds)", timeout)
-            logger.error("The following files are still unaccounted for: %s",
-                         ",".join([e.section.rh.container.localpath()
-                                   for e in self.expected.values()]))
+            logger.error(
+                "The following files are still unaccounted for: %s",
+                ",".join(
+                    [
+                        e.section.rh.container.localpath()
+                        for e in self.expected.values()
+                    ]
+                ),
+            )
             rc = True
         if rc and exception is not None:
             raise exception("The waiting loop timed-out")
@@ -511,8 +594,15 @@ class BasicInputMonitor(ManualInputMonitor):
 
     _mcontainer = OrderedDict
 
-    def __init__(self, context, role=None, kind=None,
-                 caching_freq=20, crawling_threshold=100, mute=False):
+    def __init__(
+        self,
+        context,
+        role=None,
+        kind=None,
+        caching_freq=20,
+        crawling_threshold=100,
+        mute=False,
+    ):
         """
         If the list of inputs is too long (see the *crawling_threshold*
         option), not all of the inputs will be checked at once: The first
@@ -540,13 +630,19 @@ class BasicInputMonitor(ManualInputMonitor):
         self._role = role
         self._kind = kind
         assert not (self._role is None and self._kind is None)
-        ManualInputMonitor.__init__(self, context,
-                                    [InputMonitorEntry(x)
-                                     for x in context.sequence.filtered_inputs(role=self._role,
-                                                                               kind=self._kind)],
-                                    caching_freq=caching_freq,
-                                    crawling_threshold=crawling_threshold,
-                                    mute=mute)
+        ManualInputMonitor.__init__(
+            self,
+            context,
+            [
+                InputMonitorEntry(x)
+                for x in context.sequence.filtered_inputs(
+                    role=self._role, kind=self._kind
+                )
+            ],
+            caching_freq=caching_freq,
+            crawling_threshold=crawling_threshold,
+            mute=mute,
+        )
 
 
 class _Gang(observer.Observer, _StateFull, _StateFullMembersList):
@@ -580,10 +676,11 @@ class _Gang(observer.Observer, _StateFull, _StateFullMembersList):
     def nickname(self):
         """A fancy representation of the Gang's motive."""
         if not self.info:
-            return 'Anonymous'
+            return "Anonymous"
         else:
-            return ", ".join(['{:s}={!s}'.format(k, v)
-                              for k, v in self.info.items()])
+            return ", ".join(
+                ["{:s}={!s}".format(k, v) for k, v in self.info.items()]
+            )
 
     def add_member(self, *members):
         """Introduce one or several members to the Gang."""
@@ -603,8 +700,8 @@ class _Gang(observer.Observer, _StateFull, _StateFullMembersList):
         with self._t_lock:
             observer.Observer.updobsitem(self, item, info)
             # Move the item around
-            self._members[info['previous_state']].remove(item)
-            self._members[info['state']].add(item)
+            self._members[info["previous_state"]].remove(item)
+            self._members[info["state"]].add(item)
             # Update my own state
             self._refresh_state()
 
@@ -665,20 +762,29 @@ class BasicGang(_Gang):
         # Remove the waitlimit timer
         if self._waitlimit_timer is not None and not self._ufo_members:
             self._waitlimit_timer.cancel()
-            logger.debug('Waitlimit Timer thread canceled: %s (Gang: %s)',
-                         self._waitlimit_timer, self.nickname)
+            logger.debug(
+                "Waitlimit Timer thread canceled: %s (Gang: %s)",
+                self._waitlimit_timer,
+                self.nickname,
+            )
             self._waitlimit_timer = None
         # Print some diagnosis data
         if self.info and new != self._mystates.ufo:
-            msg = ("State changed from {:s} to {:s} for Gang: {:s}"
-                   .format(previous, new, self.nickname))
+            msg = "State changed from {:s} to {:s} for Gang: {:s}".format(
+                previous, new, self.nickname
+            )
             if new == self._mystates.pcollectable:
                 if self._ufo_members:
-                    logger.warning("%s\nSome of the Gang's members are still expected " +
-                                   "but the %d seconds waitlimit is exhausted.",
-                                   msg, self.waitlimit)
+                    logger.warning(
+                        "%s\nSome of the Gang's members are still expected "
+                        + "but the %d seconds waitlimit is exhausted.",
+                        msg,
+                        self.waitlimit,
+                    )
                 else:
-                    logger.warning("%s\nSome of the Gang's members have failed.", msg)
+                    logger.warning(
+                        "%s\nSome of the Gang's members have failed.", msg
+                    )
             else:
                 logger.info(msg)
 
@@ -688,30 +794,40 @@ class BasicGang(_Gang):
             def _waitlimit_check():
                 with self._t_lock:
                     self._refresh_state()
-                    logger.debug('Waitlimit Timer thread done: %s (Gang: %s)',
-                                 self._waitlimit_timer, self.nickname)
+                    logger.debug(
+                        "Waitlimit Timer thread done: %s (Gang: %s)",
+                        self._waitlimit_timer,
+                        self.nickname,
+                    )
                     self._waitlimit_timer = None
 
-            self._waitlimit_timer = threading.Timer(self.waitlimit + 1,
-                                                    _waitlimit_check)
+            self._waitlimit_timer = threading.Timer(
+                self.waitlimit + 1, _waitlimit_check
+            )
             self._waitlimit_timer.daemon = True
             self._waitlimit_timer.start()
-            logger.debug('Waitlimit Timer thread started: %s (Gang: %s)',
-                         self._waitlimit_timer, self.nickname)
+            logger.debug(
+                "Waitlimit Timer thread started: %s (Gang: %s)",
+                self._waitlimit_timer,
+                self.nickname,
+            )
 
     def add_member(self, *members):
         with self._t_lock:
             super().add_member(*members)
-            if self._firstseen is None and any([m.state == self._mstates.available
-                                                for m in members]):
+            if self._firstseen is None and any(
+                [m.state == self._mstates.available for m in members]
+            ):
                 self._firstseen = time.time()
                 self._set_waitlimit_timer()
 
     def updobsitem(self, item, info):
         with self._t_lock:
             super().updobsitem(item, info)
-            if (self._firstseen is None and
-                    info['state'] == self._mstates.available):
+            if (
+                self._firstseen is None
+                and info["state"] == self._mstates.available
+            ):
                 self._firstseen = time.time()
                 self._set_waitlimit_timer()
 
@@ -723,21 +839,29 @@ class BasicGang(_Gang):
     @property
     def _ufo_members(self):
         """The number of ufo members (from a Gang point of view)."""
-        return len(self._members[self._mstates.ufo]) + len(self._members[self._mstates.expected])
+        return len(self._members[self._mstates.ufo]) + len(
+            self._members[self._mstates.expected]
+        )
 
     def _is_collectable(self):
         return len(self._members[self._mstates.available]) == len(self)
 
     def _is_pcollectable(self):
-        return (len(self._members[self._mstates.available]) >= self._eff_minsize and
-                (self._ufo_members == 0 or
-                 (self._firstseen is not None and
-                  time.time() - self._firstseen > self.waitlimit > 0)
-                 )
-                )
+        return len(
+            self._members[self._mstates.available]
+        ) >= self._eff_minsize and (
+            self._ufo_members == 0
+            or (
+                self._firstseen is not None
+                and time.time() - self._firstseen > self.waitlimit > 0
+            )
+        )
 
     def _is_undecided(self):
-        return len(self._members[self._mstates.available]) + self._ufo_members >= self._eff_minsize
+        return (
+            len(self._members[self._mstates.available]) + self._ufo_members
+            >= self._eff_minsize
+        )
 
 
 class MetaGang(_Gang):
@@ -764,19 +888,24 @@ class MetaGang(_Gang):
 
     def has_pcollectable(self):
         """Is there at least one collectable or collectable_partial member ?"""
-        return (len(self._members[self._mstates.pcollectable]) +
-                len(self._members[self._mstates.collectable]))
+        return len(self._members[self._mstates.pcollectable]) + len(
+            self._members[self._mstates.collectable]
+        )
 
     def pop_collectable(self):
         """Retrieve a collectable member."""
-        return self._unregister_i(self._members[self._mstates.collectable].pop())
+        return self._unregister_i(
+            self._members[self._mstates.collectable].pop()
+        )
 
     def pop_pcollectable(self):
         """Retrieve a collectable or a collectable_partial member."""
         if self.has_collectable():
             return self.pop_collectable()
         else:
-            return self._unregister_i(self._members[self._mstates.pcollectable].pop())
+            return self._unregister_i(
+                self._members[self._mstates.pcollectable].pop()
+            )
 
     def consume_colectable(self):
         """Retriece all collectable members (as a generator)."""
@@ -792,8 +921,10 @@ class MetaGang(_Gang):
         return len(self._members[self._mstates.collectable]) == len(self)
 
     def _is_pcollectable(self):
-        return (len(self._members[self._mstates.collectable]) +
-                len(self._members[self._mstates.pcollectable])) == len(self)
+        return (
+            len(self._members[self._mstates.collectable])
+            + len(self._members[self._mstates.pcollectable])
+        ) == len(self)
 
     def _is_undecided(self):
         return len(self._members[self._mstates.failed]) == 0
@@ -821,13 +952,18 @@ class AutoMetaGang(MetaGang):
         # Initialise the gangs
         mdict = defaultdict(list)
         for entry in bm.memberslist:
-            entryid = tuple([entry.section.rh.wide_key_lookup(key)
-                             for key in grouping_keys])
+            entryid = tuple(
+                [
+                    entry.section.rh.wide_key_lookup(key)
+                    for key in grouping_keys
+                ]
+            )
             mdict[entryid].append(entry)
         # Finalise the Gangs setup and use them...
         for entryid, members in mdict.items():
-            gang = BasicGang(waitlimit=waitlimit,
-                             minsize=len(members) - allowmissing)
-            gang.add_member(* members)
+            gang = BasicGang(
+                waitlimit=waitlimit, minsize=len(members) - allowmissing
+            )
+            gang.add_member(*members)
             gang.info = {k: v for k, v in zip(grouping_keys, entryid)}
             self.add_member(gang)