swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/obj/diskfile.py

@@ -65,7 +65,8 @@ from swift.common.utils import mkdirs, Timestamp, \
     get_md5_socket, F_SETPIPE_SZ, decode_timestamps, encode_timestamps, \
     MD5_OF_EMPTY_STRING, link_fd_to_path, \
     O_TMPFILE, makedirs_count, replace_partition_in_path, remove_directory, \
-    md5, is_file_older, non_negative_float
+    md5, is_file_older, non_negative_float, config_fallocate_value, \
+    fs_has_free_space, CooperativeIterator
 from swift.common.splice import splice, tee
 from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
     DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \
@@ -428,19 +429,23 @@ def consolidate_hashes(partition_dir):
     with lock_path(partition_dir):
         hashes = read_hashes(partition_dir)
 
-        found_invalidation_entry = False
+        found_invalidation_entry = hashes_updated = False
         try:
             with open(invalidations_file, 'r') as inv_fh:
                 for line in inv_fh:
                     found_invalidation_entry = True
                     suffix = line.strip()
+                    if not valid_suffix(suffix):
+                        continue
+                    hashes_updated = True
                     hashes[suffix] = None
         except (IOError, OSError) as e:
             if e.errno != errno.ENOENT:
                 raise
 
-        if found_invalidation_entry:
+        if hashes_updated:
             write_hashes(partition_dir, hashes)
+        if found_invalidation_entry:
             # Now that all the invalidations are reflected in hashes.pkl, it's
             # safe to clear out the invalidations file.
             with open(invalidations_file, 'wb') as inv_fh:
@@ -595,7 +600,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
             try:
                 suffixes = listdir(part_path)
             except OSError as e:
-                if e.errno not in (errno.ENOTDIR, errno.ENODATA):
+                if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                   errno.EUCLEAN):
                     raise
                 continue
             for asuffix in suffixes:
@@ -603,7 +609,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
                 try:
                     hashes = listdir(suff_path)
                 except OSError as e:
-                    if e.errno not in (errno.ENOTDIR, errno.ENODATA):
+                    if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                       errno.EUCLEAN):
                         raise
                     continue
                 for hsh in hashes:
@@ -751,6 +758,8 @@ class BaseDiskFileManager(object):
                 replication_concurrency_per_device)
         self.replication_lock_timeout = int(conf.get(
             'replication_lock_timeout', 15))
+        self.fallocate_reserve, self.fallocate_is_percent = \
+            config_fallocate_value(conf.get('fallocate_reserve', '1%'))
 
         self.use_splice = False
         self.pipe_size = None
@@ -1207,7 +1216,7 @@ class BaseDiskFileManager(object):
                         'it is not a directory', {'hsh_path': hsh_path,
                                                   'quar_path': quar_path})
                     continue
-                elif err.errno == errno.ENODATA:
+                elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                     try:
                         # We've seen cases where bad sectors lead to ENODATA
                         # here; use a similar hack as above
@@ -1562,7 +1571,7 @@ class BaseDiskFileManager(object):
                     'it is not a directory', {'object_path': object_path,
                                               'quar_path': quar_path})
                 raise DiskFileNotExist()
-            elif err.errno == errno.ENODATA:
+            elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                 try:
                     # We've seen cases where bad sectors lead to ENODATA here;
                     # use a similar hack as above
@@ -1793,10 +1802,12 @@ class BaseDiskFileWriter(object):
     :param bytes_per_sync: number bytes written between sync calls
     :param diskfile: the diskfile creating this DiskFileWriter instance
     :param next_part_power: the next partition power to be used
+    :param extension: the file extension to be used; may be used internally
+                      to distinguish between PUT/POST/DELETE operations
     """
 
     def __init__(self, name, datadir, size, bytes_per_sync, diskfile,
-                 next_part_power):
+                 next_part_power, extension='.data'):
         # Parameter tracking
         self._name = name
         self._datadir = datadir
@@ -1807,11 +1818,11 @@ class BaseDiskFileWriter(object):
         self._bytes_per_sync = bytes_per_sync
         self._diskfile = diskfile
         self.next_part_power = next_part_power
+        self._extension = extension
 
         # Internal attributes
         self._upload_size = 0
         self._last_sync = 0
-        self._extension = '.data'
         self._put_succeeded = False
 
     @property
@@ -1856,13 +1867,26 @@ class BaseDiskFileWriter(object):
                 # No more inodes in filesystem
                 raise DiskFileNoSpace()
             raise
-        if self._size is not None and self._size > 0:
+        if self._extension == '.ts':
+            # DELETEs always bypass any free-space reserve checks
+            pass
+        elif self._size:
             try:
                 fallocate(self._fd, self._size)
             except OSError as err:
                 if err.errno in (errno.ENOSPC, errno.EDQUOT):
                     raise DiskFileNoSpace()
                 raise
+        else:
+            # If we don't know the size (i.e. self._size is None) or the size
+            # is known to be zero, we still want to block writes once we're
+            # past the reserve threshold.
+            if not fs_has_free_space(
+                    self._fd,
+                    self.manager.fallocate_reserve,
+                    self.manager.fallocate_is_percent
+            ):
+                raise DiskFileNoSpace()
         return self
 
     def close(self):
@@ -2088,11 +2112,13 @@ class BaseDiskFileReader(object):
     :param pipe_size: size of pipe buffer used in zero-copy operations
     :param diskfile: the diskfile creating this DiskFileReader instance
     :param keep_cache: should resulting reads be kept in the buffer cache
+    :param cooperative_period: the period parameter when does cooperative
+                               yielding during file read
     """
     def __init__(self, fp, data_file, obj_size, etag,
                  disk_chunk_size, keep_cache_size, device_path, logger,
                  quarantine_hook, use_splice, pipe_size, diskfile,
-                 keep_cache=False):
+                 keep_cache=False, cooperative_period=0):
         # Parameter tracking
         self._fp = fp
         self._data_file = data_file
@@ -2111,6 +2137,7 @@ class BaseDiskFileReader(object):
             self._keep_cache = obj_size < keep_cache_size
         else:
             self._keep_cache = False
+        self._cooperative_period = cooperative_period
 
         # Internal Attributes
         self._iter_etag = None
@@ -2135,6 +2162,10 @@ class BaseDiskFileReader(object):
             self._iter_etag.update(chunk)
 
     def __iter__(self):
+        return CooperativeIterator(
+            self._inner_iter(), period=self._cooperative_period)
+
+    def _inner_iter(self):
         """Returns an iterator over the data file."""
         try:
             dropped_cache = 0
@@ -2581,7 +2612,7 @@ class BaseDiskFile(object):
                     # want this one file and not its parent.
                     os.path.join(self._datadir, "made-up-filename"),
                     "Expected directory, found file at %s" % self._datadir)
-            elif err.errno == errno.ENODATA:
+            elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                 try:
                     # We've seen cases where bad sectors lead to ENODATA here
                     raise self._quarantine(
@@ -2611,7 +2642,7 @@ class BaseDiskFile(object):
             self._fp = self._construct_from_data_file(
                 current_time=current_time, modernize=modernize, **file_info)
         except IOError as e:
-            if e.errno == errno.ENODATA:
+            if e.errno in (errno.ENODATA, errno.EUCLEAN):
                 raise self._quarantine(
                     file_info['data_file'],
                     "Failed to open %s: %s" % (file_info['data_file'], e))
@@ -2953,7 +2984,7 @@ class BaseDiskFile(object):
         with self.open(current_time=current_time):
             return self.get_metadata()
 
-    def reader(self, keep_cache=False,
+    def reader(self, keep_cache=False, cooperative_period=0,
                _quarantine_hook=lambda m: None):
         """
         Return a :class:`swift.common.swob.Response` class compatible
@@ -2965,6 +2996,8 @@ class BaseDiskFile(object):
 
         :param keep_cache: caller's preference for keeping data read in the
                            OS buffer cache
+        :param cooperative_period: the period parameter for cooperative
+                                   yielding during file read
         :param _quarantine_hook: 1-arg callable called when obj quarantined;
                                  the arg is the reason for quarantine.
                                  Default is to ignore it.
@@ -2976,19 +3009,23 @@ class BaseDiskFile(object):
             self._metadata['ETag'], self._disk_chunk_size,
             self._manager.keep_cache_size, self._device_path, self._logger,
             use_splice=self._use_splice, quarantine_hook=_quarantine_hook,
-            pipe_size=self._pipe_size, diskfile=self, keep_cache=keep_cache)
+            pipe_size=self._pipe_size, diskfile=self, keep_cache=keep_cache,
+            cooperative_period=cooperative_period)
         # At this point the reader object is now responsible for closing
         # the file pointer.
         self._fp = None
         return dr
 
-    def writer(self, size=None):
+    def _writer(self, size, extension):
         return self.writer_cls(self._name, self._datadir, size,
                                self._bytes_per_sync, self,
-                               self.next_part_power)
+                               self.next_part_power, extension=extension)
+
+    def writer(self, size=None):
+        return self._writer(size, '.data')
 
     @contextmanager
-    def create(self, size=None):
+    def create(self, size=None, extension='.data'):
         """
         Context manager to create a file. We create a temporary file first, and
         then return a DiskFileWriter object to encapsulate the state.
@@ -3001,9 +3038,11 @@ class BaseDiskFile(object):
 
         :param size: optional initial size of file to explicitly allocate on
                      disk
+        :param extension: file extension to use for the newly-created file;
+                          defaults to ``.data`` for the sake of tests
         :raises DiskFileNoSpace: if a size is specified and allocation fails
         """
-        dfw = self.writer(size)
+        dfw = self._writer(size, extension)
         try:
             yield dfw.open()
         finally:
@@ -3019,8 +3058,7 @@ class BaseDiskFile(object):
         :raises DiskFileError: this implementation will raise the same
                             errors as the `create()` method.
         """
-        with self.create() as writer:
-            writer._extension = '.meta'
+        with self.create(extension='.meta') as writer:
             writer.put(metadata)
 
     def delete(self, timestamp):
@@ -3042,8 +3080,7 @@ class BaseDiskFile(object):
         """
         # this is dumb, only tests send in strings
         timestamp = Timestamp(timestamp)
-        with self.create() as deleter:
-            deleter._extension = '.ts'
+        with self.create(extension='.ts') as deleter:
             deleter.put({'X-Timestamp': timestamp.internal})
 
 
@@ -3136,11 +3173,12 @@ class ECDiskFileReader(BaseDiskFileReader):
     def __init__(self, fp, data_file, obj_size, etag,
                  disk_chunk_size, keep_cache_size, device_path, logger,
                  quarantine_hook, use_splice, pipe_size, diskfile,
-                 keep_cache=False):
+                 keep_cache=False, cooperative_period=0):
         super(ECDiskFileReader, self).__init__(
             fp, data_file, obj_size, etag,
             disk_chunk_size, keep_cache_size, device_path, logger,
-            quarantine_hook, use_splice, pipe_size, diskfile, keep_cache)
+            quarantine_hook, use_splice, pipe_size, diskfile, keep_cache,
+            cooperative_period)
         self.frag_buf = None
         self.frag_offset = 0
         self.frag_size = self._diskfile.policy.fragment_size

swift/obj/expirer.py

@@ -14,9 +14,11 @@
 # limitations under the License.
 
 import six
+from six.moves import urllib
 
 from random import random
 from time import time
+from optparse import OptionParser
 from os.path import join
 from collections import defaultdict, deque
 
@@ -24,11 +26,12 @@ from eventlet import sleep, Timeout
 from eventlet.greenpool import GreenPool
 
 from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.internal_client import InternalClient, UnexpectedResponse
 from swift.common.utils import get_logger, dump_recon_cache, split_path, \
     Timestamp, config_true_value, normalize_delete_at_timestamp, \
-    RateLimitedIterator, md5
+    RateLimitedIterator, md5, non_negative_float, non_negative_int, \
+    parse_content_type, parse_options
 from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
     HTTP_PRECONDITION_FAILED
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -36,6 +39,7 @@ from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.container.reconciler import direct_delete_container_entry
 
 MAX_OBJECTS_TO_CACHE = 100000
+X_DELETE_TYPE = 'text/plain'
 ASYNC_DELETE_TYPE = 'application/async-deleted'
 
 
@@ -66,6 +70,80 @@ def parse_task_obj(task_obj):
     return timestamp, target_account, target_container, target_obj
 
 
+def extract_expirer_bytes_from_ctype(content_type):
+    """
+    Parse a content-type and return the number of bytes.
+
+    :param content_type: a content-type string
+    :return: int or None
+    """
+    content_type, params = parse_content_type(content_type)
+    bytes_size = None
+    for k, v in params:
+        if k == 'swift_expirer_bytes':
+            bytes_size = int(v)
+    return bytes_size
+
+
+def embed_expirer_bytes_in_ctype(content_type, metadata):
+    """
+    Embed number of bytes into content-type.  The bytes should come from
+    content-length on regular objects, but future extensions to "bytes in
+    expirer queue" monitoring may want to more closely consider expiration of
+    large multipart object manifests.
+
+    :param content_type: a content-type string
+    :param metadata: a dict, from Diskfile metadata
+    :return: str
+    """
+    # as best I can tell this key is required by df.open
+    report_bytes = metadata['Content-Length']
+    return "%s;swift_expirer_bytes=%d" % (content_type, int(report_bytes))
+
+
+def read_conf_for_delay_reaping_times(conf):
+    delay_reaping_times = {}
+    for conf_key in conf:
+        delay_reaping_prefix = "delay_reaping_"
+        if not conf_key.startswith(delay_reaping_prefix):
+            continue
+        delay_reaping_key = urllib.parse.unquote(
+            conf_key[len(delay_reaping_prefix):])
+        if delay_reaping_key.strip('/') != delay_reaping_key:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(leading or trailing "/" is not allowed)' % conf_key)
+        try:
+            # If split_path fails, have multiple '/' or
+            # account name is invalid
+            account, container = split_path(
+                '/' + delay_reaping_key, 1, 2
+            )
+        except ValueError:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(at most one "/" is allowed)' % conf_key)
+        try:
+            delay_reaping_times[(account, container)] = non_negative_float(
+                conf.get(conf_key)
+            )
+        except ValueError:
+            raise ValueError(
+                '%s must be a float '
+                'greater than or equal to 0' % conf_key)
+    return delay_reaping_times
+
+
+def get_delay_reaping(delay_reaping_times, target_account, target_container):
+    return delay_reaping_times.get(
+        (target_account, target_container),
+        delay_reaping_times.get((target_account, None), 0.0))
+
+
 class ObjectExpirer(Daemon):
     """
     Daemon that queries the internal hidden task accounts to discover objects
@@ -89,16 +167,8 @@ class ObjectExpirer(Daemon):
         self.dequeue_from_legacy = \
             True if is_legacy_conf else \
             config_true_value(conf.get('dequeue_from_legacy', 'false'))
-
-        if is_legacy_conf:
-            self.ic_conf_path = self.conf_path
-        else:
-            self.ic_conf_path = \
-                self.conf.get('internal_client_conf_path') or \
-                '/etc/swift/internal-client.conf'
-
-        self.read_conf_for_queue_access(swift)
-
+        self.swift = swift or self._make_internal_client(is_legacy_conf)
+        self.read_conf_for_queue_access()
         self.report_interval = float(conf.get('report_interval') or 300)
         self.report_first_time = self.report_last_time = time()
         self.report_objects = 0
@@ -113,35 +183,32 @@ class ObjectExpirer(Daemon):
         # with the tombstone reclaim age in the consistency engine.
         self.reclaim_age = int(conf.get('reclaim_age', 604800))
 
-    def read_conf_for_queue_access(self, swift):
-        if self.conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            auto_create_account_prefix = \
-                self.conf['auto_create_account_prefix']
+        self.delay_reaping_times = read_conf_for_delay_reaping_times(conf)
+
+    def _make_internal_client(self, is_legacy_conf):
+        if is_legacy_conf:
+            ic_conf_path = self.conf_path
         else:
-            auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+            ic_conf_path = \
+                self.conf.get('internal_client_conf_path') or \
+                '/etc/swift/internal-client.conf'
+        request_tries = int(self.conf.get('request_tries') or 3)
+        return InternalClient(
+            ic_conf_path, 'Swift Object Expirer', request_tries,
+            use_replication_network=True,
+            global_conf={'log_name': '%s-ic' % self.conf.get(
+                'log_name', self.log_route)})
 
-        self.expiring_objects_account = auto_create_account_prefix + \
+    def read_conf_for_queue_access(self):
+        self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \
             (self.conf.get('expiring_objects_account_name') or
              'expiring_objects')
 
         # This is for common parameter with general task queue in future
         self.task_container_prefix = ''
-
-        request_tries = int(self.conf.get('request_tries') or 3)
-        self.swift = swift or InternalClient(
-            self.ic_conf_path, 'Swift Object Expirer', request_tries,
-            use_replication_network=True,
-            global_conf={'log_name': '%s-ic' % self.conf.get(
-                'log_name', self.log_route)})
-
-        self.processes = int(self.conf.get('processes', 0))
-        self.process = int(self.conf.get('process', 0))
+        self.processes = non_negative_int(self.conf.get('processes', 0))
+        self.process = non_negative_int(self.conf.get('process', 0))
+        self._validate_processes_config()
 
     def report(self, final=False):
         """
@@ -258,6 +325,10 @@ class ObjectExpirer(Daemon):
                 break
             yield task_container
 
+    def get_delay_reaping(self, target_account, target_container):
+        return get_delay_reaping(self.delay_reaping_times, target_account,
+                                 target_container)
+
     def iter_task_to_expire(self, task_account_container_list,
                             my_index, divisor):
         """
@@ -278,18 +349,30 @@ class ObjectExpirer(Daemon):
                 except ValueError:
                     self.logger.exception('Unexcepted error handling task %r' %
                                           task_object)
+                    self.logger.increment('tasks.parse_errors')
                     continue
+                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+                delay_reaping = self.get_delay_reaping(target_account,
+                                                       target_container)
+
                 if delete_timestamp > Timestamp.now():
-                    # we shouldn't yield the object that doesn't reach
+                    # we shouldn't yield ANY more objects that can't reach
                     # the expiration date yet.
                     break
 
-                # Only one expirer daemon assigned for one task
+                # Only one expirer daemon assigned for each task
                 if self.hash_mod('%s/%s' % (task_container, task_object),
                                  divisor) != my_index:
+                    self.logger.increment('tasks.skipped')
                     continue
 
-                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+                if delete_timestamp > Timestamp(time() - delay_reaping) \
+                        and not is_async:
+                    # we shouldn't yield the object during the delay
+                    self.logger.increment('tasks.delayed')
+                    continue
+
+                self.logger.increment('tasks.assigned')
                 yield {'task_account': task_account,
                        'task_container': task_container,
                        'task_object': task_object,
@@ -320,6 +403,9 @@ class ObjectExpirer(Daemon):
                        These will override the values from the config file if
                        provided.
         """
+        # these config values are available to override at the command line,
+        # blow-up now if they're wrong
+        self.override_proceses_config_from_command_line(**kwargs)
         # This if-clause will be removed when general task queue feature is
         # implemented.
         if not self.dequeue_from_legacy:
@@ -330,7 +416,6 @@ class ObjectExpirer(Daemon):
                              'with dequeue_from_legacy == true.')
             return
 
-        self.get_process_values(kwargs)
         pool = GreenPool(self.concurrency)
         self.report_first_time = self.report_last_time = time()
         self.report_objects = 0
@@ -385,6 +470,9 @@ class ObjectExpirer(Daemon):
         :param kwargs: Extra keyword args to fulfill the Daemon interface; this
                        daemon has no additional keyword args.
         """
+        # these config values are available to override at the command line
+        # blow-up now if they're wrong
+        self.override_proceses_config_from_command_line(**kwargs)
         sleep(random() * self.interval)
         while True:
             begin = time()
@@ -396,7 +484,7 @@ class ObjectExpirer(Daemon):
             if elapsed < self.interval:
                 sleep(random() * (self.interval - elapsed))
 
-    def get_process_values(self, kwargs):
+    def override_proceses_config_from_command_line(self, **kwargs):
         """
         Sets self.processes and self.process from the kwargs if those
         values exist, otherwise, leaves those values as they were set in
@@ -407,19 +495,20 @@ class ObjectExpirer(Daemon):
                        line when the daemon is run.
         """
         if kwargs.get('processes') is not None:
-            self.processes = int(kwargs['processes'])
+            self.processes = non_negative_int(kwargs['processes'])
 
         if kwargs.get('process') is not None:
-            self.process = int(kwargs['process'])
+            self.process = non_negative_int(kwargs['process'])
 
-        if self.process < 0:
-            raise ValueError(
-                'process must be an integer greater than or equal to 0')
+        self._validate_processes_config()
 
-        if self.processes < 0:
-            raise ValueError(
-                'processes must be an integer greater than or equal to 0')
+    def _validate_processes_config(self):
+        """
+        Used in constructor and in override_proceses_config_from_command_line
+        to validate the processes configuration requirements.
 
+        :raiess: ValueError if processes config is invalid
+        """
         if self.processes and self.process >= self.processes:
             raise ValueError(
                 'process must be less than processes')
@@ -496,3 +585,21 @@ class ObjectExpirer(Daemon):
         self.swift.delete_object(*split_path('/' + actual_obj, 3, 3, True),
                                  headers=headers,
                                  acceptable_statuses=acceptable_statuses)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('--processes', dest='processes',
+                      help="Number of processes to use to do the work, don't "
+                      "use this option to do all the work in one process")
+    parser.add_option('--process', dest='process',
+                      help="Process number for this process, don't use "
+                      "this option to do all the work in one process, this "
+                      "is used to determine which part of the work this "
+                      "process should do")
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectExpirer, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/mem_diskfile.py

@@ -426,13 +426,14 @@ class DiskFile(object):
         with self.open(current_time=current_time):
             return self.get_metadata()
 
-    def reader(self, keep_cache=False):
+    def reader(self, keep_cache=False, cooperative_period=0):
         """
         Return a swift.common.swob.Response class compatible "app_iter"
         object. The responsibility of closing the open file is passed to the
         DiskFileReader object.
 
         :param keep_cache:
+        :param cooperative_period:
         """
         dr = DiskFileReader(self._name, self._fp,
                             int(self._metadata['Content-Length']),

swift/obj/mem_server.py

@@ -32,6 +32,7 @@ class ObjectController(server.ObjectController):
         :param conf: WSGI configuration parameter
         """
         self._filesystem = InMemoryFileSystem()
+        self.fallocate_reserve = 0
 
     def get_diskfile(self, device, partition, account, container, obj,
                      **kwargs):