swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +183 -29
  17. swift/cli/manage_shard_ranges.py +708 -37
  18. swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +198 -127
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +396 -147
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -81
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +52 -19
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +192 -174
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2191 -2762
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +555 -536
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +552 -227
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +397 -176
  135. swift/container/sharder.py +1580 -639
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +213 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +653 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +452 -86
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1009 -490
  156. swift/proxy/server.py +185 -112
  157. swift-2.35.0.dist-info/AUTHORS +501 -0
  158. swift-2.35.0.dist-info/LICENSE +202 -0
  159. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
  160. swift-2.35.0.dist-info/RECORD +201 -0
  161. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  162. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  163. swift-2.35.0.dist-info/pbr.json +1 -0
  164. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  165. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  166. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  167. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  168. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  169. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  170. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  171. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  172. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  173. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  174. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  175. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  176. swift-2.23.2.data/scripts/swift-account-auditor +0 -23
  177. swift-2.23.2.data/scripts/swift-account-info +0 -51
  178. swift-2.23.2.data/scripts/swift-account-reaper +0 -23
  179. swift-2.23.2.data/scripts/swift-account-replicator +0 -34
  180. swift-2.23.2.data/scripts/swift-account-server +0 -23
  181. swift-2.23.2.data/scripts/swift-container-auditor +0 -23
  182. swift-2.23.2.data/scripts/swift-container-info +0 -51
  183. swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
  184. swift-2.23.2.data/scripts/swift-container-replicator +0 -34
  185. swift-2.23.2.data/scripts/swift-container-sharder +0 -33
  186. swift-2.23.2.data/scripts/swift-container-sync +0 -23
  187. swift-2.23.2.data/scripts/swift-container-updater +0 -23
  188. swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
  189. swift-2.23.2.data/scripts/swift-form-signature +0 -20
  190. swift-2.23.2.data/scripts/swift-init +0 -119
  191. swift-2.23.2.data/scripts/swift-object-auditor +0 -29
  192. swift-2.23.2.data/scripts/swift-object-expirer +0 -33
  193. swift-2.23.2.data/scripts/swift-object-info +0 -60
  194. swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
  195. swift-2.23.2.data/scripts/swift-object-relinker +0 -41
  196. swift-2.23.2.data/scripts/swift-object-replicator +0 -37
  197. swift-2.23.2.data/scripts/swift-object-server +0 -27
  198. swift-2.23.2.data/scripts/swift-object-updater +0 -23
  199. swift-2.23.2.data/scripts/swift-proxy-server +0 -23
  200. swift-2.23.2.data/scripts/swift-recon +0 -24
  201. swift-2.23.2.data/scripts/swift-ring-builder +0 -24
  202. swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
  203. swift-2.23.2.data/scripts/swift-ring-composer +0 -22
  204. swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
  205. swift-2.23.2.dist-info/RECORD +0 -220
  206. swift-2.23.2.dist-info/metadata.json +0 -1
  207. swift-2.23.2.dist-info/pbr.json +0 -1
  208. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,3 @@
1
- #!python
2
1
  # Licensed under the Apache License, Version 2.0 (the "License");
3
2
  # you may not use this file except in compliance with the License.
4
3
  # You may obtain a copy of the License at
@@ -12,27 +11,25 @@
12
11
  # See the License for the specific language governing permissions and
13
12
  # limitations under the License.
14
13
 
15
- """
16
- swift-recon-cron.py
17
- """
18
-
19
14
  import os
20
15
  import sys
16
+ import time
21
17
 
22
18
  from eventlet import Timeout
23
19
 
24
20
  from swift.common.utils import get_logger, dump_recon_cache, readconf, \
25
- lock_path
21
+ lock_path, listdir
22
+ from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
26
23
  from swift.obj.diskfile import ASYNCDIR_BASE
27
24
 
28
25
 
29
- def get_async_count(device_dir, logger):
26
+ def get_async_count(device_dir):
30
27
  async_count = 0
31
- for i in os.listdir(device_dir):
28
+ for i in listdir(device_dir):
32
29
  device = os.path.join(device_dir, i)
33
30
  if not os.path.isdir(device):
34
31
  continue
35
- for asyncdir in os.listdir(device):
32
+ for asyncdir in listdir(device):
36
33
  # skip stuff like "accounts", "containers", etc.
37
34
  if not (asyncdir == ASYNCDIR_BASE or
38
35
  asyncdir.startswith(ASYNCDIR_BASE + '-')):
@@ -40,10 +37,10 @@ def get_async_count(device_dir, logger):
40
37
  async_pending = os.path.join(device, asyncdir)
41
38
 
42
39
  if os.path.isdir(async_pending):
43
- for entry in os.listdir(async_pending):
40
+ for entry in listdir(async_pending):
44
41
  if os.path.isdir(os.path.join(async_pending, entry)):
45
42
  async_hdir = os.path.join(async_pending, entry)
46
- async_count += len(os.listdir(async_hdir))
43
+ async_count += len(listdir(async_hdir))
47
44
  return async_count
48
45
 
49
46
 
@@ -53,24 +50,24 @@ def main():
53
50
  except Exception:
54
51
  print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1])
55
52
  print("ex: swift-recon-cron /etc/swift/object-server.conf")
56
- sys.exit(1)
53
+ return 1
57
54
  conf = readconf(conf_path, 'filter:recon')
58
55
  device_dir = conf.get('devices', '/srv/node')
59
- recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift')
56
+ recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH)
60
57
  recon_lock_path = conf.get('recon_lock_path', '/var/lock')
61
- cache_file = os.path.join(recon_cache_path, "object.recon")
58
+ cache_file = os.path.join(recon_cache_path, RECON_OBJECT_FILE)
62
59
  lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron")
63
60
  conf['log_name'] = conf.get('log_name', 'recon-cron')
64
61
  logger = get_logger(conf, log_route='recon-cron')
65
62
  try:
66
63
  with lock_path(lock_dir):
67
- asyncs = get_async_count(device_dir, logger)
68
- dump_recon_cache({'async_pending': asyncs}, cache_file, logger)
64
+ asyncs = get_async_count(device_dir)
65
+ dump_recon_cache({
66
+ 'async_pending': asyncs,
67
+ 'async_pending_last': time.time(),
68
+ }, cache_file, logger)
69
69
  except (Exception, Timeout) as err:
70
70
  msg = 'Exception during recon-cron while accessing devices'
71
71
  logger.exception(msg)
72
72
  print('%s: %s' % (msg, err))
73
- sys.exit(1)
74
-
75
- if __name__ == '__main__':
76
- main()
73
+ return 1
@@ -1,4 +1,4 @@
1
- #!python
1
+ #!/usr/bin/env python
2
2
  # Licensed under the Apache License, Version 2.0 (the "License");
3
3
  # you may not use this file except in compliance with the License.
4
4
  # You may obtain a copy of the License at
@@ -11,12 +11,10 @@
11
11
  # implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- from __future__ import print_function
15
14
  import sys
16
15
  from optparse import OptionParser
17
16
 
18
17
  import eventlet.debug
19
- eventlet.debug.hub_exceptions(True)
20
18
 
21
19
  from swift.common.ring import Ring
22
20
  from swift.common.utils import split_path
@@ -50,6 +48,7 @@ parser.add_option('-f', '--force', action='store_true',
50
48
 
51
49
 
52
50
  def main():
51
+ eventlet.debug.hub_exceptions(True)
53
52
  options, args = parser.parse_args()
54
53
  try:
55
54
  policy_index, path, timestamp = args