swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.2.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.2.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.2.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +183 -29
  17. swift/cli/manage_shard_ranges.py +708 -37
  18. swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.2.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.2.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +198 -127
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +396 -147
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -81
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +52 -19
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +192 -174
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.2.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2191 -2762
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +555 -536
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +552 -227
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +397 -176
  135. swift/container/sharder.py +1580 -639
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +213 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +653 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +452 -86
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1009 -490
  156. swift/proxy/server.py +185 -112
  157. swift-2.35.0.dist-info/AUTHORS +501 -0
  158. swift-2.35.0.dist-info/LICENSE +202 -0
  159. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/METADATA +52 -61
  160. swift-2.35.0.dist-info/RECORD +201 -0
  161. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  162. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  163. swift-2.35.0.dist-info/pbr.json +1 -0
  164. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  165. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  166. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  167. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  168. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  169. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  170. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  171. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  172. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  173. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  174. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  175. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  176. swift-2.23.2.data/scripts/swift-account-auditor +0 -23
  177. swift-2.23.2.data/scripts/swift-account-info +0 -51
  178. swift-2.23.2.data/scripts/swift-account-reaper +0 -23
  179. swift-2.23.2.data/scripts/swift-account-replicator +0 -34
  180. swift-2.23.2.data/scripts/swift-account-server +0 -23
  181. swift-2.23.2.data/scripts/swift-container-auditor +0 -23
  182. swift-2.23.2.data/scripts/swift-container-info +0 -51
  183. swift-2.23.2.data/scripts/swift-container-reconciler +0 -21
  184. swift-2.23.2.data/scripts/swift-container-replicator +0 -34
  185. swift-2.23.2.data/scripts/swift-container-sharder +0 -33
  186. swift-2.23.2.data/scripts/swift-container-sync +0 -23
  187. swift-2.23.2.data/scripts/swift-container-updater +0 -23
  188. swift-2.23.2.data/scripts/swift-dispersion-report +0 -24
  189. swift-2.23.2.data/scripts/swift-form-signature +0 -20
  190. swift-2.23.2.data/scripts/swift-init +0 -119
  191. swift-2.23.2.data/scripts/swift-object-auditor +0 -29
  192. swift-2.23.2.data/scripts/swift-object-expirer +0 -33
  193. swift-2.23.2.data/scripts/swift-object-info +0 -60
  194. swift-2.23.2.data/scripts/swift-object-reconstructor +0 -33
  195. swift-2.23.2.data/scripts/swift-object-relinker +0 -41
  196. swift-2.23.2.data/scripts/swift-object-replicator +0 -37
  197. swift-2.23.2.data/scripts/swift-object-server +0 -27
  198. swift-2.23.2.data/scripts/swift-object-updater +0 -23
  199. swift-2.23.2.data/scripts/swift-proxy-server +0 -23
  200. swift-2.23.2.data/scripts/swift-recon +0 -24
  201. swift-2.23.2.data/scripts/swift-ring-builder +0 -24
  202. swift-2.23.2.data/scripts/swift-ring-builder-analyzer +0 -22
  203. swift-2.23.2.data/scripts/swift-ring-composer +0 -22
  204. swift-2.23.2.dist-info/DESCRIPTION.rst +0 -166
  205. swift-2.23.2.dist-info/RECORD +0 -220
  206. swift-2.23.2.dist-info/metadata.json +0 -1
  207. swift-2.23.2.dist-info/pbr.json +0 -1
  208. {swift-2.23.2.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -13,113 +13,27 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
- import os
17
- import time
18
- from swift import gettext_ as _
19
- from random import random
20
16
 
21
- from eventlet import Timeout
17
+ from swift.container.backend import ContainerBroker
18
+ from swift.common.daemon import run_daemon
19
+ from swift.common.db_auditor import DatabaseAuditor
20
+ from swift.common.utils import parse_options
22
21
 
23
- import swift.common.db
24
- from swift.container.backend import ContainerBroker, DATADIR
25
- from swift.common.utils import get_logger, audit_location_generator, \
26
- config_true_value, dump_recon_cache, ratelimit_sleep
27
- from swift.common.daemon import Daemon
28
22
 
29
-
30
- class ContainerAuditor(Daemon):
23
+ class ContainerAuditor(DatabaseAuditor):
31
24
  """Audit containers."""
32
25
 
33
- def __init__(self, conf, logger=None):
34
- self.conf = conf
35
- self.logger = logger or get_logger(conf, log_route='container-auditor')
36
- self.devices = conf.get('devices', '/srv/node')
37
- self.mount_check = config_true_value(conf.get('mount_check', 'true'))
38
- self.interval = int(conf.get('interval', 1800))
39
- self.container_passes = 0
40
- self.container_failures = 0
41
- self.containers_running_time = 0
42
- self.max_containers_per_second = \
43
- float(conf.get('containers_per_second', 200))
44
- swift.common.db.DB_PREALLOCATION = \
45
- config_true_value(conf.get('db_preallocation', 'f'))
46
- self.recon_cache_path = conf.get('recon_cache_path',
47
- '/var/cache/swift')
48
- self.rcache = os.path.join(self.recon_cache_path, "container.recon")
26
+ server_type = "container"
27
+ broker_class = ContainerBroker
49
28
 
50
- def _one_audit_pass(self, reported):
51
- all_locs = audit_location_generator(self.devices, DATADIR, '.db',
52
- mount_check=self.mount_check,
53
- logger=self.logger)
54
- for path, device, partition in all_locs:
55
- self.container_audit(path)
56
- if time.time() - reported >= 3600: # once an hour
57
- self.logger.info(
58
- _('Since %(time)s: Container audits: %(pass)s passed '
59
- 'audit, %(fail)s failed audit'),
60
- {'time': time.ctime(reported),
61
- 'pass': self.container_passes,
62
- 'fail': self.container_failures})
63
- dump_recon_cache(
64
- {'container_audits_since': reported,
65
- 'container_audits_passed': self.container_passes,
66
- 'container_audits_failed': self.container_failures},
67
- self.rcache, self.logger)
68
- reported = time.time()
69
- self.container_passes = 0
70
- self.container_failures = 0
71
- self.containers_running_time = ratelimit_sleep(
72
- self.containers_running_time, self.max_containers_per_second)
73
- return reported
29
+ def _audit(self, job, broker):
30
+ return None
74
31
 
75
- def run_forever(self, *args, **kwargs):
76
- """Run the container audit until stopped."""
77
- reported = time.time()
78
- time.sleep(random() * self.interval)
79
- while True:
80
- self.logger.info(_('Begin container audit pass.'))
81
- begin = time.time()
82
- try:
83
- reported = self._one_audit_pass(reported)
84
- except (Exception, Timeout):
85
- self.logger.increment('errors')
86
- self.logger.exception(_('ERROR auditing'))
87
- elapsed = time.time() - begin
88
- if elapsed < self.interval:
89
- time.sleep(self.interval - elapsed)
90
- self.logger.info(
91
- _('Container audit pass completed: %.02fs'), elapsed)
92
- dump_recon_cache({'container_auditor_pass_completed': elapsed},
93
- self.rcache, self.logger)
94
32
 
95
- def run_once(self, *args, **kwargs):
96
- """Run the container audit once."""
97
- self.logger.info(_('Begin container audit "once" mode'))
98
- begin = reported = time.time()
99
- self._one_audit_pass(reported)
100
- elapsed = time.time() - begin
101
- self.logger.info(
102
- _('Container audit "once" mode completed: %.02fs'), elapsed)
103
- dump_recon_cache({'container_auditor_pass_completed': elapsed},
104
- self.rcache, self.logger)
33
+ def main():
34
+ conf_file, options = parse_options(once=True)
35
+ run_daemon(ContainerAuditor, conf_file, **options)
105
36
 
106
- def container_audit(self, path):
107
- """
108
- Audits the given container path
109
37
 
110
- :param path: the path to a container db
111
- """
112
- start_time = time.time()
113
- try:
114
- broker = ContainerBroker(path, logger=self.logger)
115
- if not broker.is_deleted():
116
- broker.get_info()
117
- self.logger.increment('passes')
118
- self.container_passes += 1
119
- self.logger.debug('Audit passed for %s', broker)
120
- except (Exception, Timeout):
121
- self.logger.increment('failures')
122
- self.container_failures += 1
123
- self.logger.exception(_('ERROR Could not get container info %s'),
124
- path)
125
- self.logger.timing_since('timing', start_time)
38
+ if __name__ == '__main__':
39
+ main()