swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swift/__init__.py +29 -50
- swift/account/auditor.py +21 -118
- swift/account/backend.py +33 -28
- swift/account/reaper.py +37 -28
- swift/account/replicator.py +22 -0
- swift/account/server.py +60 -26
- swift/account/utils.py +28 -11
- swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
- swift-2.23.3.data/scripts/swift-config → swift/cli/config.py +2 -2
- swift/cli/container_deleter.py +5 -11
- swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
- swift/cli/dispersion_report.py +10 -9
- swift-2.23.3.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
- swift/cli/form_signature.py +3 -7
- swift-2.23.3.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
- swift/cli/info.py +154 -14
- swift/cli/manage_shard_ranges.py +705 -37
- swift-2.23.3.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
- swift-2.23.3.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
- swift/cli/recon.py +196 -67
- swift-2.23.3.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
- swift-2.23.3.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
- swift/cli/relinker.py +807 -126
- swift/cli/reload.py +135 -0
- swift/cli/ringbuilder.py +217 -20
- swift/cli/ringcomposer.py +0 -1
- swift/cli/shard-info.py +4 -3
- swift/common/base_storage_server.py +9 -20
- swift/common/bufferedhttp.py +48 -74
- swift/common/constraints.py +20 -15
- swift/common/container_sync_realms.py +9 -11
- swift/common/daemon.py +25 -8
- swift/common/db.py +195 -128
- swift/common/db_auditor.py +168 -0
- swift/common/db_replicator.py +95 -55
- swift/common/digest.py +141 -0
- swift/common/direct_client.py +144 -33
- swift/common/error_limiter.py +93 -0
- swift/common/exceptions.py +25 -1
- swift/common/header_key_dict.py +2 -9
- swift/common/http_protocol.py +373 -0
- swift/common/internal_client.py +129 -59
- swift/common/linkat.py +3 -4
- swift/common/manager.py +284 -67
- swift/common/memcached.py +390 -145
- swift/common/middleware/__init__.py +4 -0
- swift/common/middleware/account_quotas.py +211 -46
- swift/common/middleware/acl.py +3 -8
- swift/common/middleware/backend_ratelimit.py +230 -0
- swift/common/middleware/bulk.py +22 -34
- swift/common/middleware/catch_errors.py +1 -3
- swift/common/middleware/cname_lookup.py +6 -11
- swift/common/middleware/container_quotas.py +1 -1
- swift/common/middleware/container_sync.py +39 -17
- swift/common/middleware/copy.py +12 -0
- swift/common/middleware/crossdomain.py +22 -9
- swift/common/middleware/crypto/__init__.py +2 -1
- swift/common/middleware/crypto/crypto_utils.py +11 -15
- swift/common/middleware/crypto/decrypter.py +28 -11
- swift/common/middleware/crypto/encrypter.py +12 -17
- swift/common/middleware/crypto/keymaster.py +8 -15
- swift/common/middleware/crypto/kms_keymaster.py +2 -1
- swift/common/middleware/dlo.py +15 -11
- swift/common/middleware/domain_remap.py +5 -4
- swift/common/middleware/etag_quoter.py +128 -0
- swift/common/middleware/formpost.py +73 -70
- swift/common/middleware/gatekeeper.py +8 -1
- swift/common/middleware/keystoneauth.py +33 -3
- swift/common/middleware/list_endpoints.py +4 -4
- swift/common/middleware/listing_formats.py +85 -49
- swift/common/middleware/memcache.py +4 -95
- swift/common/middleware/name_check.py +3 -2
- swift/common/middleware/proxy_logging.py +160 -92
- swift/common/middleware/ratelimit.py +17 -10
- swift/common/middleware/read_only.py +6 -4
- swift/common/middleware/recon.py +59 -22
- swift/common/middleware/s3api/acl_handlers.py +25 -3
- swift/common/middleware/s3api/acl_utils.py +6 -1
- swift/common/middleware/s3api/controllers/__init__.py +6 -0
- swift/common/middleware/s3api/controllers/acl.py +3 -2
- swift/common/middleware/s3api/controllers/bucket.py +242 -137
- swift/common/middleware/s3api/controllers/logging.py +2 -2
- swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
- swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
- swift/common/middleware/s3api/controllers/obj.py +112 -8
- swift/common/middleware/s3api/controllers/object_lock.py +44 -0
- swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
- swift/common/middleware/s3api/controllers/tagging.py +57 -0
- swift/common/middleware/s3api/controllers/versioning.py +36 -7
- swift/common/middleware/s3api/etree.py +22 -9
- swift/common/middleware/s3api/exception.py +0 -4
- swift/common/middleware/s3api/s3api.py +113 -41
- swift/common/middleware/s3api/s3request.py +384 -218
- swift/common/middleware/s3api/s3response.py +126 -23
- swift/common/middleware/s3api/s3token.py +16 -17
- swift/common/middleware/s3api/schema/delete.rng +1 -1
- swift/common/middleware/s3api/subresource.py +7 -10
- swift/common/middleware/s3api/utils.py +27 -10
- swift/common/middleware/slo.py +665 -358
- swift/common/middleware/staticweb.py +64 -37
- swift/common/middleware/symlink.py +51 -18
- swift/common/middleware/tempauth.py +76 -58
- swift/common/middleware/tempurl.py +191 -173
- swift/common/middleware/versioned_writes/__init__.py +51 -0
- swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
- swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
- swift/common/middleware/x_profile/exceptions.py +1 -4
- swift/common/middleware/x_profile/html_viewer.py +18 -19
- swift/common/middleware/x_profile/profile_model.py +1 -2
- swift/common/middleware/xprofile.py +10 -10
- swift-2.23.3.data/scripts/swift-container-server → swift/common/recon.py +13 -8
- swift/common/registry.py +147 -0
- swift/common/request_helpers.py +324 -57
- swift/common/ring/builder.py +67 -25
- swift/common/ring/composite_builder.py +1 -1
- swift/common/ring/ring.py +177 -51
- swift/common/ring/utils.py +1 -1
- swift/common/splice.py +10 -6
- swift/common/statsd_client.py +205 -0
- swift/common/storage_policy.py +49 -44
- swift/common/swob.py +86 -102
- swift/common/{utils.py → utils/__init__.py} +2163 -2772
- swift/common/utils/base.py +131 -0
- swift/common/utils/config.py +433 -0
- swift/common/utils/ipaddrs.py +256 -0
- swift/common/utils/libc.py +345 -0
- swift/common/utils/logs.py +859 -0
- swift/common/utils/timestamp.py +412 -0
- swift/common/wsgi.py +553 -535
- swift/container/auditor.py +14 -100
- swift/container/backend.py +490 -231
- swift/container/reconciler.py +126 -37
- swift/container/replicator.py +96 -22
- swift/container/server.py +358 -165
- swift/container/sharder.py +1540 -684
- swift/container/sync.py +94 -88
- swift/container/updater.py +53 -32
- swift/obj/auditor.py +153 -35
- swift/obj/diskfile.py +466 -217
- swift/obj/expirer.py +406 -124
- swift/obj/mem_diskfile.py +7 -4
- swift/obj/mem_server.py +1 -0
- swift/obj/reconstructor.py +523 -262
- swift/obj/replicator.py +249 -188
- swift/obj/server.py +207 -122
- swift/obj/ssync_receiver.py +145 -85
- swift/obj/ssync_sender.py +113 -54
- swift/obj/updater.py +652 -139
- swift/obj/watchers/__init__.py +0 -0
- swift/obj/watchers/dark_data.py +213 -0
- swift/proxy/controllers/account.py +11 -11
- swift/proxy/controllers/base.py +848 -604
- swift/proxy/controllers/container.py +433 -92
- swift/proxy/controllers/info.py +3 -2
- swift/proxy/controllers/obj.py +1000 -489
- swift/proxy/server.py +185 -112
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/AUTHORS +58 -11
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/METADATA +51 -56
- swift-2.35.0.dist-info/RECORD +201 -0
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
- swift-2.35.0.dist-info/pbr.json +1 -0
- swift/locale/de/LC_MESSAGES/swift.po +0 -1216
- swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
- swift/locale/es/LC_MESSAGES/swift.po +0 -1085
- swift/locale/fr/LC_MESSAGES/swift.po +0 -909
- swift/locale/it/LC_MESSAGES/swift.po +0 -894
- swift/locale/ja/LC_MESSAGES/swift.po +0 -965
- swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
- swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
- swift/locale/ru/LC_MESSAGES/swift.po +0 -891
- swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
- swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
- swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
- swift-2.23.3.data/scripts/swift-account-auditor +0 -23
- swift-2.23.3.data/scripts/swift-account-info +0 -51
- swift-2.23.3.data/scripts/swift-account-reaper +0 -23
- swift-2.23.3.data/scripts/swift-account-replicator +0 -34
- swift-2.23.3.data/scripts/swift-account-server +0 -23
- swift-2.23.3.data/scripts/swift-container-auditor +0 -23
- swift-2.23.3.data/scripts/swift-container-info +0 -55
- swift-2.23.3.data/scripts/swift-container-reconciler +0 -21
- swift-2.23.3.data/scripts/swift-container-replicator +0 -34
- swift-2.23.3.data/scripts/swift-container-sharder +0 -37
- swift-2.23.3.data/scripts/swift-container-sync +0 -23
- swift-2.23.3.data/scripts/swift-container-updater +0 -23
- swift-2.23.3.data/scripts/swift-dispersion-report +0 -24
- swift-2.23.3.data/scripts/swift-form-signature +0 -20
- swift-2.23.3.data/scripts/swift-init +0 -119
- swift-2.23.3.data/scripts/swift-object-auditor +0 -29
- swift-2.23.3.data/scripts/swift-object-expirer +0 -33
- swift-2.23.3.data/scripts/swift-object-info +0 -60
- swift-2.23.3.data/scripts/swift-object-reconstructor +0 -33
- swift-2.23.3.data/scripts/swift-object-relinker +0 -41
- swift-2.23.3.data/scripts/swift-object-replicator +0 -37
- swift-2.23.3.data/scripts/swift-object-server +0 -27
- swift-2.23.3.data/scripts/swift-object-updater +0 -23
- swift-2.23.3.data/scripts/swift-proxy-server +0 -23
- swift-2.23.3.data/scripts/swift-recon +0 -24
- swift-2.23.3.data/scripts/swift-ring-builder +0 -24
- swift-2.23.3.data/scripts/swift-ring-builder-analyzer +0 -22
- swift-2.23.3.data/scripts/swift-ring-composer +0 -22
- swift-2.23.3.dist-info/RECORD +0 -220
- swift-2.23.3.dist-info/pbr.json +0 -1
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/LICENSE +0 -0
- {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
swift/container/auditor.py
CHANGED
@@ -13,113 +13,27 @@
|
|
13
13
|
# See the License for the specific language governing permissions and
|
14
14
|
# limitations under the License.
|
15
15
|
|
16
|
-
import os
|
17
|
-
import time
|
18
|
-
from swift import gettext_ as _
|
19
|
-
from random import random
|
20
16
|
|
21
|
-
from
|
17
|
+
from swift.container.backend import ContainerBroker
|
18
|
+
from swift.common.daemon import run_daemon
|
19
|
+
from swift.common.db_auditor import DatabaseAuditor
|
20
|
+
from swift.common.utils import parse_options
|
22
21
|
|
23
|
-
import swift.common.db
|
24
|
-
from swift.container.backend import ContainerBroker, DATADIR
|
25
|
-
from swift.common.utils import get_logger, audit_location_generator, \
|
26
|
-
config_true_value, dump_recon_cache, ratelimit_sleep
|
27
|
-
from swift.common.daemon import Daemon
|
28
22
|
|
29
|
-
|
30
|
-
class ContainerAuditor(Daemon):
|
23
|
+
class ContainerAuditor(DatabaseAuditor):
|
31
24
|
"""Audit containers."""
|
32
25
|
|
33
|
-
|
34
|
-
|
35
|
-
self.logger = logger or get_logger(conf, log_route='container-auditor')
|
36
|
-
self.devices = conf.get('devices', '/srv/node')
|
37
|
-
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
|
38
|
-
self.interval = int(conf.get('interval', 1800))
|
39
|
-
self.container_passes = 0
|
40
|
-
self.container_failures = 0
|
41
|
-
self.containers_running_time = 0
|
42
|
-
self.max_containers_per_second = \
|
43
|
-
float(conf.get('containers_per_second', 200))
|
44
|
-
swift.common.db.DB_PREALLOCATION = \
|
45
|
-
config_true_value(conf.get('db_preallocation', 'f'))
|
46
|
-
self.recon_cache_path = conf.get('recon_cache_path',
|
47
|
-
'/var/cache/swift')
|
48
|
-
self.rcache = os.path.join(self.recon_cache_path, "container.recon")
|
26
|
+
server_type = "container"
|
27
|
+
broker_class = ContainerBroker
|
49
28
|
|
50
|
-
def
|
51
|
-
|
52
|
-
mount_check=self.mount_check,
|
53
|
-
logger=self.logger)
|
54
|
-
for path, device, partition in all_locs:
|
55
|
-
self.container_audit(path)
|
56
|
-
if time.time() - reported >= 3600: # once an hour
|
57
|
-
self.logger.info(
|
58
|
-
_('Since %(time)s: Container audits: %(pass)s passed '
|
59
|
-
'audit, %(fail)s failed audit'),
|
60
|
-
{'time': time.ctime(reported),
|
61
|
-
'pass': self.container_passes,
|
62
|
-
'fail': self.container_failures})
|
63
|
-
dump_recon_cache(
|
64
|
-
{'container_audits_since': reported,
|
65
|
-
'container_audits_passed': self.container_passes,
|
66
|
-
'container_audits_failed': self.container_failures},
|
67
|
-
self.rcache, self.logger)
|
68
|
-
reported = time.time()
|
69
|
-
self.container_passes = 0
|
70
|
-
self.container_failures = 0
|
71
|
-
self.containers_running_time = ratelimit_sleep(
|
72
|
-
self.containers_running_time, self.max_containers_per_second)
|
73
|
-
return reported
|
29
|
+
def _audit(self, job, broker):
|
30
|
+
return None
|
74
31
|
|
75
|
-
def run_forever(self, *args, **kwargs):
|
76
|
-
"""Run the container audit until stopped."""
|
77
|
-
reported = time.time()
|
78
|
-
time.sleep(random() * self.interval)
|
79
|
-
while True:
|
80
|
-
self.logger.info(_('Begin container audit pass.'))
|
81
|
-
begin = time.time()
|
82
|
-
try:
|
83
|
-
reported = self._one_audit_pass(reported)
|
84
|
-
except (Exception, Timeout):
|
85
|
-
self.logger.increment('errors')
|
86
|
-
self.logger.exception(_('ERROR auditing'))
|
87
|
-
elapsed = time.time() - begin
|
88
|
-
if elapsed < self.interval:
|
89
|
-
time.sleep(self.interval - elapsed)
|
90
|
-
self.logger.info(
|
91
|
-
_('Container audit pass completed: %.02fs'), elapsed)
|
92
|
-
dump_recon_cache({'container_auditor_pass_completed': elapsed},
|
93
|
-
self.rcache, self.logger)
|
94
32
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
begin = reported = time.time()
|
99
|
-
self._one_audit_pass(reported)
|
100
|
-
elapsed = time.time() - begin
|
101
|
-
self.logger.info(
|
102
|
-
_('Container audit "once" mode completed: %.02fs'), elapsed)
|
103
|
-
dump_recon_cache({'container_auditor_pass_completed': elapsed},
|
104
|
-
self.rcache, self.logger)
|
33
|
+
def main():
|
34
|
+
conf_file, options = parse_options(once=True)
|
35
|
+
run_daemon(ContainerAuditor, conf_file, **options)
|
105
36
|
|
106
|
-
def container_audit(self, path):
|
107
|
-
"""
|
108
|
-
Audits the given container path
|
109
37
|
|
110
|
-
|
111
|
-
|
112
|
-
start_time = time.time()
|
113
|
-
try:
|
114
|
-
broker = ContainerBroker(path, logger=self.logger)
|
115
|
-
if not broker.is_deleted():
|
116
|
-
broker.get_info()
|
117
|
-
self.logger.increment('passes')
|
118
|
-
self.container_passes += 1
|
119
|
-
self.logger.debug('Audit passed for %s', broker)
|
120
|
-
except (Exception, Timeout):
|
121
|
-
self.logger.increment('failures')
|
122
|
-
self.container_failures += 1
|
123
|
-
self.logger.exception(_('ERROR Could not get container info %s'),
|
124
|
-
path)
|
125
|
-
self.logger.timing_since('timing', start_time)
|
38
|
+
if __name__ == '__main__':
|
39
|
+
main()
|