swift 2.33.0__py2.py3-none-any.whl → 2.34.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. swift/account/auditor.py +11 -0
  2. swift/account/reaper.py +11 -1
  3. swift/account/replicator.py +22 -0
  4. swift/account/server.py +12 -1
  5. swift-2.33.0.data/scripts/swift-account-audit → swift/cli/account_audit.py +6 -2
  6. swift-2.33.0.data/scripts/swift-config → swift/cli/config.py +1 -1
  7. swift-2.33.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +6 -2
  8. swift-2.33.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +12 -3
  9. swift-2.33.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +6 -2
  10. swift/cli/info.py +103 -2
  11. swift-2.33.0.data/scripts/swift-oldies → swift/cli/oldies.py +6 -3
  12. swift-2.33.0.data/scripts/swift-orphans → swift/cli/orphans.py +7 -2
  13. swift/cli/recon_cron.py +5 -5
  14. swift-2.33.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  15. swift/cli/relinker.py +1 -1
  16. swift/cli/ringbuilder.py +24 -0
  17. swift/common/db.py +2 -1
  18. swift/common/db_auditor.py +2 -2
  19. swift/common/db_replicator.py +6 -0
  20. swift/common/exceptions.py +12 -0
  21. swift/common/manager.py +102 -0
  22. swift/common/memcached.py +6 -13
  23. swift/common/middleware/account_quotas.py +144 -43
  24. swift/common/middleware/backend_ratelimit.py +166 -24
  25. swift/common/middleware/catch_errors.py +1 -3
  26. swift/common/middleware/cname_lookup.py +3 -5
  27. swift/common/middleware/container_sync.py +6 -10
  28. swift/common/middleware/crypto/crypto_utils.py +4 -5
  29. swift/common/middleware/crypto/decrypter.py +4 -5
  30. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  31. swift/common/middleware/listing_formats.py +26 -38
  32. swift/common/middleware/proxy_logging.py +22 -16
  33. swift/common/middleware/ratelimit.py +6 -7
  34. swift/common/middleware/recon.py +6 -7
  35. swift/common/middleware/s3api/acl_handlers.py +9 -0
  36. swift/common/middleware/s3api/controllers/multi_upload.py +1 -9
  37. swift/common/middleware/s3api/controllers/obj.py +20 -1
  38. swift/common/middleware/s3api/s3api.py +2 -0
  39. swift/common/middleware/s3api/s3request.py +171 -62
  40. swift/common/middleware/s3api/s3response.py +35 -6
  41. swift/common/middleware/s3api/s3token.py +2 -2
  42. swift/common/middleware/s3api/utils.py +1 -0
  43. swift/common/middleware/slo.py +153 -52
  44. swift/common/middleware/tempauth.py +6 -4
  45. swift/common/middleware/tempurl.py +2 -2
  46. swift/common/middleware/x_profile/exceptions.py +1 -4
  47. swift/common/middleware/x_profile/html_viewer.py +10 -11
  48. swift/common/middleware/x_profile/profile_model.py +1 -2
  49. swift/common/middleware/xprofile.py +6 -2
  50. swift/common/request_helpers.py +69 -0
  51. swift/common/statsd_client.py +207 -0
  52. swift/common/utils/__init__.py +97 -1635
  53. swift/common/utils/base.py +138 -0
  54. swift/common/utils/config.py +443 -0
  55. swift/common/utils/logs.py +999 -0
  56. swift/common/wsgi.py +11 -3
  57. swift/container/auditor.py +11 -0
  58. swift/container/backend.py +10 -10
  59. swift/container/reconciler.py +11 -2
  60. swift/container/replicator.py +22 -1
  61. swift/container/server.py +12 -1
  62. swift/container/sharder.py +36 -12
  63. swift/container/sync.py +11 -1
  64. swift/container/updater.py +11 -2
  65. swift/obj/auditor.py +18 -2
  66. swift/obj/diskfile.py +8 -6
  67. swift/obj/expirer.py +155 -36
  68. swift/obj/reconstructor.py +28 -4
  69. swift/obj/replicator.py +61 -22
  70. swift/obj/server.py +64 -36
  71. swift/obj/updater.py +11 -2
  72. swift/proxy/controllers/base.py +38 -22
  73. swift/proxy/controllers/obj.py +23 -26
  74. swift/proxy/server.py +15 -1
  75. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/AUTHORS +11 -3
  76. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/METADATA +34 -35
  77. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/RECORD +82 -108
  78. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/WHEEL +1 -1
  79. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/entry_points.txt +38 -1
  80. swift-2.34.1.dist-info/pbr.json +1 -0
  81. swift-2.33.0.data/scripts/swift-account-auditor +0 -23
  82. swift-2.33.0.data/scripts/swift-account-info +0 -52
  83. swift-2.33.0.data/scripts/swift-account-reaper +0 -23
  84. swift-2.33.0.data/scripts/swift-account-replicator +0 -34
  85. swift-2.33.0.data/scripts/swift-account-server +0 -23
  86. swift-2.33.0.data/scripts/swift-container-auditor +0 -23
  87. swift-2.33.0.data/scripts/swift-container-info +0 -59
  88. swift-2.33.0.data/scripts/swift-container-reconciler +0 -21
  89. swift-2.33.0.data/scripts/swift-container-replicator +0 -34
  90. swift-2.33.0.data/scripts/swift-container-server +0 -23
  91. swift-2.33.0.data/scripts/swift-container-sharder +0 -37
  92. swift-2.33.0.data/scripts/swift-container-sync +0 -23
  93. swift-2.33.0.data/scripts/swift-container-updater +0 -23
  94. swift-2.33.0.data/scripts/swift-dispersion-report +0 -24
  95. swift-2.33.0.data/scripts/swift-form-signature +0 -20
  96. swift-2.33.0.data/scripts/swift-init +0 -119
  97. swift-2.33.0.data/scripts/swift-object-auditor +0 -29
  98. swift-2.33.0.data/scripts/swift-object-expirer +0 -33
  99. swift-2.33.0.data/scripts/swift-object-info +0 -60
  100. swift-2.33.0.data/scripts/swift-object-reconstructor +0 -33
  101. swift-2.33.0.data/scripts/swift-object-relinker +0 -23
  102. swift-2.33.0.data/scripts/swift-object-replicator +0 -37
  103. swift-2.33.0.data/scripts/swift-object-server +0 -27
  104. swift-2.33.0.data/scripts/swift-object-updater +0 -23
  105. swift-2.33.0.data/scripts/swift-proxy-server +0 -23
  106. swift-2.33.0.data/scripts/swift-recon +0 -24
  107. swift-2.33.0.data/scripts/swift-recon-cron +0 -24
  108. swift-2.33.0.data/scripts/swift-ring-builder +0 -37
  109. swift-2.33.0.data/scripts/swift-ring-builder-analyzer +0 -22
  110. swift-2.33.0.data/scripts/swift-ring-composer +0 -22
  111. swift-2.33.0.dist-info/pbr.json +0 -1
  112. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/LICENSE +0 -0
  113. {swift-2.33.0.dist-info → swift-2.34.1.dist-info}/top_level.txt +0 -0
swift/common/manager.py CHANGED
@@ -13,9 +13,11 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
+
16
17
  from __future__ import print_function
17
18
  import functools
18
19
  import errno
20
+ from optparse import OptionParser
19
21
  import os
20
22
  import resource
21
23
  import signal
@@ -23,6 +25,7 @@ import time
23
25
  import subprocess
24
26
  import re
25
27
  import six
28
+ import sys
26
29
  import tempfile
27
30
  try:
28
31
  from shutil import which
@@ -936,3 +939,102 @@ class Server(object):
936
939
 
937
940
  """
938
941
  return self.kill_running_pids(**kwargs)
942
+
943
+
944
+ USAGE = \
945
+ """%prog <server>[.<config>] [<server>[.<config>] ...] <command> [options]
946
+
947
+ where:
948
+ <server> is the name of a swift service e.g. proxy-server.
949
+ The '-server' part of the name may be omitted.
950
+ 'all', 'main' and 'rest' are reserved words that represent a
951
+ group of services.
952
+ all: Expands to all swift daemons.
953
+ main: Expands to main swift daemons.
954
+ (proxy, container, account, object)
955
+ rest: Expands to all remaining background daemons (beyond
956
+ "main").
957
+ (updater, replicator, auditor, etc)
958
+ <config> is an explicit configuration filename without the
959
+ .conf extension. If <config> is specified then <server> should
960
+ refer to a directory containing the configuration file, e.g.:
961
+
962
+ swift-init object.1 start
963
+
964
+ will start an object-server using the configuration file
965
+ /etc/swift/object-server/1.conf
966
+ <command> is a command from the list below.
967
+
968
+ Commands:
969
+ """ + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])
970
+
971
+
972
+ def main():
973
+ parser = OptionParser(USAGE)
974
+ parser.add_option('-v', '--verbose', action="store_true",
975
+ default=False, help="display verbose output")
976
+ parser.add_option('-w', '--no-wait', action="store_false", dest="wait",
977
+ default=True, help="won't wait for server to start "
978
+ "before returning")
979
+ parser.add_option('-o', '--once', action="store_true",
980
+ default=False, help="only run one pass of daemon")
981
+ # this is a negative option, default is options.daemon = True
982
+ parser.add_option('-n', '--no-daemon', action="store_false", dest="daemon",
983
+ default=True, help="start server interactively")
984
+ parser.add_option('-g', '--graceful', action="store_true",
985
+ default=False, help="send SIGHUP to supporting servers")
986
+ parser.add_option('-c', '--config-num', metavar="N", type="int",
987
+ dest="number", default=0,
988
+ help="send command to the Nth server only")
989
+ parser.add_option('-k', '--kill-wait', metavar="N", type="int",
990
+ dest="kill_wait", default=KILL_WAIT,
991
+ help="wait N seconds for processes to die (default 15)")
992
+ parser.add_option('-r', '--run-dir', type="str",
993
+ dest="run_dir", default=RUN_DIR,
994
+ help="alternative directory to store running pid files "
995
+ "default: %s" % RUN_DIR)
996
+ # Changing behaviour if missing config
997
+ parser.add_option('--strict', dest='strict', action='store_true',
998
+ help="Return non-zero status code if some config is "
999
+ "missing. Default mode if all servers are "
1000
+ "explicitly named.")
1001
+ # a negative option for strict
1002
+ parser.add_option('--non-strict', dest='strict', action='store_false',
1003
+ help="Return zero status code even if some config is "
1004
+ "missing. Default mode if any server is a glob or "
1005
+ "one of aliases `all`, `main` or `rest`.")
1006
+ # SIGKILL daemon after kill_wait period
1007
+ parser.add_option('--kill-after-timeout', dest='kill_after_timeout',
1008
+ action='store_true',
1009
+ help="Kill daemon and all children after kill-wait "
1010
+ "period.")
1011
+
1012
+ options, args = parser.parse_args()
1013
+
1014
+ if len(args) < 2:
1015
+ parser.print_help()
1016
+ print('ERROR: specify server(s) and command')
1017
+ return 1
1018
+
1019
+ command = args[-1]
1020
+ servers = args[:-1]
1021
+
1022
+ # this is just a silly swap for me cause I always try to "start main"
1023
+ commands = dict(Manager.list_commands()).keys()
1024
+ if command not in commands and servers[0] in commands:
1025
+ servers.append(command)
1026
+ command = servers.pop(0)
1027
+
1028
+ manager = Manager(servers, run_dir=options.run_dir)
1029
+ try:
1030
+ status = manager.run_command(command, **options.__dict__)
1031
+ except UnknownCommandError:
1032
+ parser.print_help()
1033
+ print('ERROR: unknown command, %s' % command)
1034
+ status = 1
1035
+
1036
+ return 1 if status else 0
1037
+
1038
+
1039
+ if __name__ == "__main__":
1040
+ sys.exit(main())
swift/common/memcached.py CHANGED
@@ -59,6 +59,8 @@ from eventlet import Timeout
59
59
  from six.moves import range
60
60
  from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
61
61
  from swift.common import utils
62
+ from swift.common.exceptions import MemcacheConnectionError, \
63
+ MemcacheIncrNotFoundError, MemcachePoolTimeout
62
64
  from swift.common.utils import md5, human_readable, config_true_value, \
63
65
  memcached_timing_stats
64
66
 
@@ -83,6 +85,9 @@ TIMING_SAMPLE_RATE_HIGH = 0.1
83
85
  TIMING_SAMPLE_RATE_MEDIUM = 0.01
84
86
  TIMING_SAMPLE_RATE_LOW = 0.001
85
87
 
88
+ # The max value of a delta expiration time.
89
+ EXPTIME_MAXDELTA = 30 * 24 * 60 * 60
90
+
86
91
 
87
92
  def md5hash(key):
88
93
  if not isinstance(key, bytes):
@@ -100,7 +105,7 @@ def sanitize_timeout(timeout):
100
105
  translates negative values to mean a delta of 30 days in seconds (and 1
101
106
  additional second), client beware.
102
107
  """
103
- if timeout > (30 * 24 * 60 * 60):
108
+ if timeout > EXPTIME_MAXDELTA:
104
109
  timeout += tm.time()
105
110
  return int(timeout)
106
111
 
@@ -119,18 +124,6 @@ def set_msg(key, flags, timeout, value):
119
124
  ]) + (b'\r\n' + value + b'\r\n')
120
125
 
121
126
 
122
- class MemcacheConnectionError(Exception):
123
- pass
124
-
125
-
126
- class MemcacheIncrNotFoundError(MemcacheConnectionError):
127
- pass
128
-
129
-
130
- class MemcachePoolTimeout(Timeout):
131
- pass
132
-
133
-
134
127
  class MemcacheConnPool(Pool):
135
128
  """
136
129
  Connection pool for Memcache Connections
@@ -18,15 +18,39 @@
18
18
  given account quota (in bytes) is exceeded while DELETE requests are still
19
19
  allowed.
20
20
 
21
- ``account_quotas`` uses the ``x-account-meta-quota-bytes`` metadata entry to
22
- store the overall account quota. Write requests to this metadata entry are
23
- only permitted for resellers. There is no overall account quota limit if
24
- ``x-account-meta-quota-bytes`` is not set.
25
-
26
- Additionally, account quotas may be set for each storage policy, using metadata
27
- of the form ``x-account-quota-bytes-policy-<policy name>``. Again, only
28
- resellers may update these metadata, and there will be no limit for a
29
- particular policy if the corresponding metadata is not set.
21
+ ``account_quotas`` uses the following metadata entries to store the account
22
+ quota
23
+
24
+ +---------------------------------------------+-------------------------------+
25
+ |Metadata | Use |
26
+ +=============================================+===============================+
27
+ | X-Account-Meta-Quota-Bytes (obsoleted) | Maximum overall bytes stored |
28
+ | | in account across containers. |
29
+ +---------------------------------------------+-------------------------------+
30
+ | X-Account-Quota-Bytes | Maximum overall bytes stored |
31
+ | | in account across containers. |
32
+ +---------------------------------------------+-------------------------------+
33
+ | X-Account-Quota-Bytes-Policy-<policyname> | Maximum overall bytes stored |
34
+ | | in account across containers, |
35
+ | | for the given policy. |
36
+ +---------------------------------------------+-------------------------------+
37
+ | X-Account-Quota-Count | Maximum object count under |
38
+ | | account. |
39
+ +---------------------------------------------+-------------------------------+
40
+ | X-Account-Quota-Count-Policy-<policyname> | Maximum object count under |
41
+ | | account, for the given policy.|
42
+ +---------------------------------------------+-------------------------------+
43
+
44
+
45
+ Write requests to those metadata entries are only permitted for resellers.
46
+ There is no overall byte or object count limit set if the corresponding
47
+ metadata entries are not set.
48
+
49
+ Additionally, account quotas, of type quota-bytes or quota-count, may be set
50
+ for each storage policy, using metadata of the form ``x-account-<quota type>-\
51
+ policy-<policy name>``. Again, only resellers may update these metadata, and
52
+ there will be no limit for a particular policy if the corresponding metadata
53
+ is not set.
30
54
 
31
55
  .. note::
32
56
  Per-policy quotas need not sum to the overall account quota, and the sum of
@@ -78,45 +102,69 @@ class AccountQuotaMiddleware(object):
78
102
  def __init__(self, app, *args, **kwargs):
79
103
  self.app = app
80
104
 
105
+ def validate_and_translate_quotas(self, request, quota_type):
106
+ new_quotas = {}
107
+ new_quotas[None] = request.headers.get(
108
+ 'X-Account-%s' % quota_type)
109
+ if request.headers.get(
110
+ 'X-Remove-Account-%s' % quota_type):
111
+ new_quotas[None] = '' # X-Remove dominates if both are present
112
+
113
+ for policy in POLICIES:
114
+ tail = 'Account-%s-Policy-%s' % (quota_type, policy.name)
115
+ if request.headers.get('X-Remove-' + tail):
116
+ new_quotas[policy.idx] = ''
117
+ else:
118
+ quota = request.headers.pop('X-' + tail, None)
119
+ new_quotas[policy.idx] = quota
120
+
121
+ if request.environ.get('reseller_request') is True:
122
+ if any(quota and not quota.isdigit()
123
+ for quota in new_quotas.values()):
124
+ raise HTTPBadRequest()
125
+ for idx, quota in new_quotas.items():
126
+ if idx is None:
127
+ hdr = 'X-Account-Sysmeta-%s' % quota_type
128
+ else:
129
+ hdr = 'X-Account-Sysmeta-%s-Policy-%d' % (quota_type, idx)
130
+ request.headers[hdr] = quota
131
+ elif any(quota is not None for quota in new_quotas.values()):
132
+ # deny quota set for non-reseller
133
+ raise HTTPForbidden()
134
+
81
135
  def handle_account(self, request):
82
136
  if request.method in ("POST", "PUT"):
137
+ # Support old meta format
138
+ for legacy_header in [
139
+ 'X-Account-Meta-Quota-Bytes',
140
+ 'X-Remove-Account-Meta-Quota-Bytes',
141
+ ]:
142
+ new_header = legacy_header.replace('-Meta-', '-')
143
+ legacy_value = request.headers.get(legacy_header)
144
+ if legacy_value is not None and not \
145
+ request.headers.get(new_header):
146
+ request.headers[new_header] = legacy_value
83
147
  # account request, so we pay attention to the quotas
84
- new_quotas = {}
85
- new_quotas[None] = request.headers.get(
86
- 'X-Account-Meta-Quota-Bytes')
87
- if request.headers.get(
88
- 'X-Remove-Account-Meta-Quota-Bytes'):
89
- new_quotas[None] = 0 # X-Remove dominates if both are present
90
-
91
- for policy in POLICIES:
92
- tail = 'Account-Quota-Bytes-Policy-%s' % policy.name
93
- if request.headers.get('X-Remove-' + tail):
94
- new_quotas[policy.idx] = 0
95
- else:
96
- quota = request.headers.pop('X-' + tail, None)
97
- new_quotas[policy.idx] = quota
98
-
99
- if request.environ.get('reseller_request') is True:
100
- if any(quota and not quota.isdigit()
101
- for quota in new_quotas.values()):
102
- return HTTPBadRequest()
103
- for idx, quota in new_quotas.items():
104
- if idx is None:
105
- continue # For legacy reasons, it's in user meta
106
- hdr = 'X-Account-Sysmeta-Quota-Bytes-Policy-%d' % idx
107
- request.headers[hdr] = quota
108
- elif any(quota is not None for quota in new_quotas.values()):
109
- # deny quota set for non-reseller
110
- return HTTPForbidden()
111
-
148
+ self.validate_and_translate_quotas(request, "Quota-Bytes")
149
+ self.validate_and_translate_quotas(request, "Quota-Count")
112
150
  resp = request.get_response(self.app)
113
151
  # Non-resellers can't update quotas, but they *can* see them
114
- for policy in POLICIES:
115
- infix = 'Quota-Bytes-Policy'
116
- value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
117
- infix, policy.idx))
152
+ # Global quotas
153
+ postfixes = ('Quota-Bytes', 'Quota-Count')
154
+ for postfix in postfixes:
155
+ value = resp.headers.get('X-Account-Sysmeta-%s' % postfix)
118
156
  if value:
119
- resp.headers['X-Account-%s-%s' % (infix, policy.name)] = value
157
+ resp.headers['X-Account-%s' % postfix] = value
158
+
159
+ # Per policy quotas
160
+ for policy in POLICIES:
161
+ infixes = ('Quota-Bytes-Policy', 'Quota-Count-Policy')
162
+ for infix in infixes:
163
+ value = resp.headers.get('X-Account-Sysmeta-%s-%d' % (
164
+ infix, policy.idx))
165
+ if value:
166
+ resp.headers['X-Account-%s-%s' % (
167
+ infix, policy.name)] = value
120
168
  return resp
121
169
 
122
170
  @wsgify
@@ -148,8 +196,14 @@ class AccountQuotaMiddleware(object):
148
196
  swift_source='AQ')
149
197
  if not account_info:
150
198
  return self.app
199
+
200
+ # Check for quota byte violation
151
201
  try:
152
- quota = int(account_info['meta'].get('quota-bytes', -1))
202
+ quota = int(
203
+ account_info["sysmeta"].get(
204
+ "quota-bytes", account_info["meta"].get("quota-bytes", -1)
205
+ )
206
+ )
153
207
  except ValueError:
154
208
  quota = -1
155
209
  if quota >= 0:
@@ -168,11 +222,34 @@ class AccountQuotaMiddleware(object):
168
222
  else:
169
223
  return resp
170
224
 
225
+ # Check for quota count violation
226
+ try:
227
+ quota = int(account_info['sysmeta'].get('quota-count', -1))
228
+ except ValueError:
229
+ quota = -1
230
+ if quota >= 0:
231
+ new_count = int(account_info['total_object_count']) + 1
232
+ if quota < new_count:
233
+ resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.')
234
+ if 'swift.authorize' in request.environ:
235
+ orig_authorize = request.environ['swift.authorize']
236
+
237
+ def reject_authorize(*args, **kwargs):
238
+ aresp = orig_authorize(*args, **kwargs)
239
+ if aresp:
240
+ return aresp
241
+ return resp
242
+ request.environ['swift.authorize'] = reject_authorize
243
+ else:
244
+ return resp
245
+
171
246
  container_info = get_container_info(request.environ, self.app,
172
247
  swift_source='AQ')
173
248
  if not container_info:
174
249
  return self.app
175
250
  policy_idx = container_info['storage_policy']
251
+
252
+ # Check quota-byte per policy
176
253
  sysmeta_key = 'quota-bytes-policy-%s' % policy_idx
177
254
  try:
178
255
  policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
@@ -196,6 +273,30 @@ class AccountQuotaMiddleware(object):
196
273
  else:
197
274
  return resp
198
275
 
276
+ # Check quota-count per policy
277
+ sysmeta_key = 'quota-count-policy-%s' % policy_idx
278
+ try:
279
+ policy_quota = int(account_info['sysmeta'].get(sysmeta_key, -1))
280
+ except ValueError:
281
+ policy_quota = -1
282
+ if policy_quota >= 0:
283
+ policy_stats = account_info['storage_policies'].get(policy_idx, {})
284
+ new_size = int(policy_stats.get('object_count', 0)) + 1
285
+ if policy_quota < new_size:
286
+ resp = HTTPRequestEntityTooLarge(
287
+ body='Upload exceeds policy quota.')
288
+ if 'swift.authorize' in request.environ:
289
+ orig_authorize = request.environ['swift.authorize']
290
+
291
+ def reject_authorize(*args, **kwargs):
292
+ aresp = orig_authorize(*args, **kwargs)
293
+ if aresp:
294
+ return aresp
295
+ return resp
296
+ request.environ['swift.authorize'] = reject_authorize
297
+ else:
298
+ return resp
299
+
199
300
  return self.app
200
301
 
201
302
 
@@ -12,47 +12,188 @@
12
12
  # implied.
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
-
15
+ import os
16
16
  import time
17
- from collections import defaultdict
18
17
 
19
18
  from swift.common.request_helpers import split_and_validate_path
20
19
  from swift.common.swob import Request, HTTPTooManyBackendRequests, \
21
20
  HTTPException
22
21
  from swift.common.utils import get_logger, non_negative_float, \
23
- EventletRateLimiter
22
+ EventletRateLimiter, readconf
24
23
 
25
24
  RATE_LIMITED_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE',
26
25
  'REPLICATE')
26
+ BACKEND_RATELIMIT_CONFIG_SECTION = 'backend_ratelimit'
27
+ DEFAULT_BACKEND_RATELIMIT_CONF_FILE = 'backend-ratelimit.conf'
28
+ DEFAULT_CONFIG_RELOAD_INTERVAL = 60.0
29
+ DEFAULT_REQUESTS_PER_DEVICE_PER_SECOND = 0.0
30
+ DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER = 1.0
27
31
 
28
32
 
29
33
  class BackendRateLimitMiddleware(object):
30
34
  """
31
35
  Backend rate-limiting middleware.
32
36
 
33
- Rate-limits requests to backend storage node devices. Each device is
34
- independently rate-limited. All requests with a 'GET', 'HEAD', 'PUT',
35
- 'POST', 'DELETE', 'UPDATE' or 'REPLICATE' method are included in a device's
36
- rate limit.
37
+ Rate-limits requests to backend storage node devices. Each (device, request
38
+ method) combination is independently rate-limited. All requests with a
39
+ 'GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'UPDATE' or 'REPLICATE' method are
40
+ rate limited on a per-device basis by both a method-specific rate and an
41
+ overall device rate limit.
37
42
 
38
- If a request would cause the rate-limit to be exceeded then a response with
39
- a 529 status code is returned.
43
+ If a request would cause the rate-limit to be exceeded for the method
44
+ and/or device then a response with a 529 status code is returned.
40
45
  """
41
- def __init__(self, app, conf, logger=None):
46
+ def __init__(self, app, filter_conf, logger=None):
42
47
  self.app = app
43
- self.logger = logger or get_logger(conf, log_route='backend_ratelimit')
44
- self.requests_per_device_per_second = non_negative_float(
45
- conf.get('requests_per_device_per_second', 0.0))
46
- self.requests_per_device_rate_buffer = non_negative_float(
47
- conf.get('requests_per_device_rate_buffer', 1.0))
48
-
49
- # map device -> RateLimiter
50
- self.rate_limiters = defaultdict(
51
- lambda: EventletRateLimiter(
52
- max_rate=self.requests_per_device_per_second,
48
+ self.filter_conf = filter_conf
49
+ self.logger = logger or get_logger(self.filter_conf,
50
+ log_route='backend_ratelimit')
51
+ self.requests_per_device_rate_buffer = \
52
+ DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER
53
+ # map (device, method) -> rate
54
+ self.requests_per_device_per_second = {}
55
+ # map (device, method) -> RateLimiter, populated on-demand
56
+ self.rate_limiters = {}
57
+
58
+ # some config options are *only* read from filter conf at startup...
59
+ default_conf_path = os.path.join(
60
+ self.filter_conf.get('swift_dir', '/etc/swift'),
61
+ DEFAULT_BACKEND_RATELIMIT_CONF_FILE)
62
+ try:
63
+ self.conf_path = self.filter_conf['backend_ratelimit_conf_path']
64
+ self.is_config_file_expected = True
65
+ except KeyError:
66
+ self.conf_path = default_conf_path
67
+ self.is_config_file_expected = False
68
+ self.config_reload_interval = non_negative_float(
69
+ filter_conf.get('config_reload_interval',
70
+ DEFAULT_CONFIG_RELOAD_INTERVAL))
71
+
72
+ # other conf options are read from filter section at startup but may
73
+ # also be overridden by options in a separate config file...
74
+ self._last_config_reload_attempt = time.time()
75
+ self._apply_config(self.filter_conf)
76
+ self._load_config_file()
77
+
78
+ def _refresh_ratelimiters(self):
79
+ # note: if we ever wanted to prune the ratelimiters (in case devices
80
+ # have been removed) we could inspect each ratelimiter's running_time
81
+ # and remove those with very old running_time
82
+ for (dev, method), rl in self.rate_limiters.items():
83
+ rl.set_max_rate(self.requests_per_device_per_second[method])
84
+ rl.set_rate_buffer(self.requests_per_device_rate_buffer)
85
+
86
+ def _apply_config(self, conf):
87
+ modified = False
88
+ reqs_per_device_rate_buffer = non_negative_float(
89
+ conf.get('requests_per_device_rate_buffer',
90
+ DEFAULT_REQUESTS_PER_DEVICE_RATE_BUFFER))
91
+
92
+ # note: 'None' key holds the aggregate per-device limit for all methods
93
+ reqs_per_device_per_second = {None: non_negative_float(
94
+ conf.get('requests_per_device_per_second', 0.0))}
95
+ for method in RATE_LIMITED_METHODS:
96
+ val = non_negative_float(
97
+ conf.get('%s_requests_per_device_per_second'
98
+ % method.lower(), 0.0))
99
+ reqs_per_device_per_second[method] = val
100
+
101
+ if reqs_per_device_rate_buffer != self.requests_per_device_rate_buffer:
102
+ self.requests_per_device_rate_buffer = reqs_per_device_rate_buffer
103
+ modified = True
104
+ if reqs_per_device_per_second != self.requests_per_device_per_second:
105
+ self.requests_per_device_per_second = reqs_per_device_per_second
106
+ self.is_any_rate_limit_configured = any(
107
+ self.requests_per_device_per_second.values())
108
+ modified = True
109
+ if modified:
110
+ self._refresh_ratelimiters()
111
+ return modified
112
+
113
+ def _load_config_file(self):
114
+ # If conf file can be read then apply its options to the filter conf
115
+ # options, discarding *all* options previously loaded from the conf
116
+ # file i.e. options deleted from the conf file will revert to the
117
+ # filter conf value or default value. If the conf file cannot be read
118
+ # or is invalid, then the current config is left unchanged.
119
+ try:
120
+ new_conf = dict(self.filter_conf) # filter_conf not current conf
121
+ new_conf.update(
122
+ readconf(self.conf_path, BACKEND_RATELIMIT_CONFIG_SECTION))
123
+ modified = self._apply_config(new_conf)
124
+ if modified:
125
+ self.logger.info('Loaded config file %s, config changed',
126
+ self.conf_path)
127
+ elif not self.is_config_file_expected:
128
+ self.logger.info('Loaded new config file %s, config unchanged',
129
+ self.conf_path)
130
+ else:
131
+ self.logger.debug(
132
+ 'Loaded existing config file %s, config unchanged',
133
+ self.conf_path)
134
+ self.is_config_file_expected = True
135
+ except IOError as err:
136
+ if self.is_config_file_expected:
137
+ self.logger.warning(
138
+ 'Failed to load config file, config unchanged: %s', err)
139
+ self.is_config_file_expected = False
140
+ except ValueError as err:
141
+ # ...but if it exists it should be valid
142
+ self.logger.warning('Invalid config file %s, config unchanged: %s',
143
+ self.conf_path, err)
144
+
145
+ def _maybe_reload_config(self):
146
+ if self.config_reload_interval:
147
+ now = time.time()
148
+ if (now - self._last_config_reload_attempt
149
+ >= self.config_reload_interval):
150
+ try:
151
+ self._load_config_file()
152
+ except Exception: # noqa
153
+ self.logger.exception('Error reloading config file')
154
+ finally:
155
+ # always reset last loaded time to avoid re-try storm
156
+ self._last_config_reload_attempt = now
157
+
158
+ def _get_ratelimiter(self, device, method=None):
159
+ """
160
+ Get a rate limiter for the (device, method) combination. If a rate
161
+ limiter does not yet exist for the given (device, method) combination
162
+ then it is created and added to the map of rate limiters.
163
+
164
+ :param: the device.
165
+ :method: the request method; if None then the aggregate rate limiter
166
+ for all requests to the device is returned.
167
+ :returns: an instance of ``EventletRateLimiter``.
168
+ """
169
+ try:
170
+ rl = self.rate_limiters[(device, method)]
171
+ except KeyError:
172
+ rl = EventletRateLimiter(
173
+ max_rate=self.requests_per_device_per_second[method],
53
174
  rate_buffer=self.requests_per_device_rate_buffer,
54
175
  running_time=time.time(),
55
- burst_after_idle=True))
176
+ burst_after_idle=True)
177
+ self.rate_limiters[(device, method)] = rl
178
+ return rl
179
+
180
+ def _is_allowed(self, device, method):
181
+ """
182
+ Evaluate backend rate-limiting policies for the incoming request.
183
+
184
+ A request is allowed when neither the per-(device, method) rate-limit
185
+ nor the per-device rate-limit has been reached.
186
+
187
+ Note: a request will be disallowed if the aggregate per-device
188
+ rate-limit has been reached, even if the per-(device, method)
189
+ rate-limit has not been reached for the request's method.
190
+
191
+ :param: the device.
192
+ :method: the request method.
193
+ :returns: boolean, is_allowed.
194
+ """
195
+ return (self._get_ratelimiter(device, None).is_allowed()
196
+ and self._get_ratelimiter(device, method).is_allowed())
56
197
 
57
198
  def __call__(self, env, start_response):
58
199
  """
@@ -61,9 +202,11 @@ class BackendRateLimitMiddleware(object):
61
202
  :param env: WSGI environment dictionary
62
203
  :param start_response: WSGI callable
63
204
  """
205
+ self._maybe_reload_config()
64
206
  req = Request(env)
65
207
  handler = self.app
66
- if req.method in RATE_LIMITED_METHODS:
208
+ if (self.is_any_rate_limit_configured
209
+ and req.method in RATE_LIMITED_METHODS):
67
210
  try:
68
211
  device, partition, _ = split_and_validate_path(req, 1, 3, True)
69
212
  int(partition) # check it's a valid partition
@@ -71,8 +214,7 @@ class BackendRateLimitMiddleware(object):
71
214
  # request may not have device/partition e.g. a healthcheck req
72
215
  pass
73
216
  else:
74
- rate_limiter = self.rate_limiters[device]
75
- if not rate_limiter.is_allowed():
217
+ if not self._is_allowed(device, req.method):
76
218
  self.logger.increment('backend.ratelimit')
77
219
  handler = HTTPTooManyBackendRequests()
78
220
  return handler(env, start_response)
@@ -13,8 +13,6 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
- from swift import gettext_ as _
17
-
18
16
  from swift.common.swob import Request, HTTPServerError
19
17
  from swift.common.utils import get_logger, generate_trans_id, close_if_possible
20
18
  from swift.common.wsgi import WSGIContext
@@ -74,7 +72,7 @@ class CatchErrorsContext(WSGIContext):
74
72
  # catch any errors in the pipeline
75
73
  resp = self._app_call(env)
76
74
  except: # noqa
77
- self.logger.exception(_('Error: An error occurred'))
75
+ self.logger.exception('Error: An error occurred')
78
76
  resp = HTTPServerError(request=Request(env),
79
77
  body=b'An error occurred',
80
78
  content_type='text/plain')