swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. swift/__init__.py +29 -50
  2. swift/account/auditor.py +21 -118
  3. swift/account/backend.py +33 -28
  4. swift/account/reaper.py +37 -28
  5. swift/account/replicator.py +22 -0
  6. swift/account/server.py +60 -26
  7. swift/account/utils.py +28 -11
  8. swift-2.23.3.data/scripts/swift-account-audit → swift/cli/account_audit.py +23 -13
  9. swift-2.23.3.data/scripts/swift-config → swift/cli/config.py +2 -2
  10. swift/cli/container_deleter.py +5 -11
  11. swift-2.23.3.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py +8 -7
  12. swift/cli/dispersion_report.py +10 -9
  13. swift-2.23.3.data/scripts/swift-drive-audit → swift/cli/drive_audit.py +63 -21
  14. swift/cli/form_signature.py +3 -7
  15. swift-2.23.3.data/scripts/swift-get-nodes → swift/cli/get_nodes.py +8 -2
  16. swift/cli/info.py +154 -14
  17. swift/cli/manage_shard_ranges.py +705 -37
  18. swift-2.23.3.data/scripts/swift-oldies → swift/cli/oldies.py +25 -14
  19. swift-2.23.3.data/scripts/swift-orphans → swift/cli/orphans.py +7 -3
  20. swift/cli/recon.py +196 -67
  21. swift-2.23.3.data/scripts/swift-recon-cron → swift/cli/recon_cron.py +17 -20
  22. swift-2.23.3.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py +2 -3
  23. swift/cli/relinker.py +807 -126
  24. swift/cli/reload.py +135 -0
  25. swift/cli/ringbuilder.py +217 -20
  26. swift/cli/ringcomposer.py +0 -1
  27. swift/cli/shard-info.py +4 -3
  28. swift/common/base_storage_server.py +9 -20
  29. swift/common/bufferedhttp.py +48 -74
  30. swift/common/constraints.py +20 -15
  31. swift/common/container_sync_realms.py +9 -11
  32. swift/common/daemon.py +25 -8
  33. swift/common/db.py +195 -128
  34. swift/common/db_auditor.py +168 -0
  35. swift/common/db_replicator.py +95 -55
  36. swift/common/digest.py +141 -0
  37. swift/common/direct_client.py +144 -33
  38. swift/common/error_limiter.py +93 -0
  39. swift/common/exceptions.py +25 -1
  40. swift/common/header_key_dict.py +2 -9
  41. swift/common/http_protocol.py +373 -0
  42. swift/common/internal_client.py +129 -59
  43. swift/common/linkat.py +3 -4
  44. swift/common/manager.py +284 -67
  45. swift/common/memcached.py +390 -145
  46. swift/common/middleware/__init__.py +4 -0
  47. swift/common/middleware/account_quotas.py +211 -46
  48. swift/common/middleware/acl.py +3 -8
  49. swift/common/middleware/backend_ratelimit.py +230 -0
  50. swift/common/middleware/bulk.py +22 -34
  51. swift/common/middleware/catch_errors.py +1 -3
  52. swift/common/middleware/cname_lookup.py +6 -11
  53. swift/common/middleware/container_quotas.py +1 -1
  54. swift/common/middleware/container_sync.py +39 -17
  55. swift/common/middleware/copy.py +12 -0
  56. swift/common/middleware/crossdomain.py +22 -9
  57. swift/common/middleware/crypto/__init__.py +2 -1
  58. swift/common/middleware/crypto/crypto_utils.py +11 -15
  59. swift/common/middleware/crypto/decrypter.py +28 -11
  60. swift/common/middleware/crypto/encrypter.py +12 -17
  61. swift/common/middleware/crypto/keymaster.py +8 -15
  62. swift/common/middleware/crypto/kms_keymaster.py +2 -1
  63. swift/common/middleware/dlo.py +15 -11
  64. swift/common/middleware/domain_remap.py +5 -4
  65. swift/common/middleware/etag_quoter.py +128 -0
  66. swift/common/middleware/formpost.py +73 -70
  67. swift/common/middleware/gatekeeper.py +8 -1
  68. swift/common/middleware/keystoneauth.py +33 -3
  69. swift/common/middleware/list_endpoints.py +4 -4
  70. swift/common/middleware/listing_formats.py +85 -49
  71. swift/common/middleware/memcache.py +4 -95
  72. swift/common/middleware/name_check.py +3 -2
  73. swift/common/middleware/proxy_logging.py +160 -92
  74. swift/common/middleware/ratelimit.py +17 -10
  75. swift/common/middleware/read_only.py +6 -4
  76. swift/common/middleware/recon.py +59 -22
  77. swift/common/middleware/s3api/acl_handlers.py +25 -3
  78. swift/common/middleware/s3api/acl_utils.py +6 -1
  79. swift/common/middleware/s3api/controllers/__init__.py +6 -0
  80. swift/common/middleware/s3api/controllers/acl.py +3 -2
  81. swift/common/middleware/s3api/controllers/bucket.py +242 -137
  82. swift/common/middleware/s3api/controllers/logging.py +2 -2
  83. swift/common/middleware/s3api/controllers/multi_delete.py +43 -20
  84. swift/common/middleware/s3api/controllers/multi_upload.py +219 -133
  85. swift/common/middleware/s3api/controllers/obj.py +112 -8
  86. swift/common/middleware/s3api/controllers/object_lock.py +44 -0
  87. swift/common/middleware/s3api/controllers/s3_acl.py +2 -2
  88. swift/common/middleware/s3api/controllers/tagging.py +57 -0
  89. swift/common/middleware/s3api/controllers/versioning.py +36 -7
  90. swift/common/middleware/s3api/etree.py +22 -9
  91. swift/common/middleware/s3api/exception.py +0 -4
  92. swift/common/middleware/s3api/s3api.py +113 -41
  93. swift/common/middleware/s3api/s3request.py +384 -218
  94. swift/common/middleware/s3api/s3response.py +126 -23
  95. swift/common/middleware/s3api/s3token.py +16 -17
  96. swift/common/middleware/s3api/schema/delete.rng +1 -1
  97. swift/common/middleware/s3api/subresource.py +7 -10
  98. swift/common/middleware/s3api/utils.py +27 -10
  99. swift/common/middleware/slo.py +665 -358
  100. swift/common/middleware/staticweb.py +64 -37
  101. swift/common/middleware/symlink.py +51 -18
  102. swift/common/middleware/tempauth.py +76 -58
  103. swift/common/middleware/tempurl.py +191 -173
  104. swift/common/middleware/versioned_writes/__init__.py +51 -0
  105. swift/common/middleware/{versioned_writes.py → versioned_writes/legacy.py} +27 -26
  106. swift/common/middleware/versioned_writes/object_versioning.py +1482 -0
  107. swift/common/middleware/x_profile/exceptions.py +1 -4
  108. swift/common/middleware/x_profile/html_viewer.py +18 -19
  109. swift/common/middleware/x_profile/profile_model.py +1 -2
  110. swift/common/middleware/xprofile.py +10 -10
  111. swift-2.23.3.data/scripts/swift-container-server → swift/common/recon.py +13 -8
  112. swift/common/registry.py +147 -0
  113. swift/common/request_helpers.py +324 -57
  114. swift/common/ring/builder.py +67 -25
  115. swift/common/ring/composite_builder.py +1 -1
  116. swift/common/ring/ring.py +177 -51
  117. swift/common/ring/utils.py +1 -1
  118. swift/common/splice.py +10 -6
  119. swift/common/statsd_client.py +205 -0
  120. swift/common/storage_policy.py +49 -44
  121. swift/common/swob.py +86 -102
  122. swift/common/{utils.py → utils/__init__.py} +2163 -2772
  123. swift/common/utils/base.py +131 -0
  124. swift/common/utils/config.py +433 -0
  125. swift/common/utils/ipaddrs.py +256 -0
  126. swift/common/utils/libc.py +345 -0
  127. swift/common/utils/logs.py +859 -0
  128. swift/common/utils/timestamp.py +412 -0
  129. swift/common/wsgi.py +553 -535
  130. swift/container/auditor.py +14 -100
  131. swift/container/backend.py +490 -231
  132. swift/container/reconciler.py +126 -37
  133. swift/container/replicator.py +96 -22
  134. swift/container/server.py +358 -165
  135. swift/container/sharder.py +1540 -684
  136. swift/container/sync.py +94 -88
  137. swift/container/updater.py +53 -32
  138. swift/obj/auditor.py +153 -35
  139. swift/obj/diskfile.py +466 -217
  140. swift/obj/expirer.py +406 -124
  141. swift/obj/mem_diskfile.py +7 -4
  142. swift/obj/mem_server.py +1 -0
  143. swift/obj/reconstructor.py +523 -262
  144. swift/obj/replicator.py +249 -188
  145. swift/obj/server.py +207 -122
  146. swift/obj/ssync_receiver.py +145 -85
  147. swift/obj/ssync_sender.py +113 -54
  148. swift/obj/updater.py +652 -139
  149. swift/obj/watchers/__init__.py +0 -0
  150. swift/obj/watchers/dark_data.py +213 -0
  151. swift/proxy/controllers/account.py +11 -11
  152. swift/proxy/controllers/base.py +848 -604
  153. swift/proxy/controllers/container.py +433 -92
  154. swift/proxy/controllers/info.py +3 -2
  155. swift/proxy/controllers/obj.py +1000 -489
  156. swift/proxy/server.py +185 -112
  157. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/AUTHORS +58 -11
  158. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/METADATA +51 -56
  159. swift-2.35.0.dist-info/RECORD +201 -0
  160. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/WHEEL +1 -1
  161. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/entry_points.txt +43 -0
  162. swift-2.35.0.dist-info/pbr.json +1 -0
  163. swift/locale/de/LC_MESSAGES/swift.po +0 -1216
  164. swift/locale/en_GB/LC_MESSAGES/swift.po +0 -1207
  165. swift/locale/es/LC_MESSAGES/swift.po +0 -1085
  166. swift/locale/fr/LC_MESSAGES/swift.po +0 -909
  167. swift/locale/it/LC_MESSAGES/swift.po +0 -894
  168. swift/locale/ja/LC_MESSAGES/swift.po +0 -965
  169. swift/locale/ko_KR/LC_MESSAGES/swift.po +0 -964
  170. swift/locale/pt_BR/LC_MESSAGES/swift.po +0 -881
  171. swift/locale/ru/LC_MESSAGES/swift.po +0 -891
  172. swift/locale/tr_TR/LC_MESSAGES/swift.po +0 -832
  173. swift/locale/zh_CN/LC_MESSAGES/swift.po +0 -833
  174. swift/locale/zh_TW/LC_MESSAGES/swift.po +0 -838
  175. swift-2.23.3.data/scripts/swift-account-auditor +0 -23
  176. swift-2.23.3.data/scripts/swift-account-info +0 -51
  177. swift-2.23.3.data/scripts/swift-account-reaper +0 -23
  178. swift-2.23.3.data/scripts/swift-account-replicator +0 -34
  179. swift-2.23.3.data/scripts/swift-account-server +0 -23
  180. swift-2.23.3.data/scripts/swift-container-auditor +0 -23
  181. swift-2.23.3.data/scripts/swift-container-info +0 -55
  182. swift-2.23.3.data/scripts/swift-container-reconciler +0 -21
  183. swift-2.23.3.data/scripts/swift-container-replicator +0 -34
  184. swift-2.23.3.data/scripts/swift-container-sharder +0 -37
  185. swift-2.23.3.data/scripts/swift-container-sync +0 -23
  186. swift-2.23.3.data/scripts/swift-container-updater +0 -23
  187. swift-2.23.3.data/scripts/swift-dispersion-report +0 -24
  188. swift-2.23.3.data/scripts/swift-form-signature +0 -20
  189. swift-2.23.3.data/scripts/swift-init +0 -119
  190. swift-2.23.3.data/scripts/swift-object-auditor +0 -29
  191. swift-2.23.3.data/scripts/swift-object-expirer +0 -33
  192. swift-2.23.3.data/scripts/swift-object-info +0 -60
  193. swift-2.23.3.data/scripts/swift-object-reconstructor +0 -33
  194. swift-2.23.3.data/scripts/swift-object-relinker +0 -41
  195. swift-2.23.3.data/scripts/swift-object-replicator +0 -37
  196. swift-2.23.3.data/scripts/swift-object-server +0 -27
  197. swift-2.23.3.data/scripts/swift-object-updater +0 -23
  198. swift-2.23.3.data/scripts/swift-proxy-server +0 -23
  199. swift-2.23.3.data/scripts/swift-recon +0 -24
  200. swift-2.23.3.data/scripts/swift-ring-builder +0 -24
  201. swift-2.23.3.data/scripts/swift-ring-builder-analyzer +0 -22
  202. swift-2.23.3.data/scripts/swift-ring-composer +0 -22
  203. swift-2.23.3.dist-info/RECORD +0 -220
  204. swift-2.23.3.dist-info/pbr.json +0 -1
  205. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/LICENSE +0 -0
  206. {swift-2.23.3.dist-info → swift-2.35.0.dist-info}/top_level.txt +0 -0
@@ -64,7 +64,7 @@ signature is generated using the HTTP method to allow (``GET``, ``PUT``,
64
64
  the full path to the object, and the key set on the account.
65
65
 
66
66
  The digest algorithm to be used may be configured by the operator. By default,
67
- HMAC-SHA1, HMAC-SHA256, and HMAC-SHA512 are supported. Check the
67
+ HMAC-SHA256 and HMAC-SHA512 are supported. Check the
68
68
  ``tempurl.allowed_digests`` entry in the cluster's capabilities response to
69
69
  see which algorithms are supported by your deployment; see
70
70
  :doc:`api/discoverability` for more information. On older clusters,
@@ -75,24 +75,25 @@ For example, here is code generating the signature for a ``GET`` for 60
75
75
  seconds on ``/v1/AUTH_account/container/object``::
76
76
 
77
77
  import hmac
78
- from hashlib import sha1
78
+ from hashlib import sha256
79
79
  from time import time
80
80
  method = 'GET'
81
81
  expires = int(time() + 60)
82
82
  path = '/v1/AUTH_account/container/object'
83
83
  key = 'mykey'
84
84
  hmac_body = '%s\n%s\n%s' % (method, expires, path)
85
- sig = hmac.new(key, hmac_body, sha1).hexdigest()
85
+ sig = hmac.new(key, hmac_body, sha256).hexdigest()
86
86
 
87
87
  Be certain to use the full path, from the ``/v1/`` onward.
88
88
 
89
89
  Let's say ``sig`` ends up equaling
90
- ``da39a3ee5e6b4b0d3255bfef95601890afd80709`` and ``expires`` ends up
91
- ``1323479485``. Then, for example, the website could provide a link to::
90
+ ``732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b`` and
91
+ ``expires`` ends up ``1512508563``. Then, for example, the website could
92
+ provide a link to::
92
93
 
93
94
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
94
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
95
- temp_url_expires=1323479485
95
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
96
+ temp_url_expires=1512508563
96
97
 
97
98
  For longer hashes, a hex encoding becomes unwieldy. Base64 encoding is also
98
99
  supported, and indicated by prefixing the signature with ``"<digest name>:"``.
@@ -124,11 +125,11 @@ Supposing that ``sig`` ends up equaling
124
125
  You may also use ISO 8601 UTC timestamps with the format
125
126
  ``"%Y-%m-%dT%H:%M:%SZ"`` instead of UNIX timestamps in the URL
126
127
  (but NOT in the code above for generating the signature!).
127
- So, the above HMAC-SHA1 URL could also be formulated as::
128
+ So, the above HMAC-SHA246 URL could also be formulated as::
128
129
 
129
130
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
130
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
131
- temp_url_expires=2011-12-10T01:11:25Z
131
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
132
+ temp_url_expires=2017-12-05T21:16:03Z
132
133
 
133
134
  If a prefix-based signature with the prefix ``pre`` is desired, set path to::
134
135
 
@@ -140,31 +141,31 @@ a query parameter called ``temp_url_prefix``. So, if ``sig`` and ``expires``
140
141
  would end up like above, following URL would be valid::
141
142
 
142
143
  https://swift-cluster.example.com/v1/AUTH_account/container/pre/object?
143
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
144
- temp_url_expires=1323479485&
144
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
145
+ temp_url_expires=1512508563&
145
146
  temp_url_prefix=pre
146
147
 
147
148
  Another valid URL::
148
149
 
149
150
  https://swift-cluster.example.com/v1/AUTH_account/container/pre/
150
151
  subfolder/another_object?
151
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
152
- temp_url_expires=1323479485&
152
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
153
+ temp_url_expires=1512508563&
153
154
  temp_url_prefix=pre
154
155
 
155
156
  If you wish to lock down the ip ranges from where the resource can be accessed
156
157
  to the ip ``1.2.3.4``::
157
158
 
158
159
  import hmac
159
- from hashlib import sha1
160
+ from hashlib import sha256
160
161
  from time import time
161
162
  method = 'GET'
162
163
  expires = int(time() + 60)
163
164
  path = '/v1/AUTH_account/container/object'
164
165
  ip_range = '1.2.3.4'
165
- key = 'mykey'
166
+ key = b'mykey'
166
167
  hmac_body = 'ip=%s\n%s\n%s\n%s' % (ip_range, method, expires, path)
167
- sig = hmac.new(key, hmac_body, sha1).hexdigest()
168
+ sig = hmac.new(key, hmac_body.encode('ascii'), sha256).hexdigest()
168
169
 
169
170
  The generated signature would only be valid from the ip ``1.2.3.4``. The
170
171
  middleware detects an ip-based temporary URL by a query parameter called
@@ -172,29 +173,29 @@ middleware detects an ip-based temporary URL by a query parameter called
172
173
  above, following URL would be valid::
173
174
 
174
175
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
175
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
176
- temp_url_expires=1323479485&
176
+ temp_url_sig=3f48476acaf5ec272acd8e99f7b5bad96c52ddba53ed27c60613711774a06f0c&
177
+ temp_url_expires=1648082711&
177
178
  temp_url_ip_range=1.2.3.4
178
179
 
179
180
  Similarly to lock down the ip to a range of ``1.2.3.X`` so starting
180
181
  from the ip ``1.2.3.0`` to ``1.2.3.255``::
181
182
 
182
183
  import hmac
183
- from hashlib import sha1
184
+ from hashlib import sha256
184
185
  from time import time
185
186
  method = 'GET'
186
187
  expires = int(time() + 60)
187
188
  path = '/v1/AUTH_account/container/object'
188
189
  ip_range = '1.2.3.0/24'
189
- key = 'mykey'
190
+ key = b'mykey'
190
191
  hmac_body = 'ip=%s\n%s\n%s\n%s' % (ip_range, method, expires, path)
191
- sig = hmac.new(key, hmac_body, sha1).hexdigest()
192
+ sig = hmac.new(key, hmac_body.encode('ascii'), sha256).hexdigest()
192
193
 
193
194
  Then the following url would be valid::
194
195
 
195
196
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
196
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
197
- temp_url_expires=1323479485&
197
+ temp_url_sig=6ff81256b8a3ba11d239da51a703b9c06a56ffddeb8caab74ca83af8f73c9c83&
198
+ temp_url_expires=1648082711&
198
199
  temp_url_ip_range=1.2.3.0/24
199
200
 
200
201
 
@@ -222,16 +223,16 @@ can override this with a filename query parameter. Modifying the
222
223
  above example::
223
224
 
224
225
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
225
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
226
- temp_url_expires=1323479485&filename=My+Test+File.pdf
226
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
227
+ temp_url_expires=1512508563&filename=My+Test+File.pdf
227
228
 
228
229
  If you do not want the object to be downloaded, you can cause
229
230
  ``Content-Disposition: inline`` to be set on the response by adding the
230
231
  ``inline`` parameter to the query string, like so::
231
232
 
232
233
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
233
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
234
- temp_url_expires=1323479485&inline
234
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
235
+ temp_url_expires=1512508563&inline
235
236
 
236
237
  In some cases, the client might not able to present the content of the object,
237
238
  but you still want the content able to save to local with the specific
@@ -240,8 +241,8 @@ set on the response by adding the ``inline&filename=...`` parameter to the
240
241
  query string, like so::
241
242
 
242
243
  https://swift-cluster.example.com/v1/AUTH_account/container/object?
243
- temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709&
244
- temp_url_expires=1323479485&inline&filename=My+Test+File.pdf
244
+ temp_url_sig=732fcac368abb10c78a4cbe95c3fab7f311584532bf779abd5074e13cbe8b88b&
245
+ temp_url_expires=1512508563&inline&filename=My+Test+File.pdf
245
246
 
246
247
  ---------------------
247
248
  Cluster Configuration
@@ -254,7 +255,7 @@ This middleware understands the following configuration settings:
254
255
  incoming requests. Names may optionally end with ``*`` to
255
256
  indicate a prefix match. ``incoming_allow_headers`` is a
256
257
  list of exceptions to these removals.
257
- Default: ``x-timestamp``
258
+ Default: ``x-timestamp x-open-expired``
258
259
 
259
260
  ``incoming_allow_headers``
260
261
  A whitespace-delimited list of the headers allowed as
@@ -288,7 +289,7 @@ This middleware understands the following configuration settings:
288
289
  A whitespace delimited list of digest algorithms that are allowed
289
290
  to be used when calculating the signature for a temporary URL.
290
291
 
291
- Default: ``sha1 sha256 sha512``
292
+ Default: ``sha256 sha512``
292
293
  """
293
294
 
294
295
  __all__ = ['TempURL', 'filter_factory',
@@ -297,25 +298,24 @@ __all__ = ['TempURL', 'filter_factory',
297
298
  'DEFAULT_OUTGOING_REMOVE_HEADERS',
298
299
  'DEFAULT_OUTGOING_ALLOW_HEADERS']
299
300
 
300
- import binascii
301
301
  from calendar import timegm
302
- import functools
303
- import hashlib
304
- import six
305
302
  from os.path import basename
306
303
  from time import time, strftime, strptime, gmtime
307
304
  from ipaddress import ip_address, ip_network
308
305
 
309
- from six.moves.urllib.parse import parse_qs
310
- from six.moves.urllib.parse import urlencode
306
+ from urllib.parse import parse_qs, urlencode
311
307
 
312
308
  from swift.proxy.controllers.base import get_account_info, get_container_info
313
309
  from swift.common.header_key_dict import HeaderKeyDict
310
+ from swift.common.http import is_success
311
+ from swift.common.digest import get_allowed_digests, \
312
+ extract_digest_and_algorithm, DEFAULT_ALLOWED_DIGESTS, get_hmac
314
313
  from swift.common.swob import header_to_environ_key, HTTPUnauthorized, \
315
314
  HTTPBadRequest, wsgi_to_str
316
- from swift.common.utils import split_path, get_valid_utf8_str, \
317
- register_swift_info, get_hmac, streq_const_time, quote, get_logger, \
318
- strict_b64decode
315
+ from swift.common.utils import split_path, \
316
+ streq_const_time, quote, get_logger, close_if_possible
317
+ from swift.common.registry import register_swift_info, register_sensitive_param
318
+ from swift.common.wsgi import WSGIContext
319
319
 
320
320
 
321
321
  DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target'
@@ -324,7 +324,7 @@ DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target'
324
324
  #: delimited list of header names and names can optionally end with '*' to
325
325
  #: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of
326
326
  #: exceptions to these removals.
327
- DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp'
327
+ DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp x-open-expired'
328
328
 
329
329
  #: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a
330
330
  #: whitespace delimited list of header names and names can optionally end with
@@ -342,9 +342,6 @@ DEFAULT_OUTGOING_REMOVE_HEADERS = 'x-object-meta-*'
342
342
  #: '*' to indicate a prefix match.
343
343
  DEFAULT_OUTGOING_ALLOW_HEADERS = 'x-object-meta-public-*'
344
344
 
345
- DEFAULT_ALLOWED_DIGESTS = 'sha1 sha256 sha512'
346
- SUPPORTED_DIGESTS = set(DEFAULT_ALLOWED_DIGESTS.split())
347
-
348
345
  CONTAINER_SCOPE = 'container'
349
346
  ACCOUNT_SCOPE = 'account'
350
347
 
@@ -362,11 +359,59 @@ def get_tempurl_keys_from_metadata(meta):
362
359
  meta = get_account_info(...)['meta']
363
360
  keys = get_tempurl_keys_from_metadata(meta)
364
361
  """
365
- return [(get_valid_utf8_str(value) if six.PY2 else value)
366
- for key, value in meta.items()
362
+ return [value for key, value in meta.items()
367
363
  if key.lower() in ('temp-url-key', 'temp-url-key-2')]
368
364
 
369
365
 
366
+ def normalize_temp_url_expires(value):
367
+ """
368
+ Returns the normalized expiration value as an int
369
+
370
+ If not None, the value is converted to an int if possible or 0
371
+ if not, and checked for expiration (returns 0 if expired).
372
+ """
373
+ if value is None:
374
+ return value
375
+ try:
376
+ temp_url_expires = int(value)
377
+ except ValueError:
378
+ try:
379
+ temp_url_expires = timegm(strptime(
380
+ value, EXPIRES_ISO8601_FORMAT))
381
+ except ValueError:
382
+ temp_url_expires = 0
383
+ if temp_url_expires < time():
384
+ temp_url_expires = 0
385
+ return temp_url_expires
386
+
387
+
388
+ def get_temp_url_info(env):
389
+ """
390
+ Returns the provided temporary URL parameters (sig, expires, prefix,
391
+ temp_url_ip_range), if given and syntactically valid.
392
+ Either sig, expires or prefix could be None if not provided.
393
+
394
+ :param env: The WSGI environment for the request.
395
+ :returns: (sig, expires, prefix, filename, inline,
396
+ temp_url_ip_range) as described above.
397
+ """
398
+ sig = expires = prefix = ip_range = filename = inline = None
399
+ qs = parse_qs(env.get('QUERY_STRING', ''), keep_blank_values=True)
400
+ if 'temp_url_ip_range' in qs:
401
+ ip_range = qs['temp_url_ip_range'][0]
402
+ if 'temp_url_sig' in qs:
403
+ sig = qs['temp_url_sig'][0]
404
+ if 'temp_url_expires' in qs:
405
+ expires = qs['temp_url_expires'][0]
406
+ if 'temp_url_prefix' in qs:
407
+ prefix = qs['temp_url_prefix'][0]
408
+ if 'filename' in qs:
409
+ filename = qs['filename'][0]
410
+ if 'inline' in qs:
411
+ inline = True
412
+ return (sig, expires, prefix, filename, inline, ip_range)
413
+
414
+
370
415
  def disposition_format(disposition_type, filename):
371
416
  # Content-Disposition in HTTP is defined in
372
417
  # https://tools.ietf.org/html/rfc6266 and references
@@ -424,11 +469,12 @@ class TempURL(object):
424
469
  :param conf: The configuration dict for the middleware.
425
470
  """
426
471
 
427
- def __init__(self, app, conf):
472
+ def __init__(self, app, conf, logger=None):
428
473
  #: The next WSGI application/filter in the paste.deploy pipeline.
429
474
  self.app = app
430
475
  #: The filter configuration dict.
431
476
  self.conf = conf
477
+ self.logger = logger or get_logger(conf, log_route='tempurl')
432
478
 
433
479
  self.allowed_digests = conf.get(
434
480
  'allowed_digests', DEFAULT_ALLOWED_DIGESTS.split())
@@ -497,36 +543,27 @@ class TempURL(object):
497
543
  """
498
544
  if env['REQUEST_METHOD'] == 'OPTIONS':
499
545
  return self.app(env, start_response)
500
- info = self._get_temp_url_info(env)
501
- temp_url_sig, temp_url_expires, temp_url_prefix, filename,\
546
+ info = get_temp_url_info(env)
547
+ temp_url_sig, client_temp_url_expires, temp_url_prefix, filename, \
502
548
  inline_disposition, temp_url_ip_range = info
549
+ temp_url_expires = normalize_temp_url_expires(client_temp_url_expires)
503
550
  if temp_url_sig is None and temp_url_expires is None:
504
551
  return self.app(env, start_response)
505
552
  if not temp_url_sig or not temp_url_expires:
506
553
  return self._invalid(env, start_response)
507
554
 
508
- if ':' in temp_url_sig:
509
- hash_algorithm, temp_url_sig = temp_url_sig.split(':', 1)
510
- if ('-' in temp_url_sig or '_' in temp_url_sig) and not (
511
- '+' in temp_url_sig or '/' in temp_url_sig):
512
- temp_url_sig = temp_url_sig.replace('-', '+').replace('_', '/')
513
- try:
514
- temp_url_sig = binascii.hexlify(strict_b64decode(
515
- temp_url_sig + '=='))
516
- if not six.PY2:
517
- temp_url_sig = temp_url_sig.decode('ascii')
518
- except ValueError:
519
- return self._invalid(env, start_response)
520
- elif len(temp_url_sig) == 40:
521
- hash_algorithm = 'sha1'
522
- elif len(temp_url_sig) == 64:
523
- hash_algorithm = 'sha256'
524
- else:
555
+ try:
556
+ hash_algorithm, temp_url_sig = extract_digest_and_algorithm(
557
+ temp_url_sig)
558
+ except ValueError:
525
559
  return self._invalid(env, start_response)
526
560
  if hash_algorithm not in self.allowed_digests:
527
561
  return self._invalid(env, start_response)
528
562
 
529
- account, container, obj = self._get_path_parts(env)
563
+ account, container, obj = self._get_path_parts(
564
+ env, allow_container_root=(
565
+ env['REQUEST_METHOD'] in ('GET', 'HEAD') and
566
+ temp_url_prefix == ""))
530
567
  if not account:
531
568
  return self._invalid(env, start_response)
532
569
 
@@ -535,8 +572,8 @@ class TempURL(object):
535
572
  if client_address is None:
536
573
  return self._invalid(env, start_response)
537
574
  try:
538
- allowed_ip_ranges = ip_network(six.u(temp_url_ip_range))
539
- if ip_address(six.u(client_address)) not in allowed_ip_ranges:
575
+ allowed_ip_ranges = ip_network(str(temp_url_ip_range))
576
+ if ip_address(str(client_address)) not in allowed_ip_ranges:
540
577
  return self._invalid(env, start_response)
541
578
  except ValueError:
542
579
  return self._invalid(env, start_response)
@@ -574,6 +611,7 @@ class TempURL(object):
574
611
  break
575
612
  if not is_valid_hmac:
576
613
  return self._invalid(env, start_response)
614
+ self.logger.increment('tempurl.digests.%s' % hash_algorithm)
577
615
  # disallowed headers prevent accidentally allowing upload of a pointer
578
616
  # to data that the PUT tempurl would not otherwise allow access for.
579
617
  # It should be safe to provide a GET tempurl for data that an
@@ -591,116 +629,102 @@ class TempURL(object):
591
629
  env['swift.authorize_override'] = True
592
630
  env['REMOTE_USER'] = '.wsgi.tempurl'
593
631
  qs = {'temp_url_sig': temp_url_sig,
594
- 'temp_url_expires': temp_url_expires}
632
+ 'temp_url_expires': client_temp_url_expires}
595
633
  if temp_url_prefix is not None:
596
634
  qs['temp_url_prefix'] = temp_url_prefix
597
635
  if filename:
598
636
  qs['filename'] = filename
599
637
  env['QUERY_STRING'] = urlencode(qs)
600
638
 
601
- def _start_response(status, headers, exc_info=None):
602
- headers = self._clean_outgoing_headers(headers)
603
- if env['REQUEST_METHOD'] in ('GET', 'HEAD') and status[0] == '2':
604
- # figure out the right value for content-disposition
605
- # 1) use the value from the query string
606
- # 2) use the value from the object metadata
607
- # 3) use the object name (default)
608
- out_headers = []
609
- existing_disposition = None
610
- for h, v in headers:
611
- if h.lower() != 'content-disposition':
612
- out_headers.append((h, v))
613
- else:
614
- existing_disposition = v
615
- if inline_disposition:
616
- if filename:
617
- disposition_value = disposition_format('inline',
618
- filename)
619
- else:
620
- disposition_value = 'inline'
621
- elif filename:
622
- disposition_value = disposition_format('attachment',
639
+ ctx = WSGIContext(self.app)
640
+ app_iter = ctx._app_call(env)
641
+ ctx._response_headers = self._clean_outgoing_headers(
642
+ ctx._response_headers)
643
+ if env['REQUEST_METHOD'] in ('GET', 'HEAD') and \
644
+ is_success(ctx._get_status_int()):
645
+ # figure out the right value for content-disposition
646
+ # 1) use the value from the query string
647
+ # 2) use the value from the object metadata
648
+ # 3) use the object name (default)
649
+ out_headers = []
650
+ existing_disposition = None
651
+ content_generator = None
652
+ for h, v in ctx._response_headers:
653
+ if h.lower() == 'x-backend-content-generator':
654
+ content_generator = v
655
+
656
+ if h.lower() != 'content-disposition':
657
+ out_headers.append((h, v))
658
+ else:
659
+ existing_disposition = v
660
+ if content_generator == 'staticweb':
661
+ inline_disposition = True
662
+ elif obj == "":
663
+ # Generally, tempurl requires an object. We carved out an
664
+ # exception to allow GETs at the container root for the sake
665
+ # of staticweb, but we can't tell whether we'll have a
666
+ # staticweb response or not until after we call the app
667
+ close_if_possible(app_iter)
668
+ return self._invalid(env, start_response)
669
+
670
+ if inline_disposition:
671
+ if filename:
672
+ disposition_value = disposition_format('inline',
623
673
  filename)
624
- elif existing_disposition:
625
- disposition_value = existing_disposition
626
674
  else:
627
- name = basename(wsgi_to_str(env['PATH_INFO']).rstrip('/'))
628
- disposition_value = disposition_format('attachment',
629
- name)
630
- # this is probably just paranoia, I couldn't actually get a
631
- # newline into existing_disposition
632
- value = disposition_value.replace('\n', '%0A')
633
- out_headers.append(('Content-Disposition', value))
634
-
635
- # include Expires header for better cache-control
636
- out_headers.append(('Expires', strftime(
637
- "%a, %d %b %Y %H:%M:%S GMT",
638
- gmtime(temp_url_expires))))
639
- headers = out_headers
640
- return start_response(status, headers, exc_info)
641
-
642
- return self.app(env, _start_response)
643
-
644
- def _get_path_parts(self, env):
675
+ disposition_value = 'inline'
676
+ elif filename:
677
+ disposition_value = disposition_format('attachment',
678
+ filename)
679
+ elif existing_disposition:
680
+ disposition_value = existing_disposition
681
+ else:
682
+ name = basename(wsgi_to_str(env['PATH_INFO']).rstrip('/'))
683
+ disposition_value = disposition_format('attachment',
684
+ name)
685
+ # this is probably just paranoia, I couldn't actually get a
686
+ # newline into existing_disposition
687
+ value = disposition_value.replace('\n', '%0A')
688
+ out_headers.append(('Content-Disposition', value))
689
+
690
+ # include Expires header for better cache-control
691
+ out_headers.append(('Expires', strftime(
692
+ "%a, %d %b %Y %H:%M:%S GMT",
693
+ gmtime(temp_url_expires))))
694
+ ctx._response_headers = out_headers
695
+ start_response(
696
+ ctx._response_status,
697
+ ctx._response_headers,
698
+ ctx._response_exc_info)
699
+ return app_iter
700
+
701
+ def _get_path_parts(self, env, allow_container_root=False):
645
702
  """
646
703
  Return the account, container and object name for the request,
647
704
  if it's an object request and one of the configured methods;
648
705
  otherwise, None is returned.
649
706
 
707
+ If it's a container request and allow_root_container is true,
708
+ the object name returned will be the empty string.
709
+
650
710
  :param env: The WSGI environment for the request.
711
+ :param allow_container_root: Whether requests to the root of a
712
+ container should be allowed.
651
713
  :returns: (Account str, container str, object str) or
652
714
  (None, None, None).
653
715
  """
654
716
  if env['REQUEST_METHOD'] in self.conf['methods']:
655
717
  try:
656
- ver, acc, cont, obj = split_path(env['PATH_INFO'], 4, 4, True)
718
+ ver, acc, cont, obj = split_path(
719
+ env['PATH_INFO'], 3 if allow_container_root else 4,
720
+ 4, True)
657
721
  except ValueError:
658
722
  return (None, None, None)
659
- if ver == 'v1' and obj.strip('/'):
660
- return (wsgi_to_str(acc), wsgi_to_str(cont), wsgi_to_str(obj))
723
+ if ver == 'v1' and (allow_container_root or obj.strip('/')):
724
+ return (wsgi_to_str(acc), wsgi_to_str(cont),
725
+ wsgi_to_str(obj) if obj else '')
661
726
  return (None, None, None)
662
727
 
663
- def _get_temp_url_info(self, env):
664
- """
665
- Returns the provided temporary URL parameters (sig, expires, prefix,
666
- temp_url_ip_range), if given and syntactically valid.
667
- Either sig, expires or prefix could be None if not provided.
668
- If provided, expires is also converted to an int if possible or 0
669
- if not, and checked for expiration (returns 0 if expired).
670
-
671
- :param env: The WSGI environment for the request.
672
- :returns: (sig, expires, prefix, filename, inline,
673
- temp_url_ip_range) as described above.
674
- """
675
- temp_url_sig = temp_url_expires = temp_url_prefix = filename =\
676
- inline = None
677
- temp_url_ip_range = None
678
- qs = parse_qs(env.get('QUERY_STRING', ''), keep_blank_values=True)
679
- if 'temp_url_ip_range' in qs:
680
- temp_url_ip_range = qs['temp_url_ip_range'][0]
681
- if 'temp_url_sig' in qs:
682
- temp_url_sig = qs['temp_url_sig'][0]
683
- if 'temp_url_expires' in qs:
684
- try:
685
- temp_url_expires = int(qs['temp_url_expires'][0])
686
- except ValueError:
687
- try:
688
- temp_url_expires = timegm(strptime(
689
- qs['temp_url_expires'][0],
690
- EXPIRES_ISO8601_FORMAT))
691
- except ValueError:
692
- temp_url_expires = 0
693
- if temp_url_expires < time():
694
- temp_url_expires = 0
695
- if 'temp_url_prefix' in qs:
696
- temp_url_prefix = qs['temp_url_prefix'][0]
697
- if 'filename' in qs:
698
- filename = qs['filename'][0]
699
- if 'inline' in qs:
700
- inline = True
701
- return (temp_url_sig, temp_url_expires, temp_url_prefix, filename,
702
- inline, temp_url_ip_range)
703
-
704
728
  def _get_keys(self, env):
705
729
  """
706
730
  Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
@@ -750,12 +774,10 @@ class TempURL(object):
750
774
  if not request_method:
751
775
  request_method = env['REQUEST_METHOD']
752
776
 
753
- digest = functools.partial(hashlib.new, hash_algorithm)
754
-
755
777
  return [
756
778
  (get_hmac(
757
779
  request_method, path, expires, key,
758
- digest=digest, ip_range=ip_range
780
+ digest=hash_algorithm, ip_range=ip_range
759
781
  ), scope)
760
782
  for (key, scope) in scoped_keys]
761
783
 
@@ -847,30 +869,26 @@ def filter_factory(global_conf, **local_conf):
847
869
  conf = global_conf.copy()
848
870
  conf.update(local_conf)
849
871
 
872
+ logger = get_logger(conf, log_route='tempurl')
873
+
850
874
  defaults = {
851
875
  'methods': 'GET HEAD PUT POST DELETE',
852
876
  'incoming_remove_headers': DEFAULT_INCOMING_REMOVE_HEADERS,
853
877
  'incoming_allow_headers': DEFAULT_INCOMING_ALLOW_HEADERS,
854
878
  'outgoing_remove_headers': DEFAULT_OUTGOING_REMOVE_HEADERS,
855
879
  'outgoing_allow_headers': DEFAULT_OUTGOING_ALLOW_HEADERS,
856
- 'allowed_digests': DEFAULT_ALLOWED_DIGESTS,
857
880
  }
858
881
  info_conf = {k: conf.get(k, v).split() for k, v in defaults.items()}
859
882
 
860
- allowed_digests = set(digest.lower()
861
- for digest in info_conf['allowed_digests'])
862
- not_supported = allowed_digests - SUPPORTED_DIGESTS
863
- if not_supported:
864
- logger = get_logger(conf, log_route='tempurl')
865
- logger.warning('The following digest algorithms are configured but '
866
- 'not supported: %s', ', '.join(not_supported))
867
- allowed_digests -= not_supported
868
- if not allowed_digests:
869
- raise ValueError('No valid digest algorithms are configured '
870
- 'for tempurls')
883
+ allowed_digests, deprecated_digests = get_allowed_digests(
884
+ conf.get('allowed_digests', '').split(), logger)
871
885
  info_conf['allowed_digests'] = sorted(allowed_digests)
886
+ if deprecated_digests:
887
+ info_conf['deprecated_digests'] = sorted(deprecated_digests)
872
888
 
873
889
  register_swift_info('tempurl', **info_conf)
874
890
  conf.update(info_conf)
875
891
 
876
- return lambda app: TempURL(app, conf)
892
+ register_sensitive_param('temp_url_sig')
893
+
894
+ return lambda app: TempURL(app, conf, logger)
@@ -0,0 +1,51 @@
1
+ # Copyright (c) 2019 OpenStack Foundation
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
+ # implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """
16
+ Implements middleware for object versioning which comprises an instance of a
17
+ :class:`~swift.common.middleware.versioned_writes.legacy.
18
+ VersionedWritesMiddleware` combined with an instance of an
19
+ :class:`~swift.common.middleware.versioned_writes.object_versioning.
20
+ ObjectVersioningMiddleware`.
21
+ """
22
+ from swift.common.middleware.versioned_writes. \
23
+ legacy import CLIENT_VERSIONS_LOC, CLIENT_HISTORY_LOC, \
24
+ VersionedWritesMiddleware
25
+ from swift.common.middleware.versioned_writes. \
26
+ object_versioning import ObjectVersioningMiddleware
27
+
28
+ from swift.common.utils import config_true_value
29
+ from swift.common.registry import register_swift_info, get_swift_info
30
+
31
+
32
+ def filter_factory(global_conf, **local_conf):
33
+ """Provides a factory function for loading versioning middleware."""
34
+ conf = global_conf.copy()
35
+ conf.update(local_conf)
36
+ if config_true_value(conf.get('allow_versioned_writes')):
37
+ register_swift_info('versioned_writes', allowed_flags=(
38
+ CLIENT_VERSIONS_LOC, CLIENT_HISTORY_LOC))
39
+
40
+ allow_object_versioning = config_true_value(conf.get(
41
+ 'allow_object_versioning'))
42
+ if allow_object_versioning:
43
+ register_swift_info('object_versioning')
44
+
45
+ def versioning_filter(app):
46
+ if allow_object_versioning:
47
+ if 'symlink' not in get_swift_info():
48
+ raise ValueError('object versioning requires symlinks')
49
+ app = ObjectVersioningMiddleware(app, conf)
50
+ return VersionedWritesMiddleware(app, conf)
51
+ return versioning_filter