plone.recipe.vinylcache 9.0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plone/recipe/vinylcache/__init__.py +9 -0
- plone/recipe/vinylcache/recipe.py +611 -0
- plone/recipe/vinylcache/templates/start_script.jinja2 +30 -0
- plone/recipe/vinylcache/templates/varnish.vcl.jinja2 +501 -0
- plone/recipe/vinylcache/tests/__init__.py +1 -0
- plone/recipe/vinylcache/tests/recipe.rst +353 -0
- plone/recipe/vinylcache/tests/test_recipe.py +62 -0
- plone/recipe/vinylcache/tests/test_vclgen.py +25 -0
- plone/recipe/vinylcache/tests/vclgen.rst +431 -0
- plone/recipe/vinylcache/vclgen.py +149 -0
- plone.recipe.vinylcache-9.0.3.0-py3.12-nspkg.pth +2 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/METADATA +974 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/RECORD +18 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/WHEEL +5 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/entry_points.txt +5 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/licenses/LICENSE.rst +33 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/namespace_packages.txt +2 -0
- plone_recipe_vinylcache-9.0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,611 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
from . import jinja2env
|
|
3
|
+
from zc.recipe.cmmi import Recipe as CMMIRecipe
|
|
4
|
+
from zc.recipe.cmmi import system
|
|
5
|
+
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import zc.buildout
|
|
10
|
+
|
|
11
|
+
# Vinyl Cache (the project/company formerly known as Varnish Cache/Varnish
|
|
12
|
+
# Software) 9.0.x is the current stable line this recipe targets. NOTE:
|
|
13
|
+
# despite the project rename, the actual 9.0.x *source release* is still
|
|
14
|
+
# published as a "varnish-X.Y.Z.tar.gz" tarball (PACKAGE_TARNAME is still
|
|
15
|
+
# "varnish", confirmed by inspecting the real 9.0.3 configure.ac), and it
|
|
16
|
+
# still builds/installs a binary literally called `varnishd` (confirmed via
|
|
17
|
+
# the upstream Makefile.am: `sbin_PROGRAMS = varnishd`) -- only internal
|
|
18
|
+
# source directories (e.g. bin/vinyld/) and branding/docs have been renamed
|
|
19
|
+
# so far. Re-verify both facts against a future release before assuming
|
|
20
|
+
# otherwise.
|
|
21
|
+
DOWNLOAD_URL = (
|
|
22
|
+
"https://github.com/varnish/varnish/releases/download/"
|
|
23
|
+
"varnish-9.0.3/varnish-9.0.3.tar.gz"
|
|
24
|
+
)
|
|
25
|
+
# varnish-modules (vmods like header/xkey) dropped support for Varnish 6.0
|
|
26
|
+
# LTS as of the releases targeting 7.0+; this tag targets the 9.0 line.
|
|
27
|
+
# The project itself is still hosted/named "varnish-modules" (not renamed).
|
|
28
|
+
VMODS_DOWNLOAD_URL = "https://github.com/varnish/varnish-modules/archive/0.28.0.tar.gz"
|
|
29
|
+
|
|
30
|
+
COOKIE_WHITELIST_DEFAULT = """\
|
|
31
|
+
statusmessages
|
|
32
|
+
__ac
|
|
33
|
+
_ZopeId
|
|
34
|
+
__cp
|
|
35
|
+
auth_token
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
DEFAULT_VCL_HASH = """\
|
|
39
|
+
hash_data(req.url);
|
|
40
|
+
if (req.http.host) {
|
|
41
|
+
hash_data(req.http.host);
|
|
42
|
+
} else {
|
|
43
|
+
hash_data(server.ip);
|
|
44
|
+
}
|
|
45
|
+
return(lookup);
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
COOKIE_PASS_DEFAULT = """\
|
|
49
|
+
"auth_token|__ac(|_(name|password|persistent))=":"\.(js|css|woff|woff2)$"
|
|
50
|
+
""" # noqa: W605
|
|
51
|
+
COOKIE_PASS_RE = re.compile('"(.*)":"(.*)"')
|
|
52
|
+
COOKIE_PASS_NOT_EXCLUDE_DEFAULT = "/\\+\\+resource\\+\\+zmi/"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class BaseRecipe(object):
|
|
56
|
+
def __init__(self, buildout, name, options):
|
|
57
|
+
self.name = name
|
|
58
|
+
self.options = options
|
|
59
|
+
self.buildout = buildout
|
|
60
|
+
self.logger = logging.getLogger(self.name)
|
|
61
|
+
|
|
62
|
+
def _log_and_raise(self, message):
|
|
63
|
+
"""log error first and then raise buildout Exception"""
|
|
64
|
+
self.logger.error(message)
|
|
65
|
+
raise zc.buildout.UserError(message)
|
|
66
|
+
|
|
67
|
+
def _process_bind(self):
|
|
68
|
+
self.options["bind"] = self.options.get("bind").lstrip(":")
|
|
69
|
+
bind = self.options["bind"].split(":")
|
|
70
|
+
if len(bind) == 1 and bind[0].isdigit():
|
|
71
|
+
self.options["bind-host"] = ""
|
|
72
|
+
self.options["bind-port"] = bind[0]
|
|
73
|
+
self.options["bind"] = ":" + bind[0]
|
|
74
|
+
elif len(bind) == 2 and bind[1].isdigit():
|
|
75
|
+
self.options["bind-host"] = bind[0]
|
|
76
|
+
self.options["bind-port"] = bind[1]
|
|
77
|
+
else:
|
|
78
|
+
self._log_and_raise("Invalid syntax for bind")
|
|
79
|
+
|
|
80
|
+
def get_from_section(self, part, key, default):
|
|
81
|
+
if part not in self.buildout:
|
|
82
|
+
return default
|
|
83
|
+
if key not in self.buildout[part]:
|
|
84
|
+
return default
|
|
85
|
+
return self.buildout[part][key]
|
|
86
|
+
|
|
87
|
+
def install(self):
|
|
88
|
+
pass
|
|
89
|
+
|
|
90
|
+
def update(self):
|
|
91
|
+
pass
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class BuildRecipe(CMMIRecipe, BaseRecipe):
|
|
95
|
+
def __init__(self, buildout, name, options):
|
|
96
|
+
# An inherited/extended buildout.cfg may declare "url =" with no
|
|
97
|
+
# value as a placeholder meant to be overridden downstream.
|
|
98
|
+
# setdefault() below leaves that empty string in place (the key
|
|
99
|
+
# already exists), which zc.recipe.cmmi then hands to
|
|
100
|
+
# zc.buildout.download.Download() as if it were a local path,
|
|
101
|
+
# failing with "FileNotFoundError: [Errno 2] No such file or
|
|
102
|
+
# directory: ''" instead of falling back to our default. Drop it
|
|
103
|
+
# first so setdefault() actually applies.
|
|
104
|
+
if "url" in options and not options["url"]:
|
|
105
|
+
del options["url"]
|
|
106
|
+
BaseRecipe.__init__(self, buildout, name, options)
|
|
107
|
+
self.options.setdefault("url", DOWNLOAD_URL)
|
|
108
|
+
self.options.setdefault("jobs", "4")
|
|
109
|
+
CMMIRecipe.__init__(self, buildout, name, self.options)
|
|
110
|
+
|
|
111
|
+
def build(self):
|
|
112
|
+
# build varnish (Vinyl Cache)
|
|
113
|
+
super(BuildRecipe, self).build()
|
|
114
|
+
if zc.buildout.buildout.bool_option(self.options, "compile-vmods", False):
|
|
115
|
+
if "PKG_CONFIG_PATH" not in self.environ:
|
|
116
|
+
self.environ["PKG_CONFIG_PATH"] = os.path.join(
|
|
117
|
+
self.options["location"], "lib", "pkgconfig"
|
|
118
|
+
)
|
|
119
|
+
os.environ.update(self.environ)
|
|
120
|
+
vmods_options = {
|
|
121
|
+
k[6:]: v for k, v in self.options.items() if k.startswith("vmods_")
|
|
122
|
+
}
|
|
123
|
+
# Same empty-placeholder issue as the main "url" option above.
|
|
124
|
+
if "url" in vmods_options and not vmods_options["url"]:
|
|
125
|
+
del vmods_options["url"]
|
|
126
|
+
vmods_options.setdefault("url", VMODS_DOWNLOAD_URL)
|
|
127
|
+
vmods_options.setdefault("source-directory-contains", "bootstrap")
|
|
128
|
+
vmods_options.setdefault("configure-command", "./bootstrap; ./configure")
|
|
129
|
+
vmods_recipe = CMMIRecipe(self.buildout, self.name, vmods_options)
|
|
130
|
+
vmods_recipe.build()
|
|
131
|
+
|
|
132
|
+
def cmmi(self, dest):
|
|
133
|
+
"""Do the 'configure; make; make install' command sequence.
|
|
134
|
+
|
|
135
|
+
When this is called, the current working directory is the
|
|
136
|
+
source directory. The 'dest' parameter specifies the
|
|
137
|
+
installation prefix.
|
|
138
|
+
|
|
139
|
+
This is overidden in order to enable parallel jobs in make.
|
|
140
|
+
"""
|
|
141
|
+
options = self.configure_options
|
|
142
|
+
|
|
143
|
+
if options is None:
|
|
144
|
+
options = '--prefix="%s"' % dest
|
|
145
|
+
if self.extra_options:
|
|
146
|
+
options += " %s" % self.extra_options
|
|
147
|
+
options += " --with-sphinx-build=false"
|
|
148
|
+
# C
|
|
149
|
+
system("%s %s" % (self.configure_cmd, options))
|
|
150
|
+
|
|
151
|
+
# M
|
|
152
|
+
base_make = "make"
|
|
153
|
+
if int(self.options.get("jobs")) > 1:
|
|
154
|
+
base_make += " -j {0}".format(self.options.get("jobs"))
|
|
155
|
+
system(base_make)
|
|
156
|
+
|
|
157
|
+
# MI
|
|
158
|
+
system("make install")
|
|
159
|
+
|
|
160
|
+
# set daemon location. Despite the Varnish Cache -> Vinyl Cache
|
|
161
|
+
# project rename, the installed binary is still called `varnishd`
|
|
162
|
+
# as of Vinyl Cache 9.0.x (confirmed against the upstream source:
|
|
163
|
+
# `sbin_PROGRAMS = varnishd` in bin/vinyld/Makefile.am).
|
|
164
|
+
self.options["daemon"] = os.path.join(
|
|
165
|
+
self.options["location"], "sbin", "varnishd"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class ConfigureRecipe(BaseRecipe):
|
|
170
|
+
def __init__(self, buildout, name, options):
|
|
171
|
+
super(ConfigureRecipe, self).__init__(buildout, name, options)
|
|
172
|
+
|
|
173
|
+
self.options.setdefault("build-part", "varnish-build")
|
|
174
|
+
|
|
175
|
+
self.options.setdefault(
|
|
176
|
+
"location", os.path.join(buildout["buildout"]["parts-directory"], self.name)
|
|
177
|
+
)
|
|
178
|
+
# `saint-mode` was removed from Varnish itself years ago and was
|
|
179
|
+
# already a no-op in the code this recipe was forked from; rather
|
|
180
|
+
# than silently accept it, fail loudly if it's set to anything but
|
|
181
|
+
# the (also removed) default so upgraders get clear feedback.
|
|
182
|
+
if self.options.get("saint-mode", "off") != "off":
|
|
183
|
+
self._log_and_raise(
|
|
184
|
+
"The 'saint-mode' option has been removed (saint mode no "
|
|
185
|
+
"longer exists in Varnish/Vinyl Cache); remove it from your "
|
|
186
|
+
"buildout configuration."
|
|
187
|
+
)
|
|
188
|
+
self.options.setdefault("verbose-headers", "off")
|
|
189
|
+
# collective.purgebyid compatibility. Two modes:
|
|
190
|
+
# - "ban" (also "on"): ban()-based, no vmod needed, works
|
|
191
|
+
# everywhere. Matches collective.purgebyid's own non-xkey
|
|
192
|
+
# fallback.
|
|
193
|
+
# - "xkey": uses the `xkey` vmod's secondary-key purge instead of
|
|
194
|
+
# a ban scan; more efficient, but requires
|
|
195
|
+
# `[varnish-build] compile-vmods = true` (the `xkey` import is
|
|
196
|
+
# only emitted in this mode, so "off"/"ban" never break
|
|
197
|
+
# compilation for setups that haven't built vmods).
|
|
198
|
+
self.options.setdefault("purge-by-id", "off")
|
|
199
|
+
purgebyid = self.options["purge-by-id"].strip().lower()
|
|
200
|
+
if purgebyid == "on":
|
|
201
|
+
purgebyid = "ban"
|
|
202
|
+
if purgebyid not in ("off", "ban", "xkey"):
|
|
203
|
+
self._log_and_raise(
|
|
204
|
+
"Invalid value for 'purge-by-id': {0!r}. Must be one of "
|
|
205
|
+
"'off', 'ban'/'on' or 'xkey'.".format(self.options["purge-by-id"])
|
|
206
|
+
)
|
|
207
|
+
self.options["purge-by-id"] = purgebyid
|
|
208
|
+
self.options.setdefault("balancer", "none")
|
|
209
|
+
self.options.setdefault("backends", "127.0.0.1:8080")
|
|
210
|
+
self.options.setdefault(
|
|
211
|
+
"config-file", os.path.join(self.options["location"], "varnish.vcl")
|
|
212
|
+
)
|
|
213
|
+
self.options.setdefault("connect-timeout", "0.4s")
|
|
214
|
+
self.options.setdefault("first-byte-timeout", "300s")
|
|
215
|
+
self.options.setdefault("between-bytes-timeout", "60s")
|
|
216
|
+
self.options.setdefault("purge-hosts", "")
|
|
217
|
+
self.options.setdefault("cookie-pass", COOKIE_PASS_DEFAULT)
|
|
218
|
+
self.options.setdefault(
|
|
219
|
+
"cookie-pass-not-exclude", COOKIE_PASS_NOT_EXCLUDE_DEFAULT
|
|
220
|
+
)
|
|
221
|
+
self.options.setdefault("cookie-whitelist", COOKIE_WHITELIST_DEFAULT)
|
|
222
|
+
# Set default vcl_hash function so it doesn't use the default.vcl hostname
|
|
223
|
+
self.options.setdefault("vcl_hash", DEFAULT_VCL_HASH)
|
|
224
|
+
# set and test for valid bind value
|
|
225
|
+
self.options.setdefault("bind", "127.0.0.1:8000")
|
|
226
|
+
self._process_bind()
|
|
227
|
+
|
|
228
|
+
def _process_backends(self):
|
|
229
|
+
result = []
|
|
230
|
+
raw_backends = [
|
|
231
|
+
_.rsplit(":", 2) for _ in self.options["backends"].strip().split()
|
|
232
|
+
]
|
|
233
|
+
# consistency checks
|
|
234
|
+
if len(raw_backends) > 1:
|
|
235
|
+
lengths = set([len(x) for x in raw_backends])
|
|
236
|
+
if lengths != set([3]):
|
|
237
|
+
self._log_and_raise(
|
|
238
|
+
"When using multiple backends a hostname "
|
|
239
|
+
"must be given for each client"
|
|
240
|
+
)
|
|
241
|
+
for idx, raw_backend in enumerate(raw_backends):
|
|
242
|
+
backend = {"name": "backend_{0:03d}".format(idx)}
|
|
243
|
+
try:
|
|
244
|
+
if len(raw_backend) == 3:
|
|
245
|
+
url, host, port = raw_backend
|
|
246
|
+
else:
|
|
247
|
+
host, port = raw_backend
|
|
248
|
+
url = None
|
|
249
|
+
except ValueError:
|
|
250
|
+
self._log_and_raise(
|
|
251
|
+
"Invalid syntax for backend: {0}".format(":".join(raw_backend))
|
|
252
|
+
)
|
|
253
|
+
raise zc.buildout.UserError("Invalid syntax for backends")
|
|
254
|
+
backend["url"] = url
|
|
255
|
+
backend["host"] = host
|
|
256
|
+
backend["port"] = port
|
|
257
|
+
backend["connect_timeout"] = self.options["connect-timeout"]
|
|
258
|
+
backend["first_byte_timeout"] = self.options["first-byte-timeout"]
|
|
259
|
+
backend["between_bytes_timeout"] = self.options["between-bytes-timeout"]
|
|
260
|
+
|
|
261
|
+
result.append(backend)
|
|
262
|
+
|
|
263
|
+
return result
|
|
264
|
+
|
|
265
|
+
def _process_zope_vhm_map(self, backends):
|
|
266
|
+
result = {}
|
|
267
|
+
|
|
268
|
+
vhm_external_port = self.options.get(
|
|
269
|
+
"zope2_vhm_port", self.options["bind-port"]
|
|
270
|
+
)
|
|
271
|
+
vhm_proto = "http"
|
|
272
|
+
if self.options.get("zope2_vhm_ssl", False) == "on":
|
|
273
|
+
vhm_external_port = self.options.get("zope2_vhm_ssl_port", "443")
|
|
274
|
+
vhm_proto = "https"
|
|
275
|
+
|
|
276
|
+
for line in self.options.get("zope2_vhm_map", "").split():
|
|
277
|
+
domain, location = line.split(":")
|
|
278
|
+
result[domain.strip()] = {
|
|
279
|
+
"location": location.strip(),
|
|
280
|
+
"proto": vhm_proto,
|
|
281
|
+
"external_port": vhm_external_port,
|
|
282
|
+
}
|
|
283
|
+
return result
|
|
284
|
+
|
|
285
|
+
def _process_balancers(self, balancer, backends):
|
|
286
|
+
"""if theres is a balancer configured, all backends are assigned.
|
|
287
|
+
|
|
288
|
+
this could be refactored in future to support multiple balancers.
|
|
289
|
+
"""
|
|
290
|
+
result = []
|
|
291
|
+
if balancer != "none":
|
|
292
|
+
record = {
|
|
293
|
+
"type": balancer,
|
|
294
|
+
"name": "balancer_0",
|
|
295
|
+
"backends": [_["name"] for _ in backends],
|
|
296
|
+
}
|
|
297
|
+
result.append(record)
|
|
298
|
+
return result
|
|
299
|
+
|
|
300
|
+
def install(self):
|
|
301
|
+
if "configuration-file" not in self.options:
|
|
302
|
+
if not os.path.exists(self.options["location"]):
|
|
303
|
+
os.mkdir(self.options["location"])
|
|
304
|
+
self.options.created(self.options["location"])
|
|
305
|
+
self.options["configuration-file"] = os.path.join(
|
|
306
|
+
self.options["location"], "varnish.vcl"
|
|
307
|
+
)
|
|
308
|
+
self.create_varnish_configuration()
|
|
309
|
+
return self.options.created()
|
|
310
|
+
|
|
311
|
+
def update(self):
|
|
312
|
+
self.install()
|
|
313
|
+
|
|
314
|
+
def create_varnish_configuration(self):
|
|
315
|
+
from plone.recipe.vinylcache.vclgen import VclGenerator
|
|
316
|
+
|
|
317
|
+
config = {}
|
|
318
|
+
|
|
319
|
+
# enable verbose (diagnostic) response headers: X-Cache, X-Cacheable,
|
|
320
|
+
# grace. Off by default to avoid leaking cache internals to clients.
|
|
321
|
+
config["verbose"] = self.options["verbose-headers"] == "on"
|
|
322
|
+
config["purgebyid"] = self.options["purge-by-id"]
|
|
323
|
+
# Objects larger than this (by Content-Length) are never cached.
|
|
324
|
+
# Without a limit, a single very large object can nuke a large
|
|
325
|
+
# fraction of the cache under LRU pressure to make room for
|
|
326
|
+
# itself; unset (the default) means no size limit is applied.
|
|
327
|
+
# Value must be a valid VCL BYTES literal, e.g. "50MB", "1GB".
|
|
328
|
+
config["maxcacheablesize"] = self.options.get("max-cacheable-size", None)
|
|
329
|
+
config["gracehealthy"] = self.options.get("grace-healthy", None)
|
|
330
|
+
config["gracesick"] = self.options.get("grace-sick", "600s")
|
|
331
|
+
config["healthprobeurl"] = self.options.get("health-probe-url", None)
|
|
332
|
+
config["healthprobetimeout"] = self.options.get("health-probe-timeout", None)
|
|
333
|
+
config["healthprobeinterval"] = self.options.get("health-probe-interval", None)
|
|
334
|
+
config["healthprobewindow"] = self.options.get("health-probe-window", None)
|
|
335
|
+
config["healthprobethreshold"] = self.options.get(
|
|
336
|
+
"health-probe-threshold", None
|
|
337
|
+
)
|
|
338
|
+
config["healthprobeinitial"] = self.options.get("health-probe-initial", None)
|
|
339
|
+
|
|
340
|
+
# fixup cookies for better plone caching
|
|
341
|
+
config["cookiewhitelist"] = [
|
|
342
|
+
_.strip() for _ in self.options["cookie-whitelist"].split()
|
|
343
|
+
]
|
|
344
|
+
config["cookiepass"] = []
|
|
345
|
+
for line in self.options["cookie-pass"].split():
|
|
346
|
+
line = line.strip()
|
|
347
|
+
if not line:
|
|
348
|
+
continue
|
|
349
|
+
match = COOKIE_PASS_RE.match(line)
|
|
350
|
+
mg = match.groups()
|
|
351
|
+
if not mg and len(mg) != 2:
|
|
352
|
+
continue
|
|
353
|
+
config["cookiepass"].append(dict(zip(("match", "exclude"), mg)))
|
|
354
|
+
config["cookiepassnotexclude"] = self.options["cookie-pass-not-exclude"]
|
|
355
|
+
# inject custom vcl
|
|
356
|
+
config["custom"] = {}
|
|
357
|
+
for name in (
|
|
358
|
+
"vcl_import",
|
|
359
|
+
"vcl_recv",
|
|
360
|
+
"vcl_hit",
|
|
361
|
+
"vcl_miss",
|
|
362
|
+
"vcl_backend_fetch",
|
|
363
|
+
"vcl_purge",
|
|
364
|
+
"vcl_deliver",
|
|
365
|
+
"vcl_pipe",
|
|
366
|
+
"vcl_backend_response",
|
|
367
|
+
"vcl_hash",
|
|
368
|
+
"vcl_init",
|
|
369
|
+
"vcl_pass",
|
|
370
|
+
"vcl_synth",
|
|
371
|
+
):
|
|
372
|
+
config["custom"][name] = self.options.get(name, "")
|
|
373
|
+
|
|
374
|
+
config["backends"] = self._process_backends()
|
|
375
|
+
config["directors"] = self._process_balancers(
|
|
376
|
+
self.options["balancer"].strip(), config["backends"]
|
|
377
|
+
)
|
|
378
|
+
config["zope2_vhm_map"] = self._process_zope_vhm_map(config["backends"])
|
|
379
|
+
config["code404page"] = True
|
|
380
|
+
|
|
381
|
+
# build the purge host string
|
|
382
|
+
config["purgehosts"] = set([])
|
|
383
|
+
for segment in self.options["purge-hosts"].split():
|
|
384
|
+
segment = segment.strip()
|
|
385
|
+
if segment:
|
|
386
|
+
config["purgehosts"].add(segment)
|
|
387
|
+
|
|
388
|
+
vclgenerator = VclGenerator(config)
|
|
389
|
+
filedata = vclgenerator()
|
|
390
|
+
with open(self.options["configuration-file"], "wt") as fio:
|
|
391
|
+
fio.write(filedata)
|
|
392
|
+
self.options.created(self.options["configuration-file"])
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
class ScriptRecipe(BaseRecipe):
|
|
396
|
+
def __init__(self, buildout, name, options):
|
|
397
|
+
super(ScriptRecipe, self).__init__(buildout, name, options)
|
|
398
|
+
|
|
399
|
+
self.options.setdefault("build-part", "varnish-build")
|
|
400
|
+
self.options.setdefault("configuration-part", "varnish-configuration")
|
|
401
|
+
|
|
402
|
+
self.options.setdefault(
|
|
403
|
+
"daemon",
|
|
404
|
+
self.get_from_section(self.options["build-part"], "location", "/usr")
|
|
405
|
+
+ "/sbin/varnishd",
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
self.options.setdefault(
|
|
409
|
+
"bind",
|
|
410
|
+
self.get_from_section(
|
|
411
|
+
self.options["configuration-part"], "bind", "127.0.0.1:8000"
|
|
412
|
+
),
|
|
413
|
+
)
|
|
414
|
+
self.options.setdefault(
|
|
415
|
+
"configuration-file",
|
|
416
|
+
self.get_from_section(
|
|
417
|
+
self.options["configuration-part"], "config-file", ""
|
|
418
|
+
),
|
|
419
|
+
)
|
|
420
|
+
if not self.options["configuration-file"]:
|
|
421
|
+
self._log_and_raise("No configuration file found")
|
|
422
|
+
self._process_bind()
|
|
423
|
+
|
|
424
|
+
self.options.setdefault(
|
|
425
|
+
"location", os.path.join(buildout["buildout"]["parts-directory"], self.name)
|
|
426
|
+
)
|
|
427
|
+
# Default varnishd's working directory (-n) to somewhere inside the
|
|
428
|
+
# buildout's `var` directory, not whatever system default varnishd
|
|
429
|
+
# itself picks (e.g. /var/run/varnishd, which typically requires
|
|
430
|
+
# root -- the single most common "Permission denied" trap when
|
|
431
|
+
# running varnishd unprivileged from a buildout). Deliberately
|
|
432
|
+
# *not* under `parts` (that tree is meant to be disposable/
|
|
433
|
+
# regenerated, e.g. on a varnish-build recompile, which would lose
|
|
434
|
+
# this runtime state) and keyed by this part's own name so multiple
|
|
435
|
+
# varnish instances in the same buildout don't collide. A `name`
|
|
436
|
+
# starting with "/" is used by varnishd as an absolute
|
|
437
|
+
# working-directory path directly (see the `name` option docs in
|
|
438
|
+
# README.rst).
|
|
439
|
+
self.options.setdefault(
|
|
440
|
+
"name",
|
|
441
|
+
os.path.join(buildout["buildout"]["directory"], "var", self.name),
|
|
442
|
+
)
|
|
443
|
+
self.options.setdefault("cache-type", "file")
|
|
444
|
+
self.options.setdefault("cache-size", "256M")
|
|
445
|
+
self.options.setdefault("runtime-parameters", "")
|
|
446
|
+
self.options.setdefault("secret-file", "nosecret")
|
|
447
|
+
self.options.setdefault(
|
|
448
|
+
"script-filename",
|
|
449
|
+
os.path.join(self.buildout["buildout"]["bin-directory"], self.name),
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
def install(self):
|
|
453
|
+
if "cache-location" not in self.options:
|
|
454
|
+
if not os.path.exists(self.options["location"]):
|
|
455
|
+
os.mkdir(self.options["location"])
|
|
456
|
+
self.options.created(self.options["location"])
|
|
457
|
+
self.options["cache-location"] = os.path.join(
|
|
458
|
+
self.options["location"], "storage"
|
|
459
|
+
)
|
|
460
|
+
if not os.path.exists(self.options["cache-location"]):
|
|
461
|
+
os.mkdir(self.options["cache-location"])
|
|
462
|
+
self.options.created(self.options["cache-location"])
|
|
463
|
+
|
|
464
|
+
# If `name` is an absolute path, it's used by varnishd as its
|
|
465
|
+
# working directory (-n) -- create it (and any missing parent,
|
|
466
|
+
# e.g. `var/`) ourselves, since varnishd itself only creates the
|
|
467
|
+
# leaf directory, not the full path, and fails outright
|
|
468
|
+
# ("Cannot create working directory ...: No such file or
|
|
469
|
+
# directory") when an intermediate parent is missing.
|
|
470
|
+
name = self.options.get("name")
|
|
471
|
+
if name and name.startswith("/") and not os.path.exists(name):
|
|
472
|
+
os.makedirs(name)
|
|
473
|
+
self.options.created(name)
|
|
474
|
+
|
|
475
|
+
script = self.create_varnish_script()
|
|
476
|
+
with open(self.options["script-filename"], "wt") as fio:
|
|
477
|
+
fio.write(script)
|
|
478
|
+
os.chmod(self.options["script-filename"], 0o755)
|
|
479
|
+
self.options.created(self.options["script-filename"])
|
|
480
|
+
return self.options.created()
|
|
481
|
+
|
|
482
|
+
def create_varnish_script(self):
|
|
483
|
+
# render script file
|
|
484
|
+
data = {}
|
|
485
|
+
|
|
486
|
+
data["daemon"] = self.options["daemon"]
|
|
487
|
+
data["user"] = self.options.get("user")
|
|
488
|
+
data["group"] = self.options.get("group")
|
|
489
|
+
data["cfg_file"] = self.options["configuration-file"]
|
|
490
|
+
data["pid_file"] = os.path.join(self.options["location"], "varnish.pid")
|
|
491
|
+
data["bind"] = self.options["bind"]
|
|
492
|
+
data["cache_type"] = self.options["cache-type"].lower()
|
|
493
|
+
data["cache_location"] = self.options["cache-location"]
|
|
494
|
+
data["cache_size"] = self.options["cache-size"]
|
|
495
|
+
data["mode"] = self.options.get("mode", "daemon")
|
|
496
|
+
data["name"] = self.options.get("name")
|
|
497
|
+
data["secret"] = self.options.get("secret-file", "nosecret")
|
|
498
|
+
data["telnet"] = self.options.get("telnet")
|
|
499
|
+
# `-A` (a hitch-like TLS config file) is a Vinyl Cache 9.0 addition,
|
|
500
|
+
# letting varnishd terminate TLS itself instead of needing a
|
|
501
|
+
# separate terminator in front of it.
|
|
502
|
+
data["tls_config"] = self.options.get("tls-config")
|
|
503
|
+
data["parameters"] = self.options["runtime-parameters"].strip().split()
|
|
504
|
+
|
|
505
|
+
template = jinja2env.get_template("start_script.jinja2")
|
|
506
|
+
return template.render(data)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
class SelfSignedCertRecipe(BaseRecipe):
|
|
510
|
+
"""Generate a self-signed TLS certificate plus a hitch-style config
|
|
511
|
+
file suitable for `varnishd -A` (see `ScriptRecipe`'s `tls-config`
|
|
512
|
+
option). Meant for internal/dev/testing use where a real CA-issued
|
|
513
|
+
certificate isn't warranted -- clients will need to explicitly trust
|
|
514
|
+
this certificate (or ignore validation errors) since it is
|
|
515
|
+
self-signed.
|
|
516
|
+
"""
|
|
517
|
+
|
|
518
|
+
def __init__(self, buildout, name, options):
|
|
519
|
+
super(SelfSignedCertRecipe, self).__init__(buildout, name, options)
|
|
520
|
+
|
|
521
|
+
self.options.setdefault(
|
|
522
|
+
"location", os.path.join(buildout["buildout"]["parts-directory"], self.name)
|
|
523
|
+
)
|
|
524
|
+
self.options.setdefault("common-name", "localhost")
|
|
525
|
+
self.options.setdefault("bind", "*:8443")
|
|
526
|
+
self.options.setdefault("key-size", "2048")
|
|
527
|
+
self.options.setdefault("days", "3650")
|
|
528
|
+
self.options.setdefault(
|
|
529
|
+
"key-file", os.path.join(self.options["location"], "key.pem")
|
|
530
|
+
)
|
|
531
|
+
self.options.setdefault(
|
|
532
|
+
"cert-file", os.path.join(self.options["location"], "cert.pem")
|
|
533
|
+
)
|
|
534
|
+
self.options.setdefault(
|
|
535
|
+
"combined-file", os.path.join(self.options["location"], "combined.pem")
|
|
536
|
+
)
|
|
537
|
+
self.options.setdefault(
|
|
538
|
+
"config-file", os.path.join(self.options["location"], "tls.conf")
|
|
539
|
+
)
|
|
540
|
+
self._process_bind()
|
|
541
|
+
|
|
542
|
+
def install(self):
|
|
543
|
+
if not os.path.exists(self.options["location"]):
|
|
544
|
+
# 0o700: this directory holds private key material.
|
|
545
|
+
os.mkdir(self.options["location"], 0o700)
|
|
546
|
+
self.options.created(self.options["location"])
|
|
547
|
+
|
|
548
|
+
# Idempotent: don't regenerate (and so don't rotate/invalidate) an
|
|
549
|
+
# already-present key/cert on every buildout run.
|
|
550
|
+
if not os.path.exists(self.options["key-file"]) or not os.path.exists(
|
|
551
|
+
self.options["cert-file"]
|
|
552
|
+
):
|
|
553
|
+
cmd = (
|
|
554
|
+
"openssl req -x509 -nodes "
|
|
555
|
+
"-newkey rsa:{key_size} "
|
|
556
|
+
'-keyout "{key_file}" -out "{cert_file}" '
|
|
557
|
+
"-days {days} "
|
|
558
|
+
'-subj "/CN={common_name}"'
|
|
559
|
+
).format(
|
|
560
|
+
key_size=self.options["key-size"],
|
|
561
|
+
key_file=self.options["key-file"],
|
|
562
|
+
cert_file=self.options["cert-file"],
|
|
563
|
+
days=self.options["days"],
|
|
564
|
+
common_name=self.options["common-name"],
|
|
565
|
+
)
|
|
566
|
+
self.logger.info(
|
|
567
|
+
"Generating self-signed certificate for CN=%s",
|
|
568
|
+
self.options["common-name"],
|
|
569
|
+
)
|
|
570
|
+
system(cmd)
|
|
571
|
+
# openssl writes the key with the process' default umask (often
|
|
572
|
+
# world/group-readable); it's private key material, restrict it.
|
|
573
|
+
os.chmod(self.options["key-file"], 0o600)
|
|
574
|
+
self.options.created(self.options["key-file"])
|
|
575
|
+
self.options.created(self.options["cert-file"])
|
|
576
|
+
|
|
577
|
+
# hitch/varnishd's -A pem-file directive expects cert and key
|
|
578
|
+
# concatenated into a single file. It also contains the private
|
|
579
|
+
# key, so write it with restricted permissions from the start
|
|
580
|
+
# rather than creating it world-readable and fixing it up after.
|
|
581
|
+
with open(self.options["cert-file"]) as fio:
|
|
582
|
+
cert_data = fio.read()
|
|
583
|
+
with open(self.options["key-file"]) as fio:
|
|
584
|
+
key_data = fio.read()
|
|
585
|
+
fd = os.open(
|
|
586
|
+
self.options["combined-file"], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600
|
|
587
|
+
)
|
|
588
|
+
with os.fdopen(fd, "wt") as fio:
|
|
589
|
+
fio.write(cert_data)
|
|
590
|
+
fio.write(key_data)
|
|
591
|
+
self.options.created(self.options["combined-file"])
|
|
592
|
+
|
|
593
|
+
config = (
|
|
594
|
+
"frontend = {{\n"
|
|
595
|
+
' host = "{host}"\n'
|
|
596
|
+
' port = "{port}"\n'
|
|
597
|
+
"}}\n"
|
|
598
|
+
'pem-file = "{combined_file}"\n'
|
|
599
|
+
).format(
|
|
600
|
+
host=self.options["bind-host"] or "*",
|
|
601
|
+
port=self.options["bind-port"],
|
|
602
|
+
combined_file=self.options["combined-file"],
|
|
603
|
+
)
|
|
604
|
+
with open(self.options["config-file"], "wt") as fio:
|
|
605
|
+
fio.write(config)
|
|
606
|
+
self.options.created(self.options["config-file"])
|
|
607
|
+
|
|
608
|
+
return self.options.created()
|
|
609
|
+
|
|
610
|
+
def update(self):
|
|
611
|
+
self.install()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
exec {{daemon}} \
|
|
3
|
+
{% if user %}
|
|
4
|
+
-j unix,user={{user}}{% if group %},ccgroup={{group}}{% endif %} \
|
|
5
|
+
{% endif %}
|
|
6
|
+
-f "{{cfg_file}}" \
|
|
7
|
+
-P "{{pid_file}}" \
|
|
8
|
+
-a {{bind}} \
|
|
9
|
+
{% if telnet %}
|
|
10
|
+
-T {{telnet}} \
|
|
11
|
+
{% endif %}
|
|
12
|
+
{% if tls_config %}
|
|
13
|
+
-A "{{tls_config}}" \
|
|
14
|
+
{% endif %}
|
|
15
|
+
-s {{cache_type}},{% if cache_type == 'file' %}"{{cache_location}}",{% endif %}{{cache_size}} \
|
|
16
|
+
{% if mode == 'foreground' %}
|
|
17
|
+
-F \
|
|
18
|
+
{% endif %}
|
|
19
|
+
{% if name %}
|
|
20
|
+
-n {{name}} \
|
|
21
|
+
{% endif %}
|
|
22
|
+
{% if secret.lower() == 'disabled' %}
|
|
23
|
+
-S "" \
|
|
24
|
+
{% elif secret.lower() != 'nosecret' %}
|
|
25
|
+
-S {{secret}} \
|
|
26
|
+
{% endif %}
|
|
27
|
+
{% for parameter in parameters %}
|
|
28
|
+
-p {{parameter}} \
|
|
29
|
+
{% endfor %}
|
|
30
|
+
"$@"
|