configuration-templates – Blame information for rev 18

Subversion Repositories:
Rev:
Rev Author Line No. Line
2 office 1 vcl 4.0;
16 office 2 ###########################################################################
3 ## Copyright (C) Wizardry and Steamworks 2016 - License: GNU GPLv3 ##
4 ## Please see: http://www.gnu.org/licenses/gpl.html for legal details, ##
5 ## rights of fair usage, the disclaimer and warranty conditions. ##
6 ###########################################################################
7 ## Derived from the original by Mattias Geniar as a starting template. ##
8 ###########################################################################
2 office 9  
10 import std;
11 import directors;
12  
13 backend server1 { # Define one backend
14 .host = "192.168.0.10"; # IP or Hostname of backend
15 .port = "80"; # Port Apache or whatever is listening
16 .max_connections = 300; # Number of allowed connections
17  
18 .probe = {
19 #.url = "/"; # short easy way (GET /)
20 # We prefer to only do a HEAD /
21 .request =
22 "HEAD / HTTP/1.1"
23 "Host: grimore.org"
24 "Connection: close"
25 "User-Agent: Varnish Health Probe";
26  
27 .interval = 5s; # check the health of each backend every 5 seconds
28 .timeout = 1s; # timing out after 1 second.
29 .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
30 .threshold = 3;
31 }
32  
33 .first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
34 .connect_timeout = 5s; # How long to wait for a backend connection?
35 .between_bytes_timeout = 2s; # How long to wait between bytes received from our backend?
36 }
37  
38 acl purge {
39 # ACL we'll use later to allow purges
40 "localhost";
41 "127.0.0.1";
42 /* "::1"; */
43 }
44  
45 /*
46 acl editors {
47 # ACL to honor the "Cache-Control: no-cache" header to force a refresh but only from selected IPs
48 "localhost";
49 "127.0.0.1";
50 "::1";
51 }
52 */
53  
54 sub vcl_init {
55 # Called when VCL is loaded, before any requests pass through it.
56 # Typically used to initialize VMODs.
57  
58 new vdir = directors.round_robin();
59 vdir.add_backend(server1);
60 # vdir.add_backend(server...);
61 # vdir.add_backend(servern);
62 }
63  
64 sub vcl_recv {
65 # Called at the beginning of a request, after the complete request has been received and parsed.
66 # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable,
67 # which backend to use.
68 # also used to modify the request
69  
70 set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
71  
18 office 72 ### Brotli encoding support.
73 if(req.http.Accept-Encoding ~ "br" && req.url !~ "\.(jpg|png|gif|gz|mp3|mov|avi|mpg|mp4|swf|wmf)$") {
74 set req.http.X-brotli = "true";
75 }
76  
2 office 77 # When CloudFlare is in front of varnish, send the real IP address through X-Forwarded-For
78 # otherwise if CloudFlare is not involved, set the X-Forwarded-For to the client IP address.
79 unset req.http.X-Forwarded-For;
80 if (req.http.cf-connecting-ip) {
81 set req.http.X-Forwarded-For = req.http.cf-connecting-ip;
82 } else {
83 set req.http.X-Forwarded-For = client.ip;
84 }
85  
86 # Normalize the header, remove the port (in case you're testing this on various TCP ports)
87 set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
88  
89 # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish)
90 unset req.http.proxy;
91  
92 # Normalize the query arguments
93 set req.url = std.querysort(req.url);
94  
95 # Allow purging
96 if (req.method == "PURGE") {
97 if (!client.ip ~ purge) { # purge is the ACL defined at the begining
98 # Not from an allowed IP? Then die with an error.
99 return (synth(405, "This IP is not allowed to send PURGE requests."));
100 }
101 # If you got this stage (and didn't error out above), purge the cached result
102 return (purge);
103 }
104  
105 # Only deal with "normal" types
106 if (req.method != "GET" &&
107 req.method != "HEAD" &&
108 req.method != "PUT" &&
109 req.method != "POST" &&
110 req.method != "TRACE" &&
111 req.method != "OPTIONS" &&
112 req.method != "PATCH" &&
16 office 113 req.method != "DELETE" &&
114 req.method != "PROPFIND" &&
115 req.method != "REPORT") {
2 office 116 /* Non-RFC2616 or CONNECT which is weird. */
117 return (pipe);
118 }
119  
120 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
121 if (req.http.Upgrade ~ "(?i)websocket") {
122 return (pipe);
123 }
124  
125 # Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
126 if (req.method != "GET" && req.method != "HEAD") {
127 return (pass);
128 }
129  
130 # Some generic URL manipulation, useful for all templates that follow
131 # First remove the Google Analytics added parameters, useless for our backend
132 if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
133 set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
134 set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
135 set req.url = regsub(req.url, "\?&", "?");
136 set req.url = regsub(req.url, "\?$", "");
137 }
138  
139 # Strip hash, server doesn't need it.
140 if (req.url ~ "\#") {
141 set req.url = regsub(req.url, "\#.*$", "");
142 }
143  
144 # Strip a trailing ? if it exists
145 if (req.url ~ "\?$") {
146 set req.url = regsub(req.url, "\?$", "");
147 }
148  
149 # Some generic cookie manipulation, useful for all templates that follow
150 # Remove the "has_js" cookie
151 set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
152  
16 office 153 # Remove any CloudFlare cookies
154 set req.http.Cookie = regsuball(req.http.Cookie, "cf_.=[^;]+(; )?", "");
155  
2 office 156 # Remove any Google Analytics based cookies
157 set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
158 set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
159 set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
160 set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
161 set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
162 set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
163  
164 # Remove DoubleClick offensive cookies
165 set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
166  
167 # Remove the Quant Capital cookies (added by some plugin, all __qca)
168 set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
169  
170 # Remove the AddThis cookies
171 set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
172  
173 # Remove a ";" prefix in the cookie if present
174 set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
175  
176 # Are there cookies left with only spaces or that are empty?
177 if (req.http.cookie ~ "^\s*$") {
178 unset req.http.cookie;
179 }
180  
181 if (req.http.Cache-Control ~ "(?i)no-cache") {
182 #if (req.http.Cache-Control ~ "(?i)no-cache" && client.ip ~ editors) { # create the acl editors if you want to restrict the Ctrl-F5
183 # http://varnish.projects.linpro.no/wiki/VCLExampleEnableForceRefresh
184 # Ignore requests via proxy caches and badly behaved crawlers
185 # like msnbot that send no-cache with every request.
186 if (! (req.http.Via || req.http.User-Agent ~ "(?i)bot" || req.http.X-Purge)) {
187 #set req.hash_always_miss = true; # Doesn't seems to refresh the object in the cache
188 return(purge); # Couple this with restart in vcl_purge and X-Purge header to avoid loops
189 }
190 }
191  
192 # Large static files are delivered directly to the end-user without
193 # waiting for Varnish to fully read the file first.
194 # Varnish 4 fully supports Streaming, so set do_stream in vcl_backend_response()
195 if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
196 unset req.http.Cookie;
197 return (hash);
198 }
199  
200 # Remove all cookies for static files
201 # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
202 # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
203 # Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
16 office 204 if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
205 unset req.http.Cookie;
206 return (hash);
207 }
2 office 208  
209 # Send Surrogate-Capability headers to announce ESI support to backend
210 set req.http.Surrogate-Capability = "key=ESI/1.0";
211  
212 if (req.http.Authorization) {
213 # Not cacheable by default
214 return (pass);
215 }
216  
217 return (hash);
218 }
219  
220  
221 sub vcl_pipe {
222 # Called upon entering pipe mode.
223 # In this mode, the request is passed on to the backend, and any further data from both the client
224 # and backend is passed on unaltered until either end closes the connection. Basically, Varnish will
225 # degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode,
226 # no other VCL subroutine will ever get called after vcl_pipe.
227  
228 # Note that only the first request to the backend will have
229 # X-Forwarded-For set. If you use X-Forwarded-For and want to
230 # have it set for all requests, make sure to have:
231 # set bereq.http.connection = "close";
232 # here. It is not set by default as it might break some broken web
233 # applications, like IIS with NTLM authentication.
234  
235 # set bereq.http.Connection = "Close";
236  
237 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
238 if (req.http.upgrade) {
239 set bereq.http.upgrade = req.http.upgrade;
240 }
241  
242 return (pipe);
243 }
244  
245 sub vcl_pass {
246 # Called upon entering pass mode. In this mode, the request is passed on to the backend, and the
247 # backend's response is passed on to the client, but is not entered into the cache. Subsequent
248 # requests submitted over the same client connection are handled normally.
249  
250 # return (pass);
251 }
252  
253 # The data on which the hashing will take place
254 sub vcl_hash {
255 # Called after vcl_recv to create a hash value for the request. This is used as a key
256 # to look up the object in Varnish.
257  
258 hash_data(req.url);
259  
260 if (req.http.host) {
261 hash_data(req.http.host);
262 } else {
263 hash_data(server.ip);
264 }
265  
266 # hash cookies for requests that have them
267 if (req.http.Cookie) {
268 hash_data(req.http.Cookie);
269 }
18 office 270  
271 ### Brotli encoding support.
272 if(req.http.X-brotli == "true") {
273 hash_data("brotli");
274 }
2 office 275 }
276  
277 sub vcl_hit {
278 # Called when a cache lookup is successful.
279  
280 if (obj.ttl >= 0s) {
281 # A pure unadultered hit, deliver it
282 return (deliver);
283 }
284  
285 # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
286 # When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
287 # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
288 # if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
289 # return (deliver);
290 # } else {
291 # return (fetch);
292 # }
293  
294 # We have no fresh fish. Lets look at the stale ones.
295 if (std.healthy(req.backend_hint)) {
296 # Backend is healthy. Limit age to 10s.
297 if (obj.ttl + 10s > 0s) {
298 #set req.http.grace = "normal(limited)";
299 return (deliver);
300 } else {
301 # No candidate for grace. Fetch a fresh object.
302 return(fetch);
303 }
304 } else {
305 # backend is sick - use full grace
306 if (obj.ttl + obj.grace > 0s) {
307 #set req.http.grace = "full";
308 return (deliver);
309 } else {
310 # no graced object.
311 return (fetch);
312 }
313 }
314  
315 # fetch & deliver once we get the result
316 return (fetch); # Dead code, keep as a safeguard
317 }
318  
319 sub vcl_miss {
320 # Called after a cache lookup if the requested document was not found in the cache. Its purpose
321 # is to decide whether or not to attempt to retrieve the document from the backend, and which
322 # backend to use.
323  
324 return (fetch);
325 }
326  
327 # Handle the HTTP request coming from our backend
328 sub vcl_backend_response {
329 # Called after the response headers has been successfully retrieved from the backend.
330  
331 # Pause ESI request and remove Surrogate-Control header
332 if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
333 unset beresp.http.Surrogate-Control;
334 set beresp.do_esi = true;
17 office 335 set beresp.ttl = 24h; # Set 24h ESI
2 office 336 }
18 office 337  
338 ### Brotli encoding support.
339 if(bereq.http.X-brotli == "true") {
340 set bereq.http.Accept-Encoding = "br";
341 unset bereq.http.X-brotli;
342 }
2 office 343  
16 office 344 ### Set content expiration.
345 ## You may want to disable any expiration on the backend (ie: for Apache, disable mod_expires).
346 # 1 Year
347 if(beresp.http.content-type ~ "(?i)(application/(javascript|x\-javascript))|(text/(javascript|css))|(application/(atom\+xml|rdf\+xml|rss\+xml))") {
348 set beresp.http.Expires = "" + (now + 1y);
349 }
350 # 1 Month
351 if(beresp.http.content-type ~ "(?i)(application/(vnd\.ms\-fontobject|font\-woff|x\-font\-(ttf|woff)))|(audio/ogg)|(font/(eot|opentype|woff))|(image/(bmp|gif|jp2|jpeg|png|svg\+xmlwebp))|(video/(mp4|ogg|webm))") {
352 set beresp.http.Expires = "" + (now + 24w);
353 }
354 # 1 Week
355 if(beresp.http.content-type ~ "(?i)(image/(vnd\.microsoft\.icon|x\-icon))|(application/manifest\+json)|(text/x\-cross\-domain\-policy)") {
356 set beresp.http.Expires = "" + (now + 1w);
357 }
358 # Immediately
359 # Not exactly "immediately" since we do not want to have to recache on fast reloads - ideally, set this to average page log time.
360 if(beresp.http.content-type ~ "(?i)(application/(json|ld\+json|schema\+json|vnd\.geo\+jsonxml|x\-web\-app\-manifest\+json))|(text/(xml|cache\-manifest|html))") {
361 set beresp.http.Expires = "" + (now + 3s);
362 }
363  
364 ### GZip content if not already compressed.
365 ## You may want to disable any compression on the backend (ie: for Apache, disable mod_deflate).
366 if(beresp.http.content-type ~ "(?i)(application/(atom\+xml|javascript|json|ld\+json|manifest\+json|rdf\+xml|rss\+xml|schema\+json|vnd\.geo\+json|vnd\.ms\-fontobject|x\-(font\-ttf|javascript|web\-app\-manifest\+json)|xhtml\+xml|xml)|font/(eot|opentype))|(image/(bmp|svg\+xml|vnd\.microsoft\.icon|x\-icon))|(text/(cache\-manifest|css|html|javascript|plain|vcard|vnd\.rim\.location\.xloc|vcalendar|vtt|x\-component|x\-cross\-domain\-policy|xml))") {
367 # Compress object for sending.
368 set beresp.do_gzip = true;
369 }
370  
371 ### Enable cache for all static files
372 ## The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
373 ## Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
374 if(beresp.http.content-type ~ "(?i)(application/(java\-vm|javascript|json|msword|pdf|postscript|rtf|vnd\.ms\-(excel|fontobject|powerpoint)|vnd\.oasis\.opendocument\.text|vnd\.openxmlformats\-officedocument\.(presentationml\.presentation|spreadsheetml\.sheet|wordprocessingml\.document)|x\-(7z\-compressed|bzip2|gzip|rar\-compressed|shockwave\-flash|tar)|x\-font\-(bdf|ghostscript|linux\-psf|otf|pcf|snf|ttf|type1|woff)|xz|xml|zip))|(audio/((mpeg|ogg)|(x\-(flac|matroska|ms\-wma|wav))))|(image/((bmp|gif|jp2|jpeg|png|svg\+xml|vnd\.wap\.wbmp|webp|x\-(icon|pict))))|(text/((css|csv|javascript|plain)))|(video/((mp4|quicktime|webm|x\-(flv|matroska|ms\-wmv|msvideo))))") {
2 office 375 unset beresp.http.set-cookie;
376 }
377  
378 # Large static files are delivered directly to the end-user without
379 # waiting for Varnish to fully read the file first.
380 # Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
16 office 381 if (bereq.url ~ "^[^?]*\.(7z|ace|avi|bin|bz2|flac|flv|gz|iso|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|wma|wmv|xz|zip)(\?.*)?$") {
2 office 382 unset beresp.http.set-cookie;
383 set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
384 }
385  
386 # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
387 # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
388 # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
389 # This may need finetuning on your setup.
390 #
391 # To prevent accidental replace, we only filter the 301/302 redirects for now.
392 if (beresp.status == 301 || beresp.status == 302) {
393 set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
394 }
395  
396 # Set 2min cache if unset for static files
397 if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
398 set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
399 set beresp.uncacheable = true;
400 return (deliver);
401 }
402  
403 # Don't cache 50x responses
404 if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
405 return (abandon);
406 }
407  
408 # Allow stale content, in case the backend goes down.
409 # make Varnish keep all objects for 6 hours beyond their TTL
410 set beresp.grace = 6h;
411  
412 return (deliver);
413 }
414  
415 # The routine when we deliver the HTTP request to the user
416 # Last chance to modify headers that are sent to the client
417 sub vcl_deliver {
418 # Called before a cached object is delivered to the client.
419  
420 if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
421 set resp.http.X-Cache = "HIT";
422 } else {
423 set resp.http.X-Cache = "MISS";
424 }
425  
426 # Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
427 # and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
428 # So take hits with a grain of salt
429 set resp.http.X-Cache-Hits = obj.hits;
430  
18 office 431 ###
432 ## Remove some headers: PHP version
2 office 433 unset resp.http.X-Powered-By;
18 office 434 ## Remove some headers: Apache version & OS
2 office 435 unset resp.http.Server;
436 unset resp.http.X-Drupal-Cache;
437 unset resp.http.X-Varnish;
438 unset resp.http.Via;
439 unset resp.http.Link;
440 unset resp.http.X-Generator;
18 office 441 unset resp.http.X-Powered-By;
2 office 442  
443 return (deliver);
444 }
445  
446 sub vcl_purge {
447 # Only handle actual PURGE HTTP methods, everything else is discarded
448 if (req.method != "PURGE") {
449 # restart request
450 set req.http.X-Purge = "Yes";
451 return(restart);
452 }
453 }
454  
455 sub vcl_synth {
456 if (resp.status == 720) {
457 # We use this special error status 720 to force redirects with 301 (permanent) redirects
458 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
459 set resp.http.Location = resp.reason;
460 set resp.status = 301;
461 return (deliver);
462 } elseif (resp.status == 721) {
463 # And we use error status 721 to force redirects with a 302 (temporary) redirect
464 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
465 set resp.http.Location = resp.reason;
466 set resp.status = 302;
467 return (deliver);
468 }
469  
470 return (deliver);
471 }
472  
473  
474 sub vcl_fini {
475 # Called when VCL is discarded only after all requests have exited the VCL.
476 # Typically used to clean up VMODs.
477  
478 return (ok);
479 }
480  
481 sub vcl_backend_error {
482 # Called when the backend cannot be reached.
483 # Display an HTML file located at /etc/varnish/500.html
484 set beresp.http.Content-Type = "text/html; charset=utf-8";
485 synthetic(std.fileread("/etc/varnish/500.html"));
486 return (deliver);
487 }