configuration-templates – Blame information for rev 20

Subversion Repositories:
Rev:
Rev Author Line No. Line
2 office 1 vcl 4.0;
16 office 2 ###########################################################################
3 ## Copyright (C) Wizardry and Steamworks 2016 - License: GNU GPLv3 ##
4 ## Please see: http://www.gnu.org/licenses/gpl.html for legal details, ##
5 ## rights of fair usage, the disclaimer and warranty conditions. ##
6 ###########################################################################
7 ## Derived from the original by Mattias Geniar as a starting template. ##
8 ###########################################################################
2 office 9  
10 import std;
11 import directors;
12  
13 backend server1 { # Define one backend
14 .host = "192.168.0.10"; # IP or Hostname of backend
15 .port = "80"; # Port Apache or whatever is listening
16 .max_connections = 300; # Number of allowed connections
17  
18 .probe = {
19 #.url = "/"; # short easy way (GET /)
20 # We prefer to only do a HEAD /
21 .request =
22 "HEAD / HTTP/1.1"
23 "Host: grimore.org"
24 "Connection: close"
25 "User-Agent: Varnish Health Probe";
26  
27 .interval = 5s; # check the health of each backend every 5 seconds
28 .timeout = 1s; # timing out after 1 second.
29 .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
30 .threshold = 3;
31 }
32  
33 .first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
34 .connect_timeout = 5s; # How long to wait for a backend connection?
35 .between_bytes_timeout = 2s; # How long to wait between bytes received from our backend?
36 }
37  
38 acl purge {
39 # ACL we'll use later to allow purges
40 "localhost";
41 "127.0.0.1";
42 /* "::1"; */
43 }
44  
45 /*
46 acl editors {
47 # ACL to honor the "Cache-Control: no-cache" header to force a refresh but only from selected IPs
48 "localhost";
49 "127.0.0.1";
50 "::1";
51 }
52 */
53  
54 sub vcl_init {
55 # Called when VCL is loaded, before any requests pass through it.
56 # Typically used to initialize VMODs.
57  
58 new vdir = directors.round_robin();
59 vdir.add_backend(server1);
60 # vdir.add_backend(server...);
61 # vdir.add_backend(servern);
62 }
63  
64 sub vcl_recv {
65 # Called at the beginning of a request, after the complete request has been received and parsed.
66 # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable,
67 # which backend to use.
68 # also used to modify the request
69  
70 set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
71  
18 office 72 ### Brotli encoding support.
73 if(req.http.Accept-Encoding ~ "br" && req.url !~ "\.(jpg|png|gif|gz|mp3|mov|avi|mpg|mp4|swf|wmf)$") {
74 set req.http.X-brotli = "true";
75 }
76  
2 office 77 # When CloudFlare is in front of varnish, send the real IP address through X-Forwarded-For
78 # otherwise if CloudFlare is not involved, set the X-Forwarded-For to the client IP address.
79 unset req.http.X-Forwarded-For;
80 if (req.http.cf-connecting-ip) {
81 set req.http.X-Forwarded-For = req.http.cf-connecting-ip;
82 } else {
83 set req.http.X-Forwarded-For = client.ip;
84 }
85  
86 # Normalize the header, remove the port (in case you're testing this on various TCP ports)
87 set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
88  
89 # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish)
90 unset req.http.proxy;
91  
92 # Normalize the query arguments
93 set req.url = std.querysort(req.url);
94  
95 # Allow purging
96 if (req.method == "PURGE") {
97 if (!client.ip ~ purge) { # purge is the ACL defined at the begining
98 # Not from an allowed IP? Then die with an error.
99 return (synth(405, "This IP is not allowed to send PURGE requests."));
100 }
101 # If you got this stage (and didn't error out above), purge the cached result
102 return (purge);
103 }
104  
105 # Only deal with "normal" types
106 if (req.method != "GET" &&
107 req.method != "HEAD" &&
108 req.method != "PUT" &&
109 req.method != "POST" &&
110 req.method != "TRACE" &&
111 req.method != "OPTIONS" &&
112 req.method != "PATCH" &&
16 office 113 req.method != "DELETE" &&
114 req.method != "PROPFIND" &&
115 req.method != "REPORT") {
2 office 116 /* Non-RFC2616 or CONNECT which is weird. */
117 return (pipe);
118 }
119  
120 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
121 if (req.http.Upgrade ~ "(?i)websocket") {
122 return (pipe);
123 }
124  
125 # Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
126 if (req.method != "GET" && req.method != "HEAD") {
127 return (pass);
128 }
129  
130 # Some generic URL manipulation, useful for all templates that follow
131 # First remove the Google Analytics added parameters, useless for our backend
132 if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
133 set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
134 set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
135 set req.url = regsub(req.url, "\?&", "?");
136 set req.url = regsub(req.url, "\?$", "");
137 }
138  
139 # Strip hash, server doesn't need it.
140 if (req.url ~ "\#") {
141 set req.url = regsub(req.url, "\#.*$", "");
142 }
143  
144 # Strip a trailing ? if it exists
145 if (req.url ~ "\?$") {
146 set req.url = regsub(req.url, "\?$", "");
147 }
148  
149 # Some generic cookie manipulation, useful for all templates that follow
150 # Remove the "has_js" cookie
151 set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
152  
16 office 153 # Remove any CloudFlare cookies
154 set req.http.Cookie = regsuball(req.http.Cookie, "cf_.=[^;]+(; )?", "");
155  
2 office 156 # Remove any Google Analytics based cookies
157 set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
158 set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
159 set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
160 set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
161 set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
162 set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
163  
164 # Remove DoubleClick offensive cookies
165 set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
166  
167 # Remove the Quant Capital cookies (added by some plugin, all __qca)
168 set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
169  
170 # Remove the AddThis cookies
171 set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
172  
173 # Remove a ";" prefix in the cookie if present
174 set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
175  
176 # Are there cookies left with only spaces or that are empty?
177 if (req.http.cookie ~ "^\s*$") {
178 unset req.http.cookie;
179 }
180  
181 if (req.http.Cache-Control ~ "(?i)no-cache") {
182 #if (req.http.Cache-Control ~ "(?i)no-cache" && client.ip ~ editors) { # create the acl editors if you want to restrict the Ctrl-F5
183 # http://varnish.projects.linpro.no/wiki/VCLExampleEnableForceRefresh
184 # Ignore requests via proxy caches and badly behaved crawlers
185 # like msnbot that send no-cache with every request.
186 if (! (req.http.Via || req.http.User-Agent ~ "(?i)bot" || req.http.X-Purge)) {
187 #set req.hash_always_miss = true; # Doesn't seems to refresh the object in the cache
188 return(purge); # Couple this with restart in vcl_purge and X-Purge header to avoid loops
189 }
190 }
191  
192 # Large static files are delivered directly to the end-user without
193 # waiting for Varnish to fully read the file first.
194 # Varnish 4 fully supports Streaming, so set do_stream in vcl_backend_response()
195 if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
196 unset req.http.Cookie;
197 return (hash);
198 }
199  
200 # Remove all cookies for static files
201 # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
202 # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
203 # Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
16 office 204 if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
205 unset req.http.Cookie;
206 return (hash);
207 }
2 office 208  
209 # Send Surrogate-Capability headers to announce ESI support to backend
210 set req.http.Surrogate-Capability = "key=ESI/1.0";
211  
212 if (req.http.Authorization) {
213 # Not cacheable by default
214 return (pass);
215 }
216  
217 return (hash);
218 }
219  
220  
221 sub vcl_pipe {
222 # Called upon entering pipe mode.
223 # In this mode, the request is passed on to the backend, and any further data from both the client
224 # and backend is passed on unaltered until either end closes the connection. Basically, Varnish will
225 # degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode,
226 # no other VCL subroutine will ever get called after vcl_pipe.
227  
20 office 228 ## Note that only the first request to the backend will have
229 ## X-Forwarded-For set. If you use X-Forwarded-For and want to
230 ## have it set for all requests, make sure to have:
231 ## set bereq.http.connection = "close";
232 ## here. It is not set by default as it might break some broken web
233 ## applications, like IIS with NTLM authentication.
2 office 234 # set bereq.http.Connection = "Close";
235  
236 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
237 if (req.http.upgrade) {
238 set bereq.http.upgrade = req.http.upgrade;
239 }
240  
241 return (pipe);
242 }
243  
244 sub vcl_pass {
245 # Called upon entering pass mode. In this mode, the request is passed on to the backend, and the
246 # backend's response is passed on to the client, but is not entered into the cache. Subsequent
247 # requests submitted over the same client connection are handled normally.
248  
249 # return (pass);
250 }
251  
252 # The data on which the hashing will take place
253 sub vcl_hash {
254 # Called after vcl_recv to create a hash value for the request. This is used as a key
255 # to look up the object in Varnish.
256  
257 hash_data(req.url);
258  
259 if (req.http.host) {
260 hash_data(req.http.host);
261 } else {
262 hash_data(server.ip);
263 }
264  
265 # hash cookies for requests that have them
266 if (req.http.Cookie) {
267 hash_data(req.http.Cookie);
268 }
18 office 269  
270 ### Brotli encoding support.
271 if(req.http.X-brotli == "true") {
272 hash_data("brotli");
273 }
2 office 274 }
275  
276 sub vcl_hit {
277 # Called when a cache lookup is successful.
278  
279 if (obj.ttl >= 0s) {
280 # A pure unadultered hit, deliver it
281 return (deliver);
282 }
283  
284 # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
285 # When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
286 # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
287 # if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
288 # return (deliver);
289 # } else {
290 # return (fetch);
291 # }
292  
293 # We have no fresh fish. Lets look at the stale ones.
294 if (std.healthy(req.backend_hint)) {
295 # Backend is healthy. Limit age to 10s.
296 if (obj.ttl + 10s > 0s) {
297 #set req.http.grace = "normal(limited)";
298 return (deliver);
299 } else {
300 # No candidate for grace. Fetch a fresh object.
301 return(fetch);
302 }
303 } else {
304 # backend is sick - use full grace
305 if (obj.ttl + obj.grace > 0s) {
306 #set req.http.grace = "full";
307 return (deliver);
308 } else {
309 # no graced object.
310 return (fetch);
311 }
312 }
313  
314 # fetch & deliver once we get the result
315 return (fetch); # Dead code, keep as a safeguard
316 }
317  
318 sub vcl_miss {
319 # Called after a cache lookup if the requested document was not found in the cache. Its purpose
320 # is to decide whether or not to attempt to retrieve the document from the backend, and which
321 # backend to use.
322  
323 return (fetch);
324 }
325  
326 # Handle the HTTP request coming from our backend
327 sub vcl_backend_response {
328 # Called after the response headers has been successfully retrieved from the backend.
329  
330 # Pause ESI request and remove Surrogate-Control header
331 if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
332 unset beresp.http.Surrogate-Control;
333 set beresp.do_esi = true;
17 office 334 set beresp.ttl = 24h; # Set 24h ESI
2 office 335 }
18 office 336  
337 ### Brotli encoding support.
338 if(bereq.http.X-brotli == "true") {
339 set bereq.http.Accept-Encoding = "br";
340 unset bereq.http.X-brotli;
341 }
2 office 342  
16 office 343 ### Set content expiration.
344 ## You may want to disable any expiration on the backend (ie: for Apache, disable mod_expires).
345 # 1 Year
346 if(beresp.http.content-type ~ "(?i)(application/(javascript|x\-javascript))|(text/(javascript|css))|(application/(atom\+xml|rdf\+xml|rss\+xml))") {
347 set beresp.http.Expires = "" + (now + 1y);
348 }
349 # 1 Month
350 if(beresp.http.content-type ~ "(?i)(application/(vnd\.ms\-fontobject|font\-woff|x\-font\-(ttf|woff)))|(audio/ogg)|(font/(eot|opentype|woff))|(image/(bmp|gif|jp2|jpeg|png|svg\+xmlwebp))|(video/(mp4|ogg|webm))") {
351 set beresp.http.Expires = "" + (now + 24w);
352 }
353 # 1 Week
354 if(beresp.http.content-type ~ "(?i)(image/(vnd\.microsoft\.icon|x\-icon))|(application/manifest\+json)|(text/x\-cross\-domain\-policy)") {
355 set beresp.http.Expires = "" + (now + 1w);
356 }
357 # Immediately
19 office 358 # Not exactly "immediately" since we do not want to have to recache on fast reloads - ideally, set this to average page load time.
16 office 359 if(beresp.http.content-type ~ "(?i)(application/(json|ld\+json|schema\+json|vnd\.geo\+jsonxml|x\-web\-app\-manifest\+json))|(text/(xml|cache\-manifest|html))") {
360 set beresp.http.Expires = "" + (now + 3s);
361 }
362  
363 ### GZip content if not already compressed.
364 ## You may want to disable any compression on the backend (ie: for Apache, disable mod_deflate).
365 if(beresp.http.content-type ~ "(?i)(application/(atom\+xml|javascript|json|ld\+json|manifest\+json|rdf\+xml|rss\+xml|schema\+json|vnd\.geo\+json|vnd\.ms\-fontobject|x\-(font\-ttf|javascript|web\-app\-manifest\+json)|xhtml\+xml|xml)|font/(eot|opentype))|(image/(bmp|svg\+xml|vnd\.microsoft\.icon|x\-icon))|(text/(cache\-manifest|css|html|javascript|plain|vcard|vnd\.rim\.location\.xloc|vcalendar|vtt|x\-component|x\-cross\-domain\-policy|xml))") {
366 # Compress object for sending.
367 set beresp.do_gzip = true;
368 }
369  
370 ### Enable cache for all static files
371 ## The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
372 ## Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
373 if(beresp.http.content-type ~ "(?i)(application/(java\-vm|javascript|json|msword|pdf|postscript|rtf|vnd\.ms\-(excel|fontobject|powerpoint)|vnd\.oasis\.opendocument\.text|vnd\.openxmlformats\-officedocument\.(presentationml\.presentation|spreadsheetml\.sheet|wordprocessingml\.document)|x\-(7z\-compressed|bzip2|gzip|rar\-compressed|shockwave\-flash|tar)|x\-font\-(bdf|ghostscript|linux\-psf|otf|pcf|snf|ttf|type1|woff)|xz|xml|zip))|(audio/((mpeg|ogg)|(x\-(flac|matroska|ms\-wma|wav))))|(image/((bmp|gif|jp2|jpeg|png|svg\+xml|vnd\.wap\.wbmp|webp|x\-(icon|pict))))|(text/((css|csv|javascript|plain)))|(video/((mp4|quicktime|webm|x\-(flv|matroska|ms\-wmv|msvideo))))") {
2 office 374 unset beresp.http.set-cookie;
375 }
376  
377 # Large static files are delivered directly to the end-user without
378 # waiting for Varnish to fully read the file first.
379 # Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
16 office 380 if (bereq.url ~ "^[^?]*\.(7z|ace|avi|bin|bz2|flac|flv|gz|iso|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|wma|wmv|xz|zip)(\?.*)?$") {
2 office 381 unset beresp.http.set-cookie;
382 set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
383 }
384  
385 # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
386 # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
387 # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
388 # This may need finetuning on your setup.
389 #
390 # To prevent accidental replace, we only filter the 301/302 redirects for now.
391 if (beresp.status == 301 || beresp.status == 302) {
392 set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
393 }
394  
395 # Set 2min cache if unset for static files
396 if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
397 set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
398 set beresp.uncacheable = true;
399 return (deliver);
400 }
401  
402 # Don't cache 50x responses
403 if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
404 return (abandon);
405 }
406  
407 # Allow stale content, in case the backend goes down.
408 # make Varnish keep all objects for 6 hours beyond their TTL
409 set beresp.grace = 6h;
410  
411 return (deliver);
412 }
413  
414 # The routine when we deliver the HTTP request to the user
415 # Last chance to modify headers that are sent to the client
416 sub vcl_deliver {
417 # Called before a cached object is delivered to the client.
418  
419 if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
420 set resp.http.X-Cache = "HIT";
421 } else {
422 set resp.http.X-Cache = "MISS";
423 }
424  
425 # Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
426 # and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
427 # So take hits with a grain of salt
428 set resp.http.X-Cache-Hits = obj.hits;
429  
18 office 430 ###
431 ## Remove some headers: PHP version
2 office 432 unset resp.http.X-Powered-By;
18 office 433 ## Remove some headers: Apache version & OS
2 office 434 unset resp.http.Server;
435 unset resp.http.X-Drupal-Cache;
436 unset resp.http.X-Varnish;
437 unset resp.http.Via;
438 unset resp.http.Link;
439 unset resp.http.X-Generator;
18 office 440 unset resp.http.X-Powered-By;
2 office 441  
442 return (deliver);
443 }
444  
445 sub vcl_purge {
446 # Only handle actual PURGE HTTP methods, everything else is discarded
447 if (req.method != "PURGE") {
448 # restart request
449 set req.http.X-Purge = "Yes";
450 return(restart);
451 }
452 }
453  
454 sub vcl_synth {
455 if (resp.status == 720) {
456 # We use this special error status 720 to force redirects with 301 (permanent) redirects
457 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
458 set resp.http.Location = resp.reason;
459 set resp.status = 301;
460 return (deliver);
461 } elseif (resp.status == 721) {
462 # And we use error status 721 to force redirects with a 302 (temporary) redirect
463 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
464 set resp.http.Location = resp.reason;
465 set resp.status = 302;
466 return (deliver);
467 }
468  
469 return (deliver);
470 }
471  
472  
473 sub vcl_fini {
474 # Called when VCL is discarded only after all requests have exited the VCL.
475 # Typically used to clean up VMODs.
476  
477 return (ok);
478 }
479  
480 sub vcl_backend_error {
481 # Called when the backend cannot be reached.
482 # Display an HTML file located at /etc/varnish/500.html
483 set beresp.http.Content-Type = "text/html; charset=utf-8";
484 synthetic(std.fileread("/etc/varnish/500.html"));
485 return (deliver);
486 }