configuration-templates – Blame information for rev 17

Subversion Repositories:
Rev:
Rev Author Line No. Line
2 office 1 vcl 4.0;
16 office 2 ###########################################################################
3 ## Copyright (C) Wizardry and Steamworks 2016 - License: GNU GPLv3 ##
4 ## Please see: http://www.gnu.org/licenses/gpl.html for legal details, ##
5 ## rights of fair usage, the disclaimer and warranty conditions. ##
6 ###########################################################################
7 ## Derived from the original by Mattias Geniar as a starting template. ##
8 ###########################################################################
2 office 9  
10 import std;
11 import directors;
12  
13 backend server1 { # Define one backend
14 .host = "192.168.0.10"; # IP or Hostname of backend
15 .port = "80"; # Port Apache or whatever is listening
16 .max_connections = 300; # Number of allowed connections
17  
18 .probe = {
19 #.url = "/"; # short easy way (GET /)
20 # We prefer to only do a HEAD /
21 .request =
22 "HEAD / HTTP/1.1"
23 "Host: grimore.org"
24 "Connection: close"
25 "User-Agent: Varnish Health Probe";
26  
27 .interval = 5s; # check the health of each backend every 5 seconds
28 .timeout = 1s; # timing out after 1 second.
29 .window = 5; # If 3 out of the last 5 polls succeeded the backend is considered healthy, otherwise it will be marked as sick
30 .threshold = 3;
31 }
32  
33 .first_byte_timeout = 300s; # How long to wait before we receive a first byte from our backend?
34 .connect_timeout = 5s; # How long to wait for a backend connection?
35 .between_bytes_timeout = 2s; # How long to wait between bytes received from our backend?
36 }
37  
38 acl purge {
39 # ACL we'll use later to allow purges
40 "localhost";
41 "127.0.0.1";
42 /* "::1"; */
43 }
44  
45 /*
46 acl editors {
47 # ACL to honor the "Cache-Control: no-cache" header to force a refresh but only from selected IPs
48 "localhost";
49 "127.0.0.1";
50 "::1";
51 }
52 */
53  
54 sub vcl_init {
55 # Called when VCL is loaded, before any requests pass through it.
56 # Typically used to initialize VMODs.
57  
58 new vdir = directors.round_robin();
59 vdir.add_backend(server1);
60 # vdir.add_backend(server...);
61 # vdir.add_backend(servern);
62 }
63  
64 sub vcl_recv {
65 # Called at the beginning of a request, after the complete request has been received and parsed.
66 # Its purpose is to decide whether or not to serve the request, how to do it, and, if applicable,
67 # which backend to use.
68 # also used to modify the request
69  
70 set req.backend_hint = vdir.backend(); # send all traffic to the vdir director
71  
72 # When CloudFlare is in front of varnish, send the real IP address through X-Forwarded-For
73 # otherwise if CloudFlare is not involved, set the X-Forwarded-For to the client IP address.
74 unset req.http.X-Forwarded-For;
75 if (req.http.cf-connecting-ip) {
76 set req.http.X-Forwarded-For = req.http.cf-connecting-ip;
77 } else {
78 set req.http.X-Forwarded-For = client.ip;
79 }
80  
81 # Normalize the header, remove the port (in case you're testing this on various TCP ports)
82 set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
83  
84 # Remove the proxy header (see https://httpoxy.org/#mitigate-varnish)
85 unset req.http.proxy;
86  
87 # Normalize the query arguments
88 set req.url = std.querysort(req.url);
89  
90 # Allow purging
91 if (req.method == "PURGE") {
92 if (!client.ip ~ purge) { # purge is the ACL defined at the begining
93 # Not from an allowed IP? Then die with an error.
94 return (synth(405, "This IP is not allowed to send PURGE requests."));
95 }
96 # If you got this stage (and didn't error out above), purge the cached result
97 return (purge);
98 }
99  
100 # Only deal with "normal" types
101 if (req.method != "GET" &&
102 req.method != "HEAD" &&
103 req.method != "PUT" &&
104 req.method != "POST" &&
105 req.method != "TRACE" &&
106 req.method != "OPTIONS" &&
107 req.method != "PATCH" &&
16 office 108 req.method != "DELETE" &&
109 req.method != "PROPFIND" &&
110 req.method != "REPORT") {
2 office 111 /* Non-RFC2616 or CONNECT which is weird. */
112 return (pipe);
113 }
114  
115 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
116 if (req.http.Upgrade ~ "(?i)websocket") {
117 return (pipe);
118 }
119  
120 # Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
121 if (req.method != "GET" && req.method != "HEAD") {
122 return (pass);
123 }
124  
125 # Some generic URL manipulation, useful for all templates that follow
126 # First remove the Google Analytics added parameters, useless for our backend
127 if (req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=") {
128 set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
129 set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
130 set req.url = regsub(req.url, "\?&", "?");
131 set req.url = regsub(req.url, "\?$", "");
132 }
133  
134 # Strip hash, server doesn't need it.
135 if (req.url ~ "\#") {
136 set req.url = regsub(req.url, "\#.*$", "");
137 }
138  
139 # Strip a trailing ? if it exists
140 if (req.url ~ "\?$") {
141 set req.url = regsub(req.url, "\?$", "");
142 }
143  
144 # Some generic cookie manipulation, useful for all templates that follow
145 # Remove the "has_js" cookie
146 set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
147  
16 office 148 # Remove any CloudFlare cookies
149 set req.http.Cookie = regsuball(req.http.Cookie, "cf_.=[^;]+(; )?", "");
150  
2 office 151 # Remove any Google Analytics based cookies
152 set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
153 set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
154 set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
155 set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
156 set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
157 set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
158  
159 # Remove DoubleClick offensive cookies
160 set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
161  
162 # Remove the Quant Capital cookies (added by some plugin, all __qca)
163 set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
164  
165 # Remove the AddThis cookies
166 set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
167  
168 # Remove a ";" prefix in the cookie if present
169 set req.http.Cookie = regsuball(req.http.Cookie, "^;\s*", "");
170  
171 # Are there cookies left with only spaces or that are empty?
172 if (req.http.cookie ~ "^\s*$") {
173 unset req.http.cookie;
174 }
175  
176 if (req.http.Cache-Control ~ "(?i)no-cache") {
177 #if (req.http.Cache-Control ~ "(?i)no-cache" && client.ip ~ editors) { # create the acl editors if you want to restrict the Ctrl-F5
178 # http://varnish.projects.linpro.no/wiki/VCLExampleEnableForceRefresh
179 # Ignore requests via proxy caches and badly behaved crawlers
180 # like msnbot that send no-cache with every request.
181 if (! (req.http.Via || req.http.User-Agent ~ "(?i)bot" || req.http.X-Purge)) {
182 #set req.hash_always_miss = true; # Doesn't seems to refresh the object in the cache
183 return(purge); # Couple this with restart in vcl_purge and X-Purge header to avoid loops
184 }
185 }
186  
187 # Large static files are delivered directly to the end-user without
188 # waiting for Varnish to fully read the file first.
189 # Varnish 4 fully supports Streaming, so set do_stream in vcl_backend_response()
190 if (req.url ~ "^[^?]*\.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(\?.*)?$") {
191 unset req.http.Cookie;
192 return (hash);
193 }
194  
195 # Remove all cookies for static files
196 # A valid discussion could be held on this line: do you really need to cache static files that don't cause load? Only if you have memory left.
197 # Sure, there's disk I/O, but chances are your OS will already have these files in their buffers (thus memory).
198 # Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
16 office 199 if (req.url ~ "^[^?]*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?.*)?$") {
200 unset req.http.Cookie;
201 return (hash);
202 }
2 office 203  
204 # Send Surrogate-Capability headers to announce ESI support to backend
205 set req.http.Surrogate-Capability = "key=ESI/1.0";
206  
207 if (req.http.Authorization) {
208 # Not cacheable by default
209 return (pass);
210 }
211  
212 return (hash);
213 }
214  
215  
216 sub vcl_pipe {
217 # Called upon entering pipe mode.
218 # In this mode, the request is passed on to the backend, and any further data from both the client
219 # and backend is passed on unaltered until either end closes the connection. Basically, Varnish will
220 # degrade into a simple TCP proxy, shuffling bytes back and forth. For a connection in pipe mode,
221 # no other VCL subroutine will ever get called after vcl_pipe.
222  
223 # Note that only the first request to the backend will have
224 # X-Forwarded-For set. If you use X-Forwarded-For and want to
225 # have it set for all requests, make sure to have:
226 # set bereq.http.connection = "close";
227 # here. It is not set by default as it might break some broken web
228 # applications, like IIS with NTLM authentication.
229  
230 # set bereq.http.Connection = "Close";
231  
232 # Implementing websocket support (https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html)
233 if (req.http.upgrade) {
234 set bereq.http.upgrade = req.http.upgrade;
235 }
236  
237 return (pipe);
238 }
239  
240 sub vcl_pass {
241 # Called upon entering pass mode. In this mode, the request is passed on to the backend, and the
242 # backend's response is passed on to the client, but is not entered into the cache. Subsequent
243 # requests submitted over the same client connection are handled normally.
244  
245 # return (pass);
246 }
247  
248 # The data on which the hashing will take place
249 sub vcl_hash {
250 # Called after vcl_recv to create a hash value for the request. This is used as a key
251 # to look up the object in Varnish.
252  
253 hash_data(req.url);
254  
255 if (req.http.host) {
256 hash_data(req.http.host);
257 } else {
258 hash_data(server.ip);
259 }
260  
261 # hash cookies for requests that have them
262 if (req.http.Cookie) {
263 hash_data(req.http.Cookie);
264 }
265 }
266  
267 sub vcl_hit {
268 # Called when a cache lookup is successful.
269  
270 if (obj.ttl >= 0s) {
271 # A pure unadultered hit, deliver it
272 return (deliver);
273 }
274  
275 # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
276 # When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
277 # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
278 # if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
279 # return (deliver);
280 # } else {
281 # return (fetch);
282 # }
283  
284 # We have no fresh fish. Lets look at the stale ones.
285 if (std.healthy(req.backend_hint)) {
286 # Backend is healthy. Limit age to 10s.
287 if (obj.ttl + 10s > 0s) {
288 #set req.http.grace = "normal(limited)";
289 return (deliver);
290 } else {
291 # No candidate for grace. Fetch a fresh object.
292 return(fetch);
293 }
294 } else {
295 # backend is sick - use full grace
296 if (obj.ttl + obj.grace > 0s) {
297 #set req.http.grace = "full";
298 return (deliver);
299 } else {
300 # no graced object.
301 return (fetch);
302 }
303 }
304  
305 # fetch & deliver once we get the result
306 return (fetch); # Dead code, keep as a safeguard
307 }
308  
309 sub vcl_miss {
310 # Called after a cache lookup if the requested document was not found in the cache. Its purpose
311 # is to decide whether or not to attempt to retrieve the document from the backend, and which
312 # backend to use.
313  
314 return (fetch);
315 }
316  
317 # Handle the HTTP request coming from our backend
318 sub vcl_backend_response {
319 # Called after the response headers has been successfully retrieved from the backend.
320  
321 # Pause ESI request and remove Surrogate-Control header
322 if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
323 unset beresp.http.Surrogate-Control;
324 set beresp.do_esi = true;
17 office 325 set beresp.ttl = 24h; # Set 24h ESI
2 office 326 }
327  
16 office 328 ### Set content expiration.
329 ## You may want to disable any expiration on the backend (ie: for Apache, disable mod_expires).
330 # 1 Year
331 if(beresp.http.content-type ~ "(?i)(application/(javascript|x\-javascript))|(text/(javascript|css))|(application/(atom\+xml|rdf\+xml|rss\+xml))") {
332 set beresp.http.Expires = "" + (now + 1y);
333 }
334 # 1 Month
335 if(beresp.http.content-type ~ "(?i)(application/(vnd\.ms\-fontobject|font\-woff|x\-font\-(ttf|woff)))|(audio/ogg)|(font/(eot|opentype|woff))|(image/(bmp|gif|jp2|jpeg|png|svg\+xmlwebp))|(video/(mp4|ogg|webm))") {
336 set beresp.http.Expires = "" + (now + 24w);
337 }
338 # 1 Week
339 if(beresp.http.content-type ~ "(?i)(image/(vnd\.microsoft\.icon|x\-icon))|(application/manifest\+json)|(text/x\-cross\-domain\-policy)") {
340 set beresp.http.Expires = "" + (now + 1w);
341 }
342 # Immediately
343 # Not exactly "immediately" since we do not want to have to recache on fast reloads - ideally, set this to average page log time.
344 if(beresp.http.content-type ~ "(?i)(application/(json|ld\+json|schema\+json|vnd\.geo\+jsonxml|x\-web\-app\-manifest\+json))|(text/(xml|cache\-manifest|html))") {
345 set beresp.http.Expires = "" + (now + 3s);
346 }
347  
348 ### GZip content if not already compressed.
349 ## You may want to disable any compression on the backend (ie: for Apache, disable mod_deflate).
350 if(beresp.http.content-type ~ "(?i)(application/(atom\+xml|javascript|json|ld\+json|manifest\+json|rdf\+xml|rss\+xml|schema\+json|vnd\.geo\+json|vnd\.ms\-fontobject|x\-(font\-ttf|javascript|web\-app\-manifest\+json)|xhtml\+xml|xml)|font/(eot|opentype))|(image/(bmp|svg\+xml|vnd\.microsoft\.icon|x\-icon))|(text/(cache\-manifest|css|html|javascript|plain|vcard|vnd\.rim\.location\.xloc|vcalendar|vtt|x\-component|x\-cross\-domain\-policy|xml))") {
351 # Compress object for sending.
352 set beresp.do_gzip = true;
353 }
354  
355 ### Enable cache for all static files
356 ## The same argument as the static caches from above: monitor your cache size, if you get data nuked out of it, consider giving up the static file cache.
357 ## Before you blindly enable this, have a read here: https://ma.ttias.be/stop-caching-static-files/
358 if(beresp.http.content-type ~ "(?i)(application/(java\-vm|javascript|json|msword|pdf|postscript|rtf|vnd\.ms\-(excel|fontobject|powerpoint)|vnd\.oasis\.opendocument\.text|vnd\.openxmlformats\-officedocument\.(presentationml\.presentation|spreadsheetml\.sheet|wordprocessingml\.document)|x\-(7z\-compressed|bzip2|gzip|rar\-compressed|shockwave\-flash|tar)|x\-font\-(bdf|ghostscript|linux\-psf|otf|pcf|snf|ttf|type1|woff)|xz|xml|zip))|(audio/((mpeg|ogg)|(x\-(flac|matroska|ms\-wma|wav))))|(image/((bmp|gif|jp2|jpeg|png|svg\+xml|vnd\.wap\.wbmp|webp|x\-(icon|pict))))|(text/((css|csv|javascript|plain)))|(video/((mp4|quicktime|webm|x\-(flv|matroska|ms\-wmv|msvideo))))") {
2 office 359 unset beresp.http.set-cookie;
360 }
361  
362 # Large static files are delivered directly to the end-user without
363 # waiting for Varnish to fully read the file first.
364 # Varnish 4 fully supports Streaming, so use streaming here to avoid locking.
16 office 365 if (bereq.url ~ "^[^?]*\.(7z|ace|avi|bin|bz2|flac|flv|gz|iso|mka|mkv|mov|mp3|mp4|mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|wma|wmv|xz|zip)(\?.*)?$") {
2 office 366 unset beresp.http.set-cookie;
367 set beresp.do_stream = true; # Check memory usage it'll grow in fetch_chunksize blocks (128k by default) if the backend doesn't send a Content-Length header, so only enable it for big objects
368 }
369  
370 # Sometimes, a 301 or 302 redirect formed via Apache's mod_rewrite can mess with the HTTP port that is being passed along.
371 # This often happens with simple rewrite rules in a scenario where Varnish runs on :80 and Apache on :8080 on the same box.
372 # A redirect can then often redirect the end-user to a URL on :8080, where it should be :80.
373 # This may need finetuning on your setup.
374 #
375 # To prevent accidental replace, we only filter the 301/302 redirects for now.
376 if (beresp.status == 301 || beresp.status == 302) {
377 set beresp.http.Location = regsub(beresp.http.Location, ":[0-9]+", "");
378 }
379  
380 # Set 2min cache if unset for static files
381 if (beresp.ttl <= 0s || beresp.http.Set-Cookie || beresp.http.Vary == "*") {
382 set beresp.ttl = 120s; # Important, you shouldn't rely on this, SET YOUR HEADERS in the backend
383 set beresp.uncacheable = true;
384 return (deliver);
385 }
386  
387 # Don't cache 50x responses
388 if (beresp.status == 500 || beresp.status == 502 || beresp.status == 503 || beresp.status == 504) {
389 return (abandon);
390 }
391  
392 # Allow stale content, in case the backend goes down.
393 # make Varnish keep all objects for 6 hours beyond their TTL
394 set beresp.grace = 6h;
395  
396 return (deliver);
397 }
398  
399 # The routine when we deliver the HTTP request to the user
400 # Last chance to modify headers that are sent to the client
401 sub vcl_deliver {
402 # Called before a cached object is delivered to the client.
403  
404 if (obj.hits > 0) { # Add debug header to see if it's a HIT/MISS and the number of hits, disable when not needed
405 set resp.http.X-Cache = "HIT";
406 } else {
407 set resp.http.X-Cache = "MISS";
408 }
409  
410 # Please note that obj.hits behaviour changed in 4.0, now it counts per objecthead, not per object
411 # and obj.hits may not be reset in some cases where bans are in use. See bug 1492 for details.
412 # So take hits with a grain of salt
413 set resp.http.X-Cache-Hits = obj.hits;
414  
415 # Remove some headers: PHP version
416 unset resp.http.X-Powered-By;
417  
418 # Remove some headers: Apache version & OS
419 unset resp.http.Server;
420 unset resp.http.X-Drupal-Cache;
421 unset resp.http.X-Varnish;
422 unset resp.http.Via;
423 unset resp.http.Link;
424 unset resp.http.X-Generator;
425  
426 return (deliver);
427 }
428  
429 sub vcl_purge {
430 # Only handle actual PURGE HTTP methods, everything else is discarded
431 if (req.method != "PURGE") {
432 # restart request
433 set req.http.X-Purge = "Yes";
434 return(restart);
435 }
436 }
437  
438 sub vcl_synth {
439 if (resp.status == 720) {
440 # We use this special error status 720 to force redirects with 301 (permanent) redirects
441 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
442 set resp.http.Location = resp.reason;
443 set resp.status = 301;
444 return (deliver);
445 } elseif (resp.status == 721) {
446 # And we use error status 721 to force redirects with a 302 (temporary) redirect
447 # To use this, call the following from anywhere in vcl_recv: return (synth(720, "http://host/new.html"));
448 set resp.http.Location = resp.reason;
449 set resp.status = 302;
450 return (deliver);
451 }
452  
453 return (deliver);
454 }
455  
456  
457 sub vcl_fini {
458 # Called when VCL is discarded only after all requests have exited the VCL.
459 # Typically used to clean up VMODs.
460  
461 return (ok);
462 }
463  
464 sub vcl_backend_error {
465 # Called when the backend cannot be reached.
466 # Display an HTML file located at /etc/varnish/500.html
467 set beresp.http.Content-Type = "text/html; charset=utf-8";
468 synthetic(std.fileread("/etc/varnish/500.html"));
469 return (deliver);
470 }