Varnish support

Author: paliarush

.travis.yml

@@ -33,14 +33,14 @@ install:
   - sudo apt-get update && sudo apt-get install -y socat
   # TODO Install sudo apt install nfs-kernel-server
 #  - sudo apt install -y nfs-kernel-server
+  - cat ./etc/helm/values.yaml
 
 before_script:
   - sudo bash ./scripts/host/configure_ubuntu_travis.sh
   - cat ./tests/include/configuration.sh
   - cat ./scripts/host/k_rebuild_environment.sh
   - cat ./init_project.sh
   - cat ./etc/config.yaml.dist
-  - cat ./etc/helm/values.yaml
 #  - cat ./tests/_files/* # TODO: Cannot cat because of directory, move configs to a separate directory
   - cat ./scripts/host/get_path_to_php.sh
   - while sleep 540; do echo "=====[ $SECONDS seconds still running ]====="; done &

README.md

@@ -50,7 +50,7 @@ The environment for Magento EE development is also configured.
 The [project initialization script](init_project.sh) configures a complete development environment:
 
 <!-- 1. Adds some missing software on the host -->
- 1. Configures all software necessary for Magento 2: Nginx, PHP 7.x, MySQL 5.6, Git, Composer, XDebug<!--, Rabbit MQ, Varnish-->
+ 1. Configures all software necessary for Magento 2: Nginx, PHP 7.x, MySQL 5.6, Git, Composer, XDebug, Rabbit MQ, Varnish
  1. Installs Magento 2 from Git repositories or Composer packages (can be configured via `checkout_source_from` option in [etc/config.yaml](etc/config.yaml.dist))
 <!-- 1. Configures PHP Storm project (partially at the moment)-->
 <!-- 1. Installs NodeJS, NPM, Grunt and Gulp for front end development
@@ -344,13 +344,11 @@ PHP version will be applied after "devbox reload".
 
 ### Activating Varnish
 
-Not available yet.
-<!--Set `use_varnish: 1` to use varnish along with apache in [config.yaml](etc/config.yaml.dist). Changes will be applied on `m-reinstall`.
-It will use default file etc/magento2_default_varnish.vcl.dist generated from a Magento 2.1 instance. Magento 2.2+ supports additional Varnish features and you may need to provide custom version of VCL to enable them.
-Varnish Version: 4.1
+Use the following commands to enable/disable varnish <!--without reinstalling Magento-->: `m-varnish disable` or `m-varnish enable`.
 
-Use the following commands to enable/disable varnish without reinstalling Magento: `m-varnish disable` or `m-varnish enable`.
--->
+You can also set `use_varnish: 1` in [config.yaml](etc/config.yaml.dist) to use varnish. Changes will be applied on `init_project.sh -f`.
+
+The VCL content can be found in [configmap.yaml](etc/helm/templates/configmap.yaml).
 
 ### Activating ElasticSearch
 

etc/config.yaml.dist

@@ -19,6 +19,7 @@ repository_url:
   # git@github.com:magento/magento2ee.git OR custom one
   ee: ""
   additional_repositories:
+    # [To apply changes: m-switch-to-ce -f OR m-switch-to-ee -f]
     # - "git@github.com:magento/magento2-sample-data.git"
     # - "git@github.com:magento/magento2-sample-data-ee.git"
     # - "git@github.com:magento/magento2b2b.git"

etc/helm/templates/configmap.yaml

@@ -18,11 +18,11 @@ data:
       fastcgi_read_timeout 30000s;
 
       upstream fastcgi_backend {
-        server 127.0.0.1:9001;
+        server 127.0.0.1:{{ .Values.global.monolith.service.phpFpmPort }};
       }
 
       upstream fastcgi_backend_xdebug {
-        server 127.0.0.1:9002;
+        server 127.0.0.1:{{ .Values.global.monolith.service.phpFpmXdebugPort }};
       }
 
       map $cookie_XDEBUG_SESSION $fastcgi_backend_name {
@@ -31,7 +31,7 @@ data:
       }
 
       server {
-        listen 8001;
+        listen "{{ .Values.global.monolith.service.nginxPort }}";
 
         server_name _;
         set $MAGE_ROOT {{.Values.global.monolith.volumeHostPath}}/magento;
@@ -213,3 +213,181 @@ data:
         }
       }
     }
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ template "magento.fullname" . }}-varnish-config
+data:
+  varnish.vcl: |
+    vcl 4.0;
+
+    import std;
+    # The minimal Varnish version is 4.0
+    # For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'
+
+    backend default {
+        .host = "127.0.0.1";
+        .port = "{{ .Values.global.monolith.service.nginxPort }}";
+    }
+
+    acl purge {
+        "localhost";
+        "192.0.0.0"/8;
+        "172.0.0.0"/8;
+    }
+
+    sub vcl_recv {
+        if (req.method == "PURGE") {
+            if (client.ip !~ purge) {
+                return (synth(405, "Method not allowed"));
+            }
+            if (!req.http.X-Magento-Tags-Pattern) {
+                return (synth(400, "X-Magento-Tags-Pattern header required"));
+            }
+            ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
+            return (synth(200, "Purged"));
+        }
+
+        if (req.method != "GET" &&
+            req.method != "HEAD" &&
+            req.method != "PUT" &&
+            req.method != "POST" &&
+            req.method != "TRACE" &&
+            req.method != "OPTIONS" &&
+            req.method != "DELETE") {
+              /* Non-RFC2616 or CONNECT which is weird. */
+              return (pipe);
+        }
+
+        # We only deal with GET and HEAD by default
+        if (req.method != "GET" && req.method != "HEAD") {
+            return (pass);
+        }
+
+        # Bypass shopping cart, checkout and search requests
+        if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
+            return (pass);
+        }
+
+        # normalize url in case of leading HTTP scheme and domain
+        set req.url = regsub(req.url, "^http[s]?://", "");
+
+        # collect all cookies
+        std.collect(req.http.Cookie);
+
+        # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
+        if (req.http.Accept-Encoding) {
+            if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
+                # No point in compressing these
+                unset req.http.Accept-Encoding;
+            } elsif (req.http.Accept-Encoding ~ "gzip") {
+                set req.http.Accept-Encoding = "gzip";
+            } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
+                set req.http.Accept-Encoding = "deflate";
+            } else {
+                # unkown algorithm
+                unset req.http.Accept-Encoding;
+            }
+        }
+
+        # Remove Google gclid parameters to minimize the cache objects
+        set req.url = regsuball(req.url,"\?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
+        set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
+        set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
+
+        # static files are always cacheable. remove SSL flag and cookie
+            if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
+            unset req.http.Https;
+            unset req.http.X-Forwarded-Proto;
+            unset req.http.Cookie;
+        }
+
+        return (hash);
+    }
+
+    sub vcl_hash {
+        if (req.http.cookie ~ "X-Magento-Vary=") {
+            hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
+        }
+
+        # For multi site configurations to not cache each other's content
+        if (req.http.host) {
+            hash_data(req.http.host);
+        } else {
+            hash_data(server.ip);
+        }
+
+        # To make sure http users don't see ssl warning
+        if (req.http.X-Forwarded-Proto) {
+            hash_data(req.http.X-Forwarded-Proto);
+        }
+
+    }
+
+    sub vcl_backend_response {
+        if (beresp.http.content-type ~ "text") {
+            set beresp.do_esi = true;
+        }
+
+        if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
+            set beresp.do_gzip = true;
+        }
+
+        # cache only successfully responses and 404s
+        if (beresp.status != 200 && beresp.status != 404) {
+            set beresp.ttl = 0s;
+            set beresp.uncacheable = true;
+            return (deliver);
+        } elsif (beresp.http.Cache-Control ~ "private") {
+            set beresp.uncacheable = true;
+            set beresp.ttl = 86400s;
+            return (deliver);
+        }
+
+        if (beresp.http.X-Magento-Debug) {
+            set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
+        }
+
+        # validate if we need to cache it and prevent from setting cookie
+        # images, css and js are cacheable by default so we have to remove cookie also
+        if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
+            unset beresp.http.set-cookie;
+            if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
+                set beresp.http.Pragma = "no-cache";
+                set beresp.http.Expires = "-1";
+                set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
+                set beresp.grace = 1m;
+            }
+        }
+
+       # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
+       if (beresp.ttl <= 0s ||
+            beresp.http.Surrogate-control ~ "no-store" ||
+            (!beresp.http.Surrogate-Control && beresp.http.Vary == "*")) {
+            # Mark as Hit-For-Pass for the next 2 minutes
+            set beresp.ttl = 120s;
+            set beresp.uncacheable = true;
+        }
+        return (deliver);
+    }
+
+    sub vcl_deliver {
+        if (resp.http.X-Magento-Debug) {
+            if (resp.http.x-varnish ~ " ") {
+                set resp.http.X-Magento-Cache-Debug = "HIT";
+            } else {
+                set resp.http.X-Magento-Cache-Debug = "MISS";
+            }
+        } else {
+            unset resp.http.Age;
+        }
+
+        unset resp.http.X-Magento-Debug;
+        unset resp.http.X-Magento-Tags;
+        unset resp.http.X-Powered-By;
+        unset resp.http.Server;
+        unset resp.http.X-Varnish;
+        unset resp.http.Via;
+        unset resp.http.Link;
+    }

etc/helm/templates/magento2-deployment.yaml

@@ -77,10 +77,24 @@ spec:
                       - name: nginx-config-volume
                         mountPath: /etc/nginx/nginx.conf
                         subPath: nginx.conf
+                - name: varnish
+                  image: million12/varnish
+                  env:
+                    - name: VCL_CONFIG
+                      value: /etc/varnish/magento.vcl
+                    - name: VARNISHD_PARAMS
+                      value: "-a 0.0.0.0:{{ .Values.global.monolith.service.varnishPort }}"
+                  volumeMounts:
+                      - name: varnish-config-volume
+                        mountPath: /etc/varnish/magento.vcl
+                        subPath: varnish.vcl
             volumes:
                 - name: nginx-config-volume
                   configMap:
                       name: {{ template "magento.fullname" . }}-nginx-config
+                - name: varnish-config-volume
+                  configMap:
+                      name: {{ template "magento.fullname" . }}-varnish-config
                 - name: code
                 {{- if .Values.global.persistence.nfs.enabled }}
                   persistentVolumeClaim:

etc/helm/templates/service.yaml

@@ -12,6 +12,7 @@ spec:
   type: {{ .Values.global.monolith.service.type }}
   ports:
     - port: {{ .Values.global.monolith.service.port }}
+      targetPort: {{ if .Values.global.caching.varnish.enabled }}{{ .Values.global.monolith.service.varnishPort }}{{ else }}{{ .Values.global.monolith.service.nginxPort }}{{ end }}
       nodePort: {{ .Values.global.monolith.service.nodePort }}
   selector:
     app.kubernetes.io/name: {{ include "magento.name" . }}

etc/helm/values.yaml

@@ -26,6 +26,9 @@ monolith:
     imageTag: 7.1-fpm
     imagePullPolicy: Always
 global:
+  caching:
+    varnish:
+      enabled: true # This value is overridden via <devbox_project>/etc/config.yaml
   checkout:
     enabled: false
     db:
@@ -38,7 +41,11 @@ global:
     service:
       type: NodePort
       nodePort: 30301
-      port: 8001
+      port: 8050
+      nginxPort: 8001
+      phpFpmPort: 9001
+      phpFpmXdebugPort: 9002
+      varnishPort: 6081
     mysqlHost: magento-mysql
     volumeHostPath: /Users/user/Projects/magento2-kubernetes-for-developers
   persistence:

k-ssh-varnish

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")" && devbox_dir=$PWD
+
+source "${devbox_dir}/scripts/functions.sh"
+
+loginToPodContainer "$(getMagento2PodId)" "varnish"

m-varnish

@@ -14,7 +14,10 @@ if [[ ${debug_devbox_project} -eq 1 ]]; then
     set -x
 fi
 
+arguments=$@
 cd "${devbox_dir}"
-devbox ssh -c "bash /devbox/scripts/guest/m-varnish ${@}" 2> >(logError)
+executeInMagento2Container "${devbox_dir}/scripts/guest/m-varnish" -- ${arguments} 2> >(logError)
+
+bash "${devbox_dir}/k-rebuild-environment" "$@" 2> >(logError)
 
 info "$(regular)See details in $(bold)${devbox_dir}/log/${current_script_name}.log$(regular). For debug output set $(bold)debug:devbox_project$(regular) to $(bold)1$(regular) in $(bold)etc/config.yaml$(regular)"