Add additional features to Helm Chart (#395)

* refactor helm chart

* remove duplicate command from cronjob

* Remove extra chart values header in the README

* rename persistent volume to neo-pv

* document frontEnd.baseUrl

amundsen-kube-helm/README.md

@@ -44,24 +44,34 @@ The following table lists the configurable parameters of the Amundsen charts and
 | frontEnd.OIDC_CLIENT_SECRET | string | `""` | The client secret for OIDC. |
 | frontEnd.OIDC_ORG_URL | string | `nil` | The organization URL for OIDC. |
 | frontEnd.affinity | object | `{}` | Frontend pod specific affinity. |
+| frontEnd.annotations | object | `{}` | Frontend service specific tolerations. |
+| frontEnd.baseUrl | string | `"http://localhost"` | used by notifications util to provide links to amundsen pages in emails. |
 | frontEnd.createOidcSecret | bool | `false` | OIDC needs some configuration. If you want the chart to make your secrets, set this to true and set the next four values. If you don't want to configure your secrets via helm, you can still use the amundsen-oidc-config.yaml as a template |
-| frontEnd.imageVersion | string | `"2.0.0"` | The frontend version of the metadata container. |
+| frontEnd.image | string | `"amundsendev/amundsen-frontend"` | The image of the frontend container. |
+| frontEnd.imageTag | string | `"2.0.0"` | The image tag of the frontend container. |
 | frontEnd.nodeSelector | object | `{}` | Frontend pod specific nodeSelector. |
 | frontEnd.oidcEnabled | bool | `false` | To enable auth via OIDC, set this to true. |
+| frontEnd.podAnnotations | object | `{}` | Frontend pod specific annotations. |
 | frontEnd.replicas | int | `1` | How many replicas of the frontend service to run. |
 | frontEnd.resources | object | `{}` | See pod resourcing [ref](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) |
 | frontEnd.serviceName | string | `"frontend"` | The frontend service name. |
 | frontEnd.servicePort | int | `80` | The port the frontend service will be exposed on via the loadbalancer. |
+| frontEnd.serviceType | string | `"ClusterIP"` | The frontend service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
 | frontEnd.tolerations | list | `[]` | Frontend pod specific tolerations. |
 | metadata.affinity | object | `{}` | Metadata pod specific affinity. |
-| metadata.imageVersion | string | `"2.0.0"` | The image version of the metadata container. |
+| metadata.annotations | object | `{}` | Metadata service specific tolerations. |
+| metadata.image | string | `"amundsendev/amundsen-metadata"` | The image of the metadata container. |
+| metadata.imageTag | string | `"2.0.0"` | The image tag of the metadata container. |
 | metadata.neo4jEndpoint | string | `nil` | The name of the service hosting neo4j on your cluster, if you bring your own. You should only need to change this, if you don't use the version in this chart. |
 | metadata.nodeSelector | object | `{}` | Metadata pod specific nodeSelector. |
+| metadata.podAnnotations | object | `{}` | Metadata pod specific annotations. |
 | metadata.replicas | int | `1` | How many replicas of the metadata service to run. |
 | metadata.resources | object | `{}` | See pod resourcing [ref](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) |
 | metadata.serviceName | string | `"metadata"` | The metadata service name. |
+| metadata.serviceType | string | `"ClusterIP"` | The metadata service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
 | metadata.tolerations | list | `[]` | Metadata pod specific tolerations. |
 | neo4j.affinity | object | `{}` | neo4j specific affinity. |
+| neo4j.annotations | object | `{}` | neo4j service specific tolerations. |
 | neo4j.backup | object | `{"enabled":false,"s3Path":"s3://dev/null","schedule":"0 * * * *"}` | If enabled is set to true, make sure and set the s3 path as well. |
 | neo4j.backup.s3Path | string | `"s3://dev/null"` | The s3path to write to for backups. |
 | neo4j.backup.schedule | string | `"0 * * * *"` | The schedule to run backups on. Defaults to hourly. |
@@ -73,18 +83,24 @@ The following table lists the configurable parameters of the Amundsen charts and
 | neo4j.enabled | bool | `true` | If neo4j is enabled as part of this chart, or not. Set this to false if you want to provide your own version. |
 | neo4j.nodeSelector | object | `{}` | neo4j specific nodeSelector. |
 | neo4j.persistence | object | `{}` | Neo4j persistence. Turn this on to keep your data between pod crashes, etc. This is also needed for backups. |
+| neo4j.podAnnotations | object | `{}` | neo4j pod specific annotations. |
 | neo4j.resources | object | `{}` | See pod resourcing [ref](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) |
 | neo4j.tolerations | list | `[]` | neo4j specific tolerations. |
 | neo4j.version | string | `"3.3.0"` | The neo4j application version used by amundsen. |
 | nodeSelector | object | `{}` | amundsen application wide configuration of nodeSelector. This applies to search, metadata, frontend and neo4j. Elasticsearch has it's own configuation properties for this. [ref](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) |
+| podAnnotations | object | `{}` | amundsen application wide configuration of podAnnotations. This applies to search, metadata, frontend and neo4j. Elasticsearch has it's own configuation properties for this. [ref](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) |
 | provider | string | `"aws"` | The cloud provider the app is running in. Used to construct dns hostnames (on aws only). |
 | search.affinity | object | `{}` | Search pod specific affinity. |
+| search.annotations | object | `{}` | Search service specific tolerations. |
 | search.elasticsearchEndpoint | string | `nil` | The name of the service hosting elasticsearch on your cluster, if you bring your own. You should only need to change this, if you don't use the version in this chart. |
-| search.imageVersion | string | `"2.0.0"` | The image version of the search container. |
+| search.image | string | `"amundsendev/amundsen-search"` | The image of the search container. |
+| search.imageTag | string | `"2.0.0"` | The image tag of the search container. |
 | search.nodeSelector | object | `{}` | Search pod specific nodeSelector. |
+| search.podAnnotations | object | `{}` | Search pod specific annotations. |
 | search.replicas | int | `1` | How many replicas of the search service to run. |
 | search.resources | object | `{}` | See pod resourcing [ref](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) |
 | search.serviceName | string | `"search"` | The search service name. |
+| search.serviceType | string | `"ClusterIP"` | The search service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
 | search.tolerations | list | `[]` | Search pod specific tolerations. |
 | tolerations | list | `[]` | amundsen application wide configuration of tolerations. This applies to search, metadata, frontend and neo4j. Elasticsearch has it's own configuation properties for this. [ref](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature) |
 

amundsen-kube-helm/templates/helm/Chart.yaml

 apiVersion: v1
 description: Amundsen is a metadata driven application for improving the productivity of data analysts, data scientists and engineers when interacting with data.
 name: amundsen
-version: 1.1.0
+version: 2.0.0
 icon: https://github.com/lyft/amundsen/blob/master/docs/img/logos/amundsen_logo_on_light.svg
 home: https://github.com/lyft/amundsen
 maintainers:

amundsen-kube-helm/templates/helm/templates/_helpers.tpl

+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "amundsen.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "amundsen.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "amundsen.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "amundsen.labels" -}}
+app.kubernetes.io/name: {{ include "amundsen.name" . }}
+helm.sh/chart: {{ include "amundsen.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}

amundsen-kube-helm/templates/helm/templates/amundsen-service.yaml

----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
-  labels:
-    run: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
-  annotations:
-  {{- if (eq .Values.provider "aws") }}
-    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.search.serviceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
-    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
-    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
-  {{- end }}          
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Local
-  ports:
-    - port: 5001
-      name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}-{{ .Values.environment }}-http
-      targetPort: 5001
-  selector:
-    run: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
----  
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-  labels:
-    run: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-  annotations:
-  {{- if (eq .Values.provider "aws") }}
-    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
-    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
-    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
-  {{- end }}              
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Local
-  ports:
-    - port: 5002
-      name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}-{{ .Values.environment }}-http
-      targetPort: 5002
-  selector:
-    run: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
----         
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
-  labels:
-    run: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
-  annotations:
-  {{- if (eq .Values.provider "aws") }}
-    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
-    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
-    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
-  {{- end }}       
-spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Local
-  ports:
-    - port: {{ .Values.frontEnd.servicePort }}
-      name: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}-{{ .Values.environment }}-http
-      targetPort: 5000
-  selector:
-    run: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
----            
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/neo4j-configmap.yaml → amundsen-kube-helm/templates/helm/templates/configmap-neo4j.yaml

+  
 {{ if .Values.neo4j.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: neo4j-configmap
   labels:
-    app: "neo4j"
+    app: {{ template "amundsen.name" . }}
+    component: neo4j
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 data:
   neo4j.conf: |-
     apoc.import.file.enabled=true
@@ -32,4 +37,4 @@ data:
     dbms.windows_service_name=neo4j
     apoc.export.file.enabled=true
     apoc.import.file.enabled=true
-{{ end }}
\ No newline at end of file
+{{ end }}

amundsen-kube-helm/templates/helm/templates/cronjob-neo4j-s3-backup.yaml

+{{ if and .Values.neo4j.enabled (and .Values.neo4j.backup.enabled .Values.neo4j.backup.s3Path .Values.neo4j.persistence) }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: neo4j-s3-backup
+  labels:
+    app: {{ template "amundsen.name" . }}
+    component: neo4j-s3-backup
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  schedule: {{ .Values.neo4j.backup.schedule | quote }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: {{ template "amundsen.name" . }}
+            component: neo4j-s3-backup
+            release: {{ .Release.Name }}
+      {{- with .Values.neo4j.backup.podAnnotations }}
+          annotations:
+{{ toYaml . | indent 12 }}
+      {{- end }}
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: backup-neo4j
+            image: neo4j:{{ .Values.neo4j.version }}
+            command:
+            - "/bin/sh"
+            - "-c"
+            - |
+              apk -v --update add --no-cache --quiet curl python py-pip &&
+              pip install awscli -q &&
+              NOW="$(date "+%Y-%m-%d-%H:%M:%S")" &&
+              BACKUP_SCHEMA_NAME="graph.db-backup-$NOW.schema" &&
+              BACKUP_DATA_NAME="graph.db-backup-$NOW.data" &&
+              BACKUP_NAME="graph.db-backup-$NOW" &&
+              echo "CALL apoc.export.cypher.schema('/var/lib/neo4j/data/$BACKUP_SCHEMA_NAME', {});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
+              echo "CALL apoc.export.graphml.all('/var/lib/neo4j/data/$BACKUP_DATA_NAME', {useTypes: true, readLabels: true});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
+              printf "\nTarring -> /data/$BACKUP_SCHEMA_NAME and /data/$BACKUP_DATA_NAME to /data/$BACKUP_NAME.tar" &&
+              while [ ! -f /data/$BACKUP_DATA_NAME ]; do echo "backup data file does not exist: [/data/$BACKUP_DATA_NAME] sleeping..." && ls "/data/" && sleep 30; done &&
+              tar -cvf "/data/$BACKUP_NAME.tar" "/data/$BACKUP_SCHEMA_NAME" "/data/$BACKUP_DATA_NAME" &&
+              printf "\nZipping -> /data/$BACKUP_NAME.tar.gz\n" &&
+              gzip -9 "/data/$BACKUP_NAME.tar" &&
+              printf "Pushing /data/$BACKUP_NAME.tar.gz -> $BUCKET" &&
+              aws s3 cp "/data/$BACKUP_NAME.tar.gz" "$BUCKET" &&
+              printf "Cleaning up /data/graph.db-backup*" &&
+              rm /data/graph.db-backup*
+            env:
+            - name: BUCKET
+              value: {{ .Values.neo4j.backup.s3Path }}
+            volumeMounts:
+                - name: data
+                  mountPath: /data
+          volumes:
+          - name: data
+            persistentVolumeClaim:
+              claimName: neo4j-pvc
+{{- end}}

amundsen-kube-helm/templates/helm/templates/amundsen-deployment.yaml → amundsen-kube-helm/templates/helm/templates/deployment-frontend.yaml

@@ -2,119 +2,30 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
+  name: {{ template "amundsen.fullname" . }}-{{ .Values.frontEnd.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.frontEnd.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 spec:
   selector:
     matchLabels:
-      run: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
-  replicas: {{ default 1 .Values.search.replicas }}
-  template:
-    metadata:
-      labels:
-        run: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
-    spec:
-      {{- with default .Values.nodeSelector .Values.search.nodeSelector }}
-      nodeSelector:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with default .Values.affinity .Values.search.affinity }}
-      affinity:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with default .Values.tolerations .Values.search.tolerations }}
-      tolerations:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      containers:
-      - name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
-        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.search.serviceName }}:{{ .Values.search.imageVersion }}
-        ports:
-        - containerPort: 5000  
-        env:
-        - name: PROXY_ENDPOINT
-          value: {{ if .Values.search.elasticsearchEndpoint }}{{ .Values.search.elasticsearchEndpoint }}{{ else }}{{ .Release.Namespace }}-elasticsearch-client.{{ .Release.Namespace }}.svc.cluster.local{{ end }}
-        {{- with .Values.search.resources }}
-        resources:
-{{ toYaml . | indent 10 }}
-        {{- end }}
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-spec:
-  selector:
-    matchLabels:
-      run: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-  replicas: {{ default 1 .Values.metadataReplicas }}
+      app: {{ template "amundsen.name" . }}
+      component: {{ .Values.frontEnd.serviceName }}
+      release: {{ .Release.Name }}
+  replicas: {{ default 1 .Values.frontEnd.replicas }}
   template:
     metadata:
-      labels:
-        run: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-    spec:
-      {{- with default .Values.nodeSelector .Values.metadata.nodeSelector }}
-      nodeSelector:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with default .Values.affinity .Values.metadata.affinity }}
-      affinity:
-{{ toYaml . | indent 8 }}
-      {{- end }}
-      {{- with default .Values.tolerations .Values.metadata.tolerations }}
-      tolerations:
+      {{- with default .Values.podAnnotations .Values.frontEnd.podAnnotations }}
+      annotations:
 {{ toYaml . | indent 8 }}
       {{- end }}
-      volumes:
-      {{- if .Values.frontEnd.oidcEnabled }}
-        - name: oidc-config
-          secret:
-            secretName: oidc-config
-      {{- end }}
-      containers:
-      - name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
-        {{- with .Values.metadataServiceImage }}
-        image: {{ . }}
-        {{- else }}
-        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.metadata.serviceName }}{{ if .Values.frontEnd.oidcEnabled }}-oidc{{ end }}:{{ .Values.metadata.imageVersion }}
-        {{- end }}
-        ports:
-        - containerPort: 5000
-        env:
-        - name: PROXY_HOST
-          value: {{ if .Values.metadata.neo4jEndpoint }}{{ .Values.metadata.neo4jEndpoint }}{{ else }}bolt://neo4j.{{ .Release.Namespace }}.svc.cluster.local{{ end }}
-        {{- if .Values.frontEnd.oidcEnabled }}
-        - name: FLASK_OIDC_CLIENT_SECRETS
-          value: /etc/client_secrets.json
-        - name: FLASK_OIDC_SECRET_KEY
-          valueFrom:
-            secretKeyRef:
-              name: oidc-config
-              key: OIDC_CLIENT_SECRET
-        {{- end }}
-        volumeMounts:
-        {{- if .Values.frontEnd.oidcEnabled }}
-          - name: oidc-config
-            mountPath: /etc/client_secrets.json
-            subPath: client_secrets.json
-        {{- end }}
-        {{- with .Values.metadata.resources }}
-        resources:
-{{ toYaml . | indent 10 }}
-        {{- end }}
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
-spec:
-  selector:
-    matchLabels:
-      run: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
-  replicas: {{ default 1 .Values.frontEnd.replicas }}
-  template:
-    metadata:
       labels:
-        run: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
+        app: {{ template "amundsen.name" . }}
+        component: {{ .Values.frontEnd.serviceName }}
+        release: {{ .Release.Name }}
     spec:
       {{- with default .Values.nodeSelector .Values.frontEnd.nodeSelector }}
       nodeSelector:
@@ -136,19 +47,13 @@ spec:
       {{- end }}
       containers:
       - name: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
-        {{- with .Values.frontEnd.serviceImage }}
-        image: {{ . }}
-        {{- else }}
-        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}{{ if .Values.frontEnd.oidcEnabled }}-oidc{{ end }}:{{ .Values.frontEnd.imageVersion }}
-        {{- end }}
+        image: {{ .Values.frontEnd.image }}:{{ .Values.frontEnd.imageTag }}
+        imagePullPolicy: Always
         ports:
           - containerPort: 5000
         env:
-          # FRONTEND_BASE is used by notifications util to provide links to amundsen pages in emails. If its not set, it will default to localhost.
-          {{ if .Values.frontEnd.FRONTEND_BASE }}
           - name: FRONTEND_BASE
-            value: http://{{ .Values.frontEnd.FRONTEND_BASE }}
-          {{ end }}
+            value: {{ .Values.frontEnd.baseUrl }}
           - name: SEARCHSERVICE_BASE
             value: http://{{ .Chart.Name }}-{{ .Values.search.serviceName }}:5001
           - name: METADATASERVICE_BASE
@@ -156,6 +61,16 @@ spec:
           - name: LONG_RANDOM_STRING
             value: {{ quote .Values.LONG_RANDOM_STRING }}
         {{- if .Values.frontEnd.oidcEnabled }}
+          - name: FRONTEND_SVC_CONFIG_MODULE_CLASS
+            value: amundsen_application.oidc_config.OidcConfig
+          - name: FLASK_OIDC_WHITELISTED_ENDPOINTS
+            value: status,healthcheck,health
+          - name: FLASK_OIDC_SQLALCHEMY_DATABASE_URI
+            value: sqlite:///sessions.db
+          - name: APP_WRAPPER
+            value: flaskoidc
+          - name: APP_WRAPPER_CLASS
+            value: FlaskOIDC
           - name: FLASK_OIDC_CLIENT_SECRETS
             value: /etc/client_secrets.json
           - name: FLASK_OIDC_SECRET_KEY
@@ -164,9 +79,22 @@ spec:
                 name: oidc-config
                 key: OIDC_CLIENT_SECRET
         {{- end }}
+        livenessProbe:
+          httpGet:
+            path: "/healthcheck"
+            port: 5000
+          initialDelaySeconds: 60
+          periodSeconds: 60
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 5
         volumeMounts:
         {{- if .Values.frontEnd.oidcEnabled }}
           - name: oidc-config
             mountPath: /etc/client_secrets.json
             subPath: client_secrets.json
         {{- end }}
+        {{- with .Values.metadata.resources }}
+        resources:
+{{ toYaml . | indent 10 }}
+        {{- end }}
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/deployment-metadata.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "amundsen.fullname" . }}-{{ .Values.metadata.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.metadata.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ template "amundsen.name" . }}
+      component: {{ .Values.metadata.serviceName }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      {{- with default .Values.podAnnotations .Values.metadata.podAnnotations }}
+      annotations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      labels:
+        app: {{ template "amundsen.name" . }}
+        component: {{ .Values.metadata.serviceName }}
+        release: {{ .Release.Name }}
+    spec:
+      {{- with default .Values.nodeSelector .Values.metadata.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with default .Values.affinity .Values.metadata.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with default .Values.tolerations .Values.metadata.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      volumes:
+      {{- if .Values.frontEnd.oidcEnabled }}
+        - name: oidc-config
+          secret:
+            secretName: oidc-config
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
+        image: {{ .Values.metadata.image }}:{{ .Values.metadata.imageTag }}
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 5002
+        env:
+        - name: PROXY_HOST
+          value: {{ if .Values.metadata.neo4jEndpoint }}{{ .Values.metadata.neo4jEndpoint }}{{ else }}bolt://neo4j.{{ .Release.Namespace }}.svc.cluster.local{{ end }}
+        {{- if .Values.frontEnd.oidcEnabled }}
+        - name: FLASK_OIDC_CLIENT_SECRETS
+          value: /etc/client_secrets.json
+        - name: FLASK_OIDC_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: oidc-config
+              key: OIDC_CLIENT_SECRET
+        {{- end }}
+        livenessProbe:
+          httpGet:
+            path: "/healthcheck"
+            port: 5002
+          initialDelaySeconds: 60
+          periodSeconds: 60
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 5
+        volumeMounts:
+        {{- if .Values.frontEnd.oidcEnabled }}
+          - name: oidc-config
+            mountPath: /etc/client_secrets.json
+            subPath: client_secrets.json
+        {{- end }}
+        {{- with .Values.metadata.resources }}
+        resources:
+{{ toYaml . | indent 10 }}
+        {{- end }}

amundsen-kube-helm/templates/helm/templates/neo4j-deployment.yaml → amundsen-kube-helm/templates/helm/templates/deployment-neo4j.yaml

@@ -3,15 +3,29 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: neo4j
+  labels:
+    app: {{ template "amundsen.name" . }}
+    component: neo4j
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 spec:
   selector:
     matchLabels:
-      run: neo4j
+      app: {{ template "amundsen.name" . }}
+      component: neo4j
+      release: {{ .Release.Name }}
   replicas: 1
   template:
     metadata:
+      {{- with default .Values.podAnnotations .Values.neo4j.podAnnotations }}
+      annotations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
       labels:
-        run: neo4j
+        app: {{ template "amundsen.name" . }}
+        component: neo4j
+        release: {{ .Release.Name }}
     spec:
       {{- with .Values.neo4j.nodeSelector }}
       nodeSelector:
@@ -53,8 +67,6 @@ spec:
         - containerPort: 7687
         - containerPort: 1337
         env:
-          - name: NEO4J_AUTH
-            value: "neo4j/test"
           - name: NEO4J_CONF
             value: "/conf"
         volumeMounts:
@@ -83,4 +95,4 @@ spec:
           hostPath:
             path: "/mnt/ephemeral/neo4j/plugins"
             type: DirectoryOrCreate
-{{ end }}
\ No newline at end of file
+{{ end }}

amundsen-kube-helm/templates/helm/templates/deployment-search.yaml

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "amundsen.fullname" . }}-{{ .Values.search.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.search.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ template "amundsen.name" . }}
+      component: {{ .Values.search.serviceName }}
+      release: {{ .Release.Name }}
+  replicas: {{ default 1 .Values.search.replicas }}
+  template:
+    metadata:
+      {{- with default .Values.podAnnotations .Values.search.podAnnotations }}
+      annotations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      labels:
+        app: {{ template "amundsen.name" . }}
+        component: {{ .Values.search.serviceName }}
+        release: {{ .Release.Name }}
+    spec:
+      {{- with default .Values.nodeSelector .Values.search.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with default .Values.affinity .Values.search.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      {{- with default .Values.tolerations .Values.search.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
+        image: {{ .Values.search.image }}:{{ .Values.search.imageTag }}
+        ports:
+        - containerPort: 5001
+        env:
+        - name: PROXY_ENDPOINT
+          value: {{ if .Values.search.elasticsearchEndpoint }}{{ .Values.search.elasticsearchEndpoint }}{{ else }}{{ .Release.Namespace }}-elasticsearch-client.{{ .Release.Namespace }}.svc.cluster.local{{ end }}
+        livenessProbe:
+          httpGet:
+            path: "/healthcheck"
+            port: 5001
+          initialDelaySeconds: 60
+          periodSeconds: 60
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 5
+        {{- with .Values.search.resources }}
+        resources:
+{{ toYaml . | indent 10 }}
+        {{- end }}

amundsen-kube-helm/templates/helm/templates/neo4j-pvc.yaml

-{{- if and .Values.neo4j.enabled .Values.neo4j.persistence }}
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-    name: neo4j-pvc
-spec:
-    accessModes:
-        - ReadWriteOnce
-    storageClassName: "{{ default "" .Values.neo4j.persistence.storageClass }}"
-    resources:
-        requests:
-            storage: {{ default "3Gi" .Values.neo4j.persistence.size }}
-{{- end -}}
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/neo4j-s3-backup.yaml

-{{ if and .Values.neo4j.enabled (and .Values.neo4j.backup.enabled .Values.neo4j.backup.s3Path .Values.neo4j.persistence (eq .Values.provider "aws")) }}
-apiVersion: batch/v1beta1
-kind: CronJob
-metadata:
-    name: neo4j-s3-backup
-spec:
-    schedule: {{ .Values.neo4j.backup.schedule | quote }}
-    jobTemplate:
-        spec:
-            template:
-                spec:
-                    containers:
-                        - name: backup-neo4j
-                          image: neo4j:3.3.0
-                          command:
-                              - "/bin/sh"
-                              - "-c"
-                              - |
-                                  apk -v --update add --no-cache --quiet curl python py-pip &&
-                                  pip install awscli -q &&
-                                  NOW="$(date "+%Y-%m-%d-%H:%M:%S")" &&
-                                  BACKUP_SCHEMA_NAME="graph.db-backup-$NOW.schema" &&
-                                  BACKUP_DATA_NAME="graph.db-backup-$NOW.data" &&
-                                  BACKUP_NAME="graph.db-backup-$NOW" &&
-                                  echo "CALL apoc.export.cypher.schema('/var/lib/neo4j/data/$BACKUP_SCHEMA_NAME', {});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
-                                  echo "CALL apoc.export.graphml.all('/var/lib/neo4j/data/$BACKUP_DATA_NAME', {useTypes: true, readLabels: true});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
-                                  printf "\nTarring -> /data/$BACKUP_SCHEMA_NAME and /data/$BACKUP_DATA_NAME to /data/$BACKUP_NAME.tar" &&
-                                  while [ ! -f /data/$BACKUP_DATA_NAME ]; do echo "backup data file does not exist: [/data/$BACKUP_DATA_NAME] sleeping..." && ls "/data/" && sleep 30; done &&
-                                  tar -cvf "/data/$BACKUP_NAME.tar" "/data/$BACKUP_SCHEMA_NAME" "/data/$BACKUP_DATA_NAME" &&
-                                  printf "\nZipping -> /data/$BACKUP_NAME.tar.gz\n" &&
-                                  gzip -9 "/data/$BACKUP_NAME.tar" &&
-                                  printf "Pushing /data/$BACKUP_NAME.tar.gz -> $BUCKET" &&
-                                  aws s3 cp "/data/$BACKUP_NAME.tar.gz" "$BUCKET" &&
-                                  printf "Cleaning up /data/graph.db-backup*" &&
-                                  rm /data/graph.db-backup*
-                          env:
-                              - name: BUCKET
-                                value: {{ .Values.neo4j.backup.s3Path }}
-                          volumeMounts:
-                              - name: data
-                                mountPath: /data
-                    restartPolicy: Never
-                    volumes:
-                        - name: data
-                          persistentVolumeClaim:
-                              claimName: neo4j-pvc
-{{- end}}
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/pv-neo4j.yaml

+{{- if and .Values.neo4j.enabled .Values.neo4j.persistence .Values.neo4j.persistence.efs }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: neo4j-pv
+  labels:
+    app: {{ template "amundsen.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  accessModes:
+    - ReadWriteMany
+  capacity:
+    storage: {{ default "3Gi" .Values.neo4j.persistence.size }}
+  nfs:
+    server: {{ .Values.neo4j.persistence.efs.dns }}
+    path: "/"
+{{- end }}
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/pvc-neo4j.yaml

+{{- if and .Values.neo4j.enabled .Values.neo4j.persistence }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: neo4j-pvc
+  labels:
+    app: {{ template "amundsen.fullname" . }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  accessModes:
+    - {{ default "ReadWriteOnce" .Values.neo4j.persistence.accessMode }}
+  storageClassName: "{{ default "" .Values.neo4j.persistence.storageClass }}"
+  resources:
+    requests:
+      storage: {{ default "3Gi" .Values.neo4j.persistence.size }}
+{{- end }}
\ No newline at end of file

amundsen-kube-helm/templates/helm/templates/service-frontend.yaml

+---         
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "amundsen.name" . }}-{{ .Values.frontEnd.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    run: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}
+    component: {{ .Values.frontEnd.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- with .Values.frontEnd.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end}}   
+spec:
+  type: {{ .Values.frontEnd.serviceType }}
+  selector:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.frontEnd.serviceName }}
+    release: {{ .Release.Name }}
+  ports:
+    - name: {{ .Chart.Name }}-{{ .Values.frontEnd.serviceName }}-{{ .Values.environment }}-http
+      port: 5000
+      targetPort: 5000

amundsen-kube-helm/templates/helm/templates/service-metadata.yaml

+---  
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "amundsen.name" . }}-{{ .Values.metadata.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    run: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}
+    component: {{ .Values.metadata.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- with .Values.metadata.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end}}           
+spec:
+  type: {{ .Values.metadata.serviceType }}
+  selector:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.metadata.serviceName }}
+    release: {{ .Release.Name }}
+  ports:
+    - name: {{ .Chart.Name }}-{{ .Values.metadata.serviceName }}-{{ .Values.environment }}-http
+      port: 5002
+      targetPort: 5002

amundsen-kube-helm/templates/helm/templates/neo4j-service.yaml → amundsen-kube-helm/templates/helm/templates/service-neo4j.yaml

@@ -4,16 +4,22 @@ kind: Service
 metadata:
   name: neo4j
   labels:
+    app: {{ template "amundsen.name" . }}
     run: neo4j
+    component: neo4j
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- with .Values.neo4j.annotations }}
   annotations:
-  {{- if (eq .Values.provider "aws") }}
-    external-dns.alpha.kubernetes.io/hostname: amundsen-neo4j-{{ .Values.environment }}.{{ .Values.dnsZone }}
-    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
-    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
-  {{- end }}
+{{ toYaml . | indent 4 }}
+{{- end}}   
 spec:
-  type: LoadBalancer
-  externalTrafficPolicy: Local
+  type: {{ .Values.neo4j.serviceType }}
+  selector:
+    app: {{ template "amundsen.name" . }}
+    component: neo4j
+    release: {{ .Release.Name }}
   ports:
     - port: 7473
       name: neo4j-{{ .Values.environment }}-https
@@ -27,6 +33,4 @@ spec:
     - port: 1337
       name: neo4j-{{ .Values.environment }}-shell
       targetPort: 1337
-  selector:
-    run: neo4j
-{{ end }}
\ No newline at end of file
+{{ end }}

amundsen-kube-helm/templates/helm/templates/service-search.yaml

+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "amundsen.name" . }}-{{ .Values.search.serviceName }}
+  labels:
+    app: {{ template "amundsen.name" . }}
+    run: {{ .Chart.Name }}-{{ .Values.search.serviceName }}
+    component: {{ .Values.search.serviceName }}
+    chart: {{ template "amundsen.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+{{- with .Values.search.annotations }}
+  annotations:
+{{ toYaml . | indent 4 }}
+{{- end}}   
+spec:
+  type: {{ .Values.search.serviceType }}
+  selector:
+    app: {{ template "amundsen.name" . }}
+    component: {{ .Values.search.serviceName }}
+    release: {{ .Release.Name }}
+  ports:
+    - name: {{ .Chart.Name }}-{{ .Values.search.serviceName }}-{{ .Values.environment }}-http
+      port: 5001
+      targetPort: 5001

amundsen-kube-helm/templates/helm/values.yaml

@@ -40,6 +40,10 @@ affinity: {}
 ## tolerations -- amundsen application wide configuration of tolerations. This applies to search, metadata, frontend and neo4j. Elasticsearch has it's own configuation properties for this. [ref](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature)
 ##
 tolerations: []
+##
+## podAnnotations -- amundsen application wide configuration of podAnnotations. This applies to search, metadata, frontend and neo4j. Elasticsearch has it's own configuation properties for this. [ref](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
+##
+podAnnotations: {}
 
 ##
 ## Configuration related to the search service.
@@ -50,13 +54,21 @@ search:
   ##
   serviceName: search
   ##
+  ## search.serviceType -- The search service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types)
+  ##
+  serviceType: ClusterIP
+  ##
   ## search.elasticsearchEndpoint -- The name of the service hosting elasticsearch on your cluster, if you bring your own. You should only need to change this, if you don't use the version in this chart.
   ##
   elasticsearchEndpoint:
   ##
-  ## search.imageVersion -- The image version of the search container.
+  ## search.image -- The image of the search container.
+  ##
+  image: amundsendev/amundsen-search
   ##
-  imageVersion: 2.0.0
+  ## search.imageTag -- The image tag of the search container.
+  ##
+  imageTag: 2.0.0
   ##
   ## search.replicas -- How many replicas of the search service to run.
   ##
@@ -84,6 +96,14 @@ search:
   ## search.tolerations -- Search pod specific tolerations.
   ##
   tolerations: []
+  ##
+  ## search.annotations -- Search service specific tolerations.
+  ##
+  annotations: {}
+  ##
+  ## search.podAnnotations -- Search pod specific annotations.
+  ##
+  podAnnotations: {}
 
 ##
 ## Configuration related to the metadata service.
@@ -94,13 +114,21 @@ metadata:
   ##
   serviceName: metadata
   ##
+  ## metadata.serviceType -- The metadata service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types)
+  ##
+  serviceType: ClusterIP
+  ##
   ## metadata.neo4jEndpoint -- The name of the service hosting neo4j on your cluster, if you bring your own. You should only need to change this, if you don't use the version in this chart.
   ##
   neo4jEndpoint:
   ##
-  ## metadata.imageVersion -- The image version of the metadata container.
+  ## metadata.image -- The image of the metadata container.
+  ##
+  image: amundsendev/amundsen-metadata
   ##
-  imageVersion: 2.0.0
+  ## metadata.imageTag -- The image tag of the metadata container.
+  ##
+  imageTag: 2.0.0
   ##
   ## metadata.replicas -- How many replicas of the metadata service to run.
   ##
@@ -128,6 +156,14 @@ metadata:
   ## metadata.tolerations -- Metadata pod specific tolerations.
   ##
   tolerations: []
+  ##
+  ## metadata.annotations -- Metadata service specific tolerations.
+  ##
+  annotations: {}
+  ##
+  ## metadata.podAnnotations -- Metadata pod specific annotations.
+  ##
+  podAnnotations: {}
 
 ##
 ## Configuration related to the frontEnd service.
@@ -138,9 +174,17 @@ frontEnd:
   ##
   serviceName: frontend
   ##
-  ## frontEnd.imageVersion -- The frontend version of the metadata container.
+  ## frontEnd.serviceType -- The frontend service type. See service types [ref](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types)
+  ##
+  serviceType: ClusterIP
+  ##
+  ## frontEnd.image -- The image of the frontend container.
+  ##
+  image: amundsendev/amundsen-frontend
   ##
-  imageVersion: 2.0.0
+  ## frontEnd.imageTag -- The image tag of the frontend container.
+  ##
+  imageTag: 2.0.0
   ##
   ## frontEnd.servicePort -- The port the frontend service will be exposed on via the loadbalancer.
   ##
@@ -149,7 +193,10 @@ frontEnd:
   ## frontEnd.replicas -- How many replicas of the frontend service to run.
   ##
   replicas: 1
-
+  ##
+  ## frontEnd.baseUrl -- used by notifications util to provide links to amundsen pages in emails.
+  ##
+  baseUrl: http://localhost
   ##
   ## frontEnd.oidcEnabled -- To enable auth via OIDC, set this to true.
   ##
@@ -199,6 +246,14 @@ frontEnd:
   ## frontEnd.tolerations -- Frontend pod specific tolerations.
   ##
   tolerations: []
+  ##
+  ## frontEnd.annotations -- Frontend service specific tolerations.
+  ##
+  annotations: {}
+  ##
+  ## frontEnd.podAnnotations -- Frontend pod specific annotations.
+  ##
+  podAnnotations: {}
 
 ##
 ## Configuration related to neo4j.
@@ -246,6 +301,9 @@ neo4j:
   persistence: {}
   #  storageClass: gp2
   #  size: 10Gi
+  #  accessMode: ReadWriteMany
+  #  efs:
+  #    dns:
 
   ##
   ## neo4j.backup -- If enabled is set to true, make sure and set the s3 path as well.
@@ -274,6 +332,14 @@ neo4j:
   ## neo4j.tolerations -- neo4j specific tolerations.
   ##
   tolerations: []
+  ##
+  ## neo4j.annotations -- neo4j service specific tolerations.
+  ##
+  annotations: {}
+  ##
+  ## neo4j.podAnnotations -- neo4j pod specific annotations.
+  ##
+  podAnnotations: {}
 
 ##
 ## Configuration related to elasticsearch.