(ISSUE 196) - moving neo4j backup to neo4j based image (#293)

* * setting up a pvc to allow for neo4j backups

* * allowing pvc to be turned off
* allowing pvc size to be configurable

* ISSUE 196
* added neo4j plugins
* added chart versions
* removed image pull policy never from metadata service

* * PR comment fix; using the correct maven repo urls

* (ISSUE 196) - added neo4j s3 backup cron pod

* * added missing restartPolicy

* (ISSUE 196)
* updated for PR comments
* using export schema, export graphml functions

* * using neo4j image instead of aws cli image
* updated pvc to point at /data for mount path, to avoid path collision of neo4j image
* made commands chained\dependent

* * removing full path from ls command

* * increased the sleep time to 30s

* (ISSUE 196)
* removed defaulting of persistence
* moved neo4j persistence size to dedicated block
* removed manual\host mapping of pv\pvc

* * turning on neo4j-shell and it's port
* using neo4j-shell for backups

amundsen-kube-helm/templates/helm/neo4j/Chart.yaml

 name: neo4j
 home: https://www.neo4j.com
 appVersion: 3.3.0
-version: 0.2.0
\ No newline at end of file
+version: 0.3.0
\ No newline at end of file

amundsen-kube-helm/templates/helm/neo4j/templates/configmap.yaml

@@ -11,6 +11,8 @@ data:
     dbms.connector.bolt.enabled=true
     dbms.connector.http.enabled=true
     dbms.connector.https.enabled=true
+    dbms.shell.enabled=true
+    dbms.shell.host=0.0.0.0
     dbms.connectors.default_listen_address=0.0.0.0
     dbms.directories.import=/mnt
     dbms.jvm.additional=-Djdk.tls.ephemeralDHKeySize=2048

amundsen-kube-helm/templates/helm/neo4j/templates/deployment.yaml

@@ -50,6 +50,7 @@ spec:
         ports:
         - containerPort: 7474
         - containerPort: 7687
+        - containerPort: 1337
         env:
           - name: NEO4J_AUTH
             value: "neo4j/test"
@@ -58,17 +59,17 @@ spec:
         volumeMounts:
         - name: conf
           mountPath: /conf
-        {{- if (or (kindIs "invalid" .Values.neo4jpvc) .Values.neo4jpvc)}}
+        {{- if .Values.neo4j.persistence }}
         - name: data
           mountPath: /var/lib/neo4j/data
+        {{- end}}
         - name: plugins
           mountPath: /var/lib/neo4j/plugins
-        {{- end}}
       volumes:
         - name: conf
           configMap:
             name: {{ .Chart.Name }}-configmap
-        {{- if (or (kindIs "invalid" .Values.neo4jpvc) .Values.neo4jpvc)}}
+        {{- if .Values.neo4j.persistence }}
         - name: data
           persistentVolumeClaim:
             claimName: neo4j-pvc

amundsen-kube-helm/templates/helm/neo4j/templates/neo4j-s3-backup.yaml

-{{ if and (and .Values.neo4j.backup.enabled (eq .Values.provider "aws")) (or (kindIs "invalid" .Values.neo4jpvc) .Values.neo4jpvc) }}
+{{ if and .Values.neo4j.backup.enabled .Values.neo4j.backup.s3Path .Values.neo4j.persistence (eq .Values.provider "aws") }}
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
     name: neo4j-s3-backup
 spec:
-    schedule: {{ .Values.neo4j.backup.schedule }}
+    schedule: {{ default "0 * * * *" (.Values.neo4j.backup.schedule | quote) }}
     jobTemplate:
         spec:
             template:
                 spec:
                     containers:
                         - name: backup-neo4j
-                          image: mesosphere/aws-cli:latest
+                          image: neo4j:3.3.0
                           command:
                               - "/bin/sh"
                               - "-c"
                               - |
-                                  apk -v --update add --no-cache --quiet curl
-                                  NOW="$(date "+%Y-%m-%d-%H:%M:%S")"
-                                  BACKUP_SCHEMA_NAME="graph.db-backup-$NOW.schema"
-                                  BACKUP_DATA_NAME="graph.db-backup-$NOW.data"
-                                  BACKUP_NAME="graph.db-backup-$NOW"
-                                  curl --user neo4j:test -H accept:application/json -H content-type:application/json \
-                                    -d '{"statements":[{"statement":"CALL apoc.export.cypher.schema(\"\/var\/lib\/neo4j\/data\/'$BACKUP_SCHEMA_NAME'\", {})"}]}' \
-                                    http://neo4j:7474/db/data/transaction/commit
-                                  curl --user neo4j:test -H accept:application/json -H content-type:application/json \
-                                    -d '{"statements":[{"statement":"CALL apoc.export.graphml.all(\"\/var\/lib\/neo4j\/data\/'$BACKUP_DATA_NAME'\", {useTypes: true, readLabels: true})"}]}' \
-                                    http://neo4j:7474/db/data/transaction/commit
-                                  printf "\nWaiting for APOC export..."
-                                  while [ ! -f /var/lib/neo4j/data/$BACKUP_DATA_NAME ]; do echo "backup data file does not exist, sleeping: /var/lib/neo4j/data/$BACKUP_DATA_NAME" && sleep 5; done
-                                  printf "\nTarring -> /var/lib/neo4j/data/$BACKUP_SCHEMA_NAME  and /var/lib/neo4j/data/$BACKUP_DATA_NAME to /var/lib/neo4j/data/$BACKUP_NAME.tar"
-                                  tar -cvf "/var/lib/neo4j/data/$BACKUP_NAME.tar" -C "/var/lib/neo4j/data/" $BACKUP_SCHEMA_NAME $BACKUP_DATA_NAME
-                                  printf "Zipping -> /var/lib/neo4j/data/$BACKUP_NAME.tar.gz\n"
-                                  gzip -9 "/var/lib/neo4j/data/$BACKUP_NAME.tar"
-                                  printf "Zipped backup size:"
-                                  du -hs "/var/lib/neo4j/data/$BACKUP_NAME.tar.gz"
-                                  printf "Pushing /var/lib/neo4j/data/$BACKUP_NAME.tar.gz -> $BUCKET"
-                                  aws s3 cp "/var/lib/neo4j/data/$BACKUP_NAME.tar.gz" "$BUCKET"
-                                  printf "Cleaning up /var/lib/neo4j/data/graph.db-backup*"
-                                  rm /var/lib/neo4j/data/graph.db-backup*
+                                  apk -v --update add --no-cache --quiet curl python py-pip &&
+                                  pip install awscli -q &&
+                                  NOW="$(date "+%Y-%m-%d-%H:%M:%S")" &&
+                                  BACKUP_SCHEMA_NAME="graph.db-backup-$NOW.schema" &&
+                                  BACKUP_DATA_NAME="graph.db-backup-$NOW.data" &&
+                                  BACKUP_NAME="graph.db-backup-$NOW" &&
+                                  echo "CALL apoc.export.cypher.schema('/var/lib/neo4j/data/$BACKUP_SCHEMA_NAME', {});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
+                                  echo "CALL apoc.export.graphml.all('/var/lib/neo4j/data/$BACKUP_DATA_NAME', {useTypes: true, readLabels: true});" | /var/lib/neo4j/bin/neo4j-shell -host neo4j &&
+                                  printf "\nTarring -> /data/$BACKUP_SCHEMA_NAME and /data/$BACKUP_DATA_NAME to /data/$BACKUP_NAME.tar" &&
+                                  while [ ! -f /data/$BACKUP_DATA_NAME ]; do echo "backup data file does not exist: [/data/$BACKUP_DATA_NAME] sleeping..." && ls "/data/" && sleep 30; done &&
+                                  tar -cvf "/data/$BACKUP_NAME.tar" "/data/$BACKUP_SCHEMA_NAME" "/data/$BACKUP_DATA_NAME" &&
+                                  printf "\nZipping -> /data/$BACKUP_NAME.tar.gz\n" &&
+                                  gzip -9 "/data/$BACKUP_NAME.tar" &&
+                                  printf "Pushing /data/$BACKUP_NAME.tar.gz -> $BUCKET" &&
+                                  aws s3 cp "/data/$BACKUP_NAME.tar.gz" "$BUCKET" &&
+                                  printf "Cleaning up /data/graph.db-backup*" &&
+                                  rm /data/graph.db-backup*
                           env:
                               - name: BUCKET
                                 value: {{ .Values.neo4j.backup.s3Path }}
                           volumeMounts:
                               - name: data
-                                mountPath: /var/lib/neo4j/data
+                                mountPath: /data
                     restartPolicy: Never
                     volumes:
                         - name: data

amundsen-kube-helm/templates/helm/neo4j/templates/pvc.yaml

-{{- if (or (kindIs "invalid" .Values.neo4jpvc) .Values.neo4jpvc)}}
-kind: PersistentVolume
-apiVersion: v1
-metadata:
-    name: neo4j-pv
-    labels:
-        type: local
-spec:
-    storageClassName: manual
-    capacity:
-        storage: {{ default "3Gi" .Values.neo4jpvcSize }}
-    accessModes:
-        - ReadWriteMany
-    hostPath:
-        path: "/mnt/ephemeral/neo4j/data"
----
+{{- if .Values.neo4j.persistence }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
     name: neo4j-pvc
 spec:
-    storageClassName: manual
     accessModes:
-        - ReadWriteMany
+        - ReadWriteOnce
+    storageClassName: "{{ default "" .Values.neo4j.persistence.storageClass }}"
     resources:
         requests:
-            storage: {{ default "3Gi" .Values.neo4jpvcSize }}
+            storage: {{ default "3Gi" .Values.neo4j.persistence.size }}
 {{- end -}}
\ No newline at end of file

amundsen-kube-helm/templates/helm/neo4j/templates/service.yaml

@@ -26,5 +26,9 @@ spec:
       name: {{ .Chart.Name }}-{{ .Values.environment }}-bolt
       targetPort: 7687
       nodePort: 30687
+    - port: 1337
+      name: {{ .Chart.Name }}-{{ .Values.environment }}-shell
+      targetPort: 1337
+      nodePort: 31337
   selector:
     run: {{ .Chart.Name }}
\ No newline at end of file

amundsen-kube-helm/templates/helm/neo4j/values.yaml

@@ -19,7 +19,13 @@ config:
     heap_max_size: 23000m
     pagecache_size: 26600m
 neo4j:
-    backup:
-      enabled: false
-      s3Path: "s3://my-bucket/my-prefix"
-      schedule: "0 * * * *"
+  persistence: {}
+#  persistence:
+#    storageClass: gp2
+#    size: 10Gi
+  backup:
+    enabled: false
+#  backup:
+#    enabled: true
+#    s3Path: "s3://my-bucket/my-prefix"
+#    schedule: "0 * * * *"