issue 53; k8s + helm setup for amundsen (#137)

d3ec0f4e · Shaun Elliott · Tao Feng · 43fb84f3 · d3ec0f4e · d3ec0f4e
Commit d3ec0f4e authored Oct 18, 2019 by Shaun Elliott Committed by Tao Feng Oct 18, 2019
11 changed files
--- a/amundseninfrastructure/README.md
+++ b/amundseninfrastructure/README.md
+# Amundsen Infrastructure
+
+## What is this?
+This is setup templates for deploying [amundsen](https://github.com/lyft/amundsen) on [k8s (kubernetes)](https://kubernetes.io/), using [helm.](https://helm.sh/) 
+
+## How do I get started?
+1. Make sure you have the following command line clients setup:
+    - k8s (kubectl)
+    - helm
+    - tiller
+2. Build out a cloud based k8s cluster, such as [amazon eks.](https://aws.amazon.com/eks/)
+3. Ensure you can connect to your cluster with cli tools in step 1.
+
+## How do I use this?
+You will need a values file to merge with these templates, in order to create the infrastructure. Here is an example:
+```
+environment: "dev"
+provider: aws
+dnsZone: teamname.company.com
+dockerhubImagePath: amundsendev
+searchServiceName: search
+searchImageVersion: 1.4.0
+metadataServiceName: metadata
+metadataImageVersion: 1.1.1
+frontEndServiceName: frontend
+frontEndImageVersion: 1.0.9
+frontEndServicePort: 80
+```
+
+With this values file, you can then setup amundsen with these commands:
+```
+helm install templates/helm/neo4j --values impl/helm/dev/values.yaml
+helm install templates/helm/elasticsearch --values impl/helm/dev/values.yaml
+helm install templates/helm/amundsen --values impl/helm/dev/values.yaml
+```
+
+## Other Notes
+* For aws setup, you will also need to setup the [external-dns plugin](https://github.com/kubernetes-incubator/external-dns)
+* There are exising helm charts for neo4j and elasticsearch. Future versions of amundsen may use them instead. 
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/amundsen/Chart.yaml
+++ b/amundseninfrastructure/templates/helm/amundsen/Chart.yaml
+name: amundsen
+home: https://github.com/lyft/amundsen
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/amundsen/templates/deployment.yaml
+++ b/amundseninfrastructure/templates/helm/amundsen/templates/deployment.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+spec:
+  selector:
+    matchLabels:
+      run: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.searchServiceName }}:{{ .Values.searchImageVersion }}
+        ports:
+        - containerPort: 5000  
+        env:
+        - name: PROXY_ENDPOINT
+          value: elasticsearch  
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+spec:
+  selector:
+    matchLabels:
+      run: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.metadataServiceName }}:{{ .Values.metadataImageVersion }}
+        ports:
+        - containerPort: 5000  
+        env:
+        - name: PROXY_HOST
+          value: bolt://neo4j
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+spec:
+  selector:
+    matchLabels:
+      run: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+        image: {{ .Values.dockerhubImagePath }}/{{ .Chart.Name }}-{{ .Values.frontEndServiceName }}:1.0.9
+        ports:
+        - containerPort: 5000  
+        env:
+        - name: SEARCHSERVICE_BASE
+          value: http://{{ .Chart.Name }}-{{ .Values.searchServiceName }}:5001
+        - name: METADATASERVICE_BASE
+          value: http://{{ .Chart.Name }}-{{ .Values.metadataServiceName }}:5002
+        - name: FRONTEND_SVC_CONFIG_MODULE_CLASS
+          value: amundsen_application.config.TestConfig                     
+---                          
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/amundsen/templates/service.yaml
+++ b/amundseninfrastructure/templates/helm/amundsen/templates/service.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+  labels:
+    run: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+  annotations:
+  {{- if (eq .Values.provider "aws") }}
+    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.searchServiceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
+    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
+  {{- end }}          
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+    - port: 5001
+      name: {{ .Chart.Name }}-{{ .Values.searchServiceName }}-{{ .Values.environment }}-http
+      targetPort: 5000
+      nodePort: 30001
+  selector:
+    run: {{ .Chart.Name }}-{{ .Values.searchServiceName }}
+---  
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+  labels:
+    run: {{ .Chart.Name }}-{{ .Values.metadataServiceName }} 
+  annotations:
+  {{- if (eq .Values.provider "aws") }}
+    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
+    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
+  {{- end }}              
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+    - port: 5002
+      name: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}-{{ .Values.environment }}-http
+      targetPort: 5000
+      nodePort: 30002
+  selector:
+    run: {{ .Chart.Name }}-{{ .Values.metadataServiceName }}
+---         
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+  labels:
+    run: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+  annotations:
+  {{- if (eq .Values.provider "aws") }}
+    external-dns.alpha.kubernetes.io/hostname: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
+    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
+  {{- end }}       
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+    - port: {{ .Values.frontEndServicePort }}
+      name: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}-{{ .Values.environment }}-http
+      targetPort: 5000
+      nodePort: 30003
+  selector:
+    run: {{ .Chart.Name }}-{{ .Values.frontEndServiceName }}
+---            
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/elasticsearch/Chart.yaml
+++ b/amundseninfrastructure/templates/helm/elasticsearch/Chart.yaml
+name: elasticsearch
+home: https://www.elastic.co
+appVersion: 6.7.0
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/elasticsearch/templates/deployment.yaml
+++ b/amundseninfrastructure/templates/helm/elasticsearch/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+spec:
+  selector:
+    matchLabels:
+      run: {{ .Chart.Name }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: {{ .Chart.Name }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}
+        image: {{ .Chart.Name }}:{{ .Chart.AppVersion }}
+        ports:
+        - containerPort: 9200  
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/elasticsearch/templates/service.yaml
+++ b/amundseninfrastructure/templates/helm/elasticsearch/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    run: {{ .Chart.Name }}
+  annotations:
+  {{- if (eq .Values.provider "aws") }}
+    external-dns.alpha.kubernetes.io/hostname: amundsen-{{ .Chart.Name }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
+    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
+  {{- end }}   
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+    - port: 9200
+      name: {{ .Chart.Name }}-{{ .Values.environment }}-http
+      targetPort: 9200
+      nodePort: 30200
+  selector:
+    run: {{ .Chart.Name }}
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/neo4j/Chart.yaml
+++ b/amundseninfrastructure/templates/helm/neo4j/Chart.yaml
+name: neo4j
+home: https://www.neo4j.com
+appVersion: 3.3.0
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/neo4j/templates/configmap.yaml
+++ b/amundseninfrastructure/templates/helm/neo4j/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Chart.Name }}-configmap
+  labels:
+    app: "neo4j"
+data:
+  neo4j.conf: |-
+    apoc.import.file.enabled=true
+    cypher.forbid_shortestpath_common_nodes=false
+    dbms.connector.bolt.enabled=true
+    dbms.connector.http.enabled=true
+    dbms.connector.https.enabled=true
+    dbms.connectors.default_listen_address=0.0.0.0
+    dbms.directories.import=/mnt
+    dbms.jvm.additional=-Djdk.tls.ephemeralDHKeySize=2048
+    dbms.jvm.additional=-Dunsupported.dbms.udc.source=tarball
+    dbms.jvm.additional=-XX:+AlwaysPreTouch
+    dbms.jvm.additional=-XX:+DisableExplicitGC
+    dbms.jvm.additional=-XX:+UseG1GC
+    dbms.logs.query.enabled=true
+    dbms.logs.query.rotation.keep_number=7
+    dbms.logs.query.rotation.size=20m
+    dbms.memory.heap.initial_size=23000m
+    dbms.memory.heap.max_size=23000m
+    dbms.memory.pagecache.size=26600m
+    dbms.security.allow_csv_import_from_file_urls=true
+    dbms.security.procedures.unrestricted=algo.*,apoc.*
+    dbms.windows_service_name=neo4j  
+    apoc.export.file.enabled=true
+    apoc.import.file.enabled=true
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/neo4j/templates/deployment.yaml
+++ b/amundseninfrastructure/templates/helm/neo4j/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+spec:
+  selector:
+    matchLabels:
+      run: {{ .Chart.Name }}
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        run: {{ .Chart.Name }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}
+        image: {{ .Chart.Name }}:{{ .Chart.AppVersion }}
+        ports:
+        - containerPort: 7474
+        - containerPort: 7687
+        env:
+          - name: NEO4J_AUTH
+            value: "neo4j/test"
+          - name: NEO4J_CONF
+            value: "/conf"
+        volumeMounts:
+        - name: conf
+          mountPath: /conf                 
+      volumes:
+        - name: conf
+          configMap:
+            name: {{ .Chart.Name }}-configmap
\ No newline at end of file
--- a/amundseninfrastructure/templates/helm/neo4j/templates/service.yaml
+++ b/amundseninfrastructure/templates/helm/neo4j/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    run: {{ .Chart.Name }}
+  annotations:
+  {{- if (eq .Values.provider "aws") }}
+    external-dns.alpha.kubernetes.io/hostname: amundsen-{{ .Chart.Name }}-{{ .Values.environment }}.{{ .Values.dnsZone }}
+    service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+    service.beta.kubernetes.io/aws-load-balancer-type: nlb 
+  {{- end }}
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Local
+  ports:
+    - port: 7473
+      name: {{ .Chart.Name }}-{{ .Values.environment }}-https
+      targetPort: 7473
+      nodePort: 30473
+    - port: 7474
+      name: {{ .Chart.Name }}-{{ .Values.environment }}-http
+      targetPort: 7474
+      nodePort: 30474
+    - port: 7687
+      name: {{ .Chart.Name }}-{{ .Values.environment }}-bolt
+      targetPort: 7687
+      nodePort: 30687
+  selector:
+    run: {{ .Chart.Name }}
\ No newline at end of file