updated CI/CD parts

.gitlab-ci.yml

 stages:
- - increment_version 
- - build
- - deploy
+ - build_dashboards
+ - build_alerts
+ - deploy_dashboards
+ - deploy_alerts
 
-# auto_versioning_job:
-#   stage: increment_version
-#   script:
-#     # - git pull
-#     - ./increment-version.sh
-#     - git add version.txt
-#     - git commit -m "Bump version to $(cat version.txt)"
-#     - git push
-#   only:
-#     - master
-#   tags: 
-#     - versioning
-    
-build_job:
- stage: build
+build_dashboards_job:
+ stage: build_dashboards
  before_script:
   - curl --version
   - jsonnet --version
   - tar --version
- script: 
-  - echo "Starting script for deploying to Jfrog"
-  - files_list=$(find . -type f -name "*.jsonnet" ! -path "./grafonnet-lib/*")
-  - echo "Creating directory"
-  - mkdir grafonnet
-#   - export version=$(echo "$(cat version.txt) + 0.1" | bc)
-#   - export version=$(cat version.txt)
-  - ls
-  - | 
-    while IFS= read -r jsonnet_file;
-     do
-      echo $jsonnet_file
-      filename=$(basename -- "$jsonnet_file")
-      filename="${filename%.*}"
-      echo $filename
-      JSONNET_PATH=grafonnet-lib \
-      jsonnet $jsonnet_file > grafonnet/$filename.json
-     done <<< "$files_list"
-  - tar -cvzf grafonnet.tar.gz grafonnet/
-#   - echo "version.txt is `cat version.txt` and uploading version is $version" 
-  - echo "Uploading to $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/grafonnet.tar.gz"
-  - curl -u $ARTIFACTORY_USER:$ARTIFACTORY_PASS -T grafonnet.tar.gz "$ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/grafonnet.tar.gz"
-          
-    # brew install jsonnet
-    # jsonnet -v
+ script:
+   - echo "Building updated dashboards"
+   - scripts/build_dashboards.sh
+   - echo "Promoting json artifacts to artifactory"
+   - scripts/promote_dashboard_artifacts.sh
+ rules:
+   - changes:
+       - dashboards/*
+   - when: manual
  tags:
-  - ci
+   - ci
  only:
-  - master
- 
-deploy_job:
- stage: deploy
+   - master
+
+build_alerts_job:
+  stage: build_alerts
+  before_script:
+    - curl --version
+    - jsonnet --version
+    - tar --version
+  script:
+    - echo "Building updated alerts"
+    - scripts/build_alerts.sh
+    - echo "Promoting json artifacts to artifactory"
+    - scripts/promote_alerts_artifacts.sh
+  rules:
+    - changes:
+        - alerts/*
+        - alerts-config/*
+    - when: manual
+  tags:
+    - ci
+  only:
+    - master
+
+deploy_dashboards_job:
+ stage: deploy_dashboards
  before_script:
    - curl --version
    - jq --version
    - tar --version
  script:
-#   - export version=$(echo "$(cat version.txt) + 0.1" | bc)
-#   - export version=$(cat version.txt)
-  - mkdir -p build/$CI_BUILD_ID
-  - cd build/$CI_BUILD_ID
-  - echo "Downloading artifact from $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/grafonnet.tar.gz"
-  - curl -O -u $ARTIFACTORY_USER:$ARTIFACTORY_PASS "$ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/grafonnet.tar.gz" 
-  - ls -larth
-  - pwd
-  - tar -xzvf grafonnet.tar.gz
-  - cd grafonnet
-  - ls -larth
-
-  - |
-    for FILE in *;
-     do
-      echo "$FILE \n";
-      DASHBOARD="{\"dashboard\": $(jq . $FILE), \"overwrite\": true}"
-      curl -X POST \
-       -H "Authorization: Bearer $TOKEN" \
-       -H "Content-Type: application/json" \
-       -d "$DASHBOARD" \
-       $GRAFANA_URL/api/dashboards/db
-      echo "\n"
-    done     
-     #grafonnet.tar.gz
+  - echo "Deploying dasbhoards"
+  - scripts/deploy_dashboards.sh
+ needs:
+   - job:
+       - build_dashboards
  tags:
   - cd
  only:
-  - master
\ No newline at end of file
+  - master
+
+deploy_alerts_job:
+  stage: deploy_alerts
+  before_script:
+    - curl --version
+    - jq --version
+    - tar --version
+  script:
+    - echo "Deploying alerts"
+    - scripts/deploy_alerts.sh
+  needs:
+    - job:
+        - build_alerts
+  tags:
+    - cd
+  only:
+    - master
\ No newline at end of file

scripts/build_alerts.sh

 #!/bin/bash
 
-ALERTS_JSONNET_PATH=../alerts
-ALERTS_BUILD_PATH=../build/alerts
+ALERTS_JSONNET_PATH=alerts
+ALERTS_BUILD_PATH=build/alerts
 ALERTS_TAR_NAME=alerts
-## /build/archive from the root of the project
-ARCHIVE_PATH=../archive
+ARCHIVE_PATH=build/archive
 
 CURRENT_DIR=$PWD
 
@@ -14,7 +13,11 @@ mkdir -p ${ARCHIVE_PATH}
 echo "Cleaning up previous builds"
 rm -rf ${ALERTS_BUILD_PATH}/*.json
 
-for json_file in ${ALERTS_JSONNET_PATH}/*.jsonnet;
+
+LAST_COMMIT_FILES=$(git diff --name-only HEAD^ HEAD ${ALERTS_JSONNET_PATH} | xargs -I{} basename {} )
+echo "last commit files $LAST_COMMIT_FILES"
+
+for json_file in ${LAST_COMMIT_FILES[@]};
     do
       echo "Filename is $json_file"
       filename=$(basename -- "${json_file}")
@@ -24,21 +27,6 @@ for json_file in ${ALERTS_JSONNET_PATH}/*.jsonnet;
       echo "Created new json at ${ALERTS_BUILD_PATH}/${filename}.json"
     done
 
-cd ${ALERTS_BUILD_PATH}
-
-count_file=`ls -1 *.json 2>/dev/null | wc -l`
-echo "${count_file} json files found at `pwd`"
-
-if [ $count_file != 0 ]
- then
-    tar -cvzf ${ARCHIVE_PATH}/${ALERTS_TAR_NAME}.tar.gz *.json
-    echo "Uploading to $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${ALERTS_TAR_NAME}.tar.gz"
-    curl -u $ARTIFACTORY_USER:$ARTIFACTORY_PASS -T ${ARCHIVE_PATH}/${ALERTS_TAR_NAME}.tar.gz "$ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${ALERTS_TAR_NAME}.tar.gz"
-    echo "Artifact Uploaded Successfully"
-else
-    echo  "ERROR: No json files found at ${ALERTS_BUILD_PATH}. Build process Failed"
-    ls -larth ${ALERTS_BUILD_PATH}
-    exit 1
-fi
+ls -larth ${ALERTS_BUILD_PATH}
 
 cd ${CURRENT_DIR}
\ No newline at end of file

scripts/build_dashboards.sh

 #!/bin/bash
 
-DASHBOARDS_JSONNET_PATH=../dashboards
-DASHBOARDS_BUILD_PATH=../build/dashboards
+DASHBOARDS_JSONNET_PATH=dashboards
+DASHBOARDS_BUILD_PATH=build/dashboards
 DASHBOARDS_TAR_NAME=dashboards
-## /build/archive from the root of the project
-ARCHIVE_PATH=../archive
+## build/archive from the root of the project
+ARCHIVE_PATH=build/archive
 
 CURRENT_DIR=$PWD
 
@@ -14,43 +14,20 @@ mkdir -p ${ARCHIVE_PATH}
 echo "Cleaning up previous builds"
 rm -rf ${DASHBOARDS_BUILD_PATH}/*.json
 
-DASHBOARD_JSONNET_FILES=$(find ${DASHBOARDS_JSONNET_PATH} -type f -name "*.jsonnet" ! -path "../grafonnet-lib/*") #| xargs -I{} basename {}  )
 
-LAST_COMMIT_FILES=$(git diff --name-only HEAD^ HEAD)
-echo last commit files $LAST_COMMIT_FILES
-echo dashboards files $DASHBOARD_JSONNET_FILES
+LAST_COMMIT_FILES=$(git diff --name-only HEAD^ HEAD ${DASHBOARDS_JSONNET_PATH} | xargs -I{} basename {} )
+echo "last commit files $LAST_COMMIT_FILES"
 
-UPDATED_DASHBOARDS=($(comm -12 <(echo "${LAST_COMMIT_FILES[@]}" | tr ' ' '\n' | sort) <(echo "${DASHBOARD_JSONNET_FILES[@]}" | tr ' ' '\n' | sort)))
-
-exit
-
-echo "creating jsons for ${#UPDATED_DASHBOARDS[@] } jsonnet files"
-
-for json_file in ${UPDATED_DASHBOARDS[@]};
+for json_file in ${LAST_COMMIT_FILES[@]};
     do
       echo "Filename is $json_file"
       filename=$(basename -- "${json_file}")
       filename="${filename%.*}"
-      JSONNET_PATH=../grafonnet-lib \
+      echo $filename
+      JSONNET_PATH=grafonnet-lib \
       jsonnet ${DASHBOARDS_JSONNET_PATH}/${json_file} > ${DASHBOARDS_BUILD_PATH}/${filename}.json
       echo "Created new json at ${DASHBOARDS_BUILD_PATH}/${filename}.json"
     done
-
-#cd ${DASHBOARDS_BUILD_PATH}
-#
-#count_file=`ls -1 *.json 2>/dev/null | wc -l`
-#echo "${count_file} json files found at `pwd`"
-#
-#if [ $count_file != 0 ]
-# then
-#    tar -cvzf ${ARCHIVE_PATH}/${DASHBOARDS_TAR_NAME}.tar.gz *.json
-#    echo "Uploading to $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${DASHBOARDS_TAR_NAME}.tar.gz"
-#    curl -u $ARTIFACTORY_USER:$ARTIFACTORY_PASS -T ${ARCHIVE_PATH}/${DASHBOARDS_TAR_NAME}.tar.gz "$ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${DASHBOARDS_TAR_NAME}.tar.gz"
-#    echo "Artifact Uploaded Successfully"
-#else
-#    echo  "ERROR: No json files found at ${DASHBOARDS_BUILD_PATH}. Build process Failed"
-#    ls -larth ${DASHBOARDS_BUILD_PATH}
-#    exit 1
-#fi
+ls -larth ${DASHBOARDS_BUILD_PATH}
 
 cd ${CURRENT_DIR}
\ No newline at end of file

scripts/deploy_alerts.sh

 #!/bin/bash
 
+
 ALERTS_DEPLOY_DIR=build/$CI_BUILD_ID/alerts
 ALERTS_TAR_NAME=alerts
 CURRENT_DIR=$PWD
@@ -7,7 +8,7 @@ CURRENT_DIR=$PWD
 mkdir -p ${ALERTS_DEPLOY_DIR}
 cd ${ALERTS_DEPLOY_DIR}
 
-#cleaning up previous artifacts
+echo "Cleaning up previous archives"
 rm -rf ${ALERTS_TAR_NAME}.tar.gz
 
 echo "Downloading artifact from $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${ALERTS_TAR_NAME}.tar.gz"

scripts/deploy_dashboards.sh

@@ -7,7 +7,7 @@ CURRENT_DIR=$PWD
 mkdir -p ${ALERTS_DEPLOY_DIR}
 cd ${ALERTS_DEPLOY_DIR}
 
-#cleaning up previous artifacts
+echo "Cleaning up previous archives"
 rm -rf ${DASHBOARD_TAR_NAME}.tar.gz
 
 echo "Downloading artifact from $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${DASHBOARD_TAR_NAME}.tar.gz"

scripts/promote_alerts_artifacts.sh

+#!/bin/bash
+
+ALERTS_BUILD_PATH=build/alerts
+ALERTS_TAR_NAME=alerts
+ARCHIVE_PATH=build/archive
+
+CURRENT_DIR=$PWD
+
+cd ${ALERTS_BUILD_PATH}
+
+count_file=`ls -1 *.json 2>/dev/null | wc -l`
+echo "${count_file} json files found at `pwd`"
+
+if [ $count_file != 0 ]
+ then
+    tar -cvzf ${ARCHIVE_PATH}/${ALERTS_TAR_NAME}.tar.gz *.json
+    echo "Uploading to $ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${ALERTS_TAR_NAME}.tar.gz"
+    curl -u $ARTIFACTORY_USER:$ARTIFACTORY_PASS -T ${ARCHIVE_PATH}/${ALERTS_TAR_NAME}.tar.gz "$ARTIFACTORY_URL/grafonet/$CI_PIPELINE_ID/${ALERTS_TAR_NAME}.tar.gz"
+    echo "Artifact Uploaded Successfully"
+else
+    echo  "ERROR: No json files found at ${ALERTS_BUILD_PATH}. Build process Failed"
+    ls -larth ${ALERTS_BUILD_PATH}
+    exit 1
+fi
+
+cd ${CURRENT_DIR}
\ No newline at end of file

scripts/promote_dashboard_artifacts.sh

 #!/bin/bash
-DASHBOARDS_BUILD_PATH=../build/dashboards
+
+DASHBOARDS_BUILD_PATH=build/dashboards
 DASHBOARDS_TAR_NAME=dashboards
-## /build/archive from the root of the project
-ARCHIVE_PATH=../archive
+ARCHIVE_PATH=build/archive
 
 CURRENT_DIR=$PWD