Merge pull request #1216 from tmobile/develop

✨ Enhancements to serverless framework support & elastic search upgrade

builds/delete-serverless-service-build-pack/Jenkinsfile

@@ -43,7 +43,7 @@ import groovy.json.JsonSlurperClassic
 
 node {
   try {
-	echo "Starting delete service job with params : $params"
+	echo "Starting delete service job with params: $params"
 
 	jazzBuildModuleURL = getBuildModuleUrl()
 	loadBuildModules(jazzBuildModuleURL)
@@ -79,7 +79,7 @@ node {
 	auth_token = getAuthToken()
 
 	stage("Initialization") {
-    if (params.domain && params.domain != "") {
+    	if (params.domain && params.domain != "") {
 			repo_name = params.domain + "_" + params.service_name
 		}
 		sh 'rm -rf ' + repo_name
@@ -95,7 +95,7 @@ node {
 		if (configObj.service_id) {
 			service_config = serviceMetadataLoader.loadServiceMetadata(configObj.service_id)
 		} else {
-			error "Service Id is not available."
+			error "Service id is not available."
 		}
 
 		if (!service_config) {
@@ -106,17 +106,17 @@ node {
 			accountInfo = utilModule.getAccountInfo(service_config)
 		}
 
-    if(!service_config.region) {
+		if(!service_config.region) {
 			service_config.region = configLoader.AWS.DEFAULTS.REGION
-    }
+		}
 
-    if(!service_config.accountId) {
-      service_config.accountId = configLoader.AWS.DEFAULTS.ACCOUNTID
-    }
+		if(!service_config.accountId) {
+		service_config.accountId = configLoader.AWS.DEFAULTS.ACCOUNTID
+		}
 
-    if(!service_config.provider) {
-      service_config.provider = configLoader.JAZZ.DEFAULTS.PROVIDER
-    }
+		if(!service_config.provider) {
+		service_config.provider = configLoader.JAZZ.DEFAULTS.PROVIDER
+		}
 
 		if(service_config['provider'] == 'aws'){
 			service_config.credentialId = accountInfo.CREDENTIAL_ID
@@ -150,8 +150,6 @@ node {
 
 
 		dir(repo_name){
-			echo "loadServerlessConfig......."
-
 			if (fileExists('build.api')) {
 				flowType = "API"
 				loadServerlessConfig()
@@ -326,7 +324,7 @@ node {
 				stage('Undeploy Service') {
 					for (_envId in environmentIds) {
 						current_environment_id = _envId
-						loadServerlessYml(service_config, _envId)
+						def has_serverless_yml = 	loadServerlessYml(service_config, _envId)
 						try {
 							def branch = environmentMetadataLoader.getEnvironmentBranchName(_envId)
 							if (branch && branch != 'NA') {
@@ -336,10 +334,17 @@ node {
 							}
 							events.setEnvironment(_envId)
 							events.sendStartedEvent('DELETE_ENVIRONMENT', "Environment cleanup for " + _envId + " started", null, _envId)
-							echo 'undeploying function for environment: ' + _envId
+							echo 'undeploying sls service for environment: ' + _envId
 
 							cleanupEventSourceMapping(_envId)
-							unDeployService(_envId)
+							if (has_serverless_yml) {
+								// for sls install plugins
+								installServerlessPlugins()								
+								unDeployService(_envId)
+							}
+							else {
+								unDeployCloudformationStack(_envId)
+							}
 
 							if (configLoader.CODE_QUALITY.SONAR.ENABLE_SONAR == "true" && configLoader.CODE_QUALITY.SONAR.CLEANUP == "true") {
 								cleanupCodeQualityReports()
@@ -409,6 +414,7 @@ def archiveAssetDetails(env) {
   def assets = utilModule.getAssets(assets_api, auth_token, service_config, env)
   def assetList = parseJsonMap(assets)
   for (asset in assetList.data.assets) {
+	// TODO update Asset Status to 'archived'/'deleted'?
     events.sendCompletedEvent('UPDATE_ASSET', "Environment cleanup for ${env} completed", utilModule.generateAssetMap(asset.provider, asset.provider_id , asset.asset_type, service_config), env)
   }
 }
@@ -491,7 +497,8 @@ def unDeployService(stage) {
 			}
 			def envBucketKey = "${env_key}${configLoader.JAZZ.PLATFORM.AWS.S3.BUCKET_NAME_SUFFIX}"
 			def bucketObject = utilModule.getAccountBucketName(service_config);
-			sh "serverless remove --stage ${stage} --verbose  --bucket ${bucketObject[env_key]} --profile ${credsId}"
+
+      sh "serverless remove --stage ${stage} --verbose  --bucket ${bucketObject[env_key]} --profile ${credsId}"
 
 			echo "Service undeployed"
 		} catch (ex) {
@@ -502,6 +509,42 @@ def unDeployService(stage) {
 	}
 }
 
+def installServerlessPlugins(){
+  try {
+    def serverlessyml = readFile('./serverless.yml').trim()
+    def outstandingPlugins = whiteListModule.validatePlugins(serverlessyml)
+    if(outstandingPlugins.isEmpty()) {
+      def plugins = whiteListModule.getPluginsfromYaml(serverlessyml)
+      if( plugins ) {
+        for (plugin in plugins){
+          sh "npm install ${plugin}"
+        }
+      } else {
+        echo "No plugins listed..skipping"
+      }
+    } else {
+      error "The following plugins are not allowed: ${outstandingPlugins}"
+    }
+  } catch( ex ){
+    error "Plugin Installation Failed "+ ex.getMessage();
+  }
+}
+
+def unDeployCloudformationStack(env) {
+  echo "Manualy removing the stack by using aws cli."
+  withCredentials([
+		[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: service_config.credentialId]
+	]) {
+    def cfStackName = "${instance_prefix}-${service_config['domain']}-${service_config['service']}-${env}"
+    try {
+      def res = sh(script: "aws cloudformation delete-stack --stack-name ${cfStackName} --region ${service_config['region']} ", returnStdout: true)
+      echo "Delete stack resp: ${res}"
+    } catch (ex) {
+      echo "Error occured while deleting cloudformation stack ${cfStackName} : ${ex}"
+    }
+  }
+}
+
 /**
  * Checkout Code
  * @param  repo_name
@@ -1143,9 +1186,15 @@ def loadServerlessYml(config, env){
 	environmentMetadataLoader.setEnvironmentLogicalId(env) //set Current Environment
 	//Get deployment descriptor from environments
   env_deployment_descriptor = environmentMetadataLoader.getEnvDeploymentDescriptor()
-	def deploymentDescriptor = slsBuildRules.prepareServerlessYml(config, env, configLoader, env_deployment_descriptor) // Generating the deployment descriptor
-	echo "prepareServerlessYml = ${deploymentDescriptor}"
-	writeYaml(file: './serverless.yml', data: deploymentDescriptor)
+  try {
+    def deploymentDescriptor = slsBuildRules.prepareServerlessYml(config, env, configLoader, env_deployment_descriptor, accountInfo) // Generating the deployment descriptor
+	  echo "prepareServerlessYml = ${deploymentDescriptor}"
+	  writeYaml(file: './serverless.yml', data: deploymentDescriptor)
+    return true
+  } catch (ex) {
+    echo "Failed to load the serverless.yml for ${env} environment."
+    return false
+  }
 }
 
 def loadServerlessConfig() {
@@ -1335,7 +1384,7 @@ def loadBuildModules(buildModuleUrl){
     azureDeployer.initialize(configLoader, utilModule, scmModule, events, azureUtil)
     echo "Azure deploy module init successfully."
 
-		whiteListModule = load "whitelist-validator-module.groovy"
+		whiteListModule = load "sls-app/whitelist-validator-module.groovy"
     whiteListModule.initialize()
     echo "whitelist-validator-module module loaded successfully"
 

builds/jazz-build-module/environment-deployment-metadata-loader.groovy

@@ -255,7 +255,7 @@ def getEnvironmentBranchName(logical_id) {
 	}
 }
 
-def generateEnvironmentMap(status, environment_logical_id, metadata) {
+def generateEnvironmentMap(status, environment_logical_id, metadata, deployment_descriptor = null) {
 	def env_logical_id
 	if (environment_logical_id == null) {
 		env_logical_id = getEnvironmentLogicalId()
@@ -268,6 +268,9 @@ def generateEnvironmentMap(status, environment_logical_id, metadata) {
 		branch: g_service_branch,
 		logical_id: env_logical_id
 	]
+	if (deployment_descriptor != null) {
+		serviceCtxMap.deployment_descriptor = deployment_descriptor
+	}
 	if (g_environment_endpoint != null) {
 		serviceCtxMap.endpoint = g_environment_endpoint
 	}
@@ -298,6 +301,7 @@ def generateDeploymentMap(status, environment_logical_id, gitCommitHash) {
 		scm_branch: g_service_branch,
 		request_id: g_request_id
 	]
+
 	return serviceCtxMap;
 }
 

builds/jazz-build-module/events-module.groovy

@@ -193,7 +193,6 @@ def getEventName(eventTxt) {
 	_validEvent = Event_Names[eventTxt]
 
 	if (_validEvent) {
-		echo "$_validEvent"
 		return _validEvent
 	} else {
 		error "EVENT NAME not defined- $eventTxt. Please update the EventsName table"
@@ -210,6 +209,10 @@ def getEventName(eventTxt) {
 
 def sendEvent(event_name, event_status, message, moreCxtMap){
 
+	// generate a guid used for the payload file
+	// utilModule --> utility-loader.groovy is expected to be loaded here as global variable!
+	def guid = utilModule.generateRequestId()
+
 	def context_json = []
 	def event_json = []
 	def moreCxt = moreCxtMap
@@ -240,21 +243,28 @@ def sendEvent(event_name, event_status, message, moreCxtMap){
 	]
 
 	def payload = JsonOutput.toJson(event_json)
-	echo "$event_json"
+	echo "Sending Event -- Type: ${g_event_type}, Name: ${event_name}, Status: ${event_status}"
+
+	// write payload to a file guid.json
+	writeFile(file: "./${guid}.json", text: payload)
 
 	try {
 		if (service_metadata['domain'] != "jazz") {
 			def shcmd = sh(script: "curl --silent -X POST -k -v \
 				-H \"Content-Type: application/json\" \
 					$g_events_api \
-				-d \'${payload}\'", returnStdout:true).trim()
+				-d @./${guid}.json", returnStdout:true).trim()
 
 			echo "------  Event send.........."
 		}
 	}
 	catch (e) {
 		echo "error occured when recording event: " + e.getMessage()
 	}
+	finally {
+		// delete guid.json if it exists
+		sh "rm -rf ${guid}.json"
+	}	
 }
 
 /**

builds/jazz-build-module/service-configuration-data-loader.groovy

@@ -16,7 +16,6 @@ echo "Service configuration module loaded successfully"
 @Field def accountId
 @Field def jenkins_url
 @Field def current_environment
-@Field def es_hostname
 @Field def service_name
 @Field def utilModule
 @Field def awsAPIGatewayModule
@@ -271,6 +270,7 @@ def loadServiceConfigurationData() {
 
     if ((service_name.trim() == "jazz_logs") || (service_name.trim() == "jazz_cloud-logs-streamer") || (service_name.trim() == "jazz_es-kinesis-log-streamer")) {
       updateConfigValue("{inst_elastic_search_hostname}", config_loader.JAZZ.ES_HOSTNAME)
+      updateConfigValue("{inst_kibana_search_hostname}", config_loader.JAZZ.KIBANA_HOSTNAME)
 
       if (service_name.trim() == "jazz_logs") {
         updateConfigValue("{env-prefix}", config_loader.INSTANCE_PREFIX)
@@ -402,9 +402,9 @@ def loadServiceConfigurationData() {
 }
 
 def updateConfigValue(key, val) {
-  sh "sed -i -- 's/${key}/${val}/g' ./config/dev-config.json"
-  sh "sed -i -- 's/${key}/${val}/g' ./config/stg-config.json"
-  sh "sed -i -- 's/${key}/${val}/g' ./config/prod-config.json"
+  sh "sed -i -- 's#${key}#${val}#g' ./config/dev-config.json"
+  sh "sed -i -- 's#${key}#${val}#g' ./config/stg-config.json"
+  sh "sed -i -- 's#${key}#${val}#g' ./config/prod-config.json"
 }
 
 def updateCoreAPI() {