added list of all environments and defined export function

.gitignore

@@ -1,2 +1,3 @@
+output/
 .env
 .terraform.lock.hcl

.vscode/launch.json

@@ -13,6 +13,26 @@
             "args": [
                 "TERRAFORM"
             ],
+        },
+        {
+            "name": "Python: Aktuelle Datei EMEA_PROD",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": [
+                "EMEA_PROD"
+            ],
+        },
+        {
+            "name": "Python: Aktuelle Datei EMEA_PREPROD",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": [
+                "EMEA_PREPROD"
+            ],
         }
     ]
 }

Jenkinsfile

@@ -1,21 +1,13 @@
  //not required right now as CN is reachable from EMEA as well
- def loopEnvironments(environments){
+
-     print env.JENKINS_URL 
+ environments=['EMEA_PROD', 'EMEA_PREPROD', 'NA_PROD', 'NA_PREPROD', 'CN_PROD', 'CN_PREPROD']
-     
+
-     environments.each { key, val -> 
+
-       
+ @NonCPS // has to be NonCPS or the build breaks on the call to .each
-       //Execute only if you are on the same environment
+ def export_config_all(list) {
-       //not required right now as CN is reachable from EMEA as well
+    list.each { env ->
-        if (env.JENKINS_URL == environments."${key}"[3].'jenkins') 
+        sh "python3 export.py ${env}"
-        {
+    }
-            envname = environments."${key}"[0].'name'
-            envurl = environments."${key}"[1].'env-url'
-            tokenname = environments."${key}"[2].'env-token-name'
-            
-            sh 'python createReport.py "${envname}"'
-        }
-        
-     }
  }
  pipeline {
 		options {
@@ -56,11 +48,12 @@
         // NAPREPROD_TOKEN_VAR = credentials('NAPREPROD_TOKEN_VAR')
         // CNPROD_TOKEN_VAR = credentials('CNPROD_TOKEN_VAR')
         // CNPREPROD_TOKEN_VAR = credentials('CNPREPROD_TOKEN_VAR')
-        // AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
+        AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
-		// AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
+		AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
+        AWS_S3_BUCKET="coco-dynatrace-tfstate"
+        AWS_S3_REGION="eu-central-1"
 
-    
+        //TERRAFORM_RESOURCES="dynatrace_management_zone"
-        TERRAFORM_RESOURCES="dynatrace_management_zone"
     	
         //EMEA PROD
         TF_VAR_EMEA_PROD_ENV_URL="https://xxu26128.live.dynatrace.com"
@@ -79,11 +72,11 @@
         TF_VAR_NA_PREPROD_API_TOKEN=credentials('NAPREPROD_TOKEN_VAR')
 
         //CN PROD
-        TF_VAR_CN_PROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+        TF_VAR_CN_PROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
         TF_VAR_CN_PROD_API_TOKEN=credentials('CNPROD_TOKEN_VAR')
 
         //CN PREPROD
-        TF_VAR_CN_PREPROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
+        TF_VAR_CN_PREPROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
         TF_VAR_CN_PREPROD_API_TOKEN=credentials('CNPREPROD_TOKEN_VAR')
 
         //TERRAFORM 
@@ -129,15 +122,17 @@
             }   
                
         }
-
         stage('Execute Export Script TERRAFORM') {
             steps {
                     
-                    sh 'python3 export.py TERRAFORM'
+                    export_config_all(environments)
+                    //sh 'python3 export.py EMEA_PROD'
+                    //sh 'python3 export.py TERRAFORM'
                     
                     //Only required once CN is not reachable from EMEA
                     //loopEnvironments(environments)
-            }   
+            }
+               
                
         }
         stage('Send report') {

export.py

@@ -1,15 +1,17 @@
 import os
-import subprocess
+from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
 import sys
 import time
 import shutil
 import hcl
 from dotenv import load_dotenv
 from glob import glob
+#from git import Repo
+import zipfile
+import boto3
 
 
-
+# [AA 2022.01.17] Set available resources
-    # [AA 2022.01.17] Set available resources
 
 
 if os.name == 'nt':
@@ -35,11 +37,11 @@ Resources = os.getenv("TERRAFORM_RESOURCES").split(",") if os.getenv("TERRAFORM_
          "dynatrace_database_anomalies",
          "dynatrace_custom_anomalies",
          "dynatrace_disk_anomalies",
-        # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
+         "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
          "dynatrace_service_naming",
          "dynatrace_host_naming",
          "dynatrace_processgroup_naming",
-         #"dynatrace_slo",                       # issue -> bug: whitespace issue
+         "dynatrace_slo",                       # issue -> bug: whitespace issue
          "dynatrace_span_entry_point",
          "dynatrace_span_capture_rule",
          "dynatrace_span_context_propagation",
@@ -68,33 +70,27 @@ def setEnv(env, time, path):
 
 
 # [AA 2021.12.10] Method to call process synchronously
-def runProcess(process_name, input_params):
+def runExportProcess(process_name, input_params):
-    process_names = ["Export", "Terraform init"]
+         process_names = ["Export", "Terraform init"]
-    success = False
-    #process = subprocess.Popen(input_params)
-    print("[DEBUG] Start run process: "+ ' '.join(input_params))
-    try:
-         process = subprocess.Popen(input_params)
-         process.wait(timeout=60*60)  # 10 minutes
          success = True
+         print("[DEBUG] Start run process: "+ ' '.join(input_params))
+         process = Popen(input_params,stdout=PIPE, stderr=PIPE)
+         output, error_output = process.communicate()
+         process.wait(timeout=60*60)  # 10 minutes
+         if len(output) > 0:
+              raise Exception ("Eception occured during export config: "+output.decode("utf-8"))
+         print("[DEBUG]", "Process:", process_name, "Success:", success)
+
+def runImportProcess(process_name, input_params):
+         process_names = ["Export", "Terraform init"]
+         success = True
+         print("[DEBUG] Start run process: "+ ' '.join(input_params))
+         process = Popen(input_params)
+         process.wait(timeout=60*60) 
+         if process.returncode > 0:
+            raise Exception ("Eception occured during generating state File!")
+
          print("[DEBUG]", "Process:", process_name, "Success:", success)
-         # print("[DEBUG]", "Process return code:", outs)
-    except subprocess.TimeoutExpired:
-        print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
-        print("[DEBUG]", "Killing process:", process_name)
-        process.kill()
-        success = False
-        print("[DEBUG]", "Process:", process_name, "Success:", success)
-    except Exception as err:
-        print("[DEBUG]", "Exception running export tool"+ str(err))
-        #if process_name in process_names and success == False:
-        #    print("[DEBUG]", "Process:", process_name, "Success:", success)
-        #    print("[DEBUG]", "Exiting program.")
-        #    process.kill()
-        #    success = False
-        #    sys.exit(1)
-        #else:
-        #    print("[FAILED]", input_params)
 
 # [AA 2021.12.17] Methods needed to replace the matching keys
 def replacedata(p, maplist):
@@ -106,29 +102,11 @@ def replacedata(p, maplist):
         template.seek(0)
         template.truncate()
 
-    # [AA 2021.12.14] Replace matching {$keys}
-    #for mapping in mappings[0:2]:
-
         # [AA 2021.12.17] With the values for management_zone and msid in memory
     for key, val in maplist.items():
              print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
              data = data.replace(key, val)
 
-    # # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod
-    # if os.path.basename(p) in specificfiles[0:2]:
-    #     for key, val in mappings[2][maplist].items():
-    #         print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
-    #         data = data.replace(key, val)
-
-    # # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub
-    # if os.path.basename(p) in specificfiles[2]:
-    #     for mapping in mappings[3:5]:
-    #         for key, val in mapping.items():
-    #             print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
-    #             data = data.replace(key, val[pos])
-
-    # [AA 2022.01.19] Replace key value for {}
-
     # [AA 2021.12.14] Write data from memory into file
     with open(p, 'w+') as template:
         template.write(data)
@@ -173,7 +151,10 @@ def createResourceDict():
 # [AA, EW 2022.01.17] Copy main.tf into the target folder
 def copyMainTemplate():
     shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
-    replacedata(targetFolder + "main.tf",  {"{$env}":env, "{$timestamp}":timestamp})
+    replacedata(targetFolder + "main.tf",  {"{$env}":env, 
+                                            "{$timestamp}":timestamp,
+                                            "{$S3_BUCKET}": str(os.getenv("AWS_S3_BUCKET")),
+                                            "{$S3_REGION}":str(os.getenv("AWS_S3_REGION"))})
 
 
 # [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
@@ -206,17 +187,25 @@ def editMainTF():
 def importStates():
     os.chdir(targetFolder)
     input_params = ["terraform", "init"]
-    runProcess("Terraform init",input_params)
+    runImportProcess("Terraform init",input_params)
 
     for filedir, resourceV in myDict.items():
         for resource, valueArray in resourceV.items():
             for rObject in valueArray:
                 input_params = ["terraform", "import", "module."+getModuleTag(
                     filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
-                runProcess("Import", input_params)
+                runImportProcess("Import", input_params)
     # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
     os.chdir(cwd)
 
+def zipdir(path, ziph):# ziph is zipfile handlefor root, dirs, files in os.walk(path):for file in files:
+    for root, dirs, files in os.walk(path):
+        if ".terraform" not in root:
+            for file in files:
+                if ".terraform.lock.hcl" not in file:
+                    ziph.write(os.path.join(root, file), 
+                       os.path.relpath(os.path.join(root, file), 
+                                       os.path.join(path, '..')))
 
 # [AA 2022.01.17] Arguments passed
 if(len(sys.argv) == 2):
@@ -239,7 +228,7 @@ if(len(sys.argv) == 2):
         setEnv(sys.argv[1], timestamp, outputFolder)
 
         # [AA, EW 2022.01.17] Download resource files
-        runProcess("Export", [export_tool, "export"] + Resources)
+        runExportProcess("Export", [export_tool, "export"] + Resources)
             
         # [AA, EW 2022.01.17] Create a dictionary to store information of resources
         createResourceDict()
@@ -253,8 +242,21 @@ if(len(sys.argv) == 2):
 
         # [AA, EW 2022.01.17] Import the states for each module
         importStates()
+
+        zipf = zipfile.ZipFile(outputFolder+"/"+timestamp + "_" +env+'.zip', 'w', zipfile.ZIP_DEFLATED)
+        zipdir(targetFolder, zipf)
+        zipf.close()
+
+        s3 = boto3.client('s3')
+
+        with open(outputFolder+"/"+timestamp + "_" +env+'.zip', 'rb') as data:
+            s3.upload_fileobj(data, str(os.getenv("AWS_S3_BUCKET")), 'backups/'+timestamp + "_" +env+'.zip') #TODO: Make s3 bucket name configurable over environment variables
+            
+
         print("Finished!")
         sys.exit(0)
+
+
     except Exception as err:
         print("Exception occured: "+ str(err))
         sys.exit(1) 

requirements.txt

@@ -9,3 +9,4 @@ typing
 python-dotenv
 pyhcl
 subprocess32
+boto3==1.17.0

templates/main.tf

@@ -7,9 +7,9 @@ terraform {
   }
 
   backend "s3" {
-    bucket = "coco-dynatrace-tfstate"
+    bucket = "{$S3_BUCKET}"
     key = "backup/{$env}/{$timestamp}/terraform.tfstate"
-    region = "eu-central-1"
+    region = "{$S3_REGION}"
     dynamodb_table = "coco-dynatrace-tfstate"
     encrypt = true
   }
@@ -23,6 +23,3 @@ provider "dynatrace" {
     dt_env_url    = "${var.{$env}_ENV_URL}"
     dt_api_token  = "${var.{$env}_API_TOKEN}"
 }
-
-
-