.gitignore

@@ -1,3 +1,2 @@
-output/
 .env
 .terraform.lock.hcl

.vscode/launch.json

@@ -13,26 +13,6 @@
             "args": [
                 "TERRAFORM"
             ],
-        },
-        {
-            "name": "Python: Aktuelle Datei EMEA_PROD",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal",
-            "args": [
-                "EMEA_PROD"
-            ],
-        },
-        {
-            "name": "Python: Aktuelle Datei EMEA_PREPROD",
-            "type": "python",
-            "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal",
-            "args": [
-                "EMEA_PREPROD"
-            ],
         }
     ]
 }

Jenkinsfile

@@ -1,13 +1,21 @@
  //not required right now as CN is reachable from EMEA as well
-
+ def loopEnvironments(environments){
- environments=['EMEA_PROD', 'EMEA_PREPROD', 'NA_PROD', 'NA_PREPROD', 'CN_PROD', 'CN_PREPROD']
+     print env.JENKINS_URL 
-
+     
-
+     environments.each { key, val -> 
- @NonCPS // has to be NonCPS or the build breaks on the call to .each
+       
- def export_config_all(list) {
+       //Execute only if you are on the same environment
-    list.each { env ->
+       //not required right now as CN is reachable from EMEA as well
-        sh "python3 export.py ${env}"
+        if (env.JENKINS_URL == environments."${key}"[3].'jenkins') 
-    }
+        {
+            envname = environments."${key}"[0].'name'
+            envurl = environments."${key}"[1].'env-url'
+            tokenname = environments."${key}"[2].'env-token-name'
+            
+            sh 'python createReport.py "${envname}"'
+        }
+        
+     }
  }
  pipeline {
 		options {
@@ -48,12 +56,11 @@
         // NAPREPROD_TOKEN_VAR = credentials('NAPREPROD_TOKEN_VAR')
         // CNPROD_TOKEN_VAR = credentials('CNPROD_TOKEN_VAR')
         // CNPREPROD_TOKEN_VAR = credentials('CNPREPROD_TOKEN_VAR')
-        AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
+        // AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
-		AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
+		// AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
-        AWS_S3_BUCKET="coco-dynatrace-tfstate"
-        AWS_S3_REGION="eu-central-1"
 
-        //TERRAFORM_RESOURCES="dynatrace_management_zone"
+    
+        TERRAFORM_RESOURCES="dynatrace_management_zone"
     	
         //EMEA PROD
         TF_VAR_EMEA_PROD_ENV_URL="https://xxu26128.live.dynatrace.com"
@@ -72,11 +79,11 @@
         TF_VAR_NA_PREPROD_API_TOKEN=credentials('NAPREPROD_TOKEN_VAR')
 
         //CN PROD
-        TF_VAR_CN_PROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+        TF_VAR_CN_PROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
         TF_VAR_CN_PROD_API_TOKEN=credentials('CNPROD_TOKEN_VAR')
 
         //CN PREPROD
-        TF_VAR_CN_PREPROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
+        TF_VAR_CN_PREPROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
         TF_VAR_CN_PREPROD_API_TOKEN=credentials('CNPREPROD_TOKEN_VAR')
 
         //TERRAFORM 
@@ -122,17 +129,15 @@
             }   
                
         }
+
         stage('Execute Export Script TERRAFORM') {
             steps {
                     
-                    export_config_all(environments)
+                    sh 'python3 export.py TERRAFORM'
-                    //sh 'python3 export.py EMEA_PROD'
-                    //sh 'python3 export.py TERRAFORM'
                     
                     //Only required once CN is not reachable from EMEA
                     //loopEnvironments(environments)
-            }
+            }   
-               
                
         }
         stage('Send report') {

export.py

@@ -1,17 +1,15 @@
 import os
-from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
+import subprocess
 import sys
 import time
 import shutil
 import hcl
 from dotenv import load_dotenv
 from glob import glob
-#from git import Repo
-import zipfile
-import boto3
 
 
-# [AA 2022.01.17] Set available resources
+
+    # [AA 2022.01.17] Set available resources
 
 
 if os.name == 'nt':
@@ -37,11 +35,11 @@ Resources = os.getenv("TERRAFORM_RESOURCES").split(",") if os.getenv("TERRAFORM_
          "dynatrace_database_anomalies",
          "dynatrace_custom_anomalies",
          "dynatrace_disk_anomalies",
-         "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
+        # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
          "dynatrace_service_naming",
          "dynatrace_host_naming",
          "dynatrace_processgroup_naming",
-         "dynatrace_slo",                       # issue -> bug: whitespace issue
+         #"dynatrace_slo",                       # issue -> bug: whitespace issue
          "dynatrace_span_entry_point",
          "dynatrace_span_capture_rule",
          "dynatrace_span_context_propagation",
@@ -70,27 +68,33 @@ def setEnv(env, time, path):
 
 
 # [AA 2021.12.10] Method to call process synchronously
-def runExportProcess(process_name, input_params):
+def runProcess(process_name, input_params):
-         process_names = ["Export", "Terraform init"]
+    process_names = ["Export", "Terraform init"]
-         success = True
+    success = False
-         print("[DEBUG] Start run process: "+ ' '.join(input_params))
+    #process = subprocess.Popen(input_params)
-         process = Popen(input_params,stdout=PIPE, stderr=PIPE)
+    print("[DEBUG] Start run process: "+ ' '.join(input_params))
-         output, error_output = process.communicate()
+    try:
+         process = subprocess.Popen(input_params)
          process.wait(timeout=60*60)  # 10 minutes
-         if len(output) > 0:
-              raise Exception ("Eception occured during export config: "+output.decode("utf-8"))
-         print("[DEBUG]", "Process:", process_name, "Success:", success)
-
-def runImportProcess(process_name, input_params):
-         process_names = ["Export", "Terraform init"]
          success = True
-         print("[DEBUG] Start run process: "+ ' '.join(input_params))
-         process = Popen(input_params)
-         process.wait(timeout=60*60) 
-         if process.returncode > 0:
-            raise Exception ("Eception occured during generating state File!")
-
          print("[DEBUG]", "Process:", process_name, "Success:", success)
+         # print("[DEBUG]", "Process return code:", outs)
+    except subprocess.TimeoutExpired:
+        print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
+        print("[DEBUG]", "Killing process:", process_name)
+        process.kill()
+        success = False
+        print("[DEBUG]", "Process:", process_name, "Success:", success)
+    except Exception as err:
+        print("[DEBUG]", "Exception running export tool"+ str(err))
+        #if process_name in process_names and success == False:
+        #    print("[DEBUG]", "Process:", process_name, "Success:", success)
+        #    print("[DEBUG]", "Exiting program.")
+        #    process.kill()
+        #    success = False
+        #    sys.exit(1)
+        #else:
+        #    print("[FAILED]", input_params)
 
 # [AA 2021.12.17] Methods needed to replace the matching keys
 def replacedata(p, maplist):
@@ -102,11 +106,29 @@ def replacedata(p, maplist):
         template.seek(0)
         template.truncate()
 
+    # [AA 2021.12.14] Replace matching {$keys}
+    #for mapping in mappings[0:2]:
+
         # [AA 2021.12.17] With the values for management_zone and msid in memory
     for key, val in maplist.items():
              print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
              data = data.replace(key, val)
 
+    # # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod
+    # if os.path.basename(p) in specificfiles[0:2]:
+    #     for key, val in mappings[2][maplist].items():
+    #         print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
+    #         data = data.replace(key, val)
+
+    # # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub
+    # if os.path.basename(p) in specificfiles[2]:
+    #     for mapping in mappings[3:5]:
+    #         for key, val in mapping.items():
+    #             print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
+    #             data = data.replace(key, val[pos])
+
+    # [AA 2022.01.19] Replace key value for {}
+
     # [AA 2021.12.14] Write data from memory into file
     with open(p, 'w+') as template:
         template.write(data)
@@ -151,10 +173,7 @@ def createResourceDict():
 # [AA, EW 2022.01.17] Copy main.tf into the target folder
 def copyMainTemplate():
     shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
-    replacedata(targetFolder + "main.tf",  {"{$env}":env, 
+    replacedata(targetFolder + "main.tf",  {"{$env}":env, "{$timestamp}":timestamp})
-                                            "{$timestamp}":timestamp,
-                                            "{$S3_BUCKET}": str(os.getenv("AWS_S3_BUCKET")),
-                                            "{$S3_REGION}":str(os.getenv("AWS_S3_REGION"))})
 
 
 # [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
@@ -187,25 +206,17 @@ def editMainTF():
 def importStates():
     os.chdir(targetFolder)
     input_params = ["terraform", "init"]
-    runImportProcess("Terraform init",input_params)
+    runProcess("Terraform init",input_params)
 
     for filedir, resourceV in myDict.items():
         for resource, valueArray in resourceV.items():
             for rObject in valueArray:
                 input_params = ["terraform", "import", "module."+getModuleTag(
                     filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
-                runImportProcess("Import", input_params)
+                runProcess("Import", input_params)
     # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
     os.chdir(cwd)
 
-def zipdir(path, ziph):# ziph is zipfile handlefor root, dirs, files in os.walk(path):for file in files:
-    for root, dirs, files in os.walk(path):
-        if ".terraform" not in root:
-            for file in files:
-                if ".terraform.lock.hcl" not in file:
-                    ziph.write(os.path.join(root, file), 
-                       os.path.relpath(os.path.join(root, file), 
-                                       os.path.join(path, '..')))
 
 # [AA 2022.01.17] Arguments passed
 if(len(sys.argv) == 2):
@@ -228,7 +239,7 @@ if(len(sys.argv) == 2):
         setEnv(sys.argv[1], timestamp, outputFolder)
 
         # [AA, EW 2022.01.17] Download resource files
-        runExportProcess("Export", [export_tool, "export"] + Resources)
+        runProcess("Export", [export_tool, "export"] + Resources)
             
         # [AA, EW 2022.01.17] Create a dictionary to store information of resources
         createResourceDict()
@@ -242,21 +253,8 @@ if(len(sys.argv) == 2):
 
         # [AA, EW 2022.01.17] Import the states for each module
         importStates()
-
-        zipf = zipfile.ZipFile(outputFolder+"/"+timestamp + "_" +env+'.zip', 'w', zipfile.ZIP_DEFLATED)
-        zipdir(targetFolder, zipf)
-        zipf.close()
-
-        s3 = boto3.client('s3')
-
-        with open(outputFolder+"/"+timestamp + "_" +env+'.zip', 'rb') as data:
-            s3.upload_fileobj(data, str(os.getenv("AWS_S3_BUCKET")), 'backups/'+timestamp + "_" +env+'.zip') #TODO: Make s3 bucket name configurable over environment variables
-            
-
         print("Finished!")
         sys.exit(0)
-
-
     except Exception as err:
         print("Exception occured: "+ str(err))
         sys.exit(1) 

requirements.txt

@@ -9,4 +9,3 @@ typing
 python-dotenv
 pyhcl
 subprocess32
-boto3==1.17.0

templates/main.tf

@@ -7,9 +7,9 @@ terraform {
   }
 
   backend "s3" {
-    bucket = "{$S3_BUCKET}"
+    bucket = "coco-dynatrace-tfstate"
     key = "backup/{$env}/{$timestamp}/terraform.tfstate"
-    region = "{$S3_REGION}"
+    region = "eu-central-1"
     dynamodb_table = "coco-dynatrace-tfstate"
     encrypt = true
   }
@@ -23,3 +23,6 @@ provider "dynatrace" {
     dt_env_url    = "${var.{$env}_ENV_URL}"
     dt_api_token  = "${var.{$env}_API_TOKEN}"
 }
+
+
+