Error handling subprocesses for Export & Import, Excluding terraform dir and lock file for zipping

.vscode/launch.json

@@ -13,6 +13,26 @@
             "args": [
                 "TERRAFORM"
             ],
+        },
+        {
+            "name": "Python: Aktuelle Datei EMEA_PROD",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": [
+                "EMEA_PROD"
+            ],
+        },
+        {
+            "name": "Python: Aktuelle Datei EMEA_PREPROD",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": [
+                "EMEA_PREPROD"
+            ],
         }
     ]
 }

Jenkinsfile

@@ -5,7 +5,7 @@
      environments.each { key, val -> 
        
        //Execute only if you are on the same environment
-       //not required right now as CN is reachable from EMEA as well
+       //not required right now as CN is reachable from EMEA as well 
         if (env.JENKINS_URL == environments."${key}"[3].'jenkins') 
         {
             envname = environments."${key}"[0].'name'
@@ -80,11 +80,11 @@
         TF_VAR_NA_PREPROD_API_TOKEN=credentials('NAPREPROD_TOKEN_VAR')
 
         //CN PROD
-        TF_VAR_CN_PROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+        TF_VAR_CN_PROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
         TF_VAR_CN_PROD_API_TOKEN=credentials('CNPROD_TOKEN_VAR')
 
         //CN PREPROD
-        TF_VAR_CN_PREPROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
+        TF_VAR_CN_PREPROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
         TF_VAR_CN_PREPROD_API_TOKEN=credentials('CNPREPROD_TOKEN_VAR')
 
         //TERRAFORM 

export.py

@@ -1,5 +1,5 @@
 import os
-import subprocess
+from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
 import sys
 import time
 import shutil
@@ -41,7 +41,7 @@ Resources = os.getenv("TERRAFORM_RESOURCES").split(",") if os.getenv("TERRAFORM_
          "dynatrace_service_naming",
          "dynatrace_host_naming",
          "dynatrace_processgroup_naming",
-         #"dynatrace_slo",                       # issue -> bug: whitespace issue
+         "dynatrace_slo",                       # issue -> bug: whitespace issue
          "dynatrace_span_entry_point",
          "dynatrace_span_capture_rule",
          "dynatrace_span_context_propagation",
@@ -70,33 +70,27 @@ def setEnv(env, time, path):
 
 
 # [AA 2021.12.10] Method to call process synchronously
-def runProcess(process_name, input_params):
+def runExportProcess(process_name, input_params):
-    process_names = ["Export", "Terraform init"]
+         process_names = ["Export", "Terraform init"]
-    success = False
-    #process = subprocess.Popen(input_params)
-    print("[DEBUG] Start run process: "+ ' '.join(input_params))
-    try:
-         process = subprocess.Popen(input_params)
-         process.wait(timeout=60*60)  # 10 minutes
          success = True
+         print("[DEBUG] Start run process: "+ ' '.join(input_params))
+         process = Popen(input_params,stdout=PIPE, stderr=PIPE)
+         output, error_output = process.communicate()
+         process.wait(timeout=60*60)  # 10 minutes
+         if len(output) > 0:
+              raise Exception ("Eception occured during export config: "+output.decode("utf-8"))
+         print("[DEBUG]", "Process:", process_name, "Success:", success)
+
+def runImportProcess(process_name, input_params):
+         process_names = ["Export", "Terraform init"]
+         success = True
+         print("[DEBUG] Start run process: "+ ' '.join(input_params))
+         process = Popen(input_params)
+         process.wait(timeout=60*60) 
+         if process.returncode > 0:
+            raise Exception ("Eception occured during generating state File!")
+
          print("[DEBUG]", "Process:", process_name, "Success:", success)
-         # print("[DEBUG]", "Process return code:", outs)
-    except subprocess.TimeoutExpired:
-        print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
-        print("[DEBUG]", "Killing process:", process_name)
-        process.kill()
-        success = False
-        print("[DEBUG]", "Process:", process_name, "Success:", success)
-    except Exception as err:
-        print("[DEBUG]", "Exception running export tool"+ str(err))
-        #if process_name in process_names and success == False:
-        #    print("[DEBUG]", "Process:", process_name, "Success:", success)
-        #    print("[DEBUG]", "Exiting program.")
-        #    process.kill()
-        #    success = False
-        #    sys.exit(1)
-        #else:
-        #    print("[FAILED]", input_params)
 
 # [AA 2021.12.17] Methods needed to replace the matching keys
 def replacedata(p, maplist):
@@ -108,16 +102,11 @@ def replacedata(p, maplist):
         template.seek(0)
         template.truncate()
 
-    # [AA 2021.12.14] Replace matching {$keys}
-    #for mapping in mappings[0:2]:
-
         # [AA 2021.12.17] With the values for management_zone and msid in memory
     for key, val in maplist.items():
              print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
              data = data.replace(key, val)
 
-    # [AA 2022.01.19] Replace key value for {}
-
     # [AA 2021.12.14] Write data from memory into file
     with open(p, 'w+') as template:
         template.write(data)
@@ -198,21 +187,23 @@ def editMainTF():
 def importStates():
     os.chdir(targetFolder)
     input_params = ["terraform", "init"]
-    runProcess("Terraform init",input_params)
+    runImportProcess("Terraform init",input_params)
 
     for filedir, resourceV in myDict.items():
         for resource, valueArray in resourceV.items():
             for rObject in valueArray:
                 input_params = ["terraform", "import", "module."+getModuleTag(
                     filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
-                runProcess("Import", input_params)
+                runImportProcess("Import", input_params)
     # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
     os.chdir(cwd)
 
 def zipdir(path, ziph):# ziph is zipfile handlefor root, dirs, files in os.walk(path):for file in files:
     for root, dirs, files in os.walk(path):
-        for file in files:
+        if ".terraform" not in root:
-            ziph.write(os.path.join(root, file), 
+            for file in files:
+                if ".terraform.lock.hcl" not in file:
+                    ziph.write(os.path.join(root, file), 
                        os.path.relpath(os.path.join(root, file), 
                                        os.path.join(path, '..')))
 
@@ -237,7 +228,7 @@ if(len(sys.argv) == 2):
         setEnv(sys.argv[1], timestamp, outputFolder)
 
         # [AA, EW 2022.01.17] Download resource files
-        runProcess("Export", [export_tool, "export"] + Resources)
+        runExportProcess("Export", [export_tool, "export"] + Resources)
             
         # [AA, EW 2022.01.17] Create a dictionary to store information of resources
         createResourceDict()
@@ -252,25 +243,6 @@ if(len(sys.argv) == 2):
         # [AA, EW 2022.01.17] Import the states for each module
         importStates()
 
-        #remote = "https://atc.bmwgroup.net/bitbucket/scm/opapm/coco_apm_fullbackup.git"
-        
-        #repo = Repo.init(path="C:\\Users\\wiegee\\Documents\\Projekte\\BMW\\dev\\DynatraceFullBackUpData")
-       
-        #if len(repo.index.diff("HEAD")) != 0:
-        #    repo.git.add('.')
-        #    repo.git.commit(m="testing")
-
-        #if len(repo.remotes)==0:
-        #    repo.git.remote('add', 'origin', remote)
-        
-        #repo.git.push('-u', 'origin', 'master')
-
-        #s3 = boto3.client('s3')
-        
-        # Upload the Zip file ATA.zip within the folder2 on S3with open('ATA.zip', 'rb') as data:
-
-        #s3.upload_fileobj(data, 'first-us-east-1-bucket', 'ATA.zip')g
-
         zipf = zipfile.ZipFile(outputFolder+"/"+timestamp + "_" +env+'.zip', 'w', zipfile.ZIP_DEFLATED)
         zipdir(targetFolder, zipf)
         zipf.close()