prepared for jenkins pipeline & some fixes

.vscode/launch.json

@@ -11,7 +11,7 @@
             "program": "${file}",
             "console": "integratedTerminal",
             "args": [
-                "EMEA_PROD"
+                "TERRAFORM"
             ],
         }
     ]

export.py

@@ -8,10 +8,53 @@ from dotenv import load_dotenv
 from glob import glob
 
 
+    # [AA 2022.01.17] Set available resources
+Resources = [
+        #"dynatrace_custom_service",
+         "dynatrace_dashboard",
+         "dynatrace_management_zone",
+        # "dynatrace_maintenance_window",
+        # "dynatrace_request_attribute",
+        #"dynatrace_alerting_profile",
+        # "dynatrace_notification",
+         "dynatrace_autotag"
+        # "dynatrace_aws_credentials",
+        # "dynatrace_azure_credentials",
+        # "dynatrace_k8s_credentials",
+        # "dynatrace_service_anomalies",
+        # "dynatrace_application_anomalies",
+        # "dynatrace_host_anomalies",
+        # "dynatrace_database_anomalies",
+        # "dynatrace_custom_anomalies",
+        # "dynatrace_disk_anomalies",
+        # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
+        # "dynatrace_service_naming",
+        # "dynatrace_host_naming",
+        # "dynatrace_processgroup_naming",
+        # "dynatrace_slo",                       # issue -> bug: whitespace issue
+        # "dynatrace_span_entry_point",
+        # "dynatrace_span_capture_rule",
+        # "dynatrace_span_context_propagation",
+        # "dynatrace_resource_attributes",
+        # "dynatrace_span_attribute",
+        # "dynatrace_mobile_application",
+        # "dynatrace_credentials",               #issue -> bug: unknown issue? not supported?
+        #"dynatrace_browser_monitor",
+        #"dynatrace_http_monitor",
+    ]
+
+
+
 # [AA 2021.12.10] Method to set environments
 def setEnv(env, time, path):
-    os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL"))
+    if not (os.getenv("TF_VAR_"+ env + "_ENV_URL") ):
-    os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN"))
+        raise Exception ("Environment variable missing: TF_VAR_"+ env + "_ENV_URL")
+
+    if not ( os.getenv("TF_VAR_"+ env + "_API_TOKEN")):
+        raise Exception ("Environment variable missing: TF_VAR_"+env + "_API_TOKEN")
+
+    os.environ['DYNATRACE_ENV_URL'] = str(os.getenv("TF_VAR_"+env + "_ENV_URL"))
+    os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv("TF_VAR_"+env + "_API_TOKEN"))
     os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env)
     return os.environ
 
@@ -22,7 +65,7 @@ def runProcess(process_name, input_params):
     success = False
     try:
         process = subprocess.Popen(input_params)
-        process.wait(timeout=60*10)  # 10 minutes
+        process.wait(timeout=60*60)  # 10 minutes
         success = True
         print("[DEBUG]", "Process:", process_name, "Success:", success)
         # print("[DEBUG]", "Process return code:", outs)
@@ -42,6 +85,42 @@ def runProcess(process_name, input_params):
         else:
             print("[FAILED]", input_params)
 
+# [AA 2021.12.17] Methods needed to replace the matching keys
+def replacedata(p, maplist):
+
+    # [AA 2021.12.14] Open each template to read
+    with open(p, 'r+') as template:
+        print("[DEBUG]", "Opening file at %s." % p)
+        data = template.read()
+        template.seek(0)
+        template.truncate()
+
+    # [AA 2021.12.14] Replace matching {$keys}
+    #for mapping in mappings[0:2]:
+
+        # [AA 2021.12.17] With the values for management_zone and msid in memory
+    for key, val in maplist.items():
+             print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
+             data = data.replace(key, val)
+
+    # # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod
+    # if os.path.basename(p) in specificfiles[0:2]:
+    #     for key, val in mappings[2][maplist].items():
+    #         print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
+    #         data = data.replace(key, val)
+
+    # # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub
+    # if os.path.basename(p) in specificfiles[2]:
+    #     for mapping in mappings[3:5]:
+    #         for key, val in mapping.items():
+    #             print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
+    #             data = data.replace(key, val[pos])
+
+    # [AA 2022.01.19] Replace key value for {}
+
+    # [AA 2021.12.14] Write data from memory into file
+    with open(p, 'w+') as template:
+        template.write(data)
 
 # [AA 2021.12.13] Fill dictionary
 def readFile(path):
@@ -82,6 +161,7 @@ def createResourceDict():
 # [AA, EW 2022.01.17] Copy main.tf into the target folder
 def copyMainTemplate():
     shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
+    replacedata(targetFolder + "main.tf",  {"{$env}":env, "{$timestamp}":timestamp})
 
 
 # [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
@@ -130,73 +210,46 @@ def importStates():
 # [AA 2022.01.17] Arguments passed
 if(len(sys.argv) == 2):
 
-    # [AA 2021.11.29] Load enviroment file
+    try:
-    load_dotenv()
+        # [AA 2021.11.29] Load enviroment file
+        load_dotenv()
 
-    # [AA 2022.01.17] Set available resources
+        # [AA, EW 2022.01.17] Set global variables
-    Resources = [
+        global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd, env
-        "dynatrace_custom_service",
+        env=sys.argv[1]
-        # "dynatrace_dashboard",
+        timestamp = time.strftime("%Y%m%d-%H%M%S")
-        # "dynatrace_management_zone",
+        cwd = os.getcwd()
-        # "dynatrace_maintenance_window",
+        outputFolder = "./output/"
-        # "dynatrace_request_attribute",
+        targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/"
-        "dynatrace_alerting_profile",
+        templatesFolder = "./templates/"
-        # "dynatrace_notification",
+        myDict = {}
-        # "dynatrace_autotag"
-        # "dynatrace_aws_credentials",
-        # "dynatrace_azure_credentials",
-        # "dynatrace_k8s_credentials",
-        # "dynatrace_service_anomalies",
-        # "dynatrace_application_anomalies",
-        # "dynatrace_host_anomalies",
-        # "dynatrace_database_anomalies",
-        # "dynatrace_custom_anomalies",
-        # "dynatrace_disk_anomalies",
-        # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
-        # "dynatrace_service_naming",
-        # "dynatrace_host_naming",
-        # "dynatrace_processgroup_naming",
-        # "dynatrace_slo",                       # issue -> bug: whitespace issue
-        # "dynatrace_span_entry_point",
-        # "dynatrace_span_capture_rule",
-        # "dynatrace_span_context_propagation",
-        # "dynatrace_resource_attributes",
-        # "dynatrace_span_attribute",
-        # "dynatrace_mobile_application",
-        # "dynatrace_credentials",               #issue -> bug: unknown issue? not supported?
-        "dynatrace_browser_monitor",
-        "dynatrace_http_monitor",
-    ]
 
-    # [AA, EW 2022.01.17] Set global variables
+        # [AA, EW 2022.01.17] Set env varibales
-    global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd
+        setEnv(sys.argv[1], timestamp, outputFolder)
-    timestamp = time.strftime("%Y%m%d-%H%M%S")
-    cwd = os.getcwd()
-    outputFolder = "./output/"
-    targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/"
-    templatesFolder = "./templates/"
-    myDict = {}
 
-    # [AA, EW 2022.01.17] Set env varibales
+        # [AA, EW 2022.01.17] Download resource files
-    setEnv(sys.argv[1], timestamp, outputFolder)
+        runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources)
             
-    # [AA, EW 2022.01.17] Download resource files
+        # [AA, EW 2022.01.17] Create a dictionary to store information of resources
-    runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources)
+        createResourceDict()
 
-    # [AA, EW 2022.01.17] Create a dictionary to store information of resources
+        # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files
-    createResourceDict()
+        copyMainTemplate()
+        copyModuleTemplate()
 
-    # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files
+        # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file
-    copyMainTemplate()
+        editMainTF()
-    copyModuleTemplate()
 
-    # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file
+        # [AA, EW 2022.01.17] Import the states for each module
-    editMainTF()
+        importStates()
+        print("Finished!")
+        sys.exit(0)
+    except Exception as err:
+        print("Exception occured: "+ str(err))
+        sys.exit(1) 
 
-    # [AA, EW 2022.01.17] Import the states for each module
-    importStates()
-    print("Finished!")
 else:
     print("Usage example: ")
-    print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD")
+    print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD, etc.")
-    print("python .\exportConfig.py EMEA_PROD ")
+    print("python .\export.py [Emvironment] ")
+    sys.exit(1)

templates/main.tf

@@ -5,8 +5,24 @@ terraform {
       source  = "dynatrace-oss/dynatrace"
     }
   }
+
+  backend "s3" {
+    bucket = "coco-dynatrace-tfstate"
+    key = "backup/{$env}/{$timestamp}/terraform.tfstate"
+    region = "eu-central-1"
+    dynamodb_table = "coco-dynatrace-tfstate"
+    encrypt = true
+  }
+  
 }
 
-# module "configDir" {
+variable {$env}_ENV_URL {}
-#   source = "git::https://github.com/arnauagithub/DynatraceTerraformConfiguration.git?ref=20211130-151123"
+variable {$env}_API_TOKEN {}
-# }
+
+provider "dynatrace" {
+    dt_env_url    = "${var.{$env}_ENV_URL}"
+    dt_api_token  = "${var.{$env}_API_TOKEN}"
+}
+
+
+