From 5b9784368712fb52b5411a7924b67569d40c051f Mon Sep 17 00:00:00 2001 From: ermisw Date: Wed, 26 Jan 2022 14:36:52 +0100 Subject: [PATCH] prepared for jenkins pipeline & some fixes --- .vscode/launch.json | 2 +- export.py | 181 ++++++++++++++++++++++++++++---------------- templates/main.tf | 22 +++++- 3 files changed, 137 insertions(+), 68 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8e33326..f8415c0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "program": "${file}", "console": "integratedTerminal", "args": [ - "EMEA_PROD" + "TERRAFORM" ], } ] diff --git a/export.py b/export.py index 7c1a8bc..08c22cf 100644 --- a/export.py +++ b/export.py @@ -8,10 +8,53 @@ from dotenv import load_dotenv from glob import glob + # [AA 2022.01.17] Set available resources +Resources = [ + #"dynatrace_custom_service", + "dynatrace_dashboard", + "dynatrace_management_zone", + # "dynatrace_maintenance_window", + # "dynatrace_request_attribute", + #"dynatrace_alerting_profile", + # "dynatrace_notification", + "dynatrace_autotag" + # "dynatrace_aws_credentials", + # "dynatrace_azure_credentials", + # "dynatrace_k8s_credentials", + # "dynatrace_service_anomalies", + # "dynatrace_application_anomalies", + # "dynatrace_host_anomalies", + # "dynatrace_database_anomalies", + # "dynatrace_custom_anomalies", + # "dynatrace_disk_anomalies", + # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit + # "dynatrace_service_naming", + # "dynatrace_host_naming", + # "dynatrace_processgroup_naming", + # "dynatrace_slo", # issue -> bug: whitespace issue + # "dynatrace_span_entry_point", + # "dynatrace_span_capture_rule", + # "dynatrace_span_context_propagation", + # "dynatrace_resource_attributes", + # "dynatrace_span_attribute", + # "dynatrace_mobile_application", + # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? + #"dynatrace_browser_monitor", + #"dynatrace_http_monitor", + ] + + + # [AA 2021.12.10] Method to set environments def setEnv(env, time, path): - os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL")) - os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN")) + if not (os.getenv("TF_VAR_"+ env + "_ENV_URL") ): + raise Exception ("Environment variable missing: TF_VAR_"+ env + "_ENV_URL") + + if not ( os.getenv("TF_VAR_"+ env + "_API_TOKEN")): + raise Exception ("Environment variable missing: TF_VAR_"+env + "_API_TOKEN") + + os.environ['DYNATRACE_ENV_URL'] = str(os.getenv("TF_VAR_"+env + "_ENV_URL")) + os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv("TF_VAR_"+env + "_API_TOKEN")) os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env) return os.environ @@ -22,7 +65,7 @@ def runProcess(process_name, input_params): success = False try: process = subprocess.Popen(input_params) - process.wait(timeout=60*10) # 10 minutes + process.wait(timeout=60*60) # 10 minutes success = True print("[DEBUG]", "Process:", process_name, "Success:", success) # print("[DEBUG]", "Process return code:", outs) @@ -42,6 +85,42 @@ def runProcess(process_name, input_params): else: print("[FAILED]", input_params) +# [AA 2021.12.17] Methods needed to replace the matching keys +def replacedata(p, maplist): + + # [AA 2021.12.14] Open each template to read + with open(p, 'r+') as template: + print("[DEBUG]", "Opening file at %s." % p) + data = template.read() + template.seek(0) + template.truncate() + + # [AA 2021.12.14] Replace matching {$keys} + #for mapping in mappings[0:2]: + + # [AA 2021.12.17] With the values for management_zone and msid in memory + for key, val in maplist.items(): + print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) + data = data.replace(key, val) + + # # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod + # if os.path.basename(p) in specificfiles[0:2]: + # for key, val in mappings[2][maplist].items(): + # print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) + # data = data.replace(key, val) + + # # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub + # if os.path.basename(p) in specificfiles[2]: + # for mapping in mappings[3:5]: + # for key, val in mapping.items(): + # print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) + # data = data.replace(key, val[pos]) + + # [AA 2022.01.19] Replace key value for {} + + # [AA 2021.12.14] Write data from memory into file + with open(p, 'w+') as template: + template.write(data) # [AA 2021.12.13] Fill dictionary def readFile(path): @@ -82,6 +161,7 @@ def createResourceDict(): # [AA, EW 2022.01.17] Copy main.tf into the target folder def copyMainTemplate(): shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf") + replacedata(targetFolder + "main.tf", {"{$env}":env, "{$timestamp}":timestamp}) # [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is @@ -130,73 +210,46 @@ def importStates(): # [AA 2022.01.17] Arguments passed if(len(sys.argv) == 2): - # [AA 2021.11.29] Load enviroment file - load_dotenv() + try: + # [AA 2021.11.29] Load enviroment file + load_dotenv() - # [AA 2022.01.17] Set available resources - Resources = [ - "dynatrace_custom_service", - # "dynatrace_dashboard", - # "dynatrace_management_zone", - # "dynatrace_maintenance_window", - # "dynatrace_request_attribute", - "dynatrace_alerting_profile", - # "dynatrace_notification", - # "dynatrace_autotag" - # "dynatrace_aws_credentials", - # "dynatrace_azure_credentials", - # "dynatrace_k8s_credentials", - # "dynatrace_service_anomalies", - # "dynatrace_application_anomalies", - # "dynatrace_host_anomalies", - # "dynatrace_database_anomalies", - # "dynatrace_custom_anomalies", - # "dynatrace_disk_anomalies", - # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit - # "dynatrace_service_naming", - # "dynatrace_host_naming", - # "dynatrace_processgroup_naming", - # "dynatrace_slo", # issue -> bug: whitespace issue - # "dynatrace_span_entry_point", - # "dynatrace_span_capture_rule", - # "dynatrace_span_context_propagation", - # "dynatrace_resource_attributes", - # "dynatrace_span_attribute", - # "dynatrace_mobile_application", - # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? - "dynatrace_browser_monitor", - "dynatrace_http_monitor", - ] + # [AA, EW 2022.01.17] Set global variables + global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd, env + env=sys.argv[1] + timestamp = time.strftime("%Y%m%d-%H%M%S") + cwd = os.getcwd() + outputFolder = "./output/" + targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/" + templatesFolder = "./templates/" + myDict = {} - # [AA, EW 2022.01.17] Set global variables - global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd - timestamp = time.strftime("%Y%m%d-%H%M%S") - cwd = os.getcwd() - outputFolder = "./output/" - targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/" - templatesFolder = "./templates/" - myDict = {} + # [AA, EW 2022.01.17] Set env varibales + setEnv(sys.argv[1], timestamp, outputFolder) - # [AA, EW 2022.01.17] Set env varibales - setEnv(sys.argv[1], timestamp, outputFolder) + # [AA, EW 2022.01.17] Download resource files + runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources) + + # [AA, EW 2022.01.17] Create a dictionary to store information of resources + createResourceDict() - # [AA, EW 2022.01.17] Download resource files - runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources) - - # [AA, EW 2022.01.17] Create a dictionary to store information of resources - createResourceDict() + # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files + copyMainTemplate() + copyModuleTemplate() - # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files - copyMainTemplate() - copyModuleTemplate() + # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file + editMainTF() - # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file - editMainTF() + # [AA, EW 2022.01.17] Import the states for each module + importStates() + print("Finished!") + sys.exit(0) + except Exception as err: + print("Exception occured: "+ str(err)) + sys.exit(1) - # [AA, EW 2022.01.17] Import the states for each module - importStates() - print("Finished!") else: print("Usage example: ") - print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD") - print("python .\exportConfig.py EMEA_PROD ") + print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD, etc.") + print("python .\export.py [Emvironment] ") + sys.exit(1) diff --git a/templates/main.tf b/templates/main.tf index da9b57c..e6504e9 100644 --- a/templates/main.tf +++ b/templates/main.tf @@ -5,8 +5,24 @@ terraform { source = "dynatrace-oss/dynatrace" } } + + backend "s3" { + bucket = "coco-dynatrace-tfstate" + key = "backup/{$env}/{$timestamp}/terraform.tfstate" + region = "eu-central-1" + dynamodb_table = "coco-dynatrace-tfstate" + encrypt = true + } + } -# module "configDir" { -# source = "git::https://github.com/arnauagithub/DynatraceTerraformConfiguration.git?ref=20211130-151123" -# } +variable {$env}_ENV_URL {} +variable {$env}_API_TOKEN {} + +provider "dynatrace" { + dt_env_url = "${var.{$env}_ENV_URL}" + dt_api_token = "${var.{$env}_API_TOKEN}" +} + + +