import os import subprocess import sys import time import shutil import hcl from dotenv import load_dotenv from glob import glob # [AA 2022.01.17] Set available resources if os.name == 'nt': export_tool = ".\\bin\\terraform-provider-dynatrace_v1.9.1.exe" else: export_tool = "./bin/terraform-provider-dynatrace_v1.9.1" Resources = os.getenv("TERRAFORM_RESOURCES").split(",") if os.getenv("TERRAFORM_RESOURCES") else [ "dynatrace_custom_service", "dynatrace_dashboard", "dynatrace_management_zone", "dynatrace_maintenance_window", "dynatrace_request_attribute", "dynatrace_alerting_profile", "dynatrace_notification", "dynatrace_autotag" "dynatrace_aws_credentials", "dynatrace_azure_credentials", "dynatrace_k8s_credentials", "dynatrace_service_anomalies", "dynatrace_application_anomalies", "dynatrace_host_anomalies", "dynatrace_database_anomalies", "dynatrace_custom_anomalies", "dynatrace_disk_anomalies", # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit "dynatrace_service_naming", "dynatrace_host_naming", "dynatrace_processgroup_naming", #"dynatrace_slo", # issue -> bug: whitespace issue "dynatrace_span_entry_point", "dynatrace_span_capture_rule", "dynatrace_span_context_propagation", "dynatrace_resource_attributes", "dynatrace_span_attribute", "dynatrace_mobile_application", # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? "dynatrace_browser_monitor", "dynatrace_http_monitor", ] # [AA 2021.12.10] Method to set environments def setEnv(env, time, path): if not (os.getenv("TF_VAR_"+ env + "_ENV_URL") ): raise Exception ("Environment variable missing: TF_VAR_"+ env + "_ENV_URL") if not ( os.getenv("TF_VAR_"+ env + "_API_TOKEN")): raise Exception ("Environment variable missing: TF_VAR_"+env + "_API_TOKEN") os.environ['DYNATRACE_ENV_URL'] = str(os.getenv("TF_VAR_"+env + "_ENV_URL")) os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv("TF_VAR_"+env + "_API_TOKEN")) os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env) return os.environ # [AA 2021.12.10] Method to call process synchronously def runProcess(process_name, input_params): process_names = ["Export", "Terraform init"] success = True #process = subprocess.Popen(input_params) try: process = subprocess.Popen(input_params) process.wait(timeout=60*60) # 10 minutes success = True print("[DEBUG]", "Process:", process_name, "Success:", success) # print("[DEBUG]", "Process return code:", outs) except subprocess.TimeoutExpired: print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired) print("[DEBUG]", "Killing process:", process_name) process.kill() success = False print("[DEBUG]", "Process:", process_name, "Success:", success) except Exception as err: print("[DEBUG]", "Exception running export tool"+ str(err)) #if process_name in process_names and success == False: # print("[DEBUG]", "Process:", process_name, "Success:", success) # print("[DEBUG]", "Exiting program.") # process.kill() # success = False # sys.exit(1) #else: # print("[FAILED]", input_params) # [AA 2021.12.17] Methods needed to replace the matching keys def replacedata(p, maplist): # [AA 2021.12.14] Open each template to read with open(p, 'r+') as template: print("[DEBUG]", "Opening file at %s." % p) data = template.read() template.seek(0) template.truncate() # [AA 2021.12.14] Replace matching {$keys} #for mapping in mappings[0:2]: # [AA 2021.12.17] With the values for management_zone and msid in memory for key, val in maplist.items(): print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) data = data.replace(key, val) # # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod # if os.path.basename(p) in specificfiles[0:2]: # for key, val in mappings[2][maplist].items(): # print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) # data = data.replace(key, val) # # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub # if os.path.basename(p) in specificfiles[2]: # for mapping in mappings[3:5]: # for key, val in mapping.items(): # print("[DEBUG]", "Replacing key values %s at %s." % (key, p)) # data = data.replace(key, val[pos]) # [AA 2022.01.19] Replace key value for {} # [AA 2021.12.14] Write data from memory into file with open(p, 'w+') as template: template.write(data) # [AA 2021.12.13] Fill dictionary def readFile(path): with open(path, 'r', encoding='utf8') as cfg: # [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones obj = hcl.load(cfg) # [AA, EW 2021.12.01] Store resource type and resource name of that file into a dictionary key = list(obj['resource'].keys())[0] val = list(obj['resource'][key].keys())[0] return key, val # [AA, EW 2022.01.17] Load all resources and add them to a dictionary def createResourceDict(): files = [os.path.normpath(f).replace('\\', '/') for f in glob(targetFolder + "**/**.tf", recursive=True)] for index, file in enumerate(files): filedir = "./"+("./"+os.path.dirname(file)).replace(targetFolder, "") splittedFilename = os.path.basename(file).split(".") if len(splittedFilename) == 5: resourceID = splittedFilename[1] moduleName, resourceName = readFile(file) if not filedir in myDict.keys(): myDict.setdefault(filedir, {}) if not moduleName in myDict[filedir].keys(): myDict[filedir].setdefault(moduleName, []) resourceValue = {"resourceName": resourceName, "resourceID": resourceID} myDict[filedir][moduleName].append(resourceValue) # [AA, EW 2022.01.17] Copy main.tf into the target folder def copyMainTemplate(): shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf") replacedata(targetFolder + "main.tf", {"{$env}":env, "{$timestamp}":timestamp}) # [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is def copyModuleTemplate(): dirs = glob(targetFolder + "**/", recursive=True) for index, dir in enumerate(dirs): if index != 0: shutil.copyfile(templatesFolder + "module.tf", dir + "module.tf") # [AA 2021.12.13] Append correct configuration path def writeFile(k, d): with open(".\\main.tf", "a") as mf: mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }") # [AA, EW 2022.01.17] Adjust the resource module name def getModuleTag(str): return str.replace("./", "").replace("/", "_") # [AA, EW 2022.01.17] Set the resource names def editMainTF(): with open(targetFolder + "main.tf", "a") as mf: for index, (filedir, value) in enumerate(myDict.items()): mf.writelines("\n" + "module \"" + getModuleTag(filedir) + "\" { source = \"" + filedir + "\" }") # [AA, EW 2022.01.17] Start importing def importStates(): os.chdir(targetFolder) input_params = ["terraform", "init"] runProcess("Terraform init",input_params) for filedir, resourceV in myDict.items(): for resource, valueArray in resourceV.items(): for rObject in valueArray: input_params = ["terraform", "import", "module."+getModuleTag( filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]] runProcess("Import", input_params) # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8 os.chdir(cwd) # [AA 2022.01.17] Arguments passed if(len(sys.argv) == 2): try: # [AA 2021.11.29] Load enviroment file load_dotenv() # [AA, EW 2022.01.17] Set global variables global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd, env env=sys.argv[1] timestamp = time.strftime("%Y%m%d-%H%M%S") cwd = os.getcwd() outputFolder = "./output/" targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/" templatesFolder = "./templates/" myDict = {} # [AA, EW 2022.01.17] Set env varibales setEnv(sys.argv[1], timestamp, outputFolder) # [AA, EW 2022.01.17] Download resource files runProcess("Export", [export_tool, "export"] + Resources) # [AA, EW 2022.01.17] Create a dictionary to store information of resources createResourceDict() # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files copyMainTemplate() copyModuleTemplate() # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file editMainTF() # [AA, EW 2022.01.17] Import the states for each module importStates() print("Finished!") sys.exit(0) except Exception as err: print("Exception occured: "+ str(err)) sys.exit(1) else: print("Usage example: ") print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD, etc.") print("python .\export.py [Emvironment] ") sys.exit(1)