diff --git a/export.py b/export.py index 413ab4e..7c1a8bc 100644 --- a/export.py +++ b/export.py @@ -1,44 +1,15 @@ -#!/usr/bin/env python -""" Load Dynatrace Configuration Script. -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program. If not, see . -""" - -__author__ = "Arnautovic Arnel" -__contact__ = "Arnel.Arnautovic@nttdata.com" -__copyright__ = "Copyright 2021, NTT DATA" -__credits__ = ["Ermis Wieger", "René Forstner"] -__date__ = "2021/11/29" -__deprecated__ = False -__license__ = "GPLv3" -__maintainer__ = "developer" -__status__ = "Prototype" -__version__ = "0.0.4" - -# [AA 2021.11.29] Import modules -from logging import exception -from mimetypes import init import os -from posixpath import split import subprocess import sys import time import shutil -from tkinter import FALSE, TRUE import hcl from dotenv import load_dotenv from glob import glob # [AA 2021.12.10] Method to set environments -def set_env(env, time, path): +def setEnv(env, time, path): os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL")) os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN")) os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env) @@ -46,27 +17,34 @@ def set_env(env, time, path): # [AA 2021.12.10] Method to call process synchronously -def run_export(data): - input_params= [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"]+data - return run_proc(input_params) - -def run_proc(input_params): +def runProcess(process_name, input_params): + process_names = ["Export", "Terraform init"] success = False try: - inputParam = input_params#[proc, "export"] + data - process = subprocess.Popen(inputParam) + process = subprocess.Popen(input_params) process.wait(timeout=60*10) # 10 minutes success = True + print("[DEBUG]", "Process:", process_name, "Success:", success) # print("[DEBUG]", "Process return code:", outs) except subprocess.TimeoutExpired: print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired) - print("[DEBUG]", "Killing process.") + print("[DEBUG]", "Killing process:", process_name) process.kill() success = False - return success + print("[DEBUG]", "Process:", process_name, "Success:", success) + except: + if process_name in process_names and success == False: + print("[DEBUG]", "Process:", process_name, "Success:", success) + print("[DEBUG]", "Exiting program.") + process.kill() + success = False + sys.exit(1) + else: + print("[FAILED]", input_params) + # [AA 2021.12.13] Fill dictionary -def readfile(path): +def readFile(path): with open(path, 'r', encoding='utf8') as cfg: # [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones @@ -77,139 +55,148 @@ def readfile(path): val = list(obj['resource'][key].keys())[0] return key, val -# [AA 2021.12.13] Append correct configuration path -def writefile(k, d): - with open(".\\main.tf", "a") as mf: - mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }") + +# [AA, EW 2022.01.17] Load all resources and add them to a dictionary +def createResourceDict(): + files = [os.path.normpath(f).replace('\\', '/') + for f in glob(targetFolder + "**/**.tf", recursive=True)] + for index, file in enumerate(files): + filedir = "./"+("./"+os.path.dirname(file)).replace(targetFolder, "") + splittedFilename = os.path.basename(file).split(".") + + if len(splittedFilename) == 5: + resourceID = splittedFilename[1] + moduleName, resourceName = readFile(file) + + if not filedir in myDict.keys(): + myDict.setdefault(filedir, {}) + + if not moduleName in myDict[filedir].keys(): + myDict[filedir].setdefault(moduleName, []) + + resourceValue = {"resourceName": resourceName, + "resourceID": resourceID} + myDict[filedir][moduleName].append(resourceValue) -# [AA 2021.11.29] Load enviroment file -load_dotenv() - -# [AA 2021.12.10] Set available resources -Resources = [ - "dynatrace_custom_service", - # "dynatrace_dashboard", - # "dynatrace_management_zone", - # "dynatrace_maintenance_window", - # "dynatrace_request_attribute", - "dynatrace_alerting_profile", - # "dynatrace_notification", - # "dynatrace_autotag" - # "dynatrace_aws_credentials", - # "dynatrace_azure_credentials", - # "dynatrace_k8s_credentials", - # "dynatrace_service_anomalies", - # "dynatrace_application_anomalies", - # "dynatrace_host_anomalies", - # "dynatrace_database_anomalies", - # "dynatrace_custom_anomalies", - # "dynatrace_disk_anomalies", - # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit - # "dynatrace_service_naming", - # "dynatrace_host_naming", - # "dynatrace_processgroup_naming", - # "dynatrace_slo", # issue -> bug: whitespace issue - # "dynatrace_span_entry_point", - # "dynatrace_span_capture_rule", - # "dynatrace_span_context_propagation", - # "dynatrace_resource_attributes", - # "dynatrace_span_attribute", - # "dynatrace_mobile_application", - # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? - "dynatrace_browser_monitor", - "dynatrace_http_monitor", -] - +# [AA, EW 2022.01.17] Copy main.tf into the target folder def copyMainTemplate(): shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf") +# [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is def copyModuleTemplate(): dirs = glob(targetFolder + "**/", recursive=True) for index, dir in enumerate(dirs): if index != 0: shutil.copyfile(templatesFolder + "module.tf", dir + "module.tf") -def createResourceDict(): - files = [os.path.normpath(f).replace('\\', '/') for f in glob(targetFolder + "**/**.tf", recursive=True)] - for index, file in enumerate(files): - filedir="./"+("./"+os.path.dirname(file)).replace(targetFolder,"") - splittedFilename=os.path.basename(file).split(".") - if len(splittedFilename) == 5: - resourceID=splittedFilename[1] - moduleName, resourceName = readfile(file) - - if not filedir in myDict.keys(): - myDict.setdefault(filedir, {}) - - if not moduleName in myDict[filedir].keys(): - myDict[filedir].setdefault(moduleName,[]) - - resourceValue= {"resourceName": resourceName, "resourceID": resourceID} - myDict[filedir][moduleName].append(resourceValue) +# [AA 2021.12.13] Append correct configuration path +def writeFile(k, d): + with open(".\\main.tf", "a") as mf: + mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }") +# [AA, EW 2022.01.17] Adjust the resource module name def getModuleTag(str): return str.replace("./", "").replace("/", "_") + +# [AA, EW 2022.01.17] Set the resource names def editMainTF(): with open(targetFolder + "main.tf", "a") as mf: for index, (filedir, value) in enumerate(myDict.items()): - mf.writelines("\n" + "module \"" + getModuleTag(filedir) + "\" { source = \"" + filedir + "\" }") + mf.writelines("\n" + "module \"" + getModuleTag(filedir) + + "\" { source = \"" + filedir + "\" }") -def createState(): - try: - os.chdir(targetFolder) - input_params=["terraform", "init"] - run_proc(input_params) - for filedir, resourceV in myDict.items(): - for resource, valueArray in resourceV.items(): - for rObject in valueArray: - input_params=["terraform", "import","module."+getModuleTag(filedir)+"."+resource+"."+rObject["resourceName"],rObject["resourceID"]] - run_proc(input_params) - # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8 - except: - print("Exception occured.") - finally: - os.chdir(cwd) - return None +# [AA, EW 2022.01.17] Start importing +def importStates(): + os.chdir(targetFolder) + input_params = ["terraform", "init"] + runProcess("Terraform init",input_params) -# [AA 2021.11.29] Arguments passed + for filedir, resourceV in myDict.items(): + for resource, valueArray in resourceV.items(): + for rObject in valueArray: + input_params = ["terraform", "import", "module."+getModuleTag( + filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]] + runProcess("Import", input_params) + # terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8 + os.chdir(cwd) + + +# [AA 2022.01.17] Arguments passed if(len(sys.argv) == 2): - - # set init variables + + # [AA 2021.11.29] Load enviroment file + load_dotenv() + + # [AA 2022.01.17] Set available resources + Resources = [ + "dynatrace_custom_service", + # "dynatrace_dashboard", + # "dynatrace_management_zone", + # "dynatrace_maintenance_window", + # "dynatrace_request_attribute", + "dynatrace_alerting_profile", + # "dynatrace_notification", + # "dynatrace_autotag" + # "dynatrace_aws_credentials", + # "dynatrace_azure_credentials", + # "dynatrace_k8s_credentials", + # "dynatrace_service_anomalies", + # "dynatrace_application_anomalies", + # "dynatrace_host_anomalies", + # "dynatrace_database_anomalies", + # "dynatrace_custom_anomalies", + # "dynatrace_disk_anomalies", + # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit + # "dynatrace_service_naming", + # "dynatrace_host_naming", + # "dynatrace_processgroup_naming", + # "dynatrace_slo", # issue -> bug: whitespace issue + # "dynatrace_span_entry_point", + # "dynatrace_span_capture_rule", + # "dynatrace_span_context_propagation", + # "dynatrace_resource_attributes", + # "dynatrace_span_attribute", + # "dynatrace_mobile_application", + # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? + "dynatrace_browser_monitor", + "dynatrace_http_monitor", + ] + + # [AA, EW 2022.01.17] Set global variables global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd timestamp = time.strftime("%Y%m%d-%H%M%S") - # timestamp = "20220117-143724" - templatesFolder = "./templates/" + cwd = os.getcwd() outputFolder = "./output/" targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/" + templatesFolder = "./templates/" myDict = {} - cwd = os.getcwd() - - # set env varibales - set_env(sys.argv[1], timestamp, outputFolder) - # download resource files - success = run_export(Resources) - if(success): - print("Export success.") - else: - print("Exiting program.") - sys.exit() ##Exit Code?? + # [AA, EW 2022.01.17] Set env varibales + setEnv(sys.argv[1], timestamp, outputFolder) - # copy main.tf file and add module.tf files + # [AA, EW 2022.01.17] Download resource files + runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources) + + # [AA, EW 2022.01.17] Create a dictionary to store information of resources createResourceDict() + + # [AA, EW 2022.01.17] Copy main.tf file and add module.tf files copyMainTemplate() copyModuleTemplate() + + # [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file editMainTF() - # execute import statements - createState() - + # [AA, EW 2022.01.17] Import the states for each module + importStates() + print("Finished!") else: print("Usage example: ") - print("python .\exportConfig.py EMEA_PROD") + print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD") + print("python .\exportConfig.py EMEA_PROD ") diff --git a/exportConfig.py b/exportConfig.py deleted file mode 100644 index 1f6d24b..0000000 --- a/exportConfig.py +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python -""" Load Dynatrace Configuration Script. -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program. If not, see . -""" - -__author__ = "Arnautovic Arnel" -__contact__ = "Arnel.Arnautovic@nttdata.com" -__copyright__ = "Copyright 2021, NTT DATA" -__credits__ = ["Ermis Wieger", "René Forstner"] -__date__ = "2021/11/29" -__deprecated__ = False -__license__ = "GPLv3" -__maintainer__ = "developer" -__status__ = "Prototype" -__version__ = "0.0.4" - -# [AA 2021.11.29] Import modules -from logging import exception -import os -import subprocess -import sys -import time -import shutil -import hcl -from dotenv import load_dotenv - - -# [AA 2021.12.10] Method to set environments -def set_env(env, time, path): - os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL")) - os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN")) - os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env) - return os.environ - - -# [AA 2021.12.10] Method to call process synchronously -def run_proc(proc, data): - try: - inputParam = [proc, "export"] + data - process = subprocess.Popen(inputParam) - outs = process.wait(timeout=10*60) - print("[DEBUG]", "Process return code:", outs) - except subprocess.TimeoutExpired: - print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired) - print("[DEBUG]", "Killing process.") - process.kill() - - -# [AA 2021.12.10] Simple copy methd -def copy(src, dst): - shutil.copyfile(".\\templates\\" + src, dst) - - -# [AA, EW 2021.12.13] Check if the path exists and decide between folder or file -def checkdir(path): - isFolder = False - isDeleted = False - - # [AA 2021.12.13] If the path exists and the associated path is a folder - if os.path.exists(path) and os.path.isdir(path): - isFolder = True - - # [AA 2021.12.13] If the folder exists, but is empty, delete folder - if not os.listdir(path): - isDeleted = True - - # [AA 2021.12.13] Delete Folder if empty - os.rmdir(path) - - return isFolder, isDeleted - - -# [AA 2021.12.13] Fill dictionary -def readfile(path): - with open(path, 'r', encoding='utf8') as cfg: - - # [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones - obj = hcl.load(cfg) - - # [AA, EW 2021.12.01] Store resource type and resource name of that file into a dictionary - key = list(obj['resource'].keys())[0] - val = list(obj['resource'][key].keys())[0] - return key, val - - -# [AA 2021.12.13] Add key, value to dictionary -def addentry(key, val, myDict): - myDict.setdefault(key, []) - if(val) not in myDict.get(key): - myDict[key].append(val) - return myDict - - -# [AA 2021.12.13] Append correct configuration path -def writefile(k, d): - with open(".\\main.tf", "a") as mf: - mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }") - - -# [AA 2021.12.10] Set timestamp once when script is executed -timestamp = time.strftime("%Y%m%d-%H%M%S") -#timestamp = time.strftime("") - -# [AA 2021.11.29] Load enviroment file -load_dotenv() - -# [AA 2021.12.10] Set environments/tenants -Environments = [ - "EMEA_PROD" - # "EMEA_PREPROD", - # "NA_PROD", - # "NA_PREPROD", - # "CN_PROD", - # "CN_PREPROD" -] - -# [AA 2021.12.10] Set available resources -Resources = [ - "dynatrace_custom_service", - # "dynatrace_dashboard", - # "dynatrace_management_zone", - # "dynatrace_maintenance_window", - # "dynatrace_request_attribute", - # "dynatrace_alerting_profile", - # "dynatrace_notification", - # "dynatrace_autotag" - "dynatrace_aws_credentials", - "dynatrace_azure_credentials", - "dynatrace_k8s_credentials", - "dynatrace_service_anomalies", - "dynatrace_application_anomalies", - "dynatrace_host_anomalies", - "dynatrace_database_anomalies", - "dynatrace_custom_anomalies", - "dynatrace_disk_anomalies", - # "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit - "dynatrace_service_naming", - "dynatrace_host_naming", - "dynatrace_processgroup_naming", - # "dynatrace_slo", #issue -> bug: whitespace issue - # "dynatrace_span_entry_point", - # "dynatrace_span_capture_rule", - # "dynatrace_span_context_propagation", - # "dynatrace_resource_attributes", - # "dynatrace_span_attribute", - # "dynatrace_mobile_application", - # "dynatrace_credentials", #issue -> bug: unknown issue? not supported? - # "dynatrace_browser_monitor", - # "dynatrace_http_monitor", -] - -# [AA 2021.11.29] Arguments passed -if(len(sys.argv) == 1): - - # [AA 2021.12.10] Loop through all four environments and two tenants - for e in Environments: - - # [AA 2021.12.10] Set environment variables - env = set_env(e, timestamp, "./output/") - - # [AA 2021.12.10] Run the process synchronously! - run_proc(".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", Resources) - - # [AA 2021.12.01] Create main.tf file if it does not exist, otherwise overwrite - copy("main.tf", ".\\main.tf") - - # [AA, EW 2021.11.30] Copy configuration into each configration folder - # copy("configuration.tf", env['DYNATRACE_TARGET_FOLDER'] + "\\configuration.tf") - - # [AA 2021.01.14] Set initial values - isDeleted = False - iisDeleted = False - iiisDeleted = False - - # [AA, EW 2021.11.30] Iterate trough each folder in targetfolder, f could mean file OR folder - for envdirname in os.listdir(env['DYNATRACE_TARGET_FOLDER']): - path = os.path.join(env['DYNATRACE_TARGET_FOLDER'], envdirname) - isFolder, isDeleted = checkdir(path) - - # [AA 2021.12.13] If the associated path is a folder, otherwise its the configuration.tf file - if isFolder is True and isDeleted is False: - - # [AA 2021.12.09] Create empty dictinary to store resource type and resource name - myDict = {} - - # [AA 2021.12.01] Loop through folders, eg: management_zones, autotags - # [AA 2021.12.13] Check if there is another subdirectory or a file - for dirname in os.listdir(path): - ppath = os.path.join(path, dirname) - iisFolder, iisDeleted = checkdir(ppath) - - # [AA 2021.12.14] If there is a "sub"-directory - if iisFolder is True and iisDeleted is False: - - # [AA 2021.12.13] Loop through that particular directory, where the files are! - for ddirname in os.listdir(ppath): - - # [AA 2021.12.14] But check if there is another "sub"-"sub"-directory - pppath = os.path.join(ppath, ddirname) - iiisFolder, iiisDeleted = checkdir(pppath) - - # [AA 2021.12.14] If there is a "sub"-"sub"-directory - if iiisFolder is True and iiisDeleted is False: - for file in os.listdir(pppath): - filepath = os.path.join(pppath, file) - - # [AA 2022.01.13] Try to read file otherwise catch exception - try: - key, val = readfile(filepath) - myDict = addentry(key, val, myDict) - except UnicodeDecodeError: - print("File:", filepath) - - # [AA 2021.12.13] Set resource name - resource = pppath.replace(os.sep, '/') - - # [AA 2021.12.14] If there is not a "sub"-"sub"-directory - elif iiisFolder is False and iiisDeleted is False: - filepath = pppath - try: - key, val = readfile(filepath) - myDict = addentry(key, val, myDict) - except UnicodeDecodeError: - print("File:", filepath) - - # [AA 2021.12.13] Set resource name - resource = ppath.replace(os.sep, '/') - - # [AA 2021.12.14] Manual exception will be thrown - elif iiisFolder is True and iiisDeleted is True: - print("[DEBUG]", "Deleting and skipping empty folder %s" % (ppath if pppath is None else pppath)) - - # [AA 2021.12.14] If there is not a "sub"-directory - elif iisFolder is False and iisDeleted is False: - filepath = ppath - try: - key, val = readfile(filepath) - myDict = addentry(key, val, myDict) - except UnicodeDecodeError: - print("File:", filepath) - - # [AA 2021.12.13] Set resource name - resource = path.replace(os.sep, '/') - - # [AA 2021.12.14] Manual exception will be thrown - elif iisFolder is True and iisDeleted is True: - print("[DEBUG]", "Empty folder deleted %s" % (path if ppath is None else ppath)) - - if isDeleted is False and iisDeleted is False and iiisDeleted is False: - # [AA, EW 2021.11.30] Copy module - copy("module.tf", resource + "\\module.tf") - - # [AA 2021.12.10] Print the amount of resources exported from Dynatrace - # [AA 2021.12.14] Multiple keys because they are stored withing the same subfolder - if len(myDict.keys()) > 1: - total = 0 - print("[DICTIONARY]") - for i, (key, value) in enumerate(myDict.items()): - print(" ", "resource", key, "amount", "0" if value is None else len(value)) - total += len(value) - print(" ", "total resources", i+1, "total files", total) - sys.stdout.write("\n") - else: - print("[DICTIONARY]", "resource", ", ".join(myDict.keys()), "total files", "0" if myDict.get(key) is None else len(myDict.get(key)), "\n") - - # [AA 2021.12.13] Append the necessary modules to the main.tf file - writefile(key, resource) - # print("[DEBUG]", "Set resource:", resource) - - # [AA 2021.12.14] Manual exception will be thrown - elif isFolder is True and isDeleted is True: - print("[DEBUG]", "Empty folder deleted %s" % path) - - print("[DEBUG]", "Finished.") -else: - print("Usage example: ") - print("python .\exportConfig.py") diff --git a/importStates.py b/importStates.py deleted file mode 100644 index bd8d8fe..0000000 --- a/importStates.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -""" Import Dynatrace Configuration Script. -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program. If not, see . -""" - -__author__ = "Arnautovic Arnel" -__contact__ = "Arnel.Arnautovic@nttdata.com" -__copyright__ = "Copyright 2021, NTT DATA" -__credits__ = ["Ermis Wieger", "René Forstner"] -__date__ = "2021/11/29" -__deprecated__ = False -__license__ = "GPLv3" -__maintainer__ = "developer" -__status__ = "Prototype" -__version__ = "0.0.1" - -# [AA 2021.11.30] Import modules -import os -import subprocess -import sys -import time -from dotenv import load_dotenv - -if(len(sys.argv) == 1): - # Get the folder - # folderName = sys.argv[1] - - # create main.tf - # replace - - subprocess.run(["terraform", "init"]) - subprocess.run(["terraform", "apply"]) -else: - print("Usage example: ") - print("python .\importConfig.py")