Exporter updated.

master
SLW\ARNAUA 2022-01-19 17:28:38 +01:00
parent fe809235dc
commit c9ebe96484
3 changed files with 124 additions and 465 deletions

261
export.py
View File

@ -1,44 +1,15 @@
#!/usr/bin/env python
""" Load Dynatrace Configuration Script.
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see <http://www.gnu.org/licenses/>.
"""
__author__ = "Arnautovic Arnel"
__contact__ = "Arnel.Arnautovic@nttdata.com"
__copyright__ = "Copyright 2021, NTT DATA"
__credits__ = ["Ermis Wieger", "René Forstner"]
__date__ = "2021/11/29"
__deprecated__ = False
__license__ = "GPLv3"
__maintainer__ = "developer"
__status__ = "Prototype"
__version__ = "0.0.4"
# [AA 2021.11.29] Import modules
from logging import exception
from mimetypes import init
import os
from posixpath import split
import subprocess
import sys
import time
import shutil
from tkinter import FALSE, TRUE
import hcl
from dotenv import load_dotenv
from glob import glob
# [AA 2021.12.10] Method to set environments
def set_env(env, time, path):
def setEnv(env, time, path):
os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL"))
os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN"))
os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env)
@ -46,27 +17,34 @@ def set_env(env, time, path):
# [AA 2021.12.10] Method to call process synchronously
def run_export(data):
input_params= [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"]+data
return run_proc(input_params)
def run_proc(input_params):
def runProcess(process_name, input_params):
process_names = ["Export", "Terraform init"]
success = False
try:
inputParam = input_params#[proc, "export"] + data
process = subprocess.Popen(inputParam)
process = subprocess.Popen(input_params)
process.wait(timeout=60*10) # 10 minutes
success = True
print("[DEBUG]", "Process:", process_name, "Success:", success)
# print("[DEBUG]", "Process return code:", outs)
except subprocess.TimeoutExpired:
print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
print("[DEBUG]", "Killing process.")
print("[DEBUG]", "Killing process:", process_name)
process.kill()
success = False
return success
print("[DEBUG]", "Process:", process_name, "Success:", success)
except:
if process_name in process_names and success == False:
print("[DEBUG]", "Process:", process_name, "Success:", success)
print("[DEBUG]", "Exiting program.")
process.kill()
success = False
sys.exit(1)
else:
print("[FAILED]", input_params)
# [AA 2021.12.13] Fill dictionary
def readfile(path):
def readFile(path):
with open(path, 'r', encoding='utf8') as cfg:
# [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones
@ -77,139 +55,148 @@ def readfile(path):
val = list(obj['resource'][key].keys())[0]
return key, val
# [AA 2021.12.13] Append correct configuration path
def writefile(k, d):
with open(".\\main.tf", "a") as mf:
mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }")
# [AA, EW 2022.01.17] Load all resources and add them to a dictionary
def createResourceDict():
files = [os.path.normpath(f).replace('\\', '/')
for f in glob(targetFolder + "**/**.tf", recursive=True)]
for index, file in enumerate(files):
filedir = "./"+("./"+os.path.dirname(file)).replace(targetFolder, "")
splittedFilename = os.path.basename(file).split(".")
if len(splittedFilename) == 5:
resourceID = splittedFilename[1]
moduleName, resourceName = readFile(file)
if not filedir in myDict.keys():
myDict.setdefault(filedir, {})
if not moduleName in myDict[filedir].keys():
myDict[filedir].setdefault(moduleName, [])
resourceValue = {"resourceName": resourceName,
"resourceID": resourceID}
myDict[filedir][moduleName].append(resourceValue)
# [AA 2021.11.29] Load enviroment file
load_dotenv()
# [AA 2021.12.10] Set available resources
Resources = [
"dynatrace_custom_service",
# "dynatrace_dashboard",
# "dynatrace_management_zone",
# "dynatrace_maintenance_window",
# "dynatrace_request_attribute",
"dynatrace_alerting_profile",
# "dynatrace_notification",
# "dynatrace_autotag"
# "dynatrace_aws_credentials",
# "dynatrace_azure_credentials",
# "dynatrace_k8s_credentials",
# "dynatrace_service_anomalies",
# "dynatrace_application_anomalies",
# "dynatrace_host_anomalies",
# "dynatrace_database_anomalies",
# "dynatrace_custom_anomalies",
# "dynatrace_disk_anomalies",
# "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
# "dynatrace_service_naming",
# "dynatrace_host_naming",
# "dynatrace_processgroup_naming",
# "dynatrace_slo", # issue -> bug: whitespace issue
# "dynatrace_span_entry_point",
# "dynatrace_span_capture_rule",
# "dynatrace_span_context_propagation",
# "dynatrace_resource_attributes",
# "dynatrace_span_attribute",
# "dynatrace_mobile_application",
# "dynatrace_credentials", #issue -> bug: unknown issue? not supported?
"dynatrace_browser_monitor",
"dynatrace_http_monitor",
]
# [AA, EW 2022.01.17] Copy main.tf into the target folder
def copyMainTemplate():
shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
# [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
def copyModuleTemplate():
dirs = glob(targetFolder + "**/", recursive=True)
for index, dir in enumerate(dirs):
if index != 0:
shutil.copyfile(templatesFolder + "module.tf", dir + "module.tf")
def createResourceDict():
files = [os.path.normpath(f).replace('\\', '/') for f in glob(targetFolder + "**/**.tf", recursive=True)]
for index, file in enumerate(files):
filedir="./"+("./"+os.path.dirname(file)).replace(targetFolder,"")
splittedFilename=os.path.basename(file).split(".")
if len(splittedFilename) == 5:
resourceID=splittedFilename[1]
moduleName, resourceName = readfile(file)
if not filedir in myDict.keys():
myDict.setdefault(filedir, {})
if not moduleName in myDict[filedir].keys():
myDict[filedir].setdefault(moduleName,[])
resourceValue= {"resourceName": resourceName, "resourceID": resourceID}
myDict[filedir][moduleName].append(resourceValue)
# [AA 2021.12.13] Append correct configuration path
def writeFile(k, d):
with open(".\\main.tf", "a") as mf:
mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }")
# [AA, EW 2022.01.17] Adjust the resource module name
def getModuleTag(str):
return str.replace("./", "").replace("/", "_")
# [AA, EW 2022.01.17] Set the resource names
def editMainTF():
with open(targetFolder + "main.tf", "a") as mf:
for index, (filedir, value) in enumerate(myDict.items()):
mf.writelines("\n" + "module \"" + getModuleTag(filedir) + "\" { source = \"" + filedir + "\" }")
mf.writelines("\n" + "module \"" + getModuleTag(filedir) +
"\" { source = \"" + filedir + "\" }")
def createState():
try:
os.chdir(targetFolder)
input_params=["terraform", "init"]
run_proc(input_params)
for filedir, resourceV in myDict.items():
for resource, valueArray in resourceV.items():
for rObject in valueArray:
input_params=["terraform", "import","module."+getModuleTag(filedir)+"."+resource+"."+rObject["resourceName"],rObject["resourceID"]]
run_proc(input_params)
# terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
except:
print("Exception occured.")
finally:
os.chdir(cwd)
return None
# [AA, EW 2022.01.17] Start importing
def importStates():
os.chdir(targetFolder)
input_params = ["terraform", "init"]
runProcess("Terraform init",input_params)
# [AA 2021.11.29] Arguments passed
for filedir, resourceV in myDict.items():
for resource, valueArray in resourceV.items():
for rObject in valueArray:
input_params = ["terraform", "import", "module."+getModuleTag(
filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
runProcess("Import", input_params)
# terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
os.chdir(cwd)
# [AA 2022.01.17] Arguments passed
if(len(sys.argv) == 2):
# set init variables
# [AA 2021.11.29] Load enviroment file
load_dotenv()
# [AA 2022.01.17] Set available resources
Resources = [
"dynatrace_custom_service",
# "dynatrace_dashboard",
# "dynatrace_management_zone",
# "dynatrace_maintenance_window",
# "dynatrace_request_attribute",
"dynatrace_alerting_profile",
# "dynatrace_notification",
# "dynatrace_autotag"
# "dynatrace_aws_credentials",
# "dynatrace_azure_credentials",
# "dynatrace_k8s_credentials",
# "dynatrace_service_anomalies",
# "dynatrace_application_anomalies",
# "dynatrace_host_anomalies",
# "dynatrace_database_anomalies",
# "dynatrace_custom_anomalies",
# "dynatrace_disk_anomalies",
# "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
# "dynatrace_service_naming",
# "dynatrace_host_naming",
# "dynatrace_processgroup_naming",
# "dynatrace_slo", # issue -> bug: whitespace issue
# "dynatrace_span_entry_point",
# "dynatrace_span_capture_rule",
# "dynatrace_span_context_propagation",
# "dynatrace_resource_attributes",
# "dynatrace_span_attribute",
# "dynatrace_mobile_application",
# "dynatrace_credentials", #issue -> bug: unknown issue? not supported?
"dynatrace_browser_monitor",
"dynatrace_http_monitor",
]
# [AA, EW 2022.01.17] Set global variables
global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd
timestamp = time.strftime("%Y%m%d-%H%M%S")
# timestamp = "20220117-143724"
templatesFolder = "./templates/"
cwd = os.getcwd()
outputFolder = "./output/"
targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/"
templatesFolder = "./templates/"
myDict = {}
cwd = os.getcwd()
# set env varibales
set_env(sys.argv[1], timestamp, outputFolder)
# download resource files
success = run_export(Resources)
if(success):
print("Export success.")
else:
print("Exiting program.")
sys.exit() ##Exit Code??
# [AA, EW 2022.01.17] Set env varibales
setEnv(sys.argv[1], timestamp, outputFolder)
# copy main.tf file and add module.tf files
# [AA, EW 2022.01.17] Download resource files
runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources)
# [AA, EW 2022.01.17] Create a dictionary to store information of resources
createResourceDict()
# [AA, EW 2022.01.17] Copy main.tf file and add module.tf files
copyMainTemplate()
copyModuleTemplate()
# [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file
editMainTF()
# execute import statements
createState()
# [AA, EW 2022.01.17] Import the states for each module
importStates()
print("Finished!")
else:
print("Usage example: ")
print("python .\exportConfig.py EMEA_PROD")
print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD")
print("python .\exportConfig.py EMEA_PROD ")

View File

@ -1,285 +0,0 @@
#!/usr/bin/env python
""" Load Dynatrace Configuration Script.
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see <http://www.gnu.org/licenses/>.
"""
__author__ = "Arnautovic Arnel"
__contact__ = "Arnel.Arnautovic@nttdata.com"
__copyright__ = "Copyright 2021, NTT DATA"
__credits__ = ["Ermis Wieger", "René Forstner"]
__date__ = "2021/11/29"
__deprecated__ = False
__license__ = "GPLv3"
__maintainer__ = "developer"
__status__ = "Prototype"
__version__ = "0.0.4"
# [AA 2021.11.29] Import modules
from logging import exception
import os
import subprocess
import sys
import time
import shutil
import hcl
from dotenv import load_dotenv
# [AA 2021.12.10] Method to set environments
def set_env(env, time, path):
os.environ['DYNATRACE_ENV_URL'] = str(os.getenv(env + "_ENV_URL"))
os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv(env + "_API_TOKEN"))
os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env)
return os.environ
# [AA 2021.12.10] Method to call process synchronously
def run_proc(proc, data):
try:
inputParam = [proc, "export"] + data
process = subprocess.Popen(inputParam)
outs = process.wait(timeout=10*60)
print("[DEBUG]", "Process return code:", outs)
except subprocess.TimeoutExpired:
print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
print("[DEBUG]", "Killing process.")
process.kill()
# [AA 2021.12.10] Simple copy methd
def copy(src, dst):
shutil.copyfile(".\\templates\\" + src, dst)
# [AA, EW 2021.12.13] Check if the path exists and decide between folder or file
def checkdir(path):
isFolder = False
isDeleted = False
# [AA 2021.12.13] If the path exists and the associated path is a folder
if os.path.exists(path) and os.path.isdir(path):
isFolder = True
# [AA 2021.12.13] If the folder exists, but is empty, delete folder
if not os.listdir(path):
isDeleted = True
# [AA 2021.12.13] Delete Folder if empty
os.rmdir(path)
return isFolder, isDeleted
# [AA 2021.12.13] Fill dictionary
def readfile(path):
with open(path, 'r', encoding='utf8') as cfg:
# [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones
obj = hcl.load(cfg)
# [AA, EW 2021.12.01] Store resource type and resource name of that file into a dictionary
key = list(obj['resource'].keys())[0]
val = list(obj['resource'][key].keys())[0]
return key, val
# [AA 2021.12.13] Add key, value to dictionary
def addentry(key, val, myDict):
myDict.setdefault(key, [])
if(val) not in myDict.get(key):
myDict[key].append(val)
return myDict
# [AA 2021.12.13] Append correct configuration path
def writefile(k, d):
with open(".\\main.tf", "a") as mf:
mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }")
# [AA 2021.12.10] Set timestamp once when script is executed
timestamp = time.strftime("%Y%m%d-%H%M%S")
#timestamp = time.strftime("")
# [AA 2021.11.29] Load enviroment file
load_dotenv()
# [AA 2021.12.10] Set environments/tenants
Environments = [
"EMEA_PROD"
# "EMEA_PREPROD",
# "NA_PROD",
# "NA_PREPROD",
# "CN_PROD",
# "CN_PREPROD"
]
# [AA 2021.12.10] Set available resources
Resources = [
"dynatrace_custom_service",
# "dynatrace_dashboard",
# "dynatrace_management_zone",
# "dynatrace_maintenance_window",
# "dynatrace_request_attribute",
# "dynatrace_alerting_profile",
# "dynatrace_notification",
# "dynatrace_autotag"
"dynatrace_aws_credentials",
"dynatrace_azure_credentials",
"dynatrace_k8s_credentials",
"dynatrace_service_anomalies",
"dynatrace_application_anomalies",
"dynatrace_host_anomalies",
"dynatrace_database_anomalies",
"dynatrace_custom_anomalies",
"dynatrace_disk_anomalies",
# "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
"dynatrace_service_naming",
"dynatrace_host_naming",
"dynatrace_processgroup_naming",
# "dynatrace_slo", #issue -> bug: whitespace issue
# "dynatrace_span_entry_point",
# "dynatrace_span_capture_rule",
# "dynatrace_span_context_propagation",
# "dynatrace_resource_attributes",
# "dynatrace_span_attribute",
# "dynatrace_mobile_application",
# "dynatrace_credentials", #issue -> bug: unknown issue? not supported?
# "dynatrace_browser_monitor",
# "dynatrace_http_monitor",
]
# [AA 2021.11.29] Arguments passed
if(len(sys.argv) == 1):
# [AA 2021.12.10] Loop through all four environments and two tenants
for e in Environments:
# [AA 2021.12.10] Set environment variables
env = set_env(e, timestamp, "./output/")
# [AA 2021.12.10] Run the process synchronously!
run_proc(".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", Resources)
# [AA 2021.12.01] Create main.tf file if it does not exist, otherwise overwrite
copy("main.tf", ".\\main.tf")
# [AA, EW 2021.11.30] Copy configuration into each configration folder
# copy("configuration.tf", env['DYNATRACE_TARGET_FOLDER'] + "\\configuration.tf")
# [AA 2021.01.14] Set initial values
isDeleted = False
iisDeleted = False
iiisDeleted = False
# [AA, EW 2021.11.30] Iterate trough each folder in targetfolder, f could mean file OR folder
for envdirname in os.listdir(env['DYNATRACE_TARGET_FOLDER']):
path = os.path.join(env['DYNATRACE_TARGET_FOLDER'], envdirname)
isFolder, isDeleted = checkdir(path)
# [AA 2021.12.13] If the associated path is a folder, otherwise its the configuration.tf file
if isFolder is True and isDeleted is False:
# [AA 2021.12.09] Create empty dictinary to store resource type and resource name
myDict = {}
# [AA 2021.12.01] Loop through folders, eg: management_zones, autotags
# [AA 2021.12.13] Check if there is another subdirectory or a file
for dirname in os.listdir(path):
ppath = os.path.join(path, dirname)
iisFolder, iisDeleted = checkdir(ppath)
# [AA 2021.12.14] If there is a "sub"-directory
if iisFolder is True and iisDeleted is False:
# [AA 2021.12.13] Loop through that particular directory, where the files are!
for ddirname in os.listdir(ppath):
# [AA 2021.12.14] But check if there is another "sub"-"sub"-directory
pppath = os.path.join(ppath, ddirname)
iiisFolder, iiisDeleted = checkdir(pppath)
# [AA 2021.12.14] If there is a "sub"-"sub"-directory
if iiisFolder is True and iiisDeleted is False:
for file in os.listdir(pppath):
filepath = os.path.join(pppath, file)
# [AA 2022.01.13] Try to read file otherwise catch exception
try:
key, val = readfile(filepath)
myDict = addentry(key, val, myDict)
except UnicodeDecodeError:
print("File:", filepath)
# [AA 2021.12.13] Set resource name
resource = pppath.replace(os.sep, '/')
# [AA 2021.12.14] If there is not a "sub"-"sub"-directory
elif iiisFolder is False and iiisDeleted is False:
filepath = pppath
try:
key, val = readfile(filepath)
myDict = addentry(key, val, myDict)
except UnicodeDecodeError:
print("File:", filepath)
# [AA 2021.12.13] Set resource name
resource = ppath.replace(os.sep, '/')
# [AA 2021.12.14] Manual exception will be thrown
elif iiisFolder is True and iiisDeleted is True:
print("[DEBUG]", "Deleting and skipping empty folder %s" % (ppath if pppath is None else pppath))
# [AA 2021.12.14] If there is not a "sub"-directory
elif iisFolder is False and iisDeleted is False:
filepath = ppath
try:
key, val = readfile(filepath)
myDict = addentry(key, val, myDict)
except UnicodeDecodeError:
print("File:", filepath)
# [AA 2021.12.13] Set resource name
resource = path.replace(os.sep, '/')
# [AA 2021.12.14] Manual exception will be thrown
elif iisFolder is True and iisDeleted is True:
print("[DEBUG]", "Empty folder deleted %s" % (path if ppath is None else ppath))
if isDeleted is False and iisDeleted is False and iiisDeleted is False:
# [AA, EW 2021.11.30] Copy module
copy("module.tf", resource + "\\module.tf")
# [AA 2021.12.10] Print the amount of resources exported from Dynatrace
# [AA 2021.12.14] Multiple keys because they are stored withing the same subfolder
if len(myDict.keys()) > 1:
total = 0
print("[DICTIONARY]")
for i, (key, value) in enumerate(myDict.items()):
print(" ", "resource", key, "amount", "0" if value is None else len(value))
total += len(value)
print(" ", "total resources", i+1, "total files", total)
sys.stdout.write("\n")
else:
print("[DICTIONARY]", "resource", ", ".join(myDict.keys()), "total files", "0" if myDict.get(key) is None else len(myDict.get(key)), "\n")
# [AA 2021.12.13] Append the necessary modules to the main.tf file
writefile(key, resource)
# print("[DEBUG]", "Set resource:", resource)
# [AA 2021.12.14] Manual exception will be thrown
elif isFolder is True and isDeleted is True:
print("[DEBUG]", "Empty folder deleted %s" % path)
print("[DEBUG]", "Finished.")
else:
print("Usage example: ")
print("python .\exportConfig.py")

View File

@ -1,43 +0,0 @@
#!/usr/bin/env python
""" Import Dynatrace Configuration Script.
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, either version 3 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see <http://www.gnu.org/licenses/>.
"""
__author__ = "Arnautovic Arnel"
__contact__ = "Arnel.Arnautovic@nttdata.com"
__copyright__ = "Copyright 2021, NTT DATA"
__credits__ = ["Ermis Wieger", "René Forstner"]
__date__ = "2021/11/29"
__deprecated__ = False
__license__ = "GPLv3"
__maintainer__ = "developer"
__status__ = "Prototype"
__version__ = "0.0.1"
# [AA 2021.11.30] Import modules
import os
import subprocess
import sys
import time
from dotenv import load_dotenv
if(len(sys.argv) == 1):
# Get the folder
# folderName = sys.argv[1]
# create main.tf
# replace
subprocess.run(["terraform", "init"])
subprocess.run(["terraform", "apply"])
else:
print("Usage example: ")
print("python .\importConfig.py")