arnaua
/
coco_apm_exportdtconfig
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: arnaua:317e950357f620536339bbc810734e234f206ba4
Branches Tags
arnaua:master
arnaua:production
arnaua:staging
..
compare: arnaua:0bcea9ccee06aafe85fa4e1e48a10130a8c55529
Branches Tags
arnaua:master
arnaua:production
arnaua:staging

No commits in common. "317e950357f620536339bbc810734e234f206ba4" and "0bcea9ccee06aafe85fa4e1e48a10130a8c55529" have entirely different histories.
317e950357 ... 0bcea9ccee

6 changed files with 82 additions and 98 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,2 @@
output/
.env
.terraform.lock.hcl

20
.vscode/launch.json vendored
View File

@ -13,26 +13,6 @@
"args": [
"TERRAFORM"
],
},
{
"name": "Python: Aktuelle Datei EMEA_PROD",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"EMEA_PROD"
],
},
{
"name": "Python: Aktuelle Datei EMEA_PREPROD",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"EMEA_PREPROD"
],
}
]
}

47
Jenkinsfile vendored
View File

@ -1,13 +1,21 @@
//not required right now as CN is reachable from EMEA as well
environments=['EMEA_PROD', 'EMEA_PREPROD', 'NA_PROD', 'NA_PREPROD', 'CN_PROD', 'CN_PREPROD']
@NonCPS // has to be NonCPS or the build breaks on the call to .each
def export_config_all(list) {
list.each { env ->
sh "python3 export.py ${env}"
}
def loopEnvironments(environments){
print env.JENKINS_URL
environments.each { key, val ->
//Execute only if you are on the same environment
//not required right now as CN is reachable from EMEA as well
if (env.JENKINS_URL == environments."${key}"[3].'jenkins')
{
envname = environments."${key}"[0].'name'
envurl = environments."${key}"[1].'env-url'
tokenname = environments."${key}"[2].'env-token-name'
sh 'python createReport.py "${envname}"'
}
}
}
pipeline {
options {
@ -48,12 +56,11 @@
// NAPREPROD_TOKEN_VAR = credentials('NAPREPROD_TOKEN_VAR')
// CNPROD_TOKEN_VAR = credentials('CNPROD_TOKEN_VAR')
// CNPREPROD_TOKEN_VAR = credentials('CNPREPROD_TOKEN_VAR')
AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
AWS_S3_BUCKET="coco-dynatrace-tfstate"
AWS_S3_REGION="eu-central-1"
// AWS_ACCESS_KEY_ID = credentials('AWS_TERRAFORM_KEY')
// AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
//TERRAFORM_RESOURCES="dynatrace_management_zone"
TERRAFORM_RESOURCES="dynatrace_management_zone"
//EMEA PROD
TF_VAR_EMEA_PROD_ENV_URL="https://xxu26128.live.dynatrace.com"
@ -72,11 +79,11 @@
TF_VAR_NA_PREPROD_API_TOKEN=credentials('NAPREPROD_TOKEN_VAR')
//CN PROD
TF_VAR_CN_PROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
TF_VAR_CN_PROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
TF_VAR_CN_PROD_API_TOKEN=credentials('CNPROD_TOKEN_VAR')
//CN PREPROD
TF_VAR_CN_PREPROD_ENV_URL="https://dyna-synth-cn.bmwgroup.com.cn/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
TF_VAR_CN_PREPROD_ENV_URL="https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
TF_VAR_CN_PREPROD_API_TOKEN=credentials('CNPREPROD_TOKEN_VAR')
//TERRAFORM
@ -122,17 +129,15 @@
}
}
stage('Execute Export Script TERRAFORM') {
steps {
export_config_all(environments)
//sh 'python3 export.py EMEA_PROD'
//sh 'python3 export.py TERRAFORM'
sh 'python3 export.py TERRAFORM'
//Only required once CN is not reachable from EMEA
//loopEnvironments(environments)
}
}
}
stage('Send report') {

104
export.py
View File

@ -1,17 +1,15 @@
import os
from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
import subprocess
import sys
import time
import shutil
import hcl
from dotenv import load_dotenv
from glob import glob
#from git import Repo
import zipfile
import boto3
# [AA 2022.01.17] Set available resources
# [AA 2022.01.17] Set available resources
if os.name == 'nt':
@ -37,11 +35,11 @@ Resources = os.getenv("TERRAFORM_RESOURCES").split(",") if os.getenv("TERRAFORM_
"dynatrace_database_anomalies",
"dynatrace_custom_anomalies",
"dynatrace_disk_anomalies",
"dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
# "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
"dynatrace_service_naming",
"dynatrace_host_naming",
"dynatrace_processgroup_naming",
"dynatrace_slo", # issue -> bug: whitespace issue
#"dynatrace_slo", # issue -> bug: whitespace issue
"dynatrace_span_entry_point",
"dynatrace_span_capture_rule",
"dynatrace_span_context_propagation",
@ -70,27 +68,33 @@ def setEnv(env, time, path):
# [AA 2021.12.10] Method to call process synchronously
def runExportProcess(process_name, input_params):
process_names = ["Export", "Terraform init"]
success = True
print("[DEBUG] Start run process: "+ ' '.join(input_params))
process = Popen(input_params,stdout=PIPE, stderr=PIPE)
output, error_output = process.communicate()
def runProcess(process_name, input_params):
process_names = ["Export", "Terraform init"]
success = False
#process = subprocess.Popen(input_params)
print("[DEBUG] Start run process: "+ ' '.join(input_params))
try:
process = subprocess.Popen(input_params)
process.wait(timeout=60*60) # 10 minutes
if len(output) > 0:
raise Exception ("Eception occured during export config: "+output.decode("utf-8"))
print("[DEBUG]", "Process:", process_name, "Success:", success)
def runImportProcess(process_name, input_params):
process_names = ["Export", "Terraform init"]
success = True
print("[DEBUG] Start run process: "+ ' '.join(input_params))
process = Popen(input_params)
process.wait(timeout=60*60)
if process.returncode > 0:
raise Exception ("Eception occured during generating state File!")
print("[DEBUG]", "Process:", process_name, "Success:", success)
# print("[DEBUG]", "Process return code:", outs)
except subprocess.TimeoutExpired:
print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
print("[DEBUG]", "Killing process:", process_name)
process.kill()
success = False
print("[DEBUG]", "Process:", process_name, "Success:", success)
except Exception as err:
print("[DEBUG]", "Exception running export tool"+ str(err))
#if process_name in process_names and success == False:
# print("[DEBUG]", "Process:", process_name, "Success:", success)
# print("[DEBUG]", "Exiting program.")
# process.kill()
# success = False
# sys.exit(1)
#else:
# print("[FAILED]", input_params)
# [AA 2021.12.17] Methods needed to replace the matching keys
def replacedata(p, maplist):
@ -102,11 +106,29 @@ def replacedata(p, maplist):
template.seek(0)
template.truncate()
# [AA 2021.12.14] Replace matching {$keys}
#for mapping in mappings[0:2]:
# [AA 2021.12.17] With the values for management_zone and msid in memory
for key, val in maplist.items():
print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
data = data.replace(key, val)
# # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod
# if os.path.basename(p) in specificfiles[0:2]:
# for key, val in mappings[2][maplist].items():
# print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
# data = data.replace(key, val)
# # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub
# if os.path.basename(p) in specificfiles[2]:
# for mapping in mappings[3:5]:
# for key, val in mapping.items():
# print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
# data = data.replace(key, val[pos])
# [AA 2022.01.19] Replace key value for {}
# [AA 2021.12.14] Write data from memory into file
with open(p, 'w+') as template:
template.write(data)
@ -151,10 +173,7 @@ def createResourceDict():
# [AA, EW 2022.01.17] Copy main.tf into the target folder
def copyMainTemplate():
shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
replacedata(targetFolder + "main.tf", {"{$env}":env,
"{$timestamp}":timestamp,
"{$S3_BUCKET}": str(os.getenv("AWS_S3_BUCKET")),
"{$S3_REGION}":str(os.getenv("AWS_S3_REGION"))})
replacedata(targetFolder + "main.tf", {"{$env}":env, "{$timestamp}":timestamp})
# [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
@ -187,25 +206,17 @@ def editMainTF():
def importStates():
os.chdir(targetFolder)
input_params = ["terraform", "init"]
runImportProcess("Terraform init",input_params)
runProcess("Terraform init",input_params)
for filedir, resourceV in myDict.items():
for resource, valueArray in resourceV.items():
for rObject in valueArray:
input_params = ["terraform", "import", "module."+getModuleTag(
filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
runImportProcess("Import", input_params)
runProcess("Import", input_params)
# terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
os.chdir(cwd)
def zipdir(path, ziph):# ziph is zipfile handlefor root, dirs, files in os.walk(path):for file in files:
for root, dirs, files in os.walk(path):
if ".terraform" not in root:
for file in files:
if ".terraform.lock.hcl" not in file:
ziph.write(os.path.join(root, file),
os.path.relpath(os.path.join(root, file),
os.path.join(path, '..')))
# [AA 2022.01.17] Arguments passed
if(len(sys.argv) == 2):
@ -228,7 +239,7 @@ if(len(sys.argv) == 2):
setEnv(sys.argv[1], timestamp, outputFolder)
# [AA, EW 2022.01.17] Download resource files
runExportProcess("Export", [export_tool, "export"] + Resources)
runProcess("Export", [export_tool, "export"] + Resources)
# [AA, EW 2022.01.17] Create a dictionary to store information of resources
createResourceDict()
@ -242,21 +253,8 @@ if(len(sys.argv) == 2):
# [AA, EW 2022.01.17] Import the states for each module
importStates()
zipf = zipfile.ZipFile(outputFolder+"/"+timestamp + "_" +env+'.zip', 'w', zipfile.ZIP_DEFLATED)
zipdir(targetFolder, zipf)
zipf.close()
s3 = boto3.client('s3')
with open(outputFolder+"/"+timestamp + "_" +env+'.zip', 'rb') as data:
s3.upload_fileobj(data, str(os.getenv("AWS_S3_BUCKET")), 'backups/'+timestamp + "_" +env+'.zip') #TODO: Make s3 bucket name configurable over environment variables
print("Finished!")
sys.exit(0)
except Exception as err:
print("Exception occured: "+ str(err))
sys.exit(1)

1
requirements.txt
View File

@ -9,4 +9,3 @@ typing
python-dotenv
pyhcl
subprocess32
boto3==1.17.0

7
templates/main.tf
View File

@ -7,9 +7,9 @@ terraform {
}
backend "s3" {
bucket = "{$S3_BUCKET}"
bucket = "coco-dynatrace-tfstate"
key = "backup/{$env}/{$timestamp}/terraform.tfstate"
region = "{$S3_REGION}"
region = "eu-central-1"
dynamodb_table = "coco-dynatrace-tfstate"
encrypt = true
}
@ -23,3 +23,6 @@ provider "dynatrace" {
dt_env_url = "${var.{$env}_ENV_URL}"
dt_api_token = "${var.{$env}_API_TOKEN}"
}