added export script
ermisw
commit
29d9aaf0cf
|
|
@ -0,0 +1,4 @@
|
|||
bin/
|
||||
output/
|
||||
.env
|
||||
.terraform.lock.hcl
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python: Aktuelle Datei",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "${file}",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"TERRAFORM"
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
62
README.md
62
README.md
|
|
@ -1,3 +1,4 @@
|
|||
<<<<<<< HEAD
|
||||
# Dynatrace Reporting Pipeline
|
||||
|
||||
This repository is used as a template to create automated Dynatrace reports through Jenkins (JAWS) which are sent as attachement through mail.
|
||||
|
|
@ -132,3 +133,64 @@ Do **NOT** clone this repo, create a fork instead.
|
|||
|
||||
<br>
|
||||
Your pipeline will automatically test-run for all 3 branches.
|
||||
=======
|
||||
# TerraformOnboarding
|
||||
The purpose of the exportConfig.py script is to export dynatrace specific services as terraform files. The importConfig.py script is used to additionally also export the states for each exported terraform file, since terraform does not do that by default.
|
||||
|
||||
|
||||
# Setup
|
||||
Run the following command to install all necessary dependencies:
|
||||
|
||||
```python
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
In order to ensure full functionality a `.env` file is necessary with the following format:
|
||||
|
||||
```yml
|
||||
# Environment URLs
|
||||
CN_PREPROD_ENV_URL="https://dynatracemgd-cn.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
|
||||
CN_PROD_ENV_URL="https://dynatracemgd-cn.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
|
||||
EMEA_PREPROD_ENV_URL="https://qqk70169.live.dynatrace.com"
|
||||
EMEA_PROD_ENV_URL="https://xxu26128.live.dynatrace.com"
|
||||
NA_PREPROD_ENV_URL="https://onb44935.live.dynatrace.com"
|
||||
NA_PROD_ENV_URL="https://wgv50241.live.dynatrace.com"
|
||||
|
||||
# Environment Tokens
|
||||
CN_PREPROD_API_TOKEN="<your-token>"
|
||||
CN_PROD_API_TOKEN="<your-token>"
|
||||
EMEA_PREPROD_API_TOKEN="<your-token>"
|
||||
EMEA_PROD_API_TOKEN="<your-token>"
|
||||
NA_PREPROD_API_TOKEN="<your-token>"
|
||||
NA_PROD_API_TOKEN="<your-token>"
|
||||
```
|
||||
Place the `.env` file within the root directory of the project folder:
|
||||
|
||||
```bash
|
||||
TerraformDynatrace Porter # Project Folder
|
||||
├─── res
|
||||
├─── templates
|
||||
├─── .env # Add the environment file
|
||||
├─── .gitignore
|
||||
├─── README.md
|
||||
├─── exportConfig.py
|
||||
├─── imortConfig.py
|
||||
├─── main.tf
|
||||
└─── requirements.txt
|
||||
```
|
||||
|
||||
# Run
|
||||
You can simply run the script by executing the following example command within the projects root directory:
|
||||
|
||||
```python
|
||||
python exportConfig.py
|
||||
python importConfig.py
|
||||
```
|
||||
**Note:** First run the exportConfig.py script and once that is done run the importConfig.py script.
|
||||
|
||||
|
||||
# Version
|
||||
```python
|
||||
Python 3.9.9
|
||||
```
|
||||
>>>>>>> 5b9784368712fb52b5411a7924b67569d40c051f
|
||||
|
|
|
|||
|
|
@ -0,0 +1,255 @@
|
|||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import hcl
|
||||
from dotenv import load_dotenv
|
||||
from glob import glob
|
||||
|
||||
|
||||
# [AA 2022.01.17] Set available resources
|
||||
Resources = [
|
||||
#"dynatrace_custom_service",
|
||||
"dynatrace_dashboard",
|
||||
"dynatrace_management_zone",
|
||||
# "dynatrace_maintenance_window",
|
||||
# "dynatrace_request_attribute",
|
||||
#"dynatrace_alerting_profile",
|
||||
# "dynatrace_notification",
|
||||
"dynatrace_autotag"
|
||||
# "dynatrace_aws_credentials",
|
||||
# "dynatrace_azure_credentials",
|
||||
# "dynatrace_k8s_credentials",
|
||||
# "dynatrace_service_anomalies",
|
||||
# "dynatrace_application_anomalies",
|
||||
# "dynatrace_host_anomalies",
|
||||
# "dynatrace_database_anomalies",
|
||||
# "dynatrace_custom_anomalies",
|
||||
# "dynatrace_disk_anomalies",
|
||||
# "dynatrace_calculated_service_metric", #issue -> bug: windows specific due to path length limit
|
||||
# "dynatrace_service_naming",
|
||||
# "dynatrace_host_naming",
|
||||
# "dynatrace_processgroup_naming",
|
||||
# "dynatrace_slo", # issue -> bug: whitespace issue
|
||||
# "dynatrace_span_entry_point",
|
||||
# "dynatrace_span_capture_rule",
|
||||
# "dynatrace_span_context_propagation",
|
||||
# "dynatrace_resource_attributes",
|
||||
# "dynatrace_span_attribute",
|
||||
# "dynatrace_mobile_application",
|
||||
# "dynatrace_credentials", #issue -> bug: unknown issue? not supported?
|
||||
#"dynatrace_browser_monitor",
|
||||
#"dynatrace_http_monitor",
|
||||
]
|
||||
|
||||
|
||||
|
||||
# [AA 2021.12.10] Method to set environments
|
||||
def setEnv(env, time, path):
|
||||
if not (os.getenv("TF_VAR_"+ env + "_ENV_URL") ):
|
||||
raise Exception ("Environment variable missing: TF_VAR_"+ env + "_ENV_URL")
|
||||
|
||||
if not ( os.getenv("TF_VAR_"+ env + "_API_TOKEN")):
|
||||
raise Exception ("Environment variable missing: TF_VAR_"+env + "_API_TOKEN")
|
||||
|
||||
os.environ['DYNATRACE_ENV_URL'] = str(os.getenv("TF_VAR_"+env + "_ENV_URL"))
|
||||
os.environ['DYNATRACE_API_TOKEN'] = str(os.getenv("TF_VAR_"+env + "_API_TOKEN"))
|
||||
os.environ['DYNATRACE_TARGET_FOLDER'] = str(path + time + "_" + env)
|
||||
return os.environ
|
||||
|
||||
|
||||
# [AA 2021.12.10] Method to call process synchronously
|
||||
def runProcess(process_name, input_params):
|
||||
process_names = ["Export", "Terraform init"]
|
||||
success = False
|
||||
try:
|
||||
process = subprocess.Popen(input_params)
|
||||
process.wait(timeout=60*60) # 10 minutes
|
||||
success = True
|
||||
print("[DEBUG]", "Process:", process_name, "Success:", success)
|
||||
# print("[DEBUG]", "Process return code:", outs)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
|
||||
print("[DEBUG]", "Killing process:", process_name)
|
||||
process.kill()
|
||||
success = False
|
||||
print("[DEBUG]", "Process:", process_name, "Success:", success)
|
||||
except:
|
||||
if process_name in process_names and success == False:
|
||||
print("[DEBUG]", "Process:", process_name, "Success:", success)
|
||||
print("[DEBUG]", "Exiting program.")
|
||||
process.kill()
|
||||
success = False
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("[FAILED]", input_params)
|
||||
|
||||
# [AA 2021.12.17] Methods needed to replace the matching keys
|
||||
def replacedata(p, maplist):
|
||||
|
||||
# [AA 2021.12.14] Open each template to read
|
||||
with open(p, 'r+') as template:
|
||||
print("[DEBUG]", "Opening file at %s." % p)
|
||||
data = template.read()
|
||||
template.seek(0)
|
||||
template.truncate()
|
||||
|
||||
# [AA 2021.12.14] Replace matching {$keys}
|
||||
#for mapping in mappings[0:2]:
|
||||
|
||||
# [AA 2021.12.17] With the values for management_zone and msid in memory
|
||||
for key, val in maplist.items():
|
||||
print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
|
||||
data = data.replace(key, val)
|
||||
|
||||
# # [AA 2021.12.22] Files that require a replacement for test,int and e2e,prod
|
||||
# if os.path.basename(p) in specificfiles[0:2]:
|
||||
# for key, val in mappings[2][maplist].items():
|
||||
# print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
|
||||
# data = data.replace(key, val)
|
||||
|
||||
# # [AA 2021.12.22] Replace key value for {$url} and {$env} for corresponding hub
|
||||
# if os.path.basename(p) in specificfiles[2]:
|
||||
# for mapping in mappings[3:5]:
|
||||
# for key, val in mapping.items():
|
||||
# print("[DEBUG]", "Replacing key values %s at %s." % (key, p))
|
||||
# data = data.replace(key, val[pos])
|
||||
|
||||
# [AA 2022.01.19] Replace key value for {}
|
||||
|
||||
# [AA 2021.12.14] Write data from memory into file
|
||||
with open(p, 'w+') as template:
|
||||
template.write(data)
|
||||
|
||||
# [AA 2021.12.13] Fill dictionary
|
||||
def readFile(path):
|
||||
with open(path, 'r', encoding='utf8') as cfg:
|
||||
|
||||
# [AA 2021.12.01] Load the content of the particular resource file in eg: management_zones
|
||||
obj = hcl.load(cfg)
|
||||
|
||||
# [AA, EW 2021.12.01] Store resource type and resource name of that file into a dictionary
|
||||
key = list(obj['resource'].keys())[0]
|
||||
val = list(obj['resource'][key].keys())[0]
|
||||
return key, val
|
||||
|
||||
|
||||
# [AA, EW 2022.01.17] Load all resources and add them to a dictionary
|
||||
def createResourceDict():
|
||||
files = [os.path.normpath(f).replace('\\', '/')
|
||||
for f in glob(targetFolder + "**/**.tf", recursive=True)]
|
||||
for index, file in enumerate(files):
|
||||
filedir = "./"+("./"+os.path.dirname(file)).replace(targetFolder, "")
|
||||
splittedFilename = os.path.basename(file).split(".")
|
||||
|
||||
if len(splittedFilename) == 5:
|
||||
resourceID = splittedFilename[1]
|
||||
moduleName, resourceName = readFile(file)
|
||||
|
||||
if not filedir in myDict.keys():
|
||||
myDict.setdefault(filedir, {})
|
||||
|
||||
if not moduleName in myDict[filedir].keys():
|
||||
myDict[filedir].setdefault(moduleName, [])
|
||||
|
||||
resourceValue = {"resourceName": resourceName,
|
||||
"resourceID": resourceID}
|
||||
myDict[filedir][moduleName].append(resourceValue)
|
||||
|
||||
|
||||
# [AA, EW 2022.01.17] Copy main.tf into the target folder
|
||||
def copyMainTemplate():
|
||||
shutil.copyfile(templatesFolder + "main.tf", targetFolder + "main.tf")
|
||||
replacedata(targetFolder + "main.tf", {"{$env}":env, "{$timestamp}":timestamp})
|
||||
|
||||
|
||||
# [AA 2022.01.17] Copy module.tf in all folders and subfolders except where main.tf is
|
||||
def copyModuleTemplate():
|
||||
dirs = glob(targetFolder + "**/", recursive=True)
|
||||
for index, dir in enumerate(dirs):
|
||||
if index != 0:
|
||||
shutil.copyfile(templatesFolder + "module.tf", dir + "module.tf")
|
||||
|
||||
|
||||
# [AA 2021.12.13] Append correct configuration path
|
||||
def writeFile(k, d):
|
||||
with open(".\\main.tf", "a") as mf:
|
||||
mf.writelines("\n" + "module \"" + k + "\" { source = \"" + d + "\" }")
|
||||
|
||||
|
||||
# [AA, EW 2022.01.17] Adjust the resource module name
|
||||
def getModuleTag(str):
|
||||
return str.replace("./", "").replace("/", "_")
|
||||
|
||||
|
||||
# [AA, EW 2022.01.17] Set the resource names
|
||||
def editMainTF():
|
||||
with open(targetFolder + "main.tf", "a") as mf:
|
||||
for index, (filedir, value) in enumerate(myDict.items()):
|
||||
mf.writelines("\n" + "module \"" + getModuleTag(filedir) +
|
||||
"\" { source = \"" + filedir + "\" }")
|
||||
|
||||
|
||||
# [AA, EW 2022.01.17] Start importing
|
||||
def importStates():
|
||||
os.chdir(targetFolder)
|
||||
input_params = ["terraform", "init"]
|
||||
runProcess("Terraform init",input_params)
|
||||
|
||||
for filedir, resourceV in myDict.items():
|
||||
for resource, valueArray in resourceV.items():
|
||||
for rObject in valueArray:
|
||||
input_params = ["terraform", "import", "module."+getModuleTag(
|
||||
filedir)+"."+resource+"."+rObject["resourceName"], rObject["resourceID"]]
|
||||
runProcess("Import", input_params)
|
||||
# terraform import module.alerting_profiles.dynatrace_alerting_profiles.CD_ABC 9348098098safs9f8
|
||||
os.chdir(cwd)
|
||||
|
||||
|
||||
# [AA 2022.01.17] Arguments passed
|
||||
if(len(sys.argv) == 2):
|
||||
|
||||
try:
|
||||
# [AA 2021.11.29] Load enviroment file
|
||||
load_dotenv()
|
||||
|
||||
# [AA, EW 2022.01.17] Set global variables
|
||||
global timestamp, templatesFolder, outputFolder, targetFolder, myDict, cwd, env
|
||||
env=sys.argv[1]
|
||||
timestamp = time.strftime("%Y%m%d-%H%M%S")
|
||||
cwd = os.getcwd()
|
||||
outputFolder = "./output/"
|
||||
targetFolder = outputFolder + timestamp + "_" + sys.argv[1] + "/"
|
||||
templatesFolder = "./templates/"
|
||||
myDict = {}
|
||||
|
||||
# [AA, EW 2022.01.17] Set env varibales
|
||||
setEnv(sys.argv[1], timestamp, outputFolder)
|
||||
|
||||
# [AA, EW 2022.01.17] Download resource files
|
||||
runProcess("Export", [".\\bin\\terraform-provider-dynatrace_v1.9.1.exe", "export"] + Resources)
|
||||
|
||||
# [AA, EW 2022.01.17] Create a dictionary to store information of resources
|
||||
createResourceDict()
|
||||
|
||||
# [AA, EW 2022.01.17] Copy main.tf file and add module.tf files
|
||||
copyMainTemplate()
|
||||
copyModuleTemplate()
|
||||
|
||||
# [AA, EW 2022.01.17] Print the module names with their associated module path into the main.tf file
|
||||
editMainTF()
|
||||
|
||||
# [AA, EW 2022.01.17] Import the states for each module
|
||||
importStates()
|
||||
print("Finished!")
|
||||
sys.exit(0)
|
||||
except Exception as err:
|
||||
print("Exception occured: "+ str(err))
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
print("Usage example: ")
|
||||
print("List of available environments: CN_PREPROD, CN_PROD, EMEA_PREPROD, EMEA_PROD, NA_PREPROD, NA_PROD, etc.")
|
||||
print("python .\export.py [Emvironment] ")
|
||||
sys.exit(1)
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
dynatrace = {
|
||||
version = "1.9.1"
|
||||
source = "dynatrace-oss/dynatrace"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# module "configDir" {
|
||||
# source = "git::https://github.com/arnauagithub/DynatraceTerraformConfiguration.git?ref=20211130-151123"
|
||||
# }
|
||||
|
||||
|
||||
module "m1" { source = "./output/20220117-132316_EMEA_PROD/anomalies" }
|
||||
module "m2" { source = "./output/20220117-132316_EMEA_PROD/credentials" }
|
||||
module "dynatrace_custom_service" { source = "./output/20220117-132316_EMEA_PROD/custom_services" }
|
||||
module "dynatrace_service_naming" { source = "./output/20220117-132316_EMEA_PROD/naming/services" }
|
||||
|
|
@ -1,6 +1,11 @@
|
|||
<<<<<<< HEAD
|
||||
python-decouple
|
||||
pyyaml
|
||||
pandas
|
||||
requests
|
||||
datetime
|
||||
argparse
|
||||
argparse
|
||||
=======
|
||||
python-dotenv
|
||||
pyhcl
|
||||
>>>>>>> 5b9784368712fb52b5411a7924b67569d40c051f
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
import subprocess
|
||||
import sys
|
||||
|
||||
if(len(sys.argv) == 1):
|
||||
try:
|
||||
process = subprocess.Popen(["terraform", "init"])
|
||||
outs = process.wait(timeout=10*60)
|
||||
print("[DEBUG]", "Process return code:", outs)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("[DEBUG]", "Exception occured:", subprocess.TimeoutExpired)
|
||||
print("[DEBUG]", "Killing process.")
|
||||
process.kill()
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
dynatrace = {
|
||||
version = "1.9.1"
|
||||
source = "dynatrace-oss/dynatrace"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
dynatrace = {
|
||||
version = "1.9.1"
|
||||
source = "dynatrace-oss/dynatrace"
|
||||
}
|
||||
}
|
||||
|
||||
backend "s3" {
|
||||
bucket = "coco-dynatrace-tfstate"
|
||||
key = "backup/{$env}/{$timestamp}/terraform.tfstate"
|
||||
region = "eu-central-1"
|
||||
dynamodb_table = "coco-dynatrace-tfstate"
|
||||
encrypt = true
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
variable {$env}_ENV_URL {}
|
||||
variable {$env}_API_TOKEN {}
|
||||
|
||||
provider "dynatrace" {
|
||||
dt_env_url = "${var.{$env}_ENV_URL}"
|
||||
dt_api_token = "${var.{$env}_API_TOKEN}"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
dynatrace = {
|
||||
version = "1.9.1"
|
||||
source = "dynatrace-oss/dynatrace"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue