arnaua
/
coco_apm_slo_report_host_pg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: arnaua:e63cbbf5ffc6ea8337b5e836ddeb6e0248b43d16
Branches Tags
arnaua:master
..
compare: arnaua:1952038cc0db333f7bdd4e05b8eceb2d10b14a54
Branches Tags
arnaua:master

No commits in common. "e63cbbf5ffc6ea8337b5e836ddeb6e0248b43d16" and "1952038cc0db333f7bdd4e05b8eceb2d10b14a54" have entirely different histories.
e63cbbf5ff ... 1952038cc0

10 changed files with 170 additions and 190 deletions
Show Stats Expand all files Collapse all files

5
.gitignore vendored
View File

@ -138,7 +138,4 @@ crash.log
*.xlsx
# json files
*.json
# backup files
*.bak
*.json

45
Readme.md
View File

@ -1,45 +0,0 @@
# Report Host & ProcessGroups for SLOs
This repository holds the code to generate the relevant Host and Processgroup Report.
Ouput is an excel file of all Host and Processgroups for all relevant SLOs (starting with (TP_))
# Prerequisites
##
Python Version >= 3.10.x
## Python packages
Before executing scripts, python requirements have to be satisfied. To do so, execute following command:
pip install -r requirements.txt
## .env file
To provide authentication for API calls, create ".env" file in the script directory with following definition:
<ENV NAME>=<ENV TOKEN>
<ENV NAME> is name of environment variable. This name should be passed to "environment.yaml" file as "env-token-name" parameter
Example:
environment.yaml file: "- env-token-name: "GLOBAL_CONFIG_TOKEN"
.env file: "GLOBAL_CONFIG_TOKEN=XXXXXXXXXXX"
# Usage
python create_report.py
# Files
## create_report.py
This scripts generates the report.
## environment.yaml
File containing environments to execute --auto-upload
Environment name:
name: string #name ov environment
env-url: str #url of environment
env-token-name: str #name of environment variable containing API token
## requirements.txt
File containing required python packages

285
create_report.py
View File

@ -3,8 +3,7 @@ from decouple import config
import dynatraceAPI
import pandas as pd
from pagination import Pagionation
from KRParser import krparser
from key_request_parser import krparser
from datetime import datetime, timedelta
import datetime
@ -16,19 +15,10 @@ import requests
import urllib.parse
import time
import numpy as np
def get_slo(ENV: str, DTAPIToken: str, DTENV: str) -> pd.DataFrame:
"""
Returns SLO data from dynatrace
Args:
ENV (str): Environment (euprod, naprod, cnprod)
DTAPIToken (str): Token for respective environment
DTENV (str): Full URL for the respective environment
Returns:
pd.DataFrame: Dataframe containing data from dynatrace
"""
def get_slo(ENV, DTAPIToken, DTENV) -> pd.DataFrame:
# DTENV = base url
# DTAPIToken = sec token
dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken)
@ -57,8 +47,9 @@ def build_params(params: typing.Dict) -> str:
return query_string
# TODO: remove env parameter
def get_data_from_dynatrace(
throttling_rate: float | int,
throttling: float | int,
token: str,
env_url: str,
params: typing.Dict | str,
@ -78,7 +69,9 @@ def get_data_from_dynatrace(
typing.Dict: Returns the response as
"""
time.sleep(throttling_rate)
# TODO: add nextpage key feature
time.sleep(throttling)
if type(params) is dict:
params_string = f"?{build_params(params)}"
@ -98,22 +91,10 @@ def get_data_from_dynatrace(
print(f"ERROR - {host_response.status_code}")
def check_if_service_already_exists(services: list, entity_id: str) -> bool:
"""
Requests point to the same service. This leads to double entries but we only need the data once.
Args:
services (list): List with services
entity_id (str): Entity Id for lookup
Returns:
bool: Returns True if the service is already present else False.
"""
result = False
for service in services:
if service["entityId"] == entity_id:
result = True
return result
def previous_week_range(date: datetime):
start_date = date + timedelta(-date.weekday(), weeks=-1)
end_date = date + timedelta(-date.weekday() - 1)
return start_date, end_date
def get_process_group_data(df: pd.DataFrame) -> typing.Dict:
@ -148,22 +129,19 @@ def get_process_group_data(df: pd.DataFrame) -> typing.Dict:
for hub in unique_hubs:
unique_process_groups_per_hub[hub] = {}
# hub_value = hub
process_groups_unique = df.query(f"environment == @hub")
hub_value = hub
process_groups_unique = df.query(f"environment == @hub_value")
process_groups_unique = process_groups_unique["process_group_id"].unique()
for process_group in process_groups_unique:
params = {
"entitySelector": f'type("PROCESS_GROUP"),entityId("{process_group}")',
"fields": "firstSeenTms,lastSeenTms,tags",
"fields": "firstSeenTms,tags",
}
data = get_data_from_dynatrace(
0.1, hub_data[hub]["token"], hub_data[hub]["url"], params, "entities"
)
if data is not None:
unique_process_groups_per_hub[hub][process_group] = data["entities"]
else:
print(f"{process_group} returned None")
unique_process_groups_per_hub[hub][process_group] = data["entities"]
return unique_process_groups_per_hub
@ -203,25 +181,16 @@ def build_dataframe_for_report(report_items: typing.Dict) -> pd.DataFrame:
def write_xlsx(df: pd.DataFrame) -> None:
"""
Takes in a pandas dataframe and generates writes it into a XLSX file
Args:
df (pd.DataFrame): Dataframe containing the necessary data for the report
"""
filename = f"CoCo-APM-Report_{datetime.date.today()}.xlsx"
writer = pd.ExcelWriter(filename, engine="xlsxwriter")
df.to_excel(writer, sheet_name="hosts", index=False)
writer.close()
def build_dataframe_data(data: typing.Dict) -> None:
"""
This function builds the data for the dataframe, which will be used to generate the report. Contains all data but process_groups.
def develop_load_json():
with open("test-data-with-hosts-main.json", "r") as f:
data = json.loads(f.read())
Args:
data (typing.Dict): Takes in the dictionary containing all the raw data from dynatrace.
"""
df_data = []
for hub in data:
@ -231,27 +200,6 @@ def build_dataframe_data(data: typing.Dict) -> None:
for service in data[hub][slo]["services"]:
if len(service["entities"]) > 0:
for entity in service["entities"]:
# get compass id of service here. in tags
compass_id_service = []
if "tags" in entity:
for tag in entity["tags"]:
if tag["key"] == "compass-id":
compass_id_service.append(tag["value"])
compass_id_service = ",".join(compass_id_service)
# get container name here
container_name = "None"
if "properties" in entity:
if "softwareTechnologies" in entity["properties"]:
for technology in entity["properties"][
"softwareTechnologies"
]:
if (
technology["type"] == "DOCKER"
or technology["type"] == "CONTAINERD"
):
container_name = entity["properties"][
"detectedName"
]
if "fromRelationships" in entity:
if "runsOnHost" in entity["fromRelationships"]:
for host in entity["fromRelationships"][
@ -262,45 +210,124 @@ def build_dataframe_data(data: typing.Dict) -> None:
"host_name": host["details"]["displayName"],
"host_id": host["id"],
"environment": hub,
"container_name": container_name,
"process_group_id": "",
"process_group_name": "",
"licensing_tag_host": "",
"licensing_tag_process_group": "",
"first_seen_process_group": "",
"process_group_id": "NaN",
"process_group_name": "NaN",
"licensing_tag_host": "NaN",
"licensing_tag_process_group": "NaN",
"first_seen_process_group": "NaN",
"first_seen_host": host["details"][
"firstSeenTms"
],
"last_seen_host": host["details"][
"lastSeenTms"
],
"compass_id_host": "",
"compass_id_service": compass_id_service,
}
compass_id = []
namespace = []
for tag in host["details"]["tags"]:
if tag["key"] == "Platform":
df_data_item["platform"] = tag["value"]
if tag["key"] == "Namespace":
# df_data_item["namespace"] = tag["value"]
namespace.append(tag["value"])
df_data_item["namespace"] = tag["value"]
if tag["key"] == "PaaS":
df_data_item["paas"] = tag["value"]
if tag["key"] == "compass-id":
# df_data_item["compass_id"] = tag[
# "value"
# ]
if "value" in tag:
compass_id.append(tag["value"])
df_data_item["compass_id_host"] = ",".join(
compass_id
)
# TODO: rework
if "runsOn" in entity["fromRelationships"]:
for process_group in entity[
"fromRelationships"
]["runsOn"]:
df_data_item[
"process_group_id"
] = process_group["id"]
df_data_item["namespace"] = ",".join(namespace)
df_data.append(df_data_item)
build_dataframe_for_report(df_data)
# with open("./environment.yaml") as file:
# env_doc = yaml.safe_load(file)
# for env, doc in env_doc.items():
# # DEBUG
# if env == "euprod":
# token = dict(doc[2])
# url = dict(doc[1])
# if config(token.get("env-token-name")) != "":
# print("Gather data, hold on a minute")
# DTTOKEN = config(token.get("env-token-name"))
# DTURL = url.get("env-url")
# for slo in data[env]:
# if len(data[env][slo]["services"]) == 0:
# # DEBUG
# print(f"ERROR: {slo} has no services")
# else:
# for service in data[env][slo]["services"]:
# params = {
# "entitySelector": f'type("SERVICE"),entityId("{service["entityId"]}")',
# "fields": "fromRelationships,tags",
# }
# entities = get_data_from_dynatrace(
# 0.5, DTTOKEN, DTURL, params, "entities"
# )
# # TODO: it is possible that "entities" is empty. maybe create check.
# service["entities"] = entities["entities"]
# for hosts in service["entities"]:
# if "fromRelationships" in hosts:
# if "runsOnHost" in hosts["fromRelationships"]:
# for host in hosts["fromRelationships"][
# "runsOnHost"
# ]:
# # TODO: make dynatrace call to /entites/{entityId}
# host_response = get_data_from_dynatrace(
# 0.5, DTTOKEN, DTURL, host["id"], "entities"
# )
# host["details"] = host_response
def check_if_service_already_exists(services: list, entity_id: str) -> bool:
result = False
for service in services:
if service["entityId"] == entity_id:
result = True
return result
def build_data_frame_data(data: typing.Dict) -> None:
df_data = []
for hub in data:
for slo in data[hub]:
slo_name = data[hub][slo]["sloname"]
if len(data[hub][slo]["services"]) > 0:
for service in data[hub][slo]["services"]:
if len(service["entities"]) > 0:
for entity in service["entities"]:
if "fromRelationships" in entity:
if "runsOnHost" in entity["fromRelationships"]:
for host in entity["fromRelationships"][
"runsOnHost"
]:
df_data_item = {
"slo_name": slo_name,
"host_name": host["details"]["displayName"],
"host_id": host["id"],
"environment": hub,
"process_group_id": "NaN",
"process_group_name": "NaN",
"licensing_tag_host": "NaN",
"licensing_tag_process_group": "NaN",
"first_seen_process_group": "NaN",
"first_seen_host": host["details"][
"firstSeenTms"
],
}
for tag in host["details"]["tags"]:
if tag["key"] == "Platform":
df_data_item["platform"] = tag["value"]
if tag["key"] == "Namespace":
df_data_item["namespace"] = tag["value"]
if tag["key"] == "PaaS":
df_data_item["paas"] = tag["value"]
# TODO: rework
if "runsOn" in entity["fromRelationships"]:
@ -318,10 +345,6 @@ def build_dataframe_data(data: typing.Dict) -> None:
@timer
def main() -> None:
"""
Entrypoint.
"""
throttling_rate: int | float = 0 # only tested with 0.5
reportItem = {}
with open("./environment.yaml") as file:
env_doc = yaml.safe_load(file)
@ -338,28 +361,36 @@ def main() -> None:
# krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN)
slosF = get_slo(env, DTTOKEN, DTURL)
# slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"]
slosF = slosF[slosF["name"].str.startswith("TP_")]
# parse the metric Expression to get Services and Requests
krs = []
# krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
krp = krparser.KRParser(
name=env,
options=krparser.KROption.RESOLVESERVICES
| krparser.KROption.VALIDATE_HASDATA,
config={
"threads": 10,
"serviceLookupParams": {"fields": "tags,fromRelationships"},
"extendResultObjects": {"env": env},
},
DTAPIURL=DTURL,
DTAPIToken=DTTOKEN,
krparser.KROption.VALIDATE_EXISTS
# | krparser.KROption.VALIDATE_HASDATA
| krparser.KROption.RESOLVESERVICES,
DTURL,
DTTOKEN,
)
krs = krp.parse(slosF)
for index, row in slosF.iterrows():
krs.append(krp.parseBySLO(row))
# x = 0
# SLO Name | SERVICE | PROCESS GRUPPE | TAGS
# {"sloname": {
# "sloname":$sloname$,
# "services":[{
# "serviceName": "$servicename$"
# }]
# },
# "sloname": {
# "sloname":$sloname$,
# "services":[{
# "serviceName": "$servicename$"
# }]
reportItem[str(env)] = {}
@ -392,7 +423,9 @@ def main() -> None:
== 0
):
if not check_if_service_already_exists(
reportItem[env][kr.metadata["sloName"]]["services"],
reportItem[str(env)][kr.metadata["sloName"]][
"services"
],
service["entityId"],
):
reportItem[str(env)][kr.metadata["sloName"]][
@ -404,7 +437,6 @@ def main() -> None:
"entityId": service["entityId"],
}
)
if (
len(
reportItem[str(env)][kr.metadata["sloName"]][
@ -423,16 +455,11 @@ def main() -> None:
]["services"]:
params = {
"entitySelector": f'type("SERVICE"),entityId("{service["entityId"]}")',
"fields": "fromRelationships,tags,properties",
"fields": "fromRelationships,tags",
}
entities = get_data_from_dynatrace(
throttling_rate,
DTTOKEN,
DTURL,
params,
"entities",
0.5, DTTOKEN, DTURL, params, "entities"
)
# print(entities["entities"])
# TODO: it is possible that "entities" is empty. maybe create check.
service["entities"] = entities["entities"]
for hosts in service["entities"]:
@ -447,7 +474,7 @@ def main() -> None:
]["runsOnHost"]:
host_response = (
get_data_from_dynatrace(
throttling_rate,
0.5,
DTTOKEN,
DTURL,
host["id"],
@ -455,8 +482,12 @@ def main() -> None:
)
)
host["details"] = host_response
build_dataframe_data(reportItem)
build_data_frame_data(reportItem)
# with open("test-data-with-hosts-main.json", "w") as f:
# f.write(json.dumps(reportItem, indent=4))
if __name__ == "__main__":
main()
# main()
develop_load_json()

20
environment.yaml
View File

@ -4,13 +4,13 @@ euprod:
- env-url: "https://xxu26128.live.dynatrace.com"
- env-token-name: "EUPROD_TOKEN_VAR"
- jenkins: "https://jaws.bmwgroup.net/opapm/"
# naprod:
# - name: "naprod"
# - env-url: "https://wgv50241.live.dynatrace.com"
# - env-token-name: "NAPROD_TOKEN_VAR"
# - jenkins: "https://jaws.bmwgroup.net/opapm/"
# cnprod:
# - name: "cnprod"
# - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
# - env-token-name: "CNPROD_TOKEN_VAR"
# - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
naprod:
- name: "naprod"
- env-url: "https://wgv50241.live.dynatrace.com"
- env-token-name: "NAPROD_TOKEN_VAR"
- jenkins: "https://jaws.bmwgroup.net/opapm/"
cnprod:
- name: "cnprod"
- env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
- env-token-name: "CNPROD_TOKEN_VAR"
- jenkins: "https://jaws-china.bmwgroup.net/opmaas/"

0
_BAK/__init__.py → key_request_parser/__init__.py
View File

0
_BAK/helper.py → key_request_parser/helper.py
View File

0
_BAK/keyrequests.py → key_request_parser/keyrequests.py
View File

0
_BAK/krparser.py → key_request_parser/krparser.py
View File

0
_BAK/patterns.py → key_request_parser/patterns.py
View File

5
requirements.txt
View File

@ -2,7 +2,4 @@
python-decouple
requests
pyyaml
pandas
xlsxwriter
jsonmerge
git+https://atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git
pandas