From 6110b3438461881e51f3b6d79ff03c04966ab482 Mon Sep 17 00:00:00 2001 From: Daniel Mikula Date: Fri, 12 May 2023 12:39:26 +0200 Subject: [PATCH] added compass id, lastseents, and container name --- .gitignore | 5 +- create_report.py | 176 ++++++++++++++++++++++------------------------- 2 files changed, 87 insertions(+), 94 deletions(-) diff --git a/.gitignore b/.gitignore index 90eebc9..0624076 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,7 @@ crash.log *.xlsx # json files -*.json \ No newline at end of file +*.json + +# backup files +*.bak \ No newline at end of file diff --git a/create_report.py b/create_report.py index 2116ecd..e193cff 100644 --- a/create_report.py +++ b/create_report.py @@ -98,10 +98,22 @@ def get_data_from_dynatrace( print(f"ERROR - {host_response.status_code}") -def previous_week_range(date: datetime): - start_date = date + timedelta(-date.weekday(), weeks=-1) - end_date = date + timedelta(-date.weekday() - 1) - return start_date, end_date +def check_if_service_already_exists(services: list, entity_id: str) -> bool: + """ + Requests point to the same service. This leads to double entries but we only need the data once. + + Args: + services (list): List with services + entity_id (str): Entity Id for lookup + + Returns: + bool: Returns True if the service is already present else False. + """ + result = False + for service in services: + if service["entityId"] == entity_id: + result = True + return result def get_process_group_data(df: pd.DataFrame) -> typing.Dict: @@ -136,14 +148,14 @@ def get_process_group_data(df: pd.DataFrame) -> typing.Dict: for hub in unique_hubs: unique_process_groups_per_hub[hub] = {} - hub_value = hub - process_groups_unique = df.query(f"environment == @hub_value") + # hub_value = hub + process_groups_unique = df.query(f"environment == @hub") process_groups_unique = process_groups_unique["process_group_id"].unique() for process_group in process_groups_unique: params = { "entitySelector": f'type("PROCESS_GROUP"),entityId("{process_group}")', - "fields": "firstSeenTms,tags", + "fields": "firstSeenTms,lastSeenTms,tags", } data = get_data_from_dynatrace( 0.1, hub_data[hub]["token"], hub_data[hub]["url"], params, "entities" @@ -200,79 +212,6 @@ def write_xlsx(df: pd.DataFrame) -> None: writer.close() -def develop_load_json(): - with open("test-data-with-hosts-main.json", "r") as f: - data = json.loads(f.read()) - - df_data = [] - - for hub in data: - for slo in data[hub]: - slo_name = data[hub][slo]["sloname"] - if len(data[hub][slo]["services"]) > 0: - for service in data[hub][slo]["services"]: - if len(service["entities"]) > 0: - for entity in service["entities"]: - if "fromRelationships" in entity: - if "runsOnHost" in entity["fromRelationships"]: - for host in entity["fromRelationships"][ - "runsOnHost" - ]: - df_data_item = { - "slo_name": slo_name, - "host_name": host["details"]["displayName"], - "host_id": host["id"], - "environment": hub, - "process_group_id": "", - "process_group_name": "", - "licensing_tag_host": "", - "licensing_tag_process_group": "", - "first_seen_process_group": "", - "first_seen_host": host["details"][ - "firstSeenTms" - ], - } - - for tag in host["details"]["tags"]: - if tag["key"] == "Platform": - df_data_item["platform"] = tag["value"] - if tag["key"] == "Namespace": - df_data_item["namespace"] = tag["value"] - if tag["key"] == "PaaS": - df_data_item["paas"] = tag["value"] - - # TODO: rework - add else. so datastructure is complete - if "runsOn" in entity["fromRelationships"]: - for process_group in entity[ - "fromRelationships" - ]["runsOn"]: - df_data_item[ - "process_group_id" - ] = process_group["id"] - - df_data.append(df_data_item) - - build_dataframe_for_report(df_data) - - -def check_if_service_already_exists(services: list, entity_id: str) -> bool: - """ - Requests point to the same service. This leads to double entries but we only need the data once. - - Args: - services (list): List with services - entity_id (str): Entity Id for lookup - - Returns: - bool: Returns True if the service is already present else False. - """ - result = False - for service in services: - if service["entityId"] == entity_id: - result = True - return result - - def build_dataframe_data(data: typing.Dict) -> None: """ This function builds the data for the dataframe, which will be used to generate the report. Contains all data but process_groups. @@ -289,6 +228,27 @@ def build_dataframe_data(data: typing.Dict) -> None: for service in data[hub][slo]["services"]: if len(service["entities"]) > 0: for entity in service["entities"]: + # get compass id of service here. in tags + compass_id_service = [] + if "tags" in entity: + for tag in entity["tags"]: + if tag["key"] == "compass-id": + compass_id_service.append(tag["value"]) + compass_id_service = ",".join(compass_id_service) + # get container name here + container_name = "None" + if "properties" in entity: + if "softwareTechnologies" in entity["properties"]: + for technology in entity["properties"][ + "softwareTechnologies" + ]: + if ( + technology["type"] == "DOCKER" + or technology["type"] == "CONTAINERD" + ): + container_name = entity["properties"][ + "detectedName" + ] if "fromRelationships" in entity: if "runsOnHost" in entity["fromRelationships"]: for host in entity["fromRelationships"][ @@ -299,6 +259,7 @@ def build_dataframe_data(data: typing.Dict) -> None: "host_name": host["details"]["displayName"], "host_id": host["id"], "environment": hub, + "container_name": container_name, "process_group_id": "", "process_group_name": "", "licensing_tag_host": "", @@ -307,15 +268,36 @@ def build_dataframe_data(data: typing.Dict) -> None: "first_seen_host": host["details"][ "firstSeenTms" ], + "last_seen_host": host["details"][ + "lastSeenTms" + ], + "compass_id_host": "", + "compass_id_service": compass_id_service, } + compass_id = [] + namespace = [] + for tag in host["details"]["tags"]: if tag["key"] == "Platform": df_data_item["platform"] = tag["value"] if tag["key"] == "Namespace": - df_data_item["namespace"] = tag["value"] + # df_data_item["namespace"] = tag["value"] + namespace.append(tag["value"]) if tag["key"] == "PaaS": df_data_item["paas"] = tag["value"] + if tag["key"] == "compass-id": + # df_data_item["compass_id"] = tag[ + # "value" + # ] + if "value" in tag: + compass_id.append(tag["value"]) + + df_data_item["compass_id_host"] = ",".join( + compass_id + ) + + df_data_item["namespace"] = ",".join(namespace) # TODO: rework if "runsOn" in entity["fromRelationships"]: @@ -336,7 +318,7 @@ def main() -> None: """ Entrypoint. """ - throttling_rate: int | float = 0.25 # only tested with 0.5 + throttling_rate: int | float = 0 # only tested with 0.5 reportItem = {} with open("./environment.yaml") as file: env_doc = yaml.safe_load(file) @@ -353,16 +335,27 @@ def main() -> None: # krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN) slosF = get_slo(env, DTTOKEN, DTURL) - #slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"] + # slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"] slosF = slosF[slosF["name"].str.startswith("TP_")] # parse the metric Expression to get Services and Requests krs = [] - krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN) + # krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN) - krs=krp.parseBySLO_Threaded(slosF) + krp = krparser.KRParser( + name=env, + options=krparser.KROption.RESOLVESERVICES, + config={ + "threads": 10, + "serviceLookupParams": {"fields": "tags,fromRelationships"}, + "extendResultObjects": {"env": env}, + }, + DTAPIURL=DTURL, + DTAPIToken=DTTOKEN, + ) + krs = krp.parse(slosF) reportItem[str(env)] = {} @@ -395,9 +388,7 @@ def main() -> None: == 0 ): if not check_if_service_already_exists( - reportItem[str(env)][kr.metadata["sloName"]][ - "services" - ], + reportItem[env][kr.metadata["sloName"]]["services"], service["entityId"], ): reportItem[str(env)][kr.metadata["sloName"]][ @@ -409,6 +400,7 @@ def main() -> None: "entityId": service["entityId"], } ) + if ( len( reportItem[str(env)][kr.metadata["sloName"]][ @@ -427,7 +419,7 @@ def main() -> None: ]["services"]: params = { "entitySelector": f'type("SERVICE"),entityId("{service["entityId"]}")', - "fields": "fromRelationships,tags", + "fields": "fromRelationships,tags,properties", } entities = get_data_from_dynatrace( throttling_rate, @@ -436,6 +428,7 @@ def main() -> None: params, "entities", ) + print(entities["entities"]) # TODO: it is possible that "entities" is empty. maybe create check. service["entities"] = entities["entities"] for hosts in service["entities"]: @@ -458,11 +451,8 @@ def main() -> None: ) ) host["details"] = host_response - build_dataframe_data(reportItem) - if __name__ == "__main__": - main() - + main()