made adjustments for new krp

.gitignore

@@ -138,4 +138,7 @@ crash.log
 *.xlsx
 
 # json files
-*.json
+*.json
+
+# backup files
+*.bak

Readme.md

@@ -0,0 +1,45 @@
+# Report Host & ProcessGroups for SLOs
+This repository holds the code to generate the relevant Host and Processgroup Report.
+
+Ouput is an excel file of all Host and Processgroups for all relevant SLOs (starting with (TP_))
+
+
+# Prerequisites
+##
+Python Version >= 3.10.x
+
+## Python packages
+Before executing scripts, python requirements have to be satisfied. To do so, execute following command:
+	pip install -r requirements.txt
+
+## .env file
+
+To provide authentication for API calls, create ".env" file in the script directory with following definition:
+
+	<ENV NAME>=<ENV TOKEN>
+		<ENV NAME> is name of environment variable. This name should be passed to "environment.yaml" file as "env-token-name" parameter
+		Example:
+		environment.yaml file: "- env-token-name: "GLOBAL_CONFIG_TOKEN"
+		.env file: "GLOBAL_CONFIG_TOKEN=XXXXXXXXXXX"
+
+# Usage
+    
+    python create_report.py
+
+# Files
+
+## create_report.py
+
+This scripts generates the report.
+
+## environment.yaml
+File containing environments to execute --auto-upload
+
+	Environment name:
+		name: string										#name ov environment
+		env-url: str										#url of environment
+		env-token-name: str								#name of environment variable containing API token
+
+## requirements.txt
+
+File containing required python packages

create_report.py

@@ -3,7 +3,8 @@ from decouple import config
 import dynatraceAPI
 import pandas as pd
 from pagination import Pagionation
-from key_request_parser import krparser
+from KRParser import krparser
+
 
 from datetime import datetime, timedelta
 import datetime
@@ -15,10 +16,19 @@ import requests
 import urllib.parse
 import time
 
-import numpy as np
 
+def get_slo(ENV: str, DTAPIToken: str, DTENV: str) -> pd.DataFrame:
+    """
+    Returns SLO data from dynatrace
 
-def get_slo(ENV, DTAPIToken, DTENV) -> pd.DataFrame:
+    Args:
+        ENV (str): Environment (euprod, naprod, cnprod)
+        DTAPIToken (str): Token for respective environment
+        DTENV (str): Full URL for the respective environment
+
+    Returns:
+        pd.DataFrame: Dataframe containing data from dynatrace
+    """
     # DTENV = base url
     # DTAPIToken = sec token
     dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken)
@@ -47,9 +57,8 @@ def build_params(params: typing.Dict) -> str:
     return query_string
 
 
-# TODO: remove env parameter
 def get_data_from_dynatrace(
-    throttling: float | int,
+    throttling_rate: float | int,
     token: str,
     env_url: str,
     params: typing.Dict | str,
@@ -69,9 +78,7 @@ def get_data_from_dynatrace(
         typing.Dict: Returns the response as
     """
 
-    # TODO: add nextpage key feature
-
-    time.sleep(throttling)
+    time.sleep(throttling_rate)
 
     if type(params) is dict:
         params_string = f"?{build_params(params)}"
@@ -91,10 +98,22 @@ def get_data_from_dynatrace(
         print(f"ERROR - {host_response.status_code}")
 
 
-def previous_week_range(date: datetime):
-    start_date = date + timedelta(-date.weekday(), weeks=-1)
-    end_date = date + timedelta(-date.weekday() - 1)
-    return start_date, end_date
+def check_if_service_already_exists(services: list, entity_id: str) -> bool:
+    """
+    Requests point to the same service. This leads to double entries but we only need the data once.
+
+    Args:
+        services (list): List with services
+        entity_id (str): Entity Id for lookup
+
+    Returns:
+        bool: Returns True if the service is already present else False.
+    """
+    result = False
+    for service in services:
+        if service["entityId"] == entity_id:
+            result = True
+    return result
 
 
 def get_process_group_data(df: pd.DataFrame) -> typing.Dict:
@@ -129,19 +148,22 @@ def get_process_group_data(df: pd.DataFrame) -> typing.Dict:
     for hub in unique_hubs:
         unique_process_groups_per_hub[hub] = {}
 
-        hub_value = hub
-        process_groups_unique = df.query(f"environment == @hub_value")
+        # hub_value = hub
+        process_groups_unique = df.query(f"environment == @hub")
 
         process_groups_unique = process_groups_unique["process_group_id"].unique()
         for process_group in process_groups_unique:
             params = {
                 "entitySelector": f'type("PROCESS_GROUP"),entityId("{process_group}")',
-                "fields": "firstSeenTms,tags",
+                "fields": "firstSeenTms,lastSeenTms,tags",
             }
             data = get_data_from_dynatrace(
                 0.1, hub_data[hub]["token"], hub_data[hub]["url"], params, "entities"
             )
-            unique_process_groups_per_hub[hub][process_group] = data["entities"]
+            if data is not None:
+                unique_process_groups_per_hub[hub][process_group] = data["entities"]
+            else:
+                print(f"{process_group} returned None")
 
     return unique_process_groups_per_hub
 
@@ -181,16 +203,25 @@ def build_dataframe_for_report(report_items: typing.Dict) -> pd.DataFrame:
 
 
 def write_xlsx(df: pd.DataFrame) -> None:
+    """
+    Takes in a pandas dataframe and generates writes it into a XLSX file
+
+    Args:
+        df (pd.DataFrame): Dataframe containing the necessary data for the report
+    """
     filename = f"CoCo-APM-Report_{datetime.date.today()}.xlsx"
     writer = pd.ExcelWriter(filename, engine="xlsxwriter")
     df.to_excel(writer, sheet_name="hosts", index=False)
     writer.close()
 
 
-def develop_load_json():
-    with open("test-data-with-hosts-main.json", "r") as f:
-        data = json.loads(f.read())
+def build_dataframe_data(data: typing.Dict) -> None:
+    """
+    This function builds the data for the dataframe, which will be used to generate the report. Contains all data but process_groups.
 
+    Args:
+        data (typing.Dict): Takes in the dictionary containing all the raw data from dynatrace.
+    """
     df_data = []
 
     for hub in data:
@@ -200,6 +231,27 @@ def develop_load_json():
                 for service in data[hub][slo]["services"]:
                     if len(service["entities"]) > 0:
                         for entity in service["entities"]:
+                            # get compass id of service here. in tags
+                            compass_id_service = []
+                            if "tags" in entity:
+                                for tag in entity["tags"]:
+                                    if tag["key"] == "compass-id":
+                                        compass_id_service.append(tag["value"])
+                            compass_id_service = ",".join(compass_id_service)
+                            # get container name here
+                            container_name = "None"
+                            if "properties" in entity:
+                                if "softwareTechnologies" in entity["properties"]:
+                                    for technology in entity["properties"][
+                                        "softwareTechnologies"
+                                    ]:
+                                        if (
+                                            technology["type"] == "DOCKER"
+                                            or technology["type"] == "CONTAINERD"
+                                        ):
+                                            container_name = entity["properties"][
+                                                "detectedName"
+                                            ]
                             if "fromRelationships" in entity:
                                 if "runsOnHost" in entity["fromRelationships"]:
                                     for host in entity["fromRelationships"][
@@ -210,124 +262,45 @@ def develop_load_json():
                                             "host_name": host["details"]["displayName"],
                                             "host_id": host["id"],
                                             "environment": hub,
-                                            "process_group_id": "NaN",
-                                            "process_group_name": "NaN",
-                                            "licensing_tag_host": "NaN",
-                                            "licensing_tag_process_group": "NaN",
-                                            "first_seen_process_group": "NaN",
+                                            "container_name": container_name,
+                                            "process_group_id": "",
+                                            "process_group_name": "",
+                                            "licensing_tag_host": "",
+                                            "licensing_tag_process_group": "",
+                                            "first_seen_process_group": "",
                                             "first_seen_host": host["details"][
                                                 "firstSeenTms"
                                             ],
+                                            "last_seen_host": host["details"][
+                                                "lastSeenTms"
+                                            ],
+                                            "compass_id_host": "",
+                                            "compass_id_service": compass_id_service,
                                         }
 
+                                        compass_id = []
+                                        namespace = []
+
                                         for tag in host["details"]["tags"]:
                                             if tag["key"] == "Platform":
                                                 df_data_item["platform"] = tag["value"]
                                             if tag["key"] == "Namespace":
-                                                df_data_item["namespace"] = tag["value"]
+                                                # df_data_item["namespace"] = tag["value"]
+                                                namespace.append(tag["value"])
                                             if tag["key"] == "PaaS":
                                                 df_data_item["paas"] = tag["value"]
+                                            if tag["key"] == "compass-id":
+                                                # df_data_item["compass_id"] = tag[
+                                                #     "value"
+                                                # ]
+                                                if "value" in tag:
+                                                    compass_id.append(tag["value"])
 
-                                        # TODO: rework
-                                        if "runsOn" in entity["fromRelationships"]:
-                                            for process_group in entity[
-                                                "fromRelationships"
-                                            ]["runsOn"]:
-                                                df_data_item[
-                                                    "process_group_id"
-                                                ] = process_group["id"]
+                                        df_data_item["compass_id_host"] = ",".join(
+                                            compass_id
+                                        )
 
-                                        df_data.append(df_data_item)
-
-    build_dataframe_for_report(df_data)
-
-    # with open("./environment.yaml") as file:
-    #     env_doc = yaml.safe_load(file)
-
-    # for env, doc in env_doc.items():
-    #     # DEBUG
-    #     if env == "euprod":
-    #         token = dict(doc[2])
-    #         url = dict(doc[1])
-
-    #         if config(token.get("env-token-name")) != "":
-    #             print("Gather data, hold on a minute")
-    #             DTTOKEN = config(token.get("env-token-name"))
-    #             DTURL = url.get("env-url")
-
-    #             for slo in data[env]:
-    #                 if len(data[env][slo]["services"]) == 0:
-    #                     # DEBUG
-    #                     print(f"ERROR: {slo} has no services")
-    #                 else:
-    #                     for service in data[env][slo]["services"]:
-    #                         params = {
-    #                             "entitySelector": f'type("SERVICE"),entityId("{service["entityId"]}")',
-    #                             "fields": "fromRelationships,tags",
-    #                         }
-    #                         entities = get_data_from_dynatrace(
-    #                             0.5, DTTOKEN, DTURL, params, "entities"
-    #                         )
-    #                         # TODO: it is possible that "entities" is empty. maybe create check.
-    #                         service["entities"] = entities["entities"]
-    #                         for hosts in service["entities"]:
-    #                             if "fromRelationships" in hosts:
-    #                                 if "runsOnHost" in hosts["fromRelationships"]:
-    #                                     for host in hosts["fromRelationships"][
-    #                                         "runsOnHost"
-    #                                     ]:
-    #                                         # TODO: make dynatrace call to /entites/{entityId}
-    #                                         host_response = get_data_from_dynatrace(
-    #                                             0.5, DTTOKEN, DTURL, host["id"], "entities"
-    #                                         )
-    #                                         host["details"] = host_response
-
-
-def check_if_service_already_exists(services: list, entity_id: str) -> bool:
-    result = False
-    for service in services:
-        if service["entityId"] == entity_id:
-            result = True
-    return result
-
-
-def build_data_frame_data(data: typing.Dict) -> None:
-    df_data = []
-
-    for hub in data:
-        for slo in data[hub]:
-            slo_name = data[hub][slo]["sloname"]
-            if len(data[hub][slo]["services"]) > 0:
-                for service in data[hub][slo]["services"]:
-                    if len(service["entities"]) > 0:
-                        for entity in service["entities"]:
-                            if "fromRelationships" in entity:
-                                if "runsOnHost" in entity["fromRelationships"]:
-                                    for host in entity["fromRelationships"][
-                                        "runsOnHost"
-                                    ]:
-                                        df_data_item = {
-                                            "slo_name": slo_name,
-                                            "host_name": host["details"]["displayName"],
-                                            "host_id": host["id"],
-                                            "environment": hub,
-                                            "process_group_id": "NaN",
-                                            "process_group_name": "NaN",
-                                            "licensing_tag_host": "NaN",
-                                            "licensing_tag_process_group": "NaN",
-                                            "first_seen_process_group": "NaN",
-                                            "first_seen_host": host["details"][
-                                                "firstSeenTms"
-                                            ],
-                                        }
-
-                                        for tag in host["details"]["tags"]:
-                                            if tag["key"] == "Platform":
-                                                df_data_item["platform"] = tag["value"]
-                                            if tag["key"] == "Namespace":
-                                                df_data_item["namespace"] = tag["value"]
-                                            if tag["key"] == "PaaS":
-                                                df_data_item["paas"] = tag["value"]
+                                        df_data_item["namespace"] = ",".join(namespace)
 
                                         # TODO: rework
                                         if "runsOn" in entity["fromRelationships"]:
@@ -345,6 +318,10 @@ def build_data_frame_data(data: typing.Dict) -> None:
 
 @timer
 def main() -> None:
+    """
+    Entrypoint.
+    """
+    throttling_rate: int | float = 0  # only tested with 0.5
     reportItem = {}
     with open("./environment.yaml") as file:
         env_doc = yaml.safe_load(file)
@@ -361,36 +338,28 @@ def main() -> None:
             # krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN)
 
             slosF = get_slo(env, DTTOKEN, DTURL)
+            # slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"]
             slosF = slosF[slosF["name"].str.startswith("TP_")]
 
             # parse the metric Expression to get Services and Requests
 
             krs = []
+            # krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+
             krp = krparser.KRParser(
-                krparser.KROption.VALIDATE_EXISTS
-                # | krparser.KROption.VALIDATE_HASDATA
-                | krparser.KROption.RESOLVESERVICES,
-                DTURL,
-                DTTOKEN,
+                name=env,
+                options=krparser.KROption.RESOLVESERVICES
+                | krparser.KROption.VALIDATE_HASDATA,
+                config={
+                    "threads": 10,
+                    "serviceLookupParams": {"fields": "tags,fromRelationships"},
+                    "extendResultObjects": {"env": env},
+                },
+                DTAPIURL=DTURL,
+                DTAPIToken=DTTOKEN,
             )
 
-            for index, row in slosF.iterrows():
-                krs.append(krp.parseBySLO(row))
-
-            # x = 0
-
-            # SLO Name | SERVICE | PROCESS GRUPPE | TAGS
-            #                   {"sloname": {
-            #                     "sloname":$sloname$,
-            #                     "services":[{
-            #                                    "serviceName": "$servicename$"
-            #                                  }]
-            #                   },
-            #                   "sloname": {
-            #                     "sloname":$sloname$,
-            #                     "services":[{
-            #                                    "serviceName": "$servicename$"
-            #                                  }]
+            krs = krp.parse(slosF)
 
             reportItem[str(env)] = {}
 
@@ -423,9 +392,7 @@ def main() -> None:
                             == 0
                         ):
                             if not check_if_service_already_exists(
-                                reportItem[str(env)][kr.metadata["sloName"]][
-                                    "services"
-                                ],
+                                reportItem[env][kr.metadata["sloName"]]["services"],
                                 service["entityId"],
                             ):
                                 reportItem[str(env)][kr.metadata["sloName"]][
@@ -437,6 +404,7 @@ def main() -> None:
                                         "entityId": service["entityId"],
                                     }
                                 )
+
                             if (
                                 len(
                                     reportItem[str(env)][kr.metadata["sloName"]][
@@ -455,11 +423,16 @@ def main() -> None:
                                 ]["services"]:
                                     params = {
                                         "entitySelector": f'type("SERVICE"),entityId("{service["entityId"]}")',
-                                        "fields": "fromRelationships,tags",
+                                        "fields": "fromRelationships,tags,properties",
                                     }
                                     entities = get_data_from_dynatrace(
-                                        0.5, DTTOKEN, DTURL, params, "entities"
+                                        throttling_rate,
+                                        DTTOKEN,
+                                        DTURL,
+                                        params,
+                                        "entities",
                                     )
+                                    # print(entities["entities"])
                                     # TODO: it is possible that "entities" is empty. maybe create check.
                                     service["entities"] = entities["entities"]
                                     for hosts in service["entities"]:
@@ -474,7 +447,7 @@ def main() -> None:
                                                     ]["runsOnHost"]:
                                                         host_response = (
                                                             get_data_from_dynatrace(
-                                                                0.5,
+                                                                throttling_rate,
                                                                 DTTOKEN,
                                                                 DTURL,
                                                                 host["id"],
@@ -482,12 +455,8 @@ def main() -> None:
                                                             )
                                                         )
                                                         host["details"] = host_response
-
-    build_data_frame_data(reportItem)
-    # with open("test-data-with-hosts-main.json", "w") as f:
-    #     f.write(json.dumps(reportItem, indent=4))
+    build_dataframe_data(reportItem)
 
 
 if __name__ == "__main__":
-    # main()
-    develop_load_json()
+    main()

environment.yaml

@@ -4,13 +4,13 @@ euprod:
     - env-url: "https://xxu26128.live.dynatrace.com"
     - env-token-name: "EUPROD_TOKEN_VAR"
     - jenkins: "https://jaws.bmwgroup.net/opapm/"
-naprod:
-    - name: "naprod"
-    - env-url: "https://wgv50241.live.dynatrace.com"
-    - env-token-name: "NAPROD_TOKEN_VAR"
-    - jenkins: "https://jaws.bmwgroup.net/opapm/"
-cnprod:
-    - name: "cnprod"
-    - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
-    - env-token-name: "CNPROD_TOKEN_VAR"
-    - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
+# naprod:
+#     - name: "naprod"
+#     - env-url: "https://wgv50241.live.dynatrace.com"
+#     - env-token-name: "NAPROD_TOKEN_VAR"
+#     - jenkins: "https://jaws.bmwgroup.net/opapm/"
+# cnprod:
+#     - name: "cnprod"
+#     - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+#     - env-token-name: "CNPROD_TOKEN_VAR"
+#     - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"

requirements.txt

@@ -2,4 +2,7 @@
 python-decouple
 requests
 pyyaml
-pandas
+pandas
+xlsxwriter
+jsonmerge
+git+https://atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git