Fix for evaluate type change and timezone

.gitignore

@@ -1,3 +1,10 @@
+dashboard_tiles_*
+\[STAGING\]*
+<<<<<<< HEAD
+shared_configuration/
+archive/
+=======
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -133,5 +140,6 @@ dmypy.json
 crash.log
 *.tfvars
 
-#excel reports
+## Excel Reports
 *.xlsx
+>>>>>>> 746e496e7a7c5e8134cda7921311f6a9ba22f8d3

Jenkinsfile

@@ -130,10 +130,8 @@
                     try {
                         emailext subject: env.JOB_NAME,
                             body: 'Please find the output of the weekly QM-Report attached',
-                            //to: 'michaela.jaeger@bmw.de, OOC-Support@bmwgroup.com, Andreas.DA.Danzer@partner.bmw.de',
-                            to: 'rene.forstner@nttdata.com, Andreas.DA.Danzer@partner.bmw.de, linnea.bickeboeller@partner.bmwgroup.com',
-                            //to: 'rene.forstner@nttdata.com, stephan.oertelt@bmw.de, Mohammed.Abadel@bmw.de, michaela.jaeger@bmw.de',
-                            //to: 'rene.forstner@nttdata.com, stephan.oertelt@bmw.de, Mohammed.Abadel@bmw.de, michaela.jaeger@bmw.de, OOC-Support@bmwgroup.com, Sonja.Yildizoglu@bmw.de, Andreas.DA.Danzer@partner.bmw.de',
+                            to: 'OOC-Support@bmwgroup.com, coco-apm@bmw.de, BMW.CoCo.Dynatrace@nttdata.com'
+                            //to: 'rene.forstner@nttdata.com',
                             replyTo: 'coco-apm@bmw.de',
                             attachmentsPattern: '*.xlsx'
                 

createReport.py

@@ -9,14 +9,30 @@ import pandas as pd
 #import openpyxl
 import argparse
 import warnings  
-
+from git import Repo
 import dynatraceAPI
 from pagination import Pagionation
+import os
 
 warnings.filterwarnings("ignore")
 #warning, there are warnings which are ignored!
 
+os.environ['TZ'] = 'Europe/Berlin' # set new timezone
+time.tzset()
 
+AUTHSTRING = config("BITBUCKET_USERNAME")+":"+config("BITBUCKET_TOKEN")
+CONFIG_REPO_URL = "https://"+AUTHSTRING+"@atc.bmwgroup.net/bitbucket/scm/opapm/shared_configuration.git"
+CONFIG_REPO_NAME = "shared_configuration"
+
+def clone_repo_if_notexist(repourl, reponame):
+    if(not os.path.isdir(reponame)):
+        repo = Repo.clone_from(repourl, reponame)
+        return repo
+    return Repo(reponame)
+
+def pull_repo(repo):
+    origin = repo.remotes.origin
+    origin.pull()
 '''
 def make_request(url, headers,verify,parameters):
     try:
@@ -68,7 +84,7 @@ def getSLO(DTAPIToken, DTENV, fromDate, toDate):
 		"from": int(fromDate),
 		"to": int(toDate),
         "timeFrame": "GTF",
-		"evaluate": True,
+		"evaluate": "true",
 		"sloSelector": "text(\"CoCo-QM-Report\")"
 		}
     print(DTAPIURL)
@@ -101,7 +117,7 @@ def getSLO(DTAPIToken, DTENV, fromDate, toDate, selector_var, selector_type):
                     'from': int(fromDate),
                     'to': int(toDate),
                     'timeFrame': 'GTF',
-                    'evaluate': True,
+                    'evaluate': "true",
                     # name = exact name, text = like
                     'sloSelector': f"""{selector_type}("{selector_var}")"""
                     }
@@ -112,8 +128,6 @@ def getSLO(DTAPIToken, DTENV, fromDate, toDate, selector_var, selector_type):
     df = pd.DataFrame(pages.elements)
     return df
 
-
-
 def get_daily_slice(start_date, end_date):
     tempstart = start_date
     days = pd.DataFrame()
@@ -248,8 +262,7 @@ def check_inputs(args):
         sys.exit()
     return fromDate, toDate
 
-
-def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_type):
+def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_type, touchpoint):
         ###Calc daily SLO
         df = pd.DataFrame()
         for index, row in slice.iterrows():
@@ -259,13 +272,14 @@ def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_ty
             temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime'], selector_var, selector_type)
             temp_df['Date'] = row['Date']
             temp_df['HUB'] = item
+            temp_df['Touchpoint'] = touchpoint
             df = pd.concat([df,temp_df],ignore_index=True)
         
         #sort columns in a try block - if API is returning columns which are non exist, this will not fail the script
-        try:
-            df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
-        except Exception as e:
-            print(f"This error was encounterted : {e}")
+        #try:
+        #    df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
+        #except Exception as e:
+        #    print(f"This error was encounterted : {e}")
         try:
             df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
         except Exception as e:
@@ -274,14 +288,15 @@ def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_ty
         print() # newline to remove \r from progress bar
         return out_df
 
-def get_slice_ytd_total(DTTOKEN,DTURL,item, start_date, end_date, time_name, time_val, out_df, selector_var, selector_type):
+def get_slice_ytd_total(DTTOKEN,DTURL,item, start_date, end_date, time_name, time_val, out_df, selector_var, selector_type, touchpoint):
     df = getSLO(DTTOKEN,DTURL,start_date,end_date, selector_var, selector_type)
     df[time_name] = time_val
     df['HUB'] = item
-    try:
-        df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
-    except Exception as e:
-        print(f"This error was encounterted : {e}")
+    df['Touchpoint'] = touchpoint
+    #try:
+    #    df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
+    #except Exception as e:
+    #    print(f"This error was encounterted : {e}")
     try:
         df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description','Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
     except Exception as e:
@@ -294,35 +309,28 @@ def load_slo_parameter(path):
     # the first part is to read a yaml and only select latest, valid config
     mandatory_fields = ['hub', 'selector_type', 'selector_var', 'yearstart']
     all_yaml_configs = []
+    
     with open(path) as file:
         slo_doc = yaml.safe_load(file)
-    for header_name, configs in slo_doc.items():
-        tmp_dict = {}
-        if not len(slo_doc[header_name]) == 4:
-            print(f"Slo Configuration {header_name} is broken")
-            continue
-        for config_line in configs:
-            tmp_dict.update(config_line)
-        if sorted(list(tmp_dict.keys())) == mandatory_fields:
-            #python 3.7+
-            #yearstart = datetime.date.fromisoformat(tmp_dict['yearstart'])
-            #python <3.7
     
-            yearstart = datetime.datetime.strptime(tmp_dict['yearstart'], "%Y-%m-%d")
+    for header_name, config in slo_doc.items():
         
-            #common code
-            yearstart = datetime.datetime(yearstart.year, yearstart.month, yearstart.day)
-            yearstart = time.mktime(yearstart.timetuple()) * 1000
+        selector_type = config["selector_type"]
+        selector_var = config["selector_var"]
+        yearstart = datetime.datetime.strptime(config['yearstart'], "%Y-%m-%d")
+        yearstart = datetime.datetime(yearstart.year, yearstart.month, yearstart.day)
+        yearstart = time.mktime(yearstart.timetuple()) * 1000
+        touchpoint = config["touchpoint"]
+        #this is quite dirty, hubs needs to be listed comma separated to be processed in this scritp
+        hubs = ""
+        if len(config["hubs"]) > 0:
+            for hub in config["hubs"]:
+                hubs = hubs + hub +","
             
-            selector_type = tmp_dict['selector_type'] # name if exact name is wanted
-            selector_var = tmp_dict['selector_var']
-            hub = tmp_dict['hub']
-            all_yaml_configs.append([hub, selector_type, selector_var, yearstart, header_name])
-        else:
-            print(f"Slo Configuration {header_name} is broken")
+            all_yaml_configs.append([hubs[:-1], selector_type, selector_var, yearstart, header_name, touchpoint])
+    print(all_yaml_configs)
     return all_yaml_configs
 
-
 def write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd):
     touchpoints = ['Vehicle' , 'Mobile']
     writer = pd.ExcelWriter("./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx")
@@ -355,6 +363,8 @@ def main(slo_path):
     print("fromDate: " + str(fromDate))
     print("toDate: " + str(toDate))
 
+    configrepo = clone_repo_if_notexist(CONFIG_REPO_URL, CONFIG_REPO_NAME)
+    pull_repo(configrepo)
 
     #days = get_daily_slice(fromDate,toDate)
     days = get_daily_slice(fromDate,toDate)
@@ -370,7 +380,7 @@ def main(slo_path):
     slo_configs = load_slo_parameter(slo_path)
     
     for one_slo_config in slo_configs:
-        hub, selector_type, selector_var, yearstart, header_name = one_slo_config
+        hub, selector_type, selector_var, yearstart, header_name, touchpoint = one_slo_config
         print(f"For the slo config was '{slo_path}' used with the config '{header_name}'.")
         for item, doc in env_doc.items():
             if not item in hub:
@@ -387,20 +397,20 @@ def main(slo_path):
                 
                 ###Calc daily SLO
                 if 'd' in str.lower(args.slices):
-                    dailyall = get_one_slice(item, DTTOKEN, DTURL, days, dailyall, selector_var, selector_type)
+                    dailyall = get_one_slice(item, DTTOKEN, DTURL, days, dailyall, selector_var, selector_type, touchpoint)
                 #Calc hourly SLO
                 if 'h' in str.lower(args.slices):
-                    hourlyall = get_one_slice(item, DTTOKEN, DTURL, hours, hourlyall, selector_var, selector_type)
+                    hourlyall = get_one_slice(item, DTTOKEN, DTURL, hours, hourlyall, selector_var, selector_type, touchpoint)
             ###Calc Overall YTD SLO
                 if 'y' in str.lower(args.slices):
-                    ytd = get_slice_ytd_total(DTTOKEN,DTURL,item, yearstart, days['endTime'].max(), 'Date', fromDate.year, ytd, selector_var, selector_type)
+                    ytd = get_slice_ytd_total(DTTOKEN,DTURL,item, yearstart, days['endTime'].max(), 'Date', fromDate.year, ytd, selector_var, selector_type, touchpoint)
             ###Calc Overall SLO
                 if 't' in str.lower(args.slices):
-                    totalall = get_slice_ytd_total(DTTOKEN,DTURL,item, days['startTime'].min(), days['endTime'].max(), 'Date', fromDate.isocalendar()[1], totalall, selector_var, selector_type)
+                    totalall = get_slice_ytd_total(DTTOKEN,DTURL,item, days['startTime'].min(), days['endTime'].max(), 'Date', fromDate.isocalendar()[1], totalall, selector_var, selector_type, touchpoint)
             else: 
                 print("token not found, skipping " + item)
             write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd)
     print("It took {} seconds to run this script".format(time.time()-start_timer))
 
 if __name__ == "__main__":
-    main('./slo_parameter.yaml')
+    main('./shared_configuration/slo_parameter.yaml')

readme.md

@@ -0,0 +1,39 @@
+# QM Report
+This repository holds the code to generate the Dynatrace QM Report.
+
+# shared configuration
+This script is using the following shared configuration:
+ - slo_parameter.yaml --> https://atc.bmwgroup.net/bitbucket/projects/OPAPM/repos/shared_configuration/browse
+
+# Files
+
+## createReport.py
+
+This script creates the repot as an .xlsx file with the following parameter
+  - -h, --help            show this help message and exit
+  - -f FROMDATE, --fromDate FROMDATE YYYY-mm-dd e.g. 2022-01-01
+  - -t TODATE, --toDate TODATE YYYY-mm-dd e.g. 2022-01-31
+  - -p PRESELECT, --preSelect PRESELECT  week | month - gathers the data for the last full week or month 
+  - -s SLICES, --slices SLICES  h | d | t | y - writes the slices hourly, daily, total or year to date into ecxel. given in any order
+
+Either -p with a given range (week or month) or the parameters -f AND -t are set
+
+The script reads the slo_parameters.yaml and crawls through all Dynatrace environments to gather the evaluated percentage and creates an excel report with one sheet for each "slice" (e.g. hourly sheet for hourly SLO evaluation, daily sheet for daily SLO evaluation etc.)
+
+## jenkinsfile
+The jenkinsfile enables this script to run within a Jenkinspipeline.
+It takes the same parameters as the python script and is currently implemented at https://jaws.bmwgroup.net/opapm/job/CoCo%20QM%20Report%202.0/
+
+Triggers
+The jenkinsfile as a timely trigger to run automatically every monday at 06:00 a.m
+
+Send Report
+Once the report is created, Jenkins sent the report as an attachement to the following recipients:
+to: 'rene.forstner@nttdata.com, ermis.wieger@nttdata.com, arnel.arnautovic@nttdata.com, patryk.gudalewicz.bp@nttdata.com, stephan.oertelt@bmw.de, Mohammed.Abadel@bmw.de, michaela.jaeger@bmw.de, OOC-Support@bmwgroup.com, Sonja.Yildizoglu@bmw.de, Andreas.DA.Danzer@partner.bmw.de',
+
+## dependencies
+The python dependencies are documented within the requirements.txt
+  - JAWS Jenkins (only standard libraries on JAWS)
+  - Up and running Bitbucket
+  - BMW SMPT Mail Server
+  - All PROD CoCo Dynatrace Environments

requirements.txt

@@ -6,3 +6,4 @@ datetime
 argparse
 openpyxl
 argparse
+git

slo_parameter.yaml

@@ -1,8 +1,9 @@
 ---
 TP_Mobile_Login:
     index: 1
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_Login'
     yearstart: "2022-01-01"
     displayname: "Login"
     department: "DE-442"
@@ -25,8 +26,9 @@ TP_Mobile_Login:
 
 TP_Mobile_Mapping:
     index: 2
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_Mapping'
     yearstart: "2022-01-01"
     displayname: "Mapping"
     department: "DE-443"
@@ -47,8 +49,9 @@ TP_Mobile_Mapping:
 
 TP_Mobile_VehicleList:
     index: 3
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_VehicleList'
     yearstart: "2022-01-01"
     displayname: "Vehicle List"
     department: "DE-43"
@@ -71,8 +74,9 @@ TP_Mobile_VehicleList:
 
 TP_Mobile_VehicleData:
     index: 4
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_VehicleData'
     yearstart: "2022-01-01"
     displayname: "Vehicle Data"
     department: "DE-733"
@@ -95,8 +99,9 @@ TP_Mobile_VehicleData:
 
 TP_Mobile_RemoteServices:
     index: 5
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_RemoteServices'
     yearstart: "2022-01-01"
     displayname: "Remote Services"
     department: "DE-723"
@@ -119,8 +124,9 @@ TP_Mobile_RemoteServices:
 
 TP_Mobile_Remote360:
     index: 6
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_Remote360'
     yearstart: "2022-01-01"
     displayname: "Remote Camera"
     department: "DE-723"
@@ -143,8 +149,9 @@ TP_Mobile_Remote360:
 
 TP_Mobile_Send2VehicleMGU:
     index: 7
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_Send2VehicleMGU'
     yearstart: "2022-01-01"
     displayname: "Send to Vehicle (MGU)"
     department: "DE-320"
@@ -167,8 +174,9 @@ TP_Mobile_Send2VehicleMGU:
   
 TP_Mobile_Send2VehicleLegacy:
     index: 8
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_Send2VehicleLegacy'
     yearstart: "2022-01-01"
     displayname: "Send to Vehicle (Legacy)"
     department: "DE-723"
@@ -191,8 +199,9 @@ TP_Mobile_Send2VehicleLegacy:
 
 TP_Mobile_PersonalFavorites:
     index: 9
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_PersonalFavorites'
     yearstart: "2022-01-01"
     displayname: "Personal Favorites"
     department: "DE-443"
@@ -215,8 +224,9 @@ TP_Mobile_PersonalFavorites:
 
 TP_Mobile_DigitalKey:
     index: 10
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_DigitalKey'
     yearstart: "2022-01-01"
     displayname: "Digital Key"
     department: "DE-730"
@@ -239,8 +249,9 @@ TP_Mobile_DigitalKey:
 
 TP_Mobile_RSU:
     index: 11
+    touchpoint: "Mobile"
     selector_type: "text"
-    selector_var: 'CoCo-QM-Report_Mobile'
+    selector_var: 'TP_Mobile_RSU'
     yearstart: "2022-01-01"
     displayname: "RSU Mobile"
     department: "DE-430"