arnaua
/
qm_report
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: arnaua:33eff7ba7d7d737617048fa5c27d5edb1f9d6bb0
Branches Tags
arnaua:main
arnaua:DailyReport
arnaua:feature/kpi-extension
arnaua:AddType
arnaua:OPMAAS-4035
arnaua:Version2
arnaua:Monthly_Report_Clone
arnaua:Version2_CurrentMonth
arnaua:feature/OPMAAS-4189
arnaua:DailyReport_99,5
arnaua:feature/datalakeReport
arnaua:DailyReport_SUMMARY
arnaua:Daily_Oracle
arnaua:DailyReport_Test_And
arnaua:master
arnaua:Version3
..
compare: arnaua:7799adf362b7d8dd263d959d2e998cda6e5ba417
Branches Tags
arnaua:main
arnaua:DailyReport
arnaua:feature/kpi-extension
arnaua:AddType
arnaua:OPMAAS-4035
arnaua:Version2
arnaua:Monthly_Report_Clone
arnaua:Version2_CurrentMonth
arnaua:feature/OPMAAS-4189
arnaua:DailyReport_99,5
arnaua:feature/datalakeReport
arnaua:DailyReport_SUMMARY
arnaua:Daily_Oracle
arnaua:DailyReport_Test_And
arnaua:master
arnaua:Version3

No commits in common. "33eff7ba7d7d737617048fa5c27d5edb1f9d6bb0" and "7799adf362b7d8dd263d959d2e998cda6e5ba417" have entirely different histories.
33eff7ba7d ... 7799adf362

3 changed files with 136 additions and 206 deletions
Show Stats Expand all files Collapse all files

6
Jenkinsfile vendored
View File

@ -38,7 +38,7 @@ def reportRetryCount = 0
//here comes the trigger according to crontabs - jenkins is in UTC //here comes the trigger according to crontabs - jenkins is in UTC
//triggers { triggers {
//every 1st of every month at 00:00 //every 1st of every month at 00:00
//cron('0 0 1 * *') //cron('0 0 1 * *')
@ -46,11 +46,11 @@ def reportRetryCount = 0
//cron('0 8 * * *') //cron('0 8 * * *')
//every monday at 06:00 //every monday at 06:00
//cron('0 5 * * 0-7') cron('0 5 * * 0-7')
//parameterizedCron('0 10 * * * %PRESELECT=week;SLICES=tdhy') //parameterizedCron('0 10 * * * %PRESELECT=week;SLICES=tdhy')
//} }
environment { environment {
//ProxySettings //ProxySettings

333
createReport.py
View File

@ -5,21 +5,19 @@ import yaml
import datetime import datetime
import time import time
import pandas as pd import pandas as pd
# import requests #import requests
# import openpyxl #import openpyxl
import argparse import argparse
import warnings import warnings
import os import os
import dynatraceAPI import dynatraceAPI
from pagination import Pagionation from pagination import Pagionation
from KRParser import krparser
warnings.filterwarnings("ignore") warnings.filterwarnings("ignore")
# warning, there are warnings which are ignored! #warning, there are warnings which are ignored!
os.environ['TZ'] = 'Europe/Berlin' # set new timezone os.environ['TZ'] = 'Europe/Berlin' # set new timezone
time.tzset() time.tzset()
''' '''
@ -39,25 +37,21 @@ def make_request(url, headers,verify,parameters):
return response return response
''' '''
def previous_day_range(date): def previous_day_range(date):
start_date = date - datetime.timedelta(days=1) start_date = date - datetime.timedelta(days=1)
end_date = date - datetime.timedelta(days=1) end_date = date - datetime.timedelta(days=1)
return start_date, end_date return start_date, end_date
def previous_week_range(date): def previous_week_range(date):
start_date = date + datetime.timedelta(-date.weekday(), weeks=-1) start_date = date + datetime.timedelta(-date.weekday(), weeks=-1)
end_date = date + datetime.timedelta(-date.weekday() - 1) end_date = date + datetime.timedelta(-date.weekday() - 1)
return start_date, end_date return start_date, end_date
def previous_month_range(date): def previous_month_range(date):
end_date = date.replace(day=1) - datetime.timedelta(days=1) end_date = date.replace(day=1) - datetime.timedelta(days=1)
start_date = end_date.replace(day=1) start_date = end_date.replace(day=1)
return start_date, end_date return start_date, end_date
''' '''
def getSLO(DTAPIToken, DTENV, fromDate, toDate): def getSLO(DTAPIToken, DTENV, fromDate, toDate):
@ -112,108 +106,99 @@ def getSLO(DTAPIToken, DTENV, fromDate, toDate, selector_var, selector_type):
# DTAPIToken = sec token # DTAPIToken = sec token
dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken) dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken)
my_params_report = {'pageSize': 25, my_params_report = {'pageSize': 25,
'from': int(fromDate), 'from': int(fromDate),
'to': int(toDate), 'to': int(toDate),
'timeFrame': 'GTF', 'timeFrame': 'GTF',
'evaluate': "true", 'evaluate': "true",
# name = exact name, text = like # name = exact name, text = like
'sloSelector': f"""{selector_type}("{selector_var}")""" 'sloSelector': f"""{selector_type}("{selector_var}")"""
} }
# gets all slos and filter later # gets all slos and filter later
api_url_report = "/api/v2/slo" api_url_report = "/api/v2/slo"
pages = dtclient.returnPageination(api_url_report, my_params_report, "slo") pages = dtclient.returnPageination(api_url_report, my_params_report, "slo")
# only_wanted = [x for x in pages.elements if str.lower(selector) in str.lower(x['description'])] #only_wanted = [x for x in pages.elements if str.lower(selector) in str.lower(x['description'])]
df = pd.DataFrame(pages.elements) df = pd.DataFrame(pages.elements)
return df return df
def get_metric(DTAPIToken, DTENV, fromDate, toDate, metricExpression):
print(f"here: {len(metricExpression)}")
print(f"{metricExpression[:-1]}:timeshift(-7d))")
return metricExpression
def get_daily_slice(start_date, end_date): def get_daily_slice(start_date, end_date):
tempstart = start_date tempstart = start_date
days = pd.DataFrame() days = pd.DataFrame()
# Add the first day #Add the first day
tempend = tempstart + datetime.timedelta(hours=24) tempend = tempstart + datetime.timedelta(hours=24)
startms = time.mktime(tempstart.timetuple()) * 1000 startms = time.mktime(tempstart.timetuple()) * 1000
endms = time.mktime(tempend.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000
row = {'Date': tempstart, 'startTime': startms, 'endTime': endms}
days = days.append(row, ignore_index=True)
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
days = days.append(row,ignore_index=True)
while tempstart < end_date: while tempstart < end_date:
tempstart = tempstart + datetime.timedelta(hours=24) tempstart = tempstart + datetime.timedelta(hours=24)
tempend = tempstart + datetime.timedelta(hours=24) tempend = tempstart + datetime.timedelta(hours=24)
startms = time.mktime(tempstart.timetuple()) * 1000 startms = time.mktime(tempstart.timetuple()) * 1000
endms = time.mktime(tempend.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000
row = {'Date': tempstart, 'startTime': startms, 'endTime': endms} row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
days = days.append(row, ignore_index=True) days = days.append(row,ignore_index=True)
return days return days
def get_hourly_slice(start_date, end_date): def get_hourly_slice(start_date, end_date):
# date object to datetime #date object to datetime
tempstart = datetime.datetime( tempstart = datetime.datetime(start_date.year,start_date.month,start_date.day)
start_date.year, start_date.month, start_date.day)
#date object to datetime
# date object to datetime final_end = datetime.datetime.combine(end_date,datetime.datetime.max.time())
final_end = datetime.datetime.combine(
end_date, datetime.datetime.max.time())
hours = pd.DataFrame() hours = pd.DataFrame()
# Add the first slice #Add the first slice
tempend = tempstart + datetime.timedelta(hours=1) tempend = tempstart + datetime.timedelta(hours=1)
startms = time.mktime(tempstart.timetuple()) * 1000 startms = time.mktime(tempstart.timetuple()) * 1000
endms = time.mktime(tempend.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000
row = {'Date': tempstart, 'startTime': startms, 'endTime': endms}
hours = hours.append(row, ignore_index=True)
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
hours = hours.append(row,ignore_index=True)
while tempstart < final_end: while tempstart < final_end:
tempstart = tempstart + datetime.timedelta(hours=1) tempstart = tempstart + datetime.timedelta(hours=1)
tempend = tempstart + datetime.timedelta(hours=1) tempend = tempstart + datetime.timedelta(hours=1)
startms = time.mktime(tempstart.timetuple()) * 1000 startms = time.mktime(tempstart.timetuple()) * 1000
endms = time.mktime(tempend.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000
row = {'Date': tempstart, 'startTime': startms, 'endTime': endms} row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
hours = hours.append(row, ignore_index=True) hours = hours.append(row,ignore_index=True)
return hours return hours
def init_argparse(): def init_argparse():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
usage="%(prog)s [--fromDate] [toDate] or [preSelect]", usage="%(prog)s [--fromDate] [toDate] or [preSelect]",
description="gather SLO in daily slices for given Timeframe" description="gather SLO in daily slices for given Timeframe"
) )
parser.add_argument( parser.add_argument(
"-f", "--fromDate", "-f","--fromDate",
help="YYYY-mm-dd e.g. 2022-01-01" help = "YYYY-mm-dd e.g. 2022-01-01"
) )
parser.add_argument( parser.add_argument(
"-t", "--toDate", "-t","--toDate",
help="YYYY-mm-dd e.g. 2022-01-31" help = "YYYY-mm-dd e.g. 2022-01-31"
) )
parser.add_argument( parser.add_argument(
"-p", "--preSelect", "-p","--preSelect",
help="day | week | month - gathers the data for the last full day, week or month" help = "day | week | month - gathers the data for the last full day, week or month"
) )
parser.add_argument( parser.add_argument(
"-s", "--slices", "-s","--slices",
help="h | d | t | y - writes the slices hourly, daily, total or year to date into ecxel. given in any order" help = "h | d | t | y - writes the slices hourly, daily, total or year to date into ecxel. given in any order"
) )
return parser return parser
def check_inputs(args): def check_inputs(args):
''' '''
This functions is the single point of true for arguments. If new arguments are added they need to be added in here. Returns from and to date. This functions is the single point of true for arguments. If new arguments are added they need to be added in here. Returns from and to date.
@ -221,36 +206,36 @@ def check_inputs(args):
if args.preSelect and (args.fromDate or args.toDate): if args.preSelect and (args.fromDate or args.toDate):
print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate") print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate")
sys.exit() sys.exit()
elif args.fromDate and not args.toDate: elif args.fromDate and not args.toDate:
print("--fromDate only in conjunction with --toDate") print("--fromDate only in conjunction with --toDate")
sys.exit() sys.exit()
elif args.toDate and not args.fromDate: elif args.toDate and not args.fromDate:
print("--toDate only in conjunction with --fromDate") print("--toDate only in conjunction with --fromDate")
sys.exit() sys.exit()
elif args.toDate and args.fromDate and not args.preSelect: elif args.toDate and args.fromDate and not args.preSelect:
try: try:
# fromDate = datetime.date.fromisoformat(args.fromDate) #fromDate = datetime.date.fromisoformat(args.fromDate)
fromDate = datetime.datetime.strptime(args.fromDate, "%Y-%m-%d") fromDate = datetime.datetime.strptime(args.fromDate, "%Y-%m-%d")
# toDate = datetime.date.fromisoformat(args.toDate) #toDate = datetime.date.fromisoformat(args.toDate)
toDate = datetime.datetime.strptime(args.toDate, "%Y-%m-%d") toDate = datetime.datetime.strptime(args.toDate, "%Y-%m-%d")
except Exception as e: except Exception as e:
print("Progam closed: " + str(e)) print("Progam closed: " + str(e))
sys.exit() sys.exit()
if toDate < fromDate: if toDate < fromDate:
print("--toDate can't be older than --fromDate") print("--toDate can't be older than --fromDate")
sys.exit() sys.exit()
if toDate > datetime.date.today() or fromDate > datetime.date.today(): if toDate > datetime.date.today() or fromDate > datetime.date.today():
print("--toDate or --fromDate can't be in the future") print("--toDate or --fromDate can't be in the future")
sys.exit() sys.exit()
elif args.preSelect and not args.fromDate and not args.toDate: elif args.preSelect and not args.fromDate and not args.toDate:
date = datetime.date.today() date = datetime.date.today()
if args.preSelect == "week": if args.preSelect == "week":
@ -258,7 +243,7 @@ def check_inputs(args):
elif args.preSelect == "month": elif args.preSelect == "month":
fromDate, toDate = previous_month_range(date) fromDate, toDate = previous_month_range(date)
elif args.preSelect == "day": elif args.preSelect == "day":
fromDate, toDate = previous_day_range(date) fromDate, toDate = previous_day_range(date)
else: else:
print("--preSelect must be week or month") print("--preSelect must be week or month")
sys.exit() sys.exit()
@ -268,71 +253,53 @@ def check_inputs(args):
if args.slices == None: if args.slices == None:
print("-s or --slices must not be null and needs at least one letter of h d t or y, lower- or uppercase.") print("-s or --slices must not be null and needs at least one letter of h d t or y, lower- or uppercase.")
sys.exit() sys.exit()
elif sum([1 if one_inp in str.lower(args.slices) else 0 for one_inp in ['h', 'd', 't', 'y']]) == 0: elif sum([1 if one_inp in str.lower(args.slices) else 0 for one_inp in ['h','d','t','y'] ]) == 0:
print("-s or --slices must has at least one letter of h d t or y, lower- or uppercase.") print("-s or --slices must has at least one letter of h d t or y, lower- or uppercase.")
sys.exit() sys.exit()
return fromDate, toDate return fromDate, toDate
def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_type): def get_one_slice(item, DTTOKEN, DTURL, slice, out_df, selector_var, selector_type):
# Calc daily SLO ###Calc daily SLO
df = pd.DataFrame() df = pd.DataFrame()
for index, row in slice.iterrows(): for index, row in slice.iterrows():
num_probs = len(slice) num_probs = len(slice)
percentage = str(round((100*(index+1))/num_probs, 2)).split(".") percentage = str(round((100*(index+1))/num_probs, 2)).split(".")
print("{:0>4d} von {:0>4d} = {:0>3d}.{:0>2d} %".format( print("{:0>4d} von {:0>4d} = {:0>3d}.{:0>2d} %".format(index+1, num_probs, int(percentage[0]), int(percentage[1]) ), end='\r')
index+1, num_probs, int(percentage[0]), int(percentage[1])), end='\r') temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime'], selector_var, selector_type)
temp_df = getSLO( temp_df['Date'] = row['Date']
DTTOKEN, DTURL, row['startTime'], row['endTime'], selector_var, selector_type) temp_df['HUB'] = item
temp_df['Date'] = row['Date'] df = pd.concat([df,temp_df],ignore_index=True)
temp_df['HUB'] = item
#sort columns in a try block - if API is returning columns which are non exist, this will not fail the script
try:
df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
except Exception as e:
print(f"This error was encounterted : {e}")
try:
df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
except Exception as e:
print("Could not rearrange columns: " + str(e))
out_df = pd.concat([out_df,df],ignore_index=True)
print() # newline to remove \r from progress bar
return out_df
# new metric expression def get_slice_ytd_total(DTTOKEN,DTURL,item, start_date, end_date, time_name, time_val, out_df, selector_var, selector_type):
# temp_df["newMetricExpression"] = get_metric( df = getSLO(DTTOKEN,DTURL,start_date,end_date, selector_var, selector_type)
# DTTOKEN, DTURL, row["startTime"], row["endTime"], temp_df["metricExpression"])
# print(temp_df)
temp_df["newMetricExpression"] = temp_df["metricExpression"].apply(
lambda x: get_metric(DTTOKEN, DTURL, row["startTime"], row["endTime"], x))
df = pd.concat([df, temp_df], ignore_index=True)
# sort columns in a try block - if API is returning columns which are non exist, this will not fail the script
try:
df[['description', 'Touchpoint']
] = df['description'].str.split('_', expand=True)
except Exception as e:
print(f"This error was encounterted : {e}")
try:
df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget',
'status', 'error', 'target', 'warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
except Exception as e:
print("Could not rearrange columns: " + str(e))
out_df = pd.concat([out_df, df], ignore_index=True)
print() # newline to remove \r from progress bar
return out_df
def get_slice_ytd_total(DTTOKEN, DTURL, item, start_date, end_date, time_name, time_val, out_df, selector_var, selector_type):
df = getSLO(DTTOKEN, DTURL, start_date, end_date,
selector_var, selector_type)
df[time_name] = time_val df[time_name] = time_val
df['HUB'] = item df['HUB'] = item
try: try:
df[['description', 'Touchpoint'] df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
] = df['description'].str.split('_', expand=True)
except Exception as e: except Exception as e:
print(f"This error was encounterted : {e}") print(f"This error was encounterted : {e}")
try: try:
df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description','Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
'status', 'error', 'target', 'warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
except Exception as e: except Exception as e:
print("Could not rearrange columns: " + str(e)) print("Could not rearrange columns: " + str(e))
out_df = pd.concat([out_df, df], ignore_index=True) out_df = pd.concat([out_df,df],ignore_index=True)
return out_df return out_df
def load_slo_parameter(path): def load_slo_parameter(path):
# the first part is to read a yaml and only select latest, valid config # the first part is to read a yaml and only select latest, valid config
mandatory_fields = ['hub', 'selector_type', 'selector_var', 'yearstart'] mandatory_fields = ['hub', 'selector_type', 'selector_var', 'yearstart']
@ -347,68 +314,54 @@ def load_slo_parameter(path):
for config_line in configs: for config_line in configs:
tmp_dict.update(config_line) tmp_dict.update(config_line)
if sorted(list(tmp_dict.keys())) == mandatory_fields: if sorted(list(tmp_dict.keys())) == mandatory_fields:
# python 3.7+ #python 3.7+
# yearstart = datetime.date.fromisoformat(tmp_dict['yearstart']) #yearstart = datetime.date.fromisoformat(tmp_dict['yearstart'])
# python <3.7 #python <3.7
yearstart = datetime.datetime.strptime( yearstart = datetime.datetime.strptime(tmp_dict['yearstart'], "%Y-%m-%d")
tmp_dict['yearstart'], "%Y-%m-%d")
#common code
# common code yearstart = datetime.datetime(yearstart.year, yearstart.month, yearstart.day)
yearstart = datetime.datetime(
yearstart.year, yearstart.month, yearstart.day)
yearstart = time.mktime(yearstart.timetuple()) * 1000 yearstart = time.mktime(yearstart.timetuple()) * 1000
# name if exact name is wanted selector_type = tmp_dict['selector_type'] # name if exact name is wanted
selector_type = tmp_dict['selector_type']
selector_var = tmp_dict['selector_var'] selector_var = tmp_dict['selector_var']
hub = tmp_dict['hub'] hub = tmp_dict['hub']
all_yaml_configs.append( all_yaml_configs.append([hub, selector_type, selector_var, yearstart, header_name])
[hub, selector_type, selector_var, yearstart, header_name])
else: else:
print(f"Slo Configuration {header_name} is broken") print(f"Slo Configuration {header_name} is broken")
return all_yaml_configs return all_yaml_configs
def write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd): def write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd):
touchpoints = ['Vehicle', 'Mobile'] touchpoints = ['Vehicle' , 'Mobile']
if args.preSelect == 'day': if args.preSelect == 'day':
today = datetime.date.today() today = datetime.date.today()
yesterday = today - datetime.timedelta(days=1) yesterday = today - datetime.timedelta(days = 1)
fileName = "./QM_Report_" + str(yesterday) + ".xlsx" fileName = "./QM_Report_"+ str(yesterday) +".xlsx"
else: else:
fileName = "./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx" fileName = "./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx"
writer = pd.ExcelWriter(fileName) writer = pd.ExcelWriter(fileName)
workbook = writer.book
if not totalall.empty and 't' in str.lower(args.slices): if not totalall.empty and 't' in str.lower(args.slices):
totalall = totalall[totalall['Touchpoint'].isin(touchpoints)] totalall = totalall[totalall['Touchpoint'].isin(touchpoints)]
totalall.to_excel(writer, sheet_name='total', index=False) totalall.to_excel(writer, sheet_name='total')
worksheet = writer.sheets['total']
worksheet.autofilter(0, 0, totalall.shape[0], totalall.shape[1])
if not dailyall.empty and 'd' in str.lower(args.slices): if not dailyall.empty and 'd' in str.lower(args.slices):
dailyall = dailyall[dailyall['Touchpoint'].isin(touchpoints)] dailyall = dailyall[dailyall['Touchpoint'].isin(touchpoints)]
dailyall.to_excel(writer, sheet_name='daily', index=False) dailyall.to_excel(writer, sheet_name='daily')
worksheet = writer.sheets['daily']
worksheet.autofilter(0, 0, dailyall.shape[0], dailyall.shape[1])
if not hourlyall.empty and 'h' in str.lower(args.slices): if not hourlyall.empty and 'h' in str.lower(args.slices):
hourlyall = hourlyall[hourlyall['Touchpoint'].isin(touchpoints)] hourlyall = hourlyall[hourlyall['Touchpoint'].isin(touchpoints)]
hourlyall.to_excel(writer, sheet_name='hourly', index=False) hourlyall.to_excel(writer, sheet_name='hourly')
worksheet = writer.sheets['hourly']
worksheet.autofilter(0, 0, hourlyall.shape[0], hourlyall.shape[1])
if not ytd.empty and 'y' in str.lower(args.slices): if not ytd.empty and 'y' in str.lower(args.slices):
ytd = ytd[ytd['Touchpoint'].isin(touchpoints)] ytd = ytd[ytd['Touchpoint'].isin(touchpoints)]
ytd.to_excel(writer, sheet_name='YTD', index=False) ytd.to_excel(writer, sheet_name='YTD')
worksheet = writer.sheets['YTD']
worksheet.autofilter(0, 0, ytd.shape[0], ytd.shape[1])
workbook.close()
writer.save() writer.save()
writer.close()
def main(slo_path): def main(slo_path):
start_timer = time.time() start_timer = time.time()
@ -419,74 +372,52 @@ def main(slo_path):
print("fromDate: " + str(fromDate)) print("fromDate: " + str(fromDate))
print("toDate: " + str(toDate)) print("toDate: " + str(toDate))
# days = get_daily_slice(fromDate,toDate)
days = get_daily_slice(fromDate, toDate) #days = get_daily_slice(fromDate,toDate)
hours = get_hourly_slice(fromDate, toDate) days = get_daily_slice(fromDate,toDate)
hours = get_hourly_slice(fromDate,toDate)
with open('./environment.yaml') as file: with open('./environment.yaml') as file:
env_doc = yaml.safe_load(file) env_doc = yaml.safe_load(file)
hourlyall = pd.DataFrame() hourlyall = pd.DataFrame()
dailyall = pd.DataFrame() dailyall = pd.DataFrame()
totalall = pd.DataFrame() totalall = pd.DataFrame()
ytd = pd.DataFrame() ytd = pd.DataFrame()
slo_configs = load_slo_parameter(slo_path) slo_configs = load_slo_parameter(slo_path)
for one_slo_config in slo_configs: for one_slo_config in slo_configs:
hub, selector_type, selector_var, yearstart, header_name = one_slo_config hub, selector_type, selector_var, yearstart, header_name = one_slo_config
print( print(f"For the slo config was '{slo_path}' used with the config '{header_name}'.")
f"For the slo config was '{slo_path}' used with the config '{header_name}'.")
for item, doc in env_doc.items(): for item, doc in env_doc.items():
if not item in hub: if not item in hub:
print( print(f"{item} will be skipped since it is not in {hub}, which was selected in {slo_path}")
f"{item} will be skipped since it is not in {hub}, which was selected in {slo_path}")
continue continue
token = dict(doc[2]) token = dict(doc[2])
url = dict(doc[1]) url = dict(doc[1])
print("Crawling through: " + item) print("Crawling through: " + item)
print("Check if token exists in environment...") print("Check if token exists in environment...")
if (config(token.get('env-token-name')) != ""): if(config(token.get('env-token-name')) != ""):
print("Gather data, hold on a minute") print("Gather data, hold on a minute")
DTTOKEN = config(token.get('env-token-name')) DTTOKEN = config(token.get('env-token-name'))
DTURL = url.get('env-url') DTURL = url.get('env-url')
# key request parser start ###Calc daily SLO
krp = krparser.KRParser(
name=item,
options=krparser.KROption.RESOLVESERVICES,
config={
"threads": 10,
"serviceLookupParams": {"fields": "tags,fromRelationships"},
"extendResultObjects": {"env": item},
},
DTAPIURL=DTURL,
DTAPIToken=DTTOKEN,
)
# TODO: Pass krp down to parse Slos and get metrics via /metrics/query
# Calc daily SLO
if 'd' in str.lower(args.slices): if 'd' in str.lower(args.slices):
dailyall = get_one_slice( dailyall = get_one_slice(item, DTTOKEN, DTURL, days, dailyall, selector_var, selector_type)
item, DTTOKEN, DTURL, days, dailyall, selector_var, selector_type) #Calc hourly SLO
# Calc hourly SLO
if 'h' in str.lower(args.slices): if 'h' in str.lower(args.slices):
hourlyall = get_one_slice( hourlyall = get_one_slice(item, DTTOKEN, DTURL, hours, hourlyall, selector_var, selector_type)
item, DTTOKEN, DTURL, hours, hourlyall, selector_var, selector_type) ###Calc Overall YTD SLO
# Calc Overall YTD SLO
if 'y' in str.lower(args.slices): if 'y' in str.lower(args.slices):
ytd = get_slice_ytd_total(DTTOKEN, DTURL, item, yearstart, days['endTime'].max( ytd = get_slice_ytd_total(DTTOKEN,DTURL,item, yearstart, days['endTime'].max(), 'Date', fromDate.year, ytd, selector_var, selector_type)
), 'Date', fromDate.year, ytd, selector_var, selector_type) ###Calc Overall SLO
# Calc Overall SLO
if 't' in str.lower(args.slices): if 't' in str.lower(args.slices):
totalall = get_slice_ytd_total(DTTOKEN, DTURL, item, days['startTime'].min(), days['endTime'].max( totalall = get_slice_ytd_total(DTTOKEN,DTURL,item, days['startTime'].min(), days['endTime'].max(), 'Date', fromDate.isocalendar()[1], totalall, selector_var, selector_type)
), 'Date', fromDate.isocalendar()[1], totalall, selector_var, selector_type) else:
else:
print("token not found, skipping " + item) print("token not found, skipping " + item)
write_slo_to_excel(args, fromDate, hourlyall, write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd)
dailyall, totalall, ytd)
print("It took {} seconds to run this script".format(time.time()-start_timer)) print("It took {} seconds to run this script".format(time.time()-start_timer))
if __name__ == "__main__": if __name__ == "__main__":
main('./slo_parameter.yaml') main('./slo_parameter.yaml')

3
requirements.txt
View File

@ -4,5 +4,4 @@ pandas
requests requests
datetime datetime
argparse argparse
openpyxl openpyxl
git+https://atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git