322 lines
12 KiB
Python
322 lines
12 KiB
Python
from tracemalloc import start
|
|
from decouple import config
|
|
import sys
|
|
import yaml
|
|
import datetime
|
|
import time
|
|
import pandas as pd
|
|
import requests
|
|
import openpyxl
|
|
import argparse
|
|
import os
|
|
|
|
os.environ['TZ'] = 'Europe/Berlin' # set new timezone
|
|
time.tzset()
|
|
|
|
def make_request(url, headers,verify,parameters):
|
|
try:
|
|
response = requests.get(url, headers=headers,verify=verify,params=parameters)
|
|
response.raise_for_status()
|
|
except requests.exceptions.HTTPError as errh:
|
|
return "An Http Error occurred:" + repr(errh)
|
|
except requests.exceptions.ConnectionError as errc:
|
|
return "An Error Connecting to the API occurred:" + repr(errc)
|
|
except requests.exceptions.Timeout as errt:
|
|
return "A Timeout Error occurred:" + repr(errt)
|
|
except requests.exceptions.RequestException as err:
|
|
return "An Unknown Error occurred" + repr(err)
|
|
|
|
return response
|
|
|
|
def previous_week_range(date):
|
|
start_date = date + datetime.timedelta(-date.weekday(), weeks=-1)
|
|
end_date = date + datetime.timedelta(-date.weekday() - 1)
|
|
return start_date, end_date
|
|
|
|
def previous_month_range(date):
|
|
end_date = date.replace(day=1) - datetime.timedelta(days=1)
|
|
start_date = end_date.replace(day=1)
|
|
return start_date, end_date
|
|
|
|
def getSLO(DTAPIToken, DTENV, fromDate, toDate):
|
|
|
|
env = DTENV
|
|
DTAPIToken = DTAPIToken
|
|
|
|
if (DTENV.find('dynatracemgd') != -1):
|
|
verify=False
|
|
if (DTENV.find('dyna-synt') != -1):
|
|
verify=False
|
|
else:
|
|
verify=True
|
|
|
|
DTAPIURL= env + "/api/v2/slo"
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Api-Token ' + DTAPIToken
|
|
}
|
|
|
|
parameters = {
|
|
"pageSize": 25,
|
|
"from": int(fromDate),
|
|
"to": int(toDate),
|
|
"timeFrame": "GTF",
|
|
"evaluate": "true",
|
|
"sloSelector": "text(\"CoCo-QM-Report\")"
|
|
}
|
|
r = make_request(DTAPIURL,headers=headers,parameters=parameters,verify=verify)
|
|
try:
|
|
cont = r.json()
|
|
#python3
|
|
#df = pd.json_normalize(cont['slo'])
|
|
|
|
#python2
|
|
df = pd.DataFrame.from_dict(cont['slo'])
|
|
except Exception as e:
|
|
print(e)
|
|
print("No SLO json returned")
|
|
print(parameters)
|
|
print(r)
|
|
df = pd.DataFrame()
|
|
|
|
|
|
return df
|
|
|
|
def get_daily_slice(start_date, end_date):
|
|
tempstart = start_date
|
|
days = pd.DataFrame()
|
|
|
|
#Add the first day
|
|
tempend = tempstart + datetime.timedelta(hours=24)
|
|
startms = time.mktime(tempstart.timetuple()) * 1000
|
|
endms = time.mktime(tempend.timetuple()) * 1000
|
|
|
|
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
|
|
days = days.append(row,ignore_index=True)
|
|
|
|
while tempstart < end_date:
|
|
tempstart = tempstart + datetime.timedelta(hours=24)
|
|
tempend = tempstart + datetime.timedelta(hours=24)
|
|
|
|
|
|
startms = time.mktime(tempstart.timetuple()) * 1000
|
|
endms = time.mktime(tempend.timetuple()) * 1000
|
|
|
|
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
|
|
days = days.append(row,ignore_index=True)
|
|
|
|
return days
|
|
|
|
def get_hourly_slice(start_date, end_date):
|
|
|
|
#date object to datetime
|
|
tempstart = datetime.datetime(start_date.year,start_date.month,start_date.day)
|
|
|
|
#date object to datetime
|
|
final_end = datetime.datetime.combine(end_date,datetime.datetime.max.time())
|
|
hours = pd.DataFrame()
|
|
|
|
#Add the first slice
|
|
tempend = tempstart + datetime.timedelta(hours=1)
|
|
startms = time.mktime(tempstart.timetuple()) * 1000
|
|
endms = time.mktime(tempend.timetuple()) * 1000
|
|
|
|
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
|
|
hours = hours.append(row,ignore_index=True)
|
|
|
|
while tempstart < final_end:
|
|
|
|
tempstart = tempstart + datetime.timedelta(hours=1)
|
|
tempend = tempstart + datetime.timedelta(hours=1)
|
|
startms = time.mktime(tempstart.timetuple()) * 1000
|
|
endms = time.mktime(tempend.timetuple()) * 1000
|
|
|
|
row = {'Date':tempstart,'startTime':startms, 'endTime':endms}
|
|
hours = hours.append(row,ignore_index=True)
|
|
|
|
return hours
|
|
|
|
def init_argparse():
|
|
parser = argparse.ArgumentParser(
|
|
usage="%(prog)s [--fromDate] [toDate] or [preSelect]",
|
|
description="gather SLO in daily slices for given Timeframe"
|
|
)
|
|
parser.add_argument(
|
|
"-f","--fromDate",
|
|
help = "YYYY-mm-dd e.g. 2022-01-01"
|
|
)
|
|
parser.add_argument(
|
|
"-t","--toDate",
|
|
help = "YYYY-mm-dd e.g. 2022-01-31"
|
|
)
|
|
parser.add_argument(
|
|
"-p","--preSelect",
|
|
help = "week | month - gathers the data for the last full week or month"
|
|
)
|
|
|
|
return parser
|
|
|
|
def main():
|
|
|
|
parser = init_argparse()
|
|
args = parser.parse_args()
|
|
|
|
if args.preSelect and (args.fromDate or args.toDate):
|
|
print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate")
|
|
sys.exit()
|
|
|
|
elif args.fromDate and not args.toDate:
|
|
print("--fromDate only in conjunction with --toDate")
|
|
sys.exit()
|
|
|
|
elif args.toDate and not args.fromDate:
|
|
print("--toDate only in conjunction with --fromDate")
|
|
sys.exit()
|
|
|
|
elif args.toDate and args.fromDate and not args.preSelect:
|
|
try:
|
|
fromDate = datetime.date.fromisoformat(args.fromDate)
|
|
toDate = datetime.date.fromisoformat(args.toDate)
|
|
except Exception as e:
|
|
print("Progam closed: " + str(e))
|
|
sys.exit()
|
|
|
|
if toDate < fromDate:
|
|
print("--toDate can't be older than --fromDate")
|
|
sys.exit()
|
|
|
|
if toDate > datetime.date.today() or fromDate > datetime.date.today():
|
|
print("--toDate or --fromDate can't be in the future")
|
|
sys.exit()
|
|
|
|
elif args.preSelect and not args.fromDate and not args.toDate:
|
|
|
|
date = datetime.date.today()
|
|
|
|
if args.preSelect == "week":
|
|
fromDate, toDate = previous_week_range(date)
|
|
elif args.preSelect == "month":
|
|
fromDate, toDate = previous_month_range(date)
|
|
else:
|
|
print("--preSelect must be week or month")
|
|
sys.exit()
|
|
|
|
else:
|
|
print("Invalid arguments, please use --help")
|
|
sys.exit()
|
|
|
|
#Adding 2 hours to be UTC+2 on the pipeline
|
|
fromDate = fromDate + datetime.timedelta(hours=2)
|
|
toDate = toDate + datetime.timedelta(hours=2)
|
|
print("fromDate: " + str(fromDate))
|
|
print("toDate: " + str(toDate))
|
|
|
|
#days = get_daily_slice(fromDate,toDate)
|
|
days = get_daily_slice(fromDate,toDate)
|
|
hours = get_hourly_slice(fromDate,toDate)
|
|
with open('./environment.yaml') as file:
|
|
doc = yaml.safe_load(file)
|
|
|
|
hourlyall = pd.DataFrame()
|
|
dailyall = pd.DataFrame()
|
|
totalall = pd.DataFrame()
|
|
ytd = pd.DataFrame()
|
|
|
|
for item, doc in doc.items():
|
|
token = dict(doc[2])
|
|
url = dict(doc[1])
|
|
print("Crawling through: " + item)
|
|
print("Check if token exists in environment...")
|
|
if(config(token.get('env-token-name')) != ""):
|
|
print("Gather data, hold on a minute")
|
|
DTTOKEN = config(token.get('env-token-name'))
|
|
DTURL = url.get('env-url')
|
|
|
|
|
|
###Calc daily SLO
|
|
df = pd.DataFrame()
|
|
for index, row in days.iterrows():
|
|
temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime'])
|
|
temp_df['Date'] = row['Date']
|
|
temp_df['HUB'] = item
|
|
df = pd.concat([df,temp_df],ignore_index=True)
|
|
|
|
#sort columns in a try block - if API is returning columns which are non exist, this will not fail the script
|
|
|
|
df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
|
|
try:
|
|
df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
|
|
except Exception as e:
|
|
print("Could not rearrange columns: " + str(e))
|
|
dailyall = pd.concat([dailyall,df],ignore_index=True)
|
|
|
|
#Calc hourly SLO
|
|
#if (args.preSelect == "week"):
|
|
df = pd.DataFrame()
|
|
for index, row in hours.iterrows():
|
|
temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime'])
|
|
temp_df['Date'] = row['Date']
|
|
temp_df['HUB'] = item
|
|
df = pd.concat([df,temp_df],ignore_index=True)
|
|
|
|
#sort columns in a try block - if API is returning columns which are non exist, this will not fail the script
|
|
|
|
df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
|
|
try:
|
|
df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
|
|
except Exception as e:
|
|
print("Could not rearrange columns: " + str(e))
|
|
hourlyall = pd.concat([hourlyall,df],ignore_index=True)
|
|
|
|
###Calc Overall YTD SLO
|
|
yearstart = toDate.replace(month=2,day=28)
|
|
yearstart = datetime.datetime(yearstart.year, yearstart.month, yearstart.day)
|
|
yearstart = time.mktime(yearstart.timetuple()) * 1000
|
|
df = getSLO(DTTOKEN,DTURL,yearstart,days['endTime'].max())
|
|
df['Year'] = fromDate.year
|
|
df['HUB'] = item
|
|
df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
|
|
try:
|
|
df = df[['Year', 'HUB', 'id', 'enabled', 'name', 'description','Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
|
|
except Exception as e:
|
|
print("Could not rearrange columns: " + str(e))
|
|
|
|
ytd = pd.concat([ytd,df],ignore_index=True)
|
|
|
|
|
|
###Calc Overall SLO
|
|
df = getSLO(DTTOKEN,DTURL,days['startTime'].min(),days['endTime'].max())
|
|
df['CW'] = fromDate.isocalendar()[1]
|
|
df['HUB'] = item
|
|
df[['description','Touchpoint']] = df['description'].str.split('_',expand=True)
|
|
try:
|
|
df = df[['CW', 'HUB', 'id', 'enabled', 'name', 'description','Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']]
|
|
except Exception as e:
|
|
print("Could not rearrange columns: " + str(e))
|
|
|
|
totalall = pd.concat([totalall,df],ignore_index=True)
|
|
|
|
else:
|
|
print("token not found, skipping " + item)
|
|
|
|
touchpoints = ['Vehicle' , 'Mobile']
|
|
|
|
if (args.preSelect == "week"):
|
|
hourlyall = hourlyall[hourlyall['Touchpoint'].isin(touchpoints)]
|
|
dailyall = dailyall[dailyall['Touchpoint'].isin(touchpoints)]
|
|
totalall = totalall[totalall['Touchpoint'].isin(touchpoints)]
|
|
ytd = ytd[ytd['Touchpoint'].isin(touchpoints)]
|
|
|
|
|
|
writer = pd.ExcelWriter("./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx")
|
|
totalall.to_excel(writer, sheet_name='total')
|
|
dailyall.to_excel(writer, sheet_name='daily')
|
|
if (args.preSelect == "week"):
|
|
hourlyall.to_excel(writer, sheet_name='hourly')
|
|
ytd.to_excel(writer, sheet_name='YTD')
|
|
writer.save()
|
|
writer.close()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|