from decouple import config import sys import yaml import datetime import time import pandas as pd import requests import openpyxl import argparse import os os.environ['TZ'] = 'Europe/Berlin' # set new timezone time.tzset() def make_request(url, headers,verify,parameters): try: response = requests.get(url, headers=headers,verify=verify,params=parameters) response.raise_for_status() except requests.exceptions.HTTPError as errh: return "An Http Error occurred:" + repr(errh) except requests.exceptions.ConnectionError as errc: return "An Error Connecting to the API occurred:" + repr(errc) except requests.exceptions.Timeout as errt: return "A Timeout Error occurred:" + repr(errt) except requests.exceptions.RequestException as err: return "An Unknown Error occurred" + repr(err) return response def previous_week_range(date): start_date = date + datetime.timedelta(-date.weekday(), weeks=-1) end_date = date + datetime.timedelta(-date.weekday() - 1) return start_date, end_date def previous_month_range(date): end_date = date.replace(day=1) - datetime.timedelta(days=1) start_date = end_date.replace(day=1) return start_date, end_date def getSLO(DTAPIToken, DTENV, fromDate, toDate): env = DTENV DTAPIToken = DTAPIToken if (DTENV.find('dynatracemgd') != -1): verify=False if (DTENV.find('dyna-synt') != -1): verify=False else: verify=True DTAPIURL= env + "/api/v2/slo" headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } parameters = { "pageSize": 25, "from": int(fromDate), "to": int(toDate), "timeFrame": "GTF", "evaluate": "true", "sloSelector": "text(\"CoCo-QM-Report\")" } r = make_request(DTAPIURL,headers=headers,parameters=parameters,verify=verify) try: cont = r.json() #python3 #df = pd.json_normalize(cont['slo']) #python2 df = pd.DataFrame.from_dict(cont['slo']) except Exception as e: print(e) print("No SLO json returned") print(parameters) print(r) df = pd.DataFrame() return df def get_daily_slice(start_date, end_date): tempstart = start_date days = pd.DataFrame() #Add the first day tempend = tempstart + datetime.timedelta(hours=24) startms = time.mktime(tempstart.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000 row = {'Date':tempstart,'startTime':startms, 'endTime':endms} days = days.append(row,ignore_index=True) while tempstart < end_date: tempstart = tempstart + datetime.timedelta(hours=24) tempend = tempstart + datetime.timedelta(hours=24) startms = time.mktime(tempstart.timetuple()) * 1000 endms = time.mktime(tempend.timetuple()) * 1000 row = {'Date':tempstart,'startTime':startms, 'endTime':endms} days = days.append(row,ignore_index=True) return days def init_argparse(): parser = argparse.ArgumentParser( usage="%(prog)s [--fromDate] [toDate] or [preSelect]", description="gather SLO in daily slices for given Timeframe" ) parser.add_argument( "-f","--fromDate", help = "YYYY-mm-dd e.g. 2022-01-01" ) parser.add_argument( "-t","--toDate", help = "YYYY-mm-dd e.g. 2022-01-31" ) parser.add_argument( "-p","--preSelect", help = "week | month - gathers the data for the last full week or month" ) return parser def main(): parser = init_argparse() args = parser.parse_args() if args.preSelect and (args.fromDate or args.toDate): print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate") sys.exit() elif args.fromDate and not args.toDate: print("--fromDate only in conjunction with --toDate") sys.exit() elif args.toDate and not args.fromDate: print("--toDate only in conjunction with --fromDate") sys.exit() elif args.toDate and args.fromDate and not args.preSelect: try: fromDate = datetime.date.fromisoformat(args.fromDate) toDate = datetime.date.fromisoformat(args.toDate) except Exception as e: print("Progam closed: " + str(e)) sys.exit() if toDate < fromDate: print("--toDate can't be older than --fromDate") sys.exit() if toDate > datetime.date.today() or fromDate > datetime.date.today(): print("--toDate or --fromDate can't be in the future") sys.exit() elif args.preSelect and not args.fromDate and not args.toDate: date = datetime.date.today() if args.preSelect == "week": fromDate, toDate = previous_week_range(date) elif args.preSelect == "month": fromDate, toDate = previous_month_range(date) else: print("--preSelect must be week or month") sys.exit() else: print("Invalid arguments, please use --help") sys.exit() print("fromDate: " + str(fromDate)) print("toDate: " + str(toDate)) days = get_daily_slice(fromDate,toDate) with open('./environment.yaml') as file: doc = yaml.safe_load(file) dailyall = pd.DataFrame() totalall = pd.DataFrame() for item, doc in doc.items(): token = dict(doc[2]) url = dict(doc[1]) print("Crawling through: " + item) print("Check if token exists in environment...") if(config(token.get('env-token-name')) != ""): print("Gather data, hold on a minute") DTTOKEN = config(token.get('env-token-name')) DTURL = url.get('env-url') ###Calc daily SLO df = pd.DataFrame() for index, row in days.iterrows(): temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime']) temp_df['Date'] = row['Date'] temp_df['HUB'] = item df = pd.concat([df,temp_df],ignore_index=True) #sort columns in a try block - if API is returning columns which are non exist, this will not fail the script df[['description','Touchpoint']] = df['description'].str.split('_',expand=True) try: df = df[['Date', 'HUB', 'id', 'enabled', 'name', 'description', 'Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']] except Exception as e: print("Could not rearrange columns: " + str(e)) dailyall = pd.concat([dailyall,df],ignore_index=True) ###Calc Overall SLO df = getSLO(DTTOKEN,DTURL,days['startTime'].min(),days['endTime'].max()) df['CW'] = fromDate.isocalendar()[1] df['HUB'] = item df[['description','Touchpoint']] = df['description'].str.split('_',expand=True) try: df = df[['CW', 'HUB', 'id', 'enabled', 'name', 'description','Touchpoint', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']] except Exception as e: print("Could not rearrange columns: " + str(e)) totalall = pd.concat([totalall,df],ignore_index=True) else: print("token not found, skipping " + item) touchpoints = ['Vehicle' , 'Mobile'] dailyall = dailyall[dailyall['Touchpoint'].isin(touchpoints)] totalall = totalall[totalall['Touchpoint'].isin(touchpoints)] writer = pd.ExcelWriter("./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx") totalall.to_excel(writer, sheet_name='total') dailyall.to_excel(writer, sheet_name='daily') writer.save() writer.close() if __name__ == "__main__": main()