diff --git a/createDash.py b/createDash.py index 827c0e5..867b940 100644 --- a/createDash.py +++ b/createDash.py @@ -5,7 +5,11 @@ import argparse import requests from datetime import datetime from git import Repo +from KRParser import krparser import os +import re +import warnings +warnings.filterwarnings("ignore") #set STAGING global dashboard name DASHBOARD_NAME = "[STAGING]Global Offboard Reliability 2.0" AUTHSTRING = config("BITBUCKET_USERNAME")+":"+config("BITBUCKET_TOKEN") @@ -13,6 +17,8 @@ CONFIG_REPO_URL = "https://"+AUTHSTRING+"@atc.bmwgroup.net/bitbucket/scm/opapm/s CONFIG_REPO_NAME = "shared_configuration" ARCHIVE_REPO_URL = "https://"+AUTHSTRING+"@atc.bmwgroup.net/bitbucket/scm/opapm/archive.git" ARCHIVE_REPO_NAME = "archive" +KEYREQ_REPO_URL = "https://"+AUTHSTRING+"@atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git" +KEYREQ_REPO_NAME = "keyrequestparser" BUSINESS_LINES = {'DE-3':'My Journey','DE-7':'Connected Vehicle Platform','DE-4':'My Life','EC-DE':'China Services'} parser = argparse.ArgumentParser(description="Generate and deploy the Dynatrace Global Dashboard as Code. Auto deployment works only for STAGING dashboard", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -69,6 +75,14 @@ def make_request(url, DTAPIToken,verify, method, jsondata): return response +def getSLO(env, envurl,sloid, DTAPIToken): + url = envurl+"/api/v2/slo/"+sloid+"?timeFrame=CURRENT" + response = make_request(url, DTAPIToken,True, "get", "") + responseobj = response.json() + responseobj["env"] = env + return responseobj + + def get_all_dashboards_withname(DTAPIToken, DTENV,name): DTAPIURL= DTENV + "api/config/v1/dashboards" r = make_request(DTAPIURL,DTAPIToken,True,"get",None) @@ -231,7 +245,7 @@ def get_DataExplorerTile_SingleValue(customName, metricSelector, remoteEnvironme } return dataExplorerTile_SingleValue -def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold ): +def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold,countMetricSelector,responseMetricSelector ): dataExplorerTile_Graph = { "name": "", "tileType": "DATA_EXPLORER", @@ -249,16 +263,40 @@ def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnv "foldTransformation": "TOTAL", "enabled": "true" - } + }, + { + "id": "B", + "timeAggregation": "DEFAULT", + + "metricSelector": countMetricSelector, + + "foldTransformation": "TOTAL", + "enabled": "true" + }, + { + "id": "C", + "timeAggregation": "DEFAULT", + + "metricSelector": responseMetricSelector, + + "foldTransformation": "TOTAL", + "enabled": "true" + } ], "visualConfig": { "type": "GRAPH_CHART", "global": { "seriesType": "LINE", "hideLegend": "true" }, - "rules": [ { "matcher": "A:", "properties": { "color": "DEFAULT", "seriesType": "LINE", "alias": "SLO" }, "seriesOverrides": [{"name": customName, "color": "#ffffff"}] } ], - "axes": { "xAxis": { "visible": "true" }, "yAxes": [{ "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" }] }, + "rules": [ { "matcher": "A:", "properties": { "color": "GREEN", "seriesType": "LINE", "alias": customName }, "seriesOverrides": [{"name": customName, "color": "#9cd575"}] }, + { "matcher": "B:", "properties": { "color": "BLUE", "seriesType": "COLUMN", "alias": "Request count - server" }, "seriesOverrides": [{"name": "Request count - server", "color": "#74cff7"}] }, + { "matcher": "C:", "properties": { "color": "PURPLE", "seriesType": "LINE", "alias": "Key request response time" }, "seriesOverrides": [{"name": "Key request response time", "color": "#c396e0"}] } ], + "axes": { "xAxis": { "visible": "true" }, "yAxes": [ + { "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" }, + { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "RIGHT", "queryIds": [ "B" ], "defaultAxis": "true" }, + { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "LEFT", "queryIds": [ "C" ], "defaultAxis": "true" } + ] }, "heatmapSettings": {}, "singleValueSettings": { "showTrend": "false", "showSparkLine": "false", "linkTileColorToThreshold": "true" }, - "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "true" } ], + "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "false" } ], "tableSettings": { "isThresholdBackgroundAppliedToCell": "false" }, "graphChartSettings": { "connectNulls": "false" } }, @@ -281,42 +319,90 @@ def create_default_tiles(): # EMEA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 7 , 20 , 2), "tileFilter": {}, "markdown": "# EMEA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 7 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 11 , 12 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 23 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 11 , 12 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 23 , 4 , 1), "tileFilter": {} }) # NORTH AMERICA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 28 , 20 , 2), "tileFilter": {}, "markdown": "# NORTH AMERICA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 28 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 32 , 12 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 44 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 32 , 12 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 44 , 4 , 1), "tileFilter": {} }) # CHINA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 49 , 20 , 2), "tileFilter": {}, "markdown": "# CHINA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 49 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 53 , 12 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 65 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 53 , 12 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 65 , 4 , 1), "tileFilter": {} }) else: # EMEA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 7 , 14 , 2), "tileFilter": {}, "markdown": "# EMEA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 7 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 11 , 6 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 17 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 11 , 6 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 17 , 4 , 1), "tileFilter": {} }) # NORTH AMERICA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 22 , 14 , 2), "tileFilter": {}, "markdown": "# NORTH AMERICA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 22 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 26 , 6 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 32 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 26 , 6 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 32 , 4 , 1), "tileFilter": {} }) # CHINA HUB newDashboardTiles.append({ "name": "Header" ,"tileType": "MARKDOWN" , "configured": "true" , "bounds": get_bounds(0 , 37 , 14 , 2), "tileFilter": {}, "markdown": "# CHINA" }) newDashboardTiles.append({ "name": "Last 1 h" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 37 , 4 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Reliability Graph (3 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 41 , 6 , 1), "tileFilter": {} }) - newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 47 , 4 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Reliability Graph (2 days)" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 41 , 6 , 1), "tileFilter": {} }) + newDashboardTiles.append({ "name": "Last 2 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 47 , 4 , 1), "tileFilter": {} }) return newDashboardTiles - +def getSloReqCountSelector(service_names): + selector = "" + if(service_names["selectortype"] == "KR"): + service_names = service_names["services"] + print("Building Keyrequest count selector for: "+service_names) + selector = "builtin:service.keyRequest.count.total:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names+"))\"))))):splitBy()" + elif(service_names["selectortype"] == "SRV"): + service_names = service_names["services"] + print("Building Service requests count selector for: "+service_names) + selector = "builtin:service.requestCount.total:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.in("+service_names+")\"))))):splitBy()" + return selector +def getSloReqTimeSelector(service_names): + selector = "" + if(service_names["selectortype"] == "KR"): + selector = "builtin:service.keyRequest.response.server:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names["services"]+"))\"))))):splitBy()" + elif(service_names["selectortype"] == "SRV"): + selector = "builtin:service.response.server:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.in("+service_names["services"]+")\"))))):splitBy()" + return selector +def getSloSrvNames(hub_config, configuration, doc, env): + hub = "" + namestr = "" + if env=="euprod": + hub = "emea" + elif env=="naprod": + hub = "na" + elif env=="cnprod": + hub = "cn" + krp = krparser.KRParser(name=env,options=krparser.KROption.RESOLVESERVICES, config={"threads":10, "serviceLookupParams":{"fields":"tags"}, "extendResultObjects":{"env":env}}, DTAPIURL=hub_config[env]["remote_url"], DTAPIToken=config(doc[env][2].get('env-token-name'))) + sloobj = getSLO(hub,hub_config[env]["remote_url"],configuration["ids"][hub],config(doc[env][2].get('env-token-name'))) + krs = krp.parse(sloobj) + slosrvnames = [] + outputslos = [] + selectortype = "" + for krslo in krs: + if("builtin:service.keyRequest" in krslo.metadata["metricExpression"]): + selectortype = "KR" + elif("builtin:service.keyRequest" not in krslo.metadata["metricExpression"]): + selectortype = "SRV" + for kr in krslo.keyRequests: + for srv in kr["services"]: + slosrvnames.append(srv["displayName"]) + slosrvnames = list(dict.fromkeys(slosrvnames)) + for srv in slosrvnames: + outputslos.append("~\""+srv+"~\"") + return {"selectortype":selectortype, "services":",".join(outputslos)} def main(slo_path): configrepo = clone_repo_if_notexist(CONFIG_REPO_URL, CONFIG_REPO_NAME) pull_repo(configrepo) archiverepo = clone_repo_if_notexist(ARCHIVE_REPO_URL, ARCHIVE_REPO_NAME) pull_repo(archiverepo) + keyreqrepo = clone_repo_if_notexist(KEYREQ_REPO_URL, KEYREQ_REPO_NAME) + pull_repo(keyreqrepo) print("Generating dashboard tiles...") + with open('./environment.yaml') as file: + doc = yaml.safe_load(file) slo_doc = load_slo_parameter(slo_path) dashboard_json = create_default_tiles() if(args.wall): @@ -362,7 +448,7 @@ def main(slo_path): if configuration['department'].startswith(blvalue) or blvalue == "ALL": if(configuration['selector_var'] == "CoCo-QM-Report_Mobile"): slorelevant = True - print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+str(boundindex) +" of "+str(rowcount)) + print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+slo_name) if rowcount > 0 and boundindex > rowcount: dashboard_json = create_default_tiles() dahboardcount = dahboardcount+1 @@ -370,7 +456,7 @@ def main(slo_path): slo_display = configuration["displayname"] slo_department = configuration["department"] #timeframe_ytd = configuration["yearstart"] + " 00:00 to now" - timeframe_ytd = "-3d" + timeframe_ytd = "-2d" slo_graphThreshold_SingleValue = get_dataExplorerTileSloThreshold(configuration["thresholds"]["single_value"]) slo_graphThreshold_Graph = get_dataExplorerTileSloThreshold(configuration["thresholds"]["graph_value"]) emeaslourl = "" @@ -389,9 +475,9 @@ def main(slo_path): dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 7 + hub_config[hub]["offset"] , 4 , 3), timeframe_actual, slo_graphThreshold_SingleValue)) if "graph" in tiles["tiles"]: if(args.wall): - dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 12 , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph)) + dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 12 , 3), timeframe_graph, "97", "100.1", slo_graphThreshold_Graph, getSloReqCountSelector(getSloSrvNames(hub_config, configuration, doc, hub)),getSloReqTimeSelector(getSloSrvNames(hub_config, configuration, doc, hub)))) else: - dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 6 , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph)) + dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 6 , 3), timeframe_graph, "97", "100.1", slo_graphThreshold_Graph, getSloReqCountSelector(getSloSrvNames(hub_config, configuration, doc, hub)),getSloReqTimeSelector(getSloSrvNames(hub_config, configuration, doc, hub)))) if "ytd" in tiles["tiles"]: if(args.wall): dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 23 + hub_config[hub]["offset"] , 4 , 3), timeframe_ytd, slo_graphThreshold_SingleValue)) @@ -410,24 +496,22 @@ def main(slo_path): if args.auto_upload: print("Getting existing STAGING dashboards from Dynatrace") - with open('./environment.yaml') as file: - doc = yaml.safe_load(file) - for item, doc in doc.items(): - token = dict(doc[2]) - url = dict(doc[1]) - print("Crawling through: " + item) - print("Gather data, hold on a minute") - DTTOKEN = config(token.get('env-token-name')) - DTURL = url.get('env-url') - print("Downloading STAGING dashboards to local repo ("+blname+")...") - existingdashboards = get_all_dashboards_withname(DTTOKEN, DTURL,DASHBOARD_NAME +blname) - print("Uploading STAGING dashboards to Dynatrace ("+blname+")...") - backup_dashboards(DTTOKEN, DTURL, existingdashboards) - now=datetime.now() - strnowdate = now.strftime("%Y%m%d") - push_repo(archiverepo, strnowdate+"_Global dashboard as code auto-upload backup") - create_or_update_dashboard(DTTOKEN, DTURL, existingdashboards, generatedfiles, blname) + if(item == "globaldashboard"): + token = dict(doc[2]) + url = dict(doc[1]) + print("Crawling through: " + item) + print("Gather data, hold on a minute") + DTTOKEN = config(token.get('env-token-name')) + DTURL = url.get('env-url') + print("Downloading STAGING dashboards to local repo ("+blname+")...") + existingdashboards = get_all_dashboards_withname(DTTOKEN, DTURL,DASHBOARD_NAME +blname) + print("Uploading STAGING dashboards to Dynatrace ("+blname+")...") + backup_dashboards(DTTOKEN, DTURL, existingdashboards) + now=datetime.now() + strnowdate = now.strftime("%Y%m%d") + push_repo(archiverepo, strnowdate+"_Global dashboard as code auto-upload backup") + create_or_update_dashboard(DTTOKEN, DTURL, existingdashboards, generatedfiles, blname) else: print("ERROR: Could not find Business line for given department.") diff --git a/environment.yaml b/environment.yaml index 6de8125..e1f5178 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,4 +1,17 @@ globaldashboard: - name: "globaldashboard" - env-url: "https://jyy23483.live.dynatrace.com/" - - env-token-name: "GLOBAL_CONFIG_TOKEN" \ No newline at end of file + - env-token-name: "GLOBAL_CONFIG_TOKEN" + +euprod: + - name: "euprod" + - env-url: "https://xxu26128.live.dynatrace.com" + - env-token-name: "EUPROD_TOKEN" +naprod: + - name: "naprod" + - env-url: "https://wgv50241.live.dynatrace.com" + - env-token-name: "NAPROD_TOKEN" +cnprod: + - name: "cnprod" + - env-url: "https://dynatrace-cn-int.bmwgroup.com:443/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" + - env-token-name: "CNPROD_TOKEN" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c8afe58..1849510 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ pyyaml requests datetime argparse -GitPython \ No newline at end of file +GitPython +keyrequestparser @ git+https://atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git \ No newline at end of file