diff --git a/createDash.py b/createDash.py index 2e8d1d2..bcddc04 100644 --- a/createDash.py +++ b/createDash.py @@ -5,7 +5,9 @@ import argparse import requests from datetime import datetime from git import Repo +from key_request_parser import krparser import os +import re #set STAGING global dashboard name DASHBOARD_NAME = "[STAGING]Global Offboard Reliability 2.0 - " AUTHSTRING = config("BITBUCKET_USERNAME")+":"+config("BITBUCKET_TOKEN") @@ -69,6 +71,14 @@ def make_request(url, DTAPIToken,verify, method, jsondata): return response +def getSLO(env, envurl,sloid, DTAPIToken): + url = envurl+"/api/v2/slo/"+sloid+"?timeFrame=CURRENT" + response = make_request(url, DTAPIToken,True, "get", "") + responseobj = response.json() + responseobj["env"] = env + return responseobj + + def get_all_dashboards_withname(DTAPIToken, DTENV,name): DTAPIURL= DTENV + "api/config/v1/dashboards" r = make_request(DTAPIURL,DTAPIToken,True,"get",None) @@ -219,7 +229,7 @@ def get_DataExplorerTile_SingleValue(customName, metricSelector, remoteEnvironme } return dataExplorerTile_SingleValue -def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold ): +def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold,countMetricSelector,responseMetricSelector ): dataExplorerTile_Graph = { "name": "", "tileType": "DATA_EXPLORER", @@ -237,16 +247,40 @@ def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnv "foldTransformation": "TOTAL", "enabled": "true" - } + }, + { + "id": "B", + "timeAggregation": "DEFAULT", + + "metricSelector": countMetricSelector, + + "foldTransformation": "TOTAL", + "enabled": "true" + }, + { + "id": "C", + "timeAggregation": "DEFAULT", + + "metricSelector": responseMetricSelector, + + "foldTransformation": "TOTAL", + "enabled": "true" + } ], "visualConfig": { "type": "GRAPH_CHART", "global": { "seriesType": "LINE", "hideLegend": "true" }, - "rules": [ { "matcher": "A:", "properties": { "color": "DEFAULT", "seriesType": "LINE", "alias": "SLO" }, "seriesOverrides": [{"name": customName, "color": "#ffffff"}] } ], - "axes": { "xAxis": { "visible": "true" }, "yAxes": [{ "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" }] }, + "rules": [ { "matcher": "A:", "properties": { "color": "GREEN", "seriesType": "LINE", "alias": customName }, "seriesOverrides": [{"name": customName, "color": "#9cd575"}] }, + { "matcher": "B:", "properties": { "color": "BLUE", "seriesType": "COLUMN", "alias": "Request count - server" }, "seriesOverrides": [{"name": "Request count - server", "color": "#74cff7"}] }, + { "matcher": "C:", "properties": { "color": "PURPLE", "seriesType": "LINE", "alias": "Key request response time" }, "seriesOverrides": [{"name": "Key request response time", "color": "#c396e0"}] } ], + "axes": { "xAxis": { "visible": "true" }, "yAxes": [ + { "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" }, + { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "RIGHT", "queryIds": [ "B" ], "defaultAxis": "true" }, + { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "LEFT", "queryIds": [ "C" ], "defaultAxis": "true" } + ] }, "heatmapSettings": {}, "singleValueSettings": { "showTrend": "false", "showSparkLine": "false", "linkTileColorToThreshold": "true" }, - "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "true" } ], + "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "false" } ], "tableSettings": { "isThresholdBackgroundAppliedToCell": "false" }, "graphChartSettings": { "connectNulls": "false" } }, @@ -282,13 +316,59 @@ def create_default_tiles(): newDashboardTiles.append({ "name": "Last 3 days" ,"tileType": "HEADER" , "configured": "true" , "bounds": get_bounds(2 , 65 , 4 , 1), "tileFilter": {} }) return newDashboardTiles - +def getSloReqCountSelector(service_names): + selector = "" + if(service_names["selectortype"] == "KR"): + service_names = service_names["namestr"] + print("Building Keyrequest count selector for: "+service_names) + selector = "builtin:service.keyRequest.count.total:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names+"))\"))))):splitBy()" + elif(service_names["selectortype"] == "SRV"): + service_names = service_names["namestr"] + print("Building Service requests count selector for: "+service_names) + selector = "builtin:service.requestCount.total:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.equals("+service_names+")\"))))):splitBy()" + return selector +def getSloReqTimeSelector(service_names): + selector = "" + if(service_names["selectortype"] == "KR"): + service_names = service_names["namestr"] + print("Building Keyrequest time selector for: "+service_names) + selector = "builtin:service.keyRequest.response.server:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names+"))\"))))):splitBy()" + elif(service_names["selectortype"] == "SRV"): + service_names = service_names["namestr"] + print("Building Service requests time selector for: "+service_names) + selector = "builtin:service.response.server:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.in("+service_names+")\"))))):splitBy()" + return selector +def getSloSrvNames(hub_config, configuration, doc, env): + hub = "" + namestr = "" + if env=="euprod": + hub = "emea" + elif env=="naprod": + hub = "na" + elif env=="cnprod": + hub = "cn" + emeasloobj = getSLO(hub,hub_config[env]["remote_url"],configuration["ids"][hub],config(doc[env][2].get('env-token-name'))) + emeaslosrvnames = [] + selectortype = "" + if("builtin:service.keyRequest" in emeasloobj["metricExpression"]): + selectortype = "KR" + rgx = re.search("type\((service|SERVICE|~\"SERVICE~\"|~SERVICE~)\),\s*entityName.(equals|in|contains)\s*\(\s*(.+?\s*\")\s*\)",emeasloobj["metricExpression"].replace("\r","").replace("\n",""),re.IGNORECASE) + if(rgx): + namestr = rgx.group(3) + elif("builtin:service.keyRequest" not in emeasloobj["metricExpression"]): + selectortype = "SRV" + rgx = re.search("type\((service|SERVICE|~\"SERVICE~\"|~SERVICE~)\),\s*entityName.(equals|in|contains)\s*\(\s*(.+?\s*\")\s*\)",emeasloobj["metricExpression"].replace("\r","").replace("\n",""),re.IGNORECASE) + if(rgx): + namestr = rgx.group(3) + return {"selectortype":selectortype, "namestr":' '.join(namestr.split())} def main(slo_path): configrepo = clone_repo_if_notexist(CONFIG_REPO_URL, CONFIG_REPO_NAME) pull_repo(configrepo) archiverepo = clone_repo_if_notexist(ARCHIVE_REPO_URL, ARCHIVE_REPO_NAME) pull_repo(archiverepo) print("Generating dashboard tiles...") + with open('./environment.yaml') as file: + doc = yaml.safe_load(file) slo_doc = load_slo_parameter(slo_path) dashboard_json = create_default_tiles() @@ -325,7 +405,7 @@ def main(slo_path): if(blname and blvalue): for slo_name, configuration in slo_doc.items(): if configuration['department'].startswith(blvalue): - print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+str(boundindex) +" of "+str(rowcount)) + print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+slo_name) if rowcount > 0 and boundindex > rowcount: dashboard_json = create_default_tiles() dahboardcount = dahboardcount+1 @@ -351,7 +431,7 @@ def main(slo_path): if 'actual' in tiles["tiles"]: dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 7 + hub_config[hub]["offset"] , 4 , 3), timeframe_actual, slo_graphThreshold_SingleValue)) if "graph" in tiles["tiles"]: - dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 12 , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph)) + dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""), hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 11 + hub_config[hub]["offset"] , 12 , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph, getSloReqCountSelector(getSloSrvNames(hub_config, configuration, doc, hub)),getSloReqTimeSelector(getSloSrvNames(hub_config, configuration, doc, hub)))) if "ytd" in tiles["tiles"]: dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"], get_bounds(((boundindex)*(3)) , 23 + hub_config[hub]["offset"] , 4 , 3), timeframe_ytd, slo_graphThreshold_SingleValue)) boundindex = boundindex+1 @@ -367,27 +447,25 @@ def main(slo_path): if args.auto_upload: print("Getting existing STAGING dashboards from Dynatrace") - with open('./environment.yaml') as file: - doc = yaml.safe_load(file) - for item, doc in doc.items(): - token = dict(doc[2]) - url = dict(doc[1]) - print("Crawling through: " + item) - print("Gather data, hold on a minute") - DTTOKEN = config(token.get('env-token-name')) - DTURL = url.get('env-url') - print("Downloading STAGING dashboards to local repo ("+blname+")...") - existingdashboards = get_all_dashboards_withname(DTTOKEN, DTURL,DASHBOARD_NAME +blname) - print("Uploading STAGING dashboards to Dynatrace ("+blname+")...") - backup_dashboards(DTTOKEN, DTURL, existingdashboards) - now=datetime.now() - strnowdate = now.strftime("%Y%m%d") - push_repo(archiverepo, strnowdate+"_Global dashboard as code auto-upload backup") - create_or_update_dashboard(DTTOKEN, DTURL, existingdashboards, generatedfiles, blname) + if(item == "globaldashboard"): + token = dict(doc[2]) + url = dict(doc[1]) + print("Crawling through: " + item) + print("Gather data, hold on a minute") + DTTOKEN = config(token.get('env-token-name')) + DTURL = url.get('env-url') + print("Downloading STAGING dashboards to local repo ("+blname+")...") + existingdashboards = get_all_dashboards_withname(DTTOKEN, DTURL,DASHBOARD_NAME +blname) + print("Uploading STAGING dashboards to Dynatrace ("+blname+")...") + backup_dashboards(DTTOKEN, DTURL, existingdashboards) + now=datetime.now() + strnowdate = now.strftime("%Y%m%d") + push_repo(archiverepo, strnowdate+"_Global dashboard as code auto-upload backup") + create_or_update_dashboard(DTTOKEN, DTURL, existingdashboards, generatedfiles, blname) else: print("ERROR: Could not find Business line for given department.") if __name__ == "__main__": - main('./shared_configuration/slo_parameter.yaml') + main('./shared_configuration/simplified_slo_parameter.yaml') diff --git a/environment.yaml b/environment.yaml index 6de8125..e1f5178 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,4 +1,17 @@ globaldashboard: - name: "globaldashboard" - env-url: "https://jyy23483.live.dynatrace.com/" - - env-token-name: "GLOBAL_CONFIG_TOKEN" \ No newline at end of file + - env-token-name: "GLOBAL_CONFIG_TOKEN" + +euprod: + - name: "euprod" + - env-url: "https://xxu26128.live.dynatrace.com" + - env-token-name: "EUPROD_TOKEN" +naprod: + - name: "naprod" + - env-url: "https://wgv50241.live.dynatrace.com" + - env-token-name: "NAPROD_TOKEN" +cnprod: + - name: "cnprod" + - env-url: "https://dynatrace-cn-int.bmwgroup.com:443/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" + - env-token-name: "CNPROD_TOKEN" \ No newline at end of file