Branch with enhanced graph tiles. Simplified SLOs are used to generate dashboards

createDash.py

@@ -5,7 +5,9 @@ import argparse
 import requests
 from datetime import datetime
 from git import Repo
+from key_request_parser import krparser
 import os
+import re
 #set STAGING global dashboard name
 DASHBOARD_NAME = "[STAGING]Global Offboard Reliability 2.0 - "
 AUTHSTRING = config("BITBUCKET_USERNAME")+":"+config("BITBUCKET_TOKEN")
@@ -69,6 +71,14 @@ def make_request(url, DTAPIToken,verify, method, jsondata):
     
     return response
 
+def getSLO(env, envurl,sloid, DTAPIToken):
+    url = envurl+"/api/v2/slo/"+sloid+"?timeFrame=CURRENT"
+    response = make_request(url, DTAPIToken,True, "get", "")
+    responseobj = response.json()
+    responseobj["env"] = env
+    return responseobj
+
+
 def get_all_dashboards_withname(DTAPIToken, DTENV,name):
     DTAPIURL= DTENV + "api/config/v1/dashboards"
     r = make_request(DTAPIURL,DTAPIToken,True,"get",None)
@@ -219,7 +229,7 @@ def get_DataExplorerTile_SingleValue(customName, metricSelector, remoteEnvironme
                                         }
     return dataExplorerTile_SingleValue
 
-def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold ):
+def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnvironmentUrl, bounds, timeframe, axisTargetMin, axisTargetMax, graphThreshold,countMetricSelector,responseMetricSelector ):
     dataExplorerTile_Graph =    { 
                                     "name": "", 
                                     "tileType": "DATA_EXPLORER", 
@@ -235,6 +245,24 @@ def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnv
 
                                                     "metricSelector": metricSelector,
                                                     
+                                                    "foldTransformation": "TOTAL", 	
+                                                    "enabled": "true" 
+                                                    },
+                                                    {
+                                                    "id": "B", 
+                                                    "timeAggregation": "DEFAULT", 
+
+                                                    "metricSelector": countMetricSelector,
+                                                    
+                                                    "foldTransformation": "TOTAL", 	
+                                                    "enabled": "true" 
+                                                    },
+                                                    {
+                                                    "id": "C", 
+                                                    "timeAggregation": "DEFAULT", 
+
+                                                    "metricSelector": responseMetricSelector,
+                                                    
                                                     "foldTransformation": "TOTAL", 	
                                                     "enabled": "true" 
                                                     }
@@ -242,11 +270,17 @@ def get_DataExplorerTile_Graph(customName, metricSelector, metricName, remoteEnv
                                         
                                     "visualConfig": { 
                                                         "type": "GRAPH_CHART", 	"global": { "seriesType": "LINE", "hideLegend": "true" }, 
-                                                        "rules": [ { "matcher": "A:", "properties": { "color": "DEFAULT", "seriesType": "LINE", "alias": "SLO" }, "seriesOverrides": [{"name": customName, "color": "#ffffff"}] } ], 
+                                                        "rules": [ { "matcher": "A:", "properties": { "color": "GREEN", "seriesType": "LINE", "alias": customName }, "seriesOverrides": [{"name": customName, "color": "#9cd575"}] },
-                                                        "axes": { "xAxis": { "visible": "true" }, "yAxes": [{ "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" }] }, 
+                                                                  { "matcher": "B:", "properties": { "color": "BLUE", "seriesType": "COLUMN", "alias": "Request count - server" }, "seriesOverrides": [{"name": "Request count - server", "color": "#74cff7"}] },
+                                                                   { "matcher": "C:", "properties": { "color": "PURPLE", "seriesType": "LINE", "alias": "Key request response time" }, "seriesOverrides": [{"name": "Key request response time", "color": "#c396e0"}] } ], 
+                                                        "axes": { "xAxis": { "visible": "true" }, "yAxes": [
+                                                                                                            { "displayName": "", "visible": "true", "min": axisTargetMin, "max": axisTargetMax, "position": "LEFT", "queryIds": [ "A" ], "defaultAxis": "true" },
+                                                                                                            { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "RIGHT", "queryIds": [ "B" ], "defaultAxis": "true" },
+                                                                                                            { "displayName": "", "visible": "true", "min": "AUTO", "max": "AUTO", "position": "LEFT", "queryIds": [ "C" ], "defaultAxis": "true" }
+                                                                                                            ] }, 
                                                         "heatmapSettings": {}, 
                                                         "singleValueSettings": { "showTrend": "false", "showSparkLine": "false", "linkTileColorToThreshold": "true" }, 
-                                                        "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "true" } ], 
+                                                        "thresholds": [ { "axisTarget": "LEFT", "rules": graphThreshold, "queryId": "", "visible": "false" } ], 
                                                         "tableSettings": { "isThresholdBackgroundAppliedToCell": "false" }, 
                                                         "graphChartSettings": { "connectNulls": "false" } }, 
                                                         
@@ -282,13 +316,59 @@ def create_default_tiles():
     newDashboardTiles.append({ "name": "Last 3 days"     ,"tileType": "HEADER"   , "configured": "true"  , "bounds": get_bounds(2   , 65   , 4      , 1), "tileFilter": {}                                       })
 
     return newDashboardTiles
-
+def getSloReqCountSelector(service_names):
+    selector = ""
+    if(service_names["selectortype"] == "KR"):
+        service_names = service_names["namestr"]
+        print("Building Keyrequest count selector for: "+service_names)
+        selector = "builtin:service.keyRequest.count.total:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names+"))\"))))):splitBy()"
+    elif(service_names["selectortype"] == "SRV"):
+        service_names = service_names["namestr"]
+        print("Building Service requests count selector for: "+service_names)
+        selector = "builtin:service.requestCount.total:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.equals("+service_names+")\"))))):splitBy()"
+    return selector
+def getSloReqTimeSelector(service_names):
+    selector = ""
+    if(service_names["selectortype"] == "KR"):
+        service_names = service_names["namestr"]
+        print("Building Keyrequest time selector for: "+service_names)
+        selector = "builtin:service.keyRequest.response.server:filter(and(or(in(\"dt.entity.service_method\",entitySelector(\"type(service_method), fromRelationship.isServiceMethodOfService( type(~\"SERVICE~\"),entityName.in("+service_names+"))\"))))):splitBy()"
+    elif(service_names["selectortype"] == "SRV"):
+        service_names = service_names["namestr"]
+        print("Building Service requests time selector for: "+service_names)
+        selector = "builtin:service.response.server:filter(and(or(in(\"dt.entity.service\",entitySelector(\"type(service),entityName.in("+service_names+")\"))))):splitBy()"
+    return selector
+def getSloSrvNames(hub_config, configuration, doc, env):
+    hub = ""
+    namestr = ""
+    if env=="euprod":
+        hub = "emea"
+    elif env=="naprod":
+        hub = "na"
+    elif env=="cnprod":
+        hub = "cn"
+    emeasloobj = getSLO(hub,hub_config[env]["remote_url"],configuration["ids"][hub],config(doc[env][2].get('env-token-name')))
+    emeaslosrvnames = []
+    selectortype = ""
+    if("builtin:service.keyRequest" in emeasloobj["metricExpression"]):
+        selectortype = "KR"
+        rgx = re.search("type\((service|SERVICE|~\"SERVICE~\"|~SERVICE~)\),\s*entityName.(equals|in|contains)\s*\(\s*(.+?\s*\")\s*\)",emeasloobj["metricExpression"].replace("\r","").replace("\n",""),re.IGNORECASE)
+        if(rgx):
+            namestr = rgx.group(3)
+    elif("builtin:service.keyRequest" not in emeasloobj["metricExpression"]):
+        selectortype = "SRV"
+        rgx = re.search("type\((service|SERVICE|~\"SERVICE~\"|~SERVICE~)\),\s*entityName.(equals|in|contains)\s*\(\s*(.+?\s*\")\s*\)",emeasloobj["metricExpression"].replace("\r","").replace("\n",""),re.IGNORECASE)
+        if(rgx):
+            namestr = rgx.group(3)
+    return {"selectortype":selectortype, "namestr":' '.join(namestr.split())}
 def main(slo_path):
     configrepo = clone_repo_if_notexist(CONFIG_REPO_URL, CONFIG_REPO_NAME)
     pull_repo(configrepo)
     archiverepo = clone_repo_if_notexist(ARCHIVE_REPO_URL, ARCHIVE_REPO_NAME)
     pull_repo(archiverepo)
     print("Generating dashboard tiles...")
+    with open('./environment.yaml') as file:
+        doc = yaml.safe_load(file)
     slo_doc = load_slo_parameter(slo_path)
     dashboard_json = create_default_tiles()
 
@@ -325,7 +405,7 @@ def main(slo_path):
     if(blname and blvalue):
         for slo_name, configuration in slo_doc.items():
             if configuration['department'].startswith(blvalue):
-                print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+str(boundindex) +" of "+str(rowcount))
+                print("Dashboard #"+str(dahboardcount)+" : Configurint SLO "+slo_name)
                 if rowcount > 0 and boundindex > rowcount:
                     dashboard_json = create_default_tiles()
                     dahboardcount = dahboardcount+1
@@ -351,7 +431,7 @@ def main(slo_path):
                         if 'actual' in tiles["tiles"]:
                             dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"],      get_bounds(((boundindex)*(3))   , 7 + hub_config[hub]["offset"]   , 4    , 3), timeframe_actual, slo_graphThreshold_SingleValue))
                         if "graph" in tiles["tiles"]:
-                            dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""),       hub_config[hub]["remote_url"],      get_bounds(((boundindex)*(3))   , 11 + hub_config[hub]["offset"]  , 12   , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph))
+                            dashboard_json.append(get_DataExplorerTile_Graph(slo_name, configuration["metric"], configuration["selector_var"].replace("~",""),       hub_config[hub]["remote_url"],      get_bounds(((boundindex)*(3))   , 11 + hub_config[hub]["offset"]  , 12   , 3), timeframe_graph, "97", "102", slo_graphThreshold_Graph, getSloReqCountSelector(getSloSrvNames(hub_config, configuration, doc, hub)),getSloReqTimeSelector(getSloSrvNames(hub_config, configuration, doc, hub))))
                         if  "ytd" in tiles["tiles"]:
                             dashboard_json.append(get_DataExplorerTile_SingleValue(slo_name, configuration["metric"], hub_config[hub]["remote_url"],      get_bounds(((boundindex)*(3))   , 23 + hub_config[hub]["offset"]  , 4    , 3), timeframe_ytd, slo_graphThreshold_SingleValue))
                 boundindex = boundindex+1
@@ -367,10 +447,8 @@ def main(slo_path):
 
         if args.auto_upload:
             print("Getting existing STAGING dashboards from Dynatrace")
-            with open('./environment.yaml') as file:
-                doc = yaml.safe_load(file)
-
             for item, doc in doc.items():
+                if(item == "globaldashboard"):
                     token = dict(doc[2])
                     url = dict(doc[1])
                     print("Crawling through: " + item)
@@ -390,4 +468,4 @@ def main(slo_path):
 
 
 if __name__ == "__main__":
-    main('./shared_configuration/slo_parameter.yaml')
+    main('./shared_configuration/simplified_slo_parameter.yaml')

environment.yaml

@@ -2,3 +2,16 @@ globaldashboard:
     - name: "globaldashboard"
     - env-url: "https://jyy23483.live.dynatrace.com/"
     - env-token-name: "GLOBAL_CONFIG_TOKEN"
+
+euprod:
+    - name: "euprod"
+    - env-url: "https://xxu26128.live.dynatrace.com"
+    - env-token-name: "EUPROD_TOKEN"
+naprod:
+    - name: "naprod"
+    - env-url: "https://wgv50241.live.dynatrace.com"
+    - env-token-name: "NAPROD_TOKEN"
+cnprod:
+    - name: "cnprod"
+    - env-url: "https://dynatrace-cn-int.bmwgroup.com:443/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+    - env-token-name: "CNPROD_TOKEN"