adapted no data to has data (makes no sense)

createKeyRequestReport.py

@@ -56,12 +56,12 @@ def getSLO(ENV, DTAPIToken, DTENV):
     return df
 
  
-def write_to_excel(ignored, notExists, noData1W):
+def write_to_excel(ignored, noData1M, noData1W):
     
     fileName = "./KeyRequest_Report_"+ str(datetime.date.today())  +".xlsx"
 
     writer = pd.ExcelWriter(fileName)
-    notExists.to_excel(writer, sheet_name='notExists')
+    noData1M.to_excel(writer, sheet_name='noData1M')
     noData1W.to_excel(writer, sheet_name='noData1W')
     ignored.to_excel(writer, sheet_name='ignored')
 
@@ -76,22 +76,29 @@ def getStats(krs):
     for kr in [a for a in krs if len(a.keyRequests) == 0]:
         tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","",kr.metadata["filter"], kr.metadata["metricExpression"]])
     
-    keyReuqestsNotExists=[]
+    
-    for kr in [s for s in krs if s.hasNotExistingKeyRequests()==True]:
+    
-         for k in kr.getNotExistingKeyRequests():
+    noData1M=[]
-             keyReuqestsNotExists.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])
+    for kr in [s for s in krs if s.hasNoData("1M")==True]:
+         for k in kr.getKeyRequestByHasData("1M"):
+             noData1M.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])
     
     noData1W=[]
-    for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]:
+    for kr in [s for s in krs if s.hasNoData("1W")==True]:
-        for k in kr.getNoData1WKeyRequests():
+         for k in kr.getKeyRequestByHasData("1W"):
              noData1W.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])         
              
+    # noData1W=[]
+    # for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]:
+    #     for k in kr.getNoData1WKeyRequests():
+    #          noData1W.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])        
+    
     columns =['slo', 'env', 'displayName','entityId', 'filter', 'metricExpression']
     ignored=pd.DataFrame(tmpIgnoredList, columns=columns, dtype = float)
-    notExists=pd.DataFrame(keyReuqestsNotExists, columns = columns, dtype = float)
+    noData_1M=pd.DataFrame(noData1M, columns = columns, dtype = float)
     noData_1W=pd.DataFrame(noData1W, columns = columns, dtype = float)
 
-    return ignored, notExists, noData_1W
+    return ignored, noData_1M, noData_1W
 
 def main(slo_path):
     
@@ -116,13 +123,13 @@ def main(slo_path):
                 slosF=getSLO(env, DTTOKEN, DTURL)
                 
                 for index, row in slosF.iterrows():
-                  #if row['id'] == "69c57b3f-725f-36fb-88ef-fb64b03eb0ce": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
+                  if row['id'] == "7a7f9c66-75c1-30b9-a505-801ffa5f9e98": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
                    krs.append(krp.parseBySLO(row))
               
                 resultSlos.extend(krs)
                 
-    ignoerd,notExists, noData1W= getStats(resultSlos)            
+    ignoerd,noData1M, noData1W= getStats(resultSlos)            
-    write_to_excel(ignoerd, notExists, noData1W)
+    write_to_excel(ignoerd, noData1M, noData1W)
 
 if __name__ == "__main__":
     main('./slo_parameter.yaml')

environment.yaml

@@ -4,21 +4,12 @@ euprod:
     - env-url: "https://xxu26128.live.dynatrace.com"
     - env-token-name: "EUPROD_TOKEN_VAR"
     - jenkins: "https://jaws.bmwgroup.net/opapm/"
-#eupreprod:
+
-    - name: "eupreprod"
+# naprod:
-    - env-url: "https://qqk70169.live.dynatrace.com"
+#     - name: "naprod"
-    - env-token-name: "EUPREPROD_TOKEN_VAR"
+#     - env-url: "https://wgv50241.live.dynatrace.com"
-    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+#     - env-token-name: "NAPROD_TOKEN_VAR"
-#napreprod:
+#     - jenkins: "https://jaws.bmwgroup.net/opapm/"
-    - name: "napreprod"
-    - env-url: "https://onb44935.live.dynatrace.com"
-    - env-token-name: "NAPREPROD_TOKEN_VAR"
-    - jenkins: "https://jaws.bmwgroup.net/opapm/"
- naprod:
-     - name: "naprod"
-     - env-url: "https://wgv50241.live.dynatrace.com"
-     - env-token-name: "NAPROD_TOKEN_VAR"
-     - jenkins: "https://jaws.bmwgroup.net/opapm/"
 # cnprod:
 #     - name: "cnprod"
 #     - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"

key_request_parser/keyrequests.py

@@ -85,23 +85,33 @@ class KeyRequestGroup(MutableSequence):
          
 class KR:
     
-     def getNotExistingKeyRequests(self):
+    #  def getNotExistingKeyRequests(self):
-         return [k for k in self.keyRequests if k['exists']==False]
+    #      return [k for k in self.keyRequests if k['exists']==False]
     
-     def hasNotExistingKeyRequests(self):
+    #  def hasNotExistingKeyRequests(self):
+    #     for k in self.keyRequests:
+    #         if k['exists']==False:
+    #             return True
+            
+    #     return False 
+    
+    #  def getNoData1WKeyRequests(self):
+    #      return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True]
+    
+    #  def hasNoData1WKeyRequests(self):
+    #     for k in self.keyRequests:
+    #         if k['hasData_1W']==False and k['exists'] == True:
+    #             return True
+            
+    #     return False 
+    
+     def getKeyRequestByHasData(self,label):
+        return [k for k in self.keyRequests if k['hasData'][label]==False]
+    
+     def hasNoData(self,label):
         for k in self.keyRequests:
-            if k['exists']==False:
+          if k['hasData'][label]==False:
-                return True
+            return True
-            
-        return False 
-    
-     def getNoData1WKeyRequests(self):
-         return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True]
-    
-     def hasNoData1WKeyRequests(self):
-        for k in self.keyRequests:
-            if k['hasData_1W']==False and k['exists'] == True:
-                return True
             
         return False 
              

key_request_parser/krparser.py

@@ -45,42 +45,7 @@ class KRParser:
         return groups 
   
     
-    # def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken):
+    def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
- 
-    #      DTAPIURL = DTAPIURL + "/api/v2/entities"
-
-    #      headers = {
-    #         'Content-Type': 'application/json',
-    #         'Authorization': 'Api-Token ' + DTAPIToken
-    #      }
-
-    #      for group in kr.matchedGroups:
-    #          params={"entitySelector": group["existsQuery"]}
-    #          response = helper.get_request(DTAPIURL, headers, params)        
-    #          entities = (response.json())['entities']
-             
-    #          for method in group["methods"]:
-    #            comparer=None
-    #            if method.startswith('SERVICE_METHOD-'):
-    #               comparer="entityId"
-    #            else:
-    #               comparer="displayName"
-
-    #            found = [x for x in entities if x[comparer] == method]
-                  
-    #            if len(found) > 0:
-    #               #Keyrequest exists 
-    #               tmp=found[0]
-    #               tmp["exists"]=True
-    #            else:
-    #               #Keyrequest not exists
-    #               tmp={"displayName":method,"type": "SERVICE_METHOD", "entityId":method, "exists":False}
-                    
-    #            kr.keyRequests.append(tmp)
-        
-    #      return kr
-     
-    def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken):
  
          DTAPIURL = DTAPIURL + "/api/v2/entities"
 
@@ -90,7 +55,7 @@ class KRParser:
          }
 
          for gid, group in enumerate(kr.matchedGroups):
-             params={"entitySelector": group["existsQuery"]}
+             params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"]}
              response = helper.get_request(DTAPIURL, headers, params)        
              entities = (response.json())['entities']
              
@@ -101,39 +66,12 @@ class KRParser:
                     found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
                  
                     if len(found) > 0:
-                         method["exists"]=True
+                         #method["exists"]=True
+                         method["hasData"][tfrom["label"]]=True
                          method["displayName"]=found[0]["displayName"]
                          method["entityId"]=found[0]["entityId"]
                     else:
-                        method["exists"]=False    
+                        method["hasData"][tfrom["label"]]=False                            
-                        
-    def checkKeyRequetsHasData(self,kr, DTAPIURL, DTAPIToken):
- 
-         DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
-
-         headers = {
-            'Content-Type': 'application/json',
-            'Authorization': 'Api-Token ' + DTAPIToken
-         }
-         
-         existingKRs=[k for k in kr.keyRequests if k["exists"]==True] 
-         
-         result=[]
-         if len(existingKRs) > 0:
-          query='builtin:service.keyRequest.count.total:filter(and(or(in("dt.entity.service_method",entitySelector("type(service_method),entityId(~"'+"~\",~\"".join([s["entityId"] for s in existingKRs])+'~")"))))):splitBy("dt.entity.service_method"):sort(value(auto,descending)):limit(20)'
-          params={"metricSelector": query, "resolution":"1d", "from":"now-1w"} 
-          response = helper.get_request(DTAPIURL, headers, params)        
-          result = (response.json())['result'][0]['data']
-          
-         for k in kr.keyRequests:
-              if k["exists"]==False:
-                  k['hasData_1W']=None
-              
-              if len(result)> 0:       
-               if len([r for r in result if k["entityId"] in r['dimensions'] and len(r['values']) > 0]) > 0:
-                  k['hasData_1W']=True
-               else:    
-                  k['hasData_1W']=False         
                             
               
                      
@@ -144,18 +82,19 @@ class KRParser:
         for gid, group in enumerate(kr.matchedGroups):
             for method in group["methods"]:
                if method.startswith('SERVICE_METHOD-'):
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":method, "exists":None, "groupId":gid}
+                    tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None,
                else:
-                    tmp={"displayName":method,"comparer": "displayName", "entityId":None, "exists":None, "groupId":gid}        
+                    tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None,       
                       
                kr.keyRequests.append(tmp)
                
         
-        if self.options and KROption.VALIDATE_EXISTS in self.options:
+        # if self.options and KROption.VALIDATE_EXISTS in self.options:
-          self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
+        #   self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
           
-          if KROption.VALIDATE_HASDATA in self.options:
+        if KROption.VALIDATE_HASDATA in self.options:
-              self.checkKeyRequetsHasData(kr,self.DTAPIURL, self.DTAPIToken)
+            self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
+            self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
         
         return kr