added dervice for every key request when possible

createKeyRequestReport.py

@@ -46,26 +46,26 @@ def getStats(krs):
     
     tmpIgnoredList=[]
     for kr in [a for a in krs if len(a.keyRequests) == 0]:
-        tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","",kr.metadata["filter"], kr.metadata["metricExpression"]])
+        tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","","",kr.metadata["filter"], kr.metadata["metricExpression"]])
     
     
     
     noData1M=[]
     for kr in [s for s in krs if s.hasNoData("1M")==True]:
          for k in kr.getKeyRequestByHasData("1M"):
-             noData1M.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])
+             noData1M.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"],k["services"][0]["displayName"] if len(k["services"]) > 0 else "", kr.metadata["filter"], kr.metadata["metricExpression"]])
     
     noData1W=[]
     for kr in [s for s in krs if s.hasNoData("1W")==True]:
          for k in kr.getKeyRequestByHasData("1W"):
-             noData1W.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])         
+             noData1W.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], k["services"][0]["displayName"] if len(k["services"]) > 0 else "", kr.metadata["filter"], kr.metadata["metricExpression"]])         
              
     # noData1W=[]
     # for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]:
     #     for k in kr.getNoData1WKeyRequests():
     #          noData1W.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])        
     
-    columns =['slo', 'env', 'displayName','entityId', 'filter', 'metricExpression']
+    columns =['slo', 'env', 'displayName','entityId','service','filter', 'metricExpression']
     ignored=pd.DataFrame(tmpIgnoredList, columns=columns, dtype = float)
     noData_1M=pd.DataFrame(noData1M, columns = columns, dtype = float)
     noData_1W=pd.DataFrame(noData1W, columns = columns, dtype = float)
@@ -90,12 +90,14 @@ def main():
                 DTURL = url.get('env-url')
 
                 krs=[]
-                krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN)
+                krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN)
                 
                 slosF=getSLO(env, DTTOKEN, DTURL)
                 
                 for index, row in slosF.iterrows():
-                  #if row['id'] == "7a7f9c66-75c1-30b9-a505-801ffa5f9e98": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
+                  #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
+                  #if row['id'] == "8fca3fbd-87a8-3005-91c8-727825ed7d99":
+                  #if str.startswith(row["name"],"TP_"):
                     krs.append(krp.parseBySLO(row))
               
                 resultSlos.extend(krs)

key_request_parser/helper.py

@@ -1,17 +1,17 @@
 import requests
 
 def get_request(url, headers, params):
-      try:
+      #try:
          response = requests.get(url, headers=headers, params=params, verify=False)
          response.raise_for_status()
-      except requests.exceptions.HTTPError as errh:
+      # except requests.exceptions.HTTPError as errh:
-         return "An Http Error occurred:" + repr(errh)
+      #    return "An Http Error occurred:" + repr(errh)
-      except requests.exceptions.ConnectionError as errc:
+      # except requests.exceptions.ConnectionError as errc:
-         return "An Error Connecting to the API occurred:" + repr(errc)
+      #    return "An Error Connecting to the API occurred:" + repr(errc)
-      except requests.exceptions.Timeout as errt:
+      # except requests.exceptions.Timeout as errt:
-         return "A Timeout Error occurred:" + repr(errt)
+      #    return "A Timeout Error occurred:" + repr(errt)
-      except requests.exceptions.RequestException as err:
+      # except requests.exceptions.RequestException as err:
-         return "An Unknown Error occurred" + repr(err)
+      #    return "An Unknown Error occurred" + repr(err)
       
          return response
 

key_request_parser/keyrequests.py

@@ -60,12 +60,22 @@ class KeyRequestGroup(MutableSequence):
          
          val["existsQuery"]= query
          
+    # def createServiceResolveQuery(self, val):
+    #     query="type(SERVICE)"
+    #     val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
    
+    #     if len(val["services"]) > 0:
+    #         if val["services"][0].startswith("SERVICE-"):
+    #             query+=",entityId(\""+'","'.join(val["services"])+"\")"
+    #         else:    
+    #             query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
+             
+    #     val["resolveServiceQuery"]= query        
 
 
     def insert(self, ii, val):
        self.createExistsQuery(val)
-
+       #self.createServiceResolveQuery(val)
        self._list.insert(ii, val)
 
            
@@ -116,9 +126,18 @@ class KR:
         return False 
     
     
+             
+
      def checkKeyRequestsHasData(self):
          pass
      
+     def mergeServices(self, listServices):
+         listOfServiceIds=[o["entityId"] for o in self.services]
+          
+         for s in listServices:
+             if s["entityId"] not in listOfServiceIds:
+                 self.services.append(s)
+
      def __init__(self,
       metadata,
       matchedGroups: KeyRequestGroup = None):
@@ -130,5 +149,6 @@ class KR:
             self.matchedGroups = keyRequests_groups
 
          self.keyRequests=[]
+         self.services=[]
       
          

key_request_parser/krparser.py

@@ -8,9 +8,11 @@ from enum import Flag, auto
 class KROption(Flag):
     VALIDATE_EXISTS = auto()
     VALIDATE_HASDATA = auto()
+    RESOLVEKEYREQUETS = auto()
+    RESOLVESERVICES = auto()
 
 class KRParser:
-    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern4()]
+    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()]
 
     def normalize(self,x):
         #tmp=x.replace("~","")
@@ -55,7 +57,7 @@ class KRParser:
          }
 
          for gid, group in enumerate(kr.matchedGroups):
-             params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"]}
+             params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
              response = helper.get_request(DTAPIURL, headers, params)        
              entities = (response.json())['entities']
              
@@ -66,13 +68,87 @@ class KRParser:
                     found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
                  
                     if len(found) > 0:
-                         #method["exists"]=True
                          method["hasData"][tfrom["label"]]=True
-                         method["displayName"]=found[0]["displayName"]
+                         #method["displayName"]=found[0]["displayName"]
-                         method["entityId"]=found[0]["entityId"]
+                         #method["entityId"]=found[0]["entityId"]
+                         #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
+                         
+                        #  for idx,o in enumerate(method["services"]):
+                        #     tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
+                        #     if len(tmpS)>0:
+                        #         method["services"][idx]=tmpS[0]
+                             
                     else:
                         method["hasData"][tfrom["label"]]=False                            
                             
+    def resolveServices(self,services, DTAPIURL, DTAPIToken):
+         #DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+         headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTAPIToken
+         }       
+         
+         for gid, service in enumerate(services):
+             query="type(SERVICE),entityId("+service["id"]+")"
+             params={"entitySelector": query,"from":"now-2y"}
+             response = helper.get_request(DTAPIURL, headers, params)        
+             entities = (response.json())['entities']     
+             
+             if len(entities)>0:
+                 services[gid]=entities[0]
+    
+    def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
+         DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+         headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTAPIToken
+         }       
+         
+         
+         for gid, k in enumerate(kr.keyRequests):
+             try:
+                query="type(service_method)"
+                group=kr.matchedGroups[k["groupId"]]
+                
+                if len(group["services"])> 0:
+                    if group["services"][0].startswith("SERVICE-"):
+                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
+                    else:    
+                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
+                
+                if k["comparer"]=="entityId":
+                    query+=",entityId("+k["entityId"]+")"
+                else:
+                    query+=",entityName.in(\""+k["displayName"]+"\")" 
+                    
+                params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"}
+                response = helper.get_request(DTAPIURL, headers, params)        
+                entities = (response.json())['entities']   
+                
+                #  if len(entities) > 1:
+                #      kr.keyRequests[gid]['foundCount']=len(entities)
+                #      print("Multiple keyrequest found: ")
+                
+                if len(entities)> 0:
+                    kr.keyRequests[gid]["found"]=True
+                    kr.keyRequests[gid]['foundCount']=len(entities)
+                    kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
+                    kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
+                    
+                    if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
+                        kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
+                
+                if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
+                    self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
+                    
+             except Exception as err:
+                  kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
+                 
+                 
+                   
+             #kr.mergeServices(entities)
              
                
                     
@@ -82,19 +158,24 @@ class KRParser:
         for gid, group in enumerate(kr.matchedGroups):
             for method in group["methods"]:
                if method.startswith('SERVICE_METHOD-'):
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None,
+                    tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
                else:
-                    tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None,       
+                    tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
                       
                kr.keyRequests.append(tmp)
                
         
         # if self.options and KROption.VALIDATE_EXISTS in self.options:
         #   self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
+        self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
           
         if KROption.VALIDATE_HASDATA in self.options:
             self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
             self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
+        # elif KROption.RESOLVEKEYREQUETS in self.options:
+        #     self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
+        # if KROption.RESOLVESERVICES in self.options:
+        #     self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
             
         return kr
         

key_request_parser/patterns.py

@@ -60,20 +60,20 @@ class Pattern3:
 
 
 
-class Pattern4:
+# class Pattern4:
 
-   def parseServicesAndMethods(self, metricExpression):
+#    def parseServicesAndMethods(self, metricExpression):
-         result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+#          result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
    
-         groups=[]
+#          groups=[]
-         methods=[]
+#          methods=[]
-         if result:
+#          if result:
-            for r in result:
+#             for r in result:
-                methods.append(r)
+#                 methods.append(r)
                 
-         groups.append({"services":[], "methods":methods})
+#          groups.append({"services":[], "methods":methods})
            
-         return groups
+#          return groups
 
 # class FilterMethodPattern: