added new patterns & threading

createKeyRequestReport.py

@@ -90,17 +90,24 @@ def main():
                 DTURL = url.get('env-url')
 
                 krs=[]
-                krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN)
+                #krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN)
+                krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
                 
                 slosF=getSLO(env, DTTOKEN, DTURL)
-                
+                #slosF=slosF.loc[slosF['id'] == "a96a4031-e201-3e98-b739-27be7dc85a09"]
-                for index, row in slosF.iterrows():
+                krs=krp.parseBySLO_Threaded(slosF)
-                  #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
+                # for index, row in slosF.iterrows():
-                  #if row['id'] == "8fca3fbd-87a8-3005-91c8-727825ed7d99":
+                #   #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
-                  #if str.startswith(row["name"],"TP_"):
+                #   #if row['id'] == "a01cc45e-8369-3623-a156-1b261e6f4c98":
-                    krs.append(krp.parseBySLO(row))
+                #   #if str.startswith(row["name"],"TP_"):
+                #     krs.append(krp.parseBySLO(row))
               
                 resultSlos.extend(krs)
+    
+    
+    
+    #  for slo in resultSlos:
+    #      if slo.key
                 
     ignoerd,noData1M, noData1W= getStats(resultSlos)            
     write_to_excel(ignoerd, noData1M, noData1W)

environment.yaml

@@ -4,16 +4,16 @@ euprod:
     - env-url: "https://xxu26128.live.dynatrace.com"
     - env-token-name: "EUPROD_TOKEN_VAR"
     - jenkins: "https://jaws.bmwgroup.net/opapm/"
-naprod:
+# naprod:
-    - name: "naprod"
+#     - name: "naprod"
-    - env-url: "https://wgv50241.live.dynatrace.com"
+#     - env-url: "https://wgv50241.live.dynatrace.com"
-    - env-token-name: "NAPROD_TOKEN_VAR"
+#     - env-token-name: "NAPROD_TOKEN_VAR"
-    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+#     - jenkins: "https://jaws.bmwgroup.net/opapm/"
-cnprod:
+# cnprod:
-    - name: "cnprod"
+#     - name: "cnprod"
-    - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+#     - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
-    - env-token-name: "CNPROD_TOKEN_VAR"
+#     - env-token-name: "CNPROD_TOKEN_VAR"
-    - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
+#     - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
 # #cnpreprod:
 #     - name: "cnpreprod"
 #     - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"

key_request_parser/helper.py

@@ -1,6 +1,7 @@
 import requests
+from requests.adapters import HTTPAdapter, Retry
 
-def get_request(url, headers, params):
+def get_requestOld(url, headers, params):
       #try:
          response = requests.get(url, headers=headers, params=params, verify=False)
          response.raise_for_status()
@@ -14,6 +15,29 @@ def get_request(url, headers, params):
       #    return "An Unknown Error occurred" + repr(err)
       
          return response
+      
+def get_request(url, headers, params):
+      #try:
+      session = requests.Session()
+      retry = Retry(connect=3, backoff_factor=10)
+      adapter = HTTPAdapter(max_retries=retry)
+      session.mount('http://', adapter)
+      session.mount('https://', adapter)
+      
+      #response = requests.get(url, headers=headers, params=params, verify=False)
+      response = session.get(url,headers=headers, params=params, verify=False)
+      
+      response.raise_for_status()
+      # except requests.exceptions.HTTPError as errh:
+      #    return "An Http Error occurred:" + repr(errh)
+      # except requests.exceptions.ConnectionError as errc:
+      #    return "An Error Connecting to the API occurred:" + repr(errc)
+      # except requests.exceptions.Timeout as errt:
+      #    return "A Timeout Error occurred:" + repr(errt)
+      # except requests.exceptions.RequestException as err:
+      #    return "An Unknown Error occurred" + repr(err)
+      
+      return response
 
 def contains(list, filter):
             for x in list:

key_request_parser/keyrequests.py

@@ -52,12 +52,13 @@ class KeyRequestGroup(MutableSequence):
                 query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
             else:    
                 query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
-     
+        
-         if val["methods"][0].startswith("SERVICE_METHOD-"):
+         if len(val["methods"]) > 0:
-            query+=",entityId(\""+'","'.join(val["methods"])+"\")"
+            if val["methods"][0].startswith("SERVICE_METHOD-"):
-         else:  
+                query+=",entityId(\""+'","'.join(val["methods"])+"\")"
-            query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
+            else:  
-         
+                query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
+            
          val["existsQuery"]= query
          
     # def createServiceResolveQuery(self, val):
@@ -75,21 +76,23 @@ class KeyRequestGroup(MutableSequence):
 
     def insert(self, ii, val):
        self.createExistsQuery(val)
-       #self.createServiceResolveQuery(val)
+       
        self._list.insert(ii, val)
 
            
 
     def append(self, val):
 
+        for g in val:
+            
          if len(self._list) == 0:
             #self._list.insert(ii, val)
-            self.insert(len(self._list), val)
+            self.insert(len(self._list), g)
             return
 
          for group in self._list:
-            if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0:
+            if len(set(group["services"]) - set(g["services"])) > 0 or len(set(group["methods"]) - set(g["methods"])) > 0:
-               self.insert(len(self._list), val)
+               self.insert(len(self._list), g)
 
          
          

key_request_parser/krparser.py

@@ -3,6 +3,11 @@ import re
 from key_request_parser import patterns, keyrequests, helper
 
 from enum import Flag, auto
+import logging
+import threading
+import concurrent.futures
+import time
+
 
 
 class KROption(Flag):
@@ -12,8 +17,10 @@ class KROption(Flag):
     RESOLVESERVICES = auto()
 
 class KRParser:
-    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()]
+    #threadLimiter = threading.BoundedSemaphore(3)
-
+    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
+    lock = threading.Lock()
+    
     def normalize(self,x):
         #tmp=x.replace("~","")
         tmp=x.replace("\n","")
@@ -123,7 +130,7 @@ class KRParser:
                 else:
                     query+=",entityName.in(\""+k["displayName"]+"\")" 
                     
-                params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"}
+                params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
                 response = helper.get_request(DTAPIURL, headers, params)        
                 entities = (response.json())['entities']   
                 
@@ -151,11 +158,38 @@ class KRParser:
              #kr.mergeServices(entities)
              
                
-                    
+    def getKeyRequestsByServices(self, services):
+        #type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD"))
+        DTAPIURL = self.DTAPIURL + "/api/v2/entities"
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + self.DTAPIToken
+         }       
+         
+        if len(services) > 0:
+            if services[0].startswith("SERVICE-"):
+               query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))"
+            else:
+               query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
+        
+        params={"entitySelector": query}
+        response = helper.get_request(DTAPIURL, headers, params)        
+        entities = (response.json())['entities'] 
+        
+        return entities  
+                            
     
     def process(self, kr):
         
         for gid, group in enumerate(kr.matchedGroups):
+            
+            if len(group["services"]) > 0 and len(group["methods"])==0:
+                tmp_methods=self.getKeyRequestsByServices(group["services"])
+                
+                for m in tmp_methods:
+                    tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"],  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
+                    kr.keyRequests.append(tmp)
+                
             for method in group["methods"]:
                if method.startswith('SERVICE_METHOD-'):
                     tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
@@ -163,15 +197,25 @@ class KRParser:
                     tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
                       
                kr.keyRequests.append(tmp)
+            
+            
                
-        
+            # for service in group["services"]:
-        # if self.options and KROption.VALIDATE_EXISTS in self.options:
+            #     if service.startswith('SERVICE-'):
-        #   self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
+            #         tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
-        self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
+            #     else:
-          
+            #         tmp={"displayName":service,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
-        if KROption.VALIDATE_HASDATA in self.options:
+                
-            self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
+            #     kr.services.append(tmp)
-            self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
+                   
+               
+        if self.options:
+            if KROption.RESOLVEKEYREQUETS in self.options:
+                self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
+            
+            if KROption.VALIDATE_HASDATA in self.options:
+                self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
+                self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
         # elif KROption.RESOLVEKEYREQUETS in self.options:
         #     self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
         # if KROption.RESOLVESERVICES in self.options:
@@ -180,32 +224,76 @@ class KRParser:
         return kr
         
 
-    def parseBySLO(self, row):
+    def parseBySLO(self,index,row):
         #normalize
-        normFilter=self.normalize(row['filter'])
+        print(index)
-        normExpresseion=self.normalize(row['metricExpression'])
         
-        tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
+        try:
-    
+            normFilter=self.normalize(row['filter'])
-
+            normExpresseion=self.normalize(row['metricExpression'])
-        #SLO with Filter
-        if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"):
-         subject=normFilter     
-        else:
-         subject=normExpresseion
-
-        groups=self.applyPatterns(subject)
-    
-        for g in groups:
-            if g["methods"] != None and len(g["methods"]) > 0:
-                tmp_KR.matchedGroups.append(g)
             
-        #self.process(tmp_KR)
+            tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
+        
 
-        return self.process(tmp_KR)
+            #SLO with Filter
+            if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
+               subject=normFilter     
+            else:
+               subject=normExpresseion
+
+            groups=self.applyPatterns(subject)
+        
+            tmp_KR.matchedGroups.append(groups)
+            # for g in groups:
+            #     #if g["methods"] != None and len(g["methods"]) > 0:
+            #        tmp_KR.matchedGroups.append(g)
+                
+            #self.process(tmp_KR)
+            kr=self.process(tmp_KR)
+            with self.lock:
+                self.krs.append(kr)
+                
+        except Exception as err:
+                  print(repr(err)) 
+        #return self.process(tmp_KR)
+    
+    def parseBySLO_Threaded(self, slosF):
+         self.krs=[]
+         #i=1
+        #  threads = list()
+        #  for index, row in slosF.iterrows():
+        #     logging.info("Main    : create and start thread %d.", index)
+        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
+        #     threads.append(x)
+        #     x.start()
+            
+        #     #krs.append(krp.parseBySLO(row))
+        
+        #  for index, thread in enumerate(threads):
+        #     logging.info("Main    : before joining thread %d.", index)
+        #     thread.join()
+        #     logging.info("Main    : thread %d done", index)
+         
+        #  #resultSlos.extend(krs)    
+         
+         
+         with concurrent.futures.ThreadPoolExecutor(10) as executor:   
+            for index, row in slosF.iterrows():
+                # if i % 25 == 0:
+                #     time.sleep(0)
+                #args={index:index, }
+                executor.submit(self.parseBySLO, index,row)
+                # print(str(i)+"\n")
+                # i=i+1
+        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
+        #     threads.append(x)
+        #     x.start()
+        
+         return self.krs
         
     
-    def __init__(self, options: KROption , DTAPIURL, DTAPIToken ):
+    def __init__(self, options: KROption=None , DTAPIURL=None, DTAPIToken=None ):
         self.DTAPIURL= DTAPIURL
         self.DTAPIToken=DTAPIToken
-        self.options=options
+        self.options=options
+        self.krs=[]

key_request_parser/patterns.py

@@ -47,42 +47,50 @@ class Pattern2:
 class Pattern3:
 
    def parseServicesAndMethods(self, metricExpression):
+         metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
          result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
          # services=[]
          # methods=[]
          groups=[]
          if result:
             for r in result:
-                methods=[s.strip() for s in r.split(",")]
+                methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
                 groups.append({"services":[], "methods":methods})
            
          return groups
 
-
+class Pattern4:
-
-# class Pattern4:
-
-#    def parseServicesAndMethods(self, metricExpression):
-#          result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
    
-#          groups=[]
+   def parseServicesAndMethods(self, metricExpression):
-#          methods=[]
+         metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-#          if result:
+         result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
-#             for r in result:
+    
-#                 methods.append(r)
+         groups=[]
-                
+         if result:
-#          groups.append({"services":[], "methods":methods})
+            for r in result:
+               if not r:
+                #methods=[s.strip() for s in r.split(",")]
+                services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
+                groups.append({"services":services, "methods":[]})
            
-#          return groups
+         return groups
 
-# class FilterMethodPattern:
+class Pattern5:
-
+   
-#    def parseServicesAndMethods(self, metricExpression):
+   def parseServicesAndMethods(self, metricExpression):
-#          result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+       
-#          services=[]
+         #Endoce
-#          methods=[]
+         metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-#          if result:
+         result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)      
-#             for r in result:
+         #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
-#                 methods=[s.strip() for s in r.split(",")] 
+        #services=[]
-           
+         #methods=[]
-#          return services, methods
+         groups=[]
+         if result:
+            for r in result:
+                if not r:
+                  services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] 
+                  #methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] 
+                  groups.append({"services":services, "methods":[]})
+         #return services, methods
+         return groups