requirements.txt edited online with Bitbucket

switching back to default import

.gitignore

@@ -0,0 +1,138 @@
+.vscode
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Terraform stuff
+**/.terraform/*
+crash.log
+*.tfvars
+
+#excel reports
+*.xlsx

KRParser/keyrequests.py

@@ -58,20 +58,24 @@ class KeyRequestGroup(MutableSequence):
                 query+=",entityId(\""+'","'.join(val["methods"])+"\")"
             else:  
                 query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
-            
+        #  query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)"
+        #  val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
+        #  val['methods'] = list(map(lambda x: x.replace("~","") , val['methods']))
+        #  if len(val["services"]) > 0:
+        #      if val["services"][0].startswith("SERVICE-"):
+        #         query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
+        #      else:
+        #         query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
+                
+        #  if len(val["methods"]) > 0:
+        #      if val["methods"][0].startswith("SERVICE_METHOD-"):
+        #          query+=",entityId(\""+'","'.join(val["methods"])+"\")"
+        #      else:  
+        #          query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
+        
+        #  query+="\")))"
          val["existsQuery"]= query
-         
+               
-    # def createServiceResolveQuery(self, val):
-    #     query="type(SERVICE)"
-    #     val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
-   
-    #     if len(val["services"]) > 0:
-    #         if val["services"][0].startswith("SERVICE-"):
-    #             query+=",entityId(\""+'","'.join(val["services"])+"\")"
-    #         else:    
-    #             query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
-             
-    #     val["resolveServiceQuery"]= query        
 
 
     def insert(self, ii, val):
@@ -127,8 +131,6 @@ class KR:
             return True
             
         return False 
-    
-    
              
 
      def checkKeyRequestsHasData(self):
@@ -140,6 +142,12 @@ class KR:
          for s in listServices:
              if s["entityId"] not in listOfServiceIds:
                  self.services.append(s)
+                 
+    #  def addKeyRequest(self, data): 
+    #     if type(data) == list:
+    #         for m in data:
+    #             self.keyRequests.append({"displayName":m["["displayName"]"], "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""})         
+     
 
      def __init__(self,
       metadata,

KRParser/krparser.py

@@ -1,14 +1,12 @@
 import re
 
 from KRParser import patterns, keyrequests, helper
-
-
 from enum import Flag, auto
-import logging
 import threading
 import concurrent.futures
-import time
 from jsonmerge import merge
+import pandas as pd
+from tqdm import *
 
 
 
@@ -19,24 +17,15 @@ class KROption(Flag):
     RESOLVESERVICES = auto()
 
 class KRParser:
-    #threadLimiter = threading.BoundedSemaphore(3)
+    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4(), patterns.Pattern5()]
-    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
     lock = threading.Lock()
     
     def normalize(self,x):
-        #tmp=x.replace("~","")
         tmp=x.replace("\n","")
-        #tmp=tmp.replace("\"/","\"")
-        #tmp=tmp.replace("\"/","") -_>was active
-        #tmp=tmp.replace("/\"","\"")
         tmp=tmp.replace("/\"","")
         tmp=tmp.replace("\"","")
         tmp=tmp.replace("\t","")
-        
         tmp=re.sub("([\s]*)\)", ")", tmp)
-        
-        
-        
         tmp=re.sub("\([\s\n\r]*", "(", tmp)
         tmp=re.sub("\,[\s\n\r]*", ",", tmp)
         tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
@@ -50,7 +39,9 @@ class KRParser:
         for p in self.patterns:        
             groups=p.parseServicesAndMethods(subject)
             
-            if len(groups) > 0:        
+            if len(groups) > 0:
+                for g in groups:
+                    g["pattern"]=str(p.__class__)
                 break
         
         return groups 
@@ -58,40 +49,54 @@ class KRParser:
     
     def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
  
-         DTAPIURL = DTAPIURL + "/api/v2/entities"
+         # DTAPIURL = DTAPIURL + "/api/v2/entities"
 
+        #  headers = {
+        #     'Content-Type': 'application/json',
+        #     'Authorization': 'Api-Token ' + DTAPIToken
+        #  }
+
+        #  for gid, group in enumerate(kr.matchedGroups):
+        #      params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
+        #      response = helper.get_request(DTAPIURL, headers, params)        
+        #      entities = (response.json())['entities']
+             
+        #      if len(entities) > 0:
+        #          y=0
+        #      for method in kr.keyRequests:
+        #          if method["groupId"] == gid:
+        #             found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
+                 
+        #             if len(found) > 0:
+        #                  method["hasData"][tfrom["label"]]=True
+        #             else:
+        #                 method["hasData"][tfrom["label"]]=False
+        
+         DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
+         
          headers = {
             'Content-Type': 'application/json',
             'Authorization': 'Api-Token ' + DTAPIToken
          }
 
          for gid, group in enumerate(kr.matchedGroups):
-             params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
+             params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]}
              response = helper.get_request(DTAPIURL, headers, params)        
-             entities = (response.json())['entities']
+             entities = (response.json())["result"][0]["data"]
              
-             if len(entities) > 0:
-                 y=0
              for method in kr.keyRequests:
                  if method["groupId"] == gid:
-                    found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
+                     
+                    found = [x for x in entities if x["dimensions"][0] == method["entityId"]] 
                  
                     if len(found) > 0:
                          method["hasData"][tfrom["label"]]=True
-                         #method["displayName"]=found[0]["displayName"]
+                         method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None])
-                         #method["entityId"]=found[0]["entityId"]
-                         #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
-                         
-                        #  for idx,o in enumerate(method["services"]):
-                        #     tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
-                        #     if len(tmpS)>0:
-                        #         method["services"][idx]=tmpS[0]
-                             
                     else:
-                        method["hasData"][tfrom["label"]]=False                            
+                        method["hasData"][tfrom["label"]]=False
+                        method["count"][tfrom["label"]]=0        
                             
     def resolveServices(self,services, DTAPIURL, DTAPIToken):
-         #DTAPIURL = DTAPIURL + "/api/v2/entities"
 
          headers = {
             'Content-Type': 'application/json',
@@ -101,14 +106,16 @@ class KRParser:
          for gid, service in enumerate(services):
              query="type(SERVICE),entityId("+service["id"]+")"
             
-             params=merge(self.serviceLookupParams,{"entitySelector": query})
+             params=merge(self.config["serviceLookupParams"],{"entitySelector": query})
-             #params={"entitySelector": query,"from":"now-2y", "fields":"tags"}
+          
              response = helper.get_request(DTAPIURL, headers, params)        
              entities = (response.json())['entities']     
              
              if len(entities)>0:
                  services[gid]=entities[0]
-    
+                 
+                 
+                 
     def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
          DTAPIURL = DTAPIURL + "/api/v2/entities"
 
@@ -117,49 +124,55 @@ class KRParser:
             'Authorization': 'Api-Token ' + DTAPIToken
          }       
          
-         
+         tmp_KR=[]
-         for gid, k in enumerate(kr.keyRequests):
+         for gid, group in enumerate(kr.matchedGroups):
-             try:
+              #try:
-                query="type(service_method)"
+                for k in group["methods"]:
-                group=kr.matchedGroups[k["groupId"]]
+                     
-                
+                     
-                if len(group["services"])> 0:
+                    #tmp_kr={"displayName":None, "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}
-                    if group["services"][0].startswith("SERVICE-"):
-                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
-                    else:    
-                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
-                
-                if k["comparer"]=="entityId":
-                    query+=",entityId("+k["entityId"]+")"
-                else:
-                    query+=",entityName.in(\""+k["displayName"]+"\")" 
                     
-                params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+                    query="type(service_method)"
-                response = helper.get_request(DTAPIURL, headers, params)        
-                entities = (response.json())['entities']   
-                
-                #  if len(entities) > 1:
-                #      kr.keyRequests[gid]['foundCount']=len(entities)
-                #      print("Multiple keyrequest found: ")
-                
-                if len(entities)> 0:
-                    kr.keyRequests[gid]["found"]=True
-                    kr.keyRequests[gid]['foundCount']=len(entities)
-                    kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
-                    kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
                     
-                    if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
+                    if len(group["services"])> 0:
-                        kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
+                        
-                
+                        if group["services"][0].startswith("SERVICE-"):
-                if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
+                            query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
-                    self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
+                        else:    
+                            query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
                     
-             except Exception as err:
+                    if k.startswith("SERVICE_METHOD-"):
-                  kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
+                        query+=",entityId("+k+")"
-                 
+                    else:
-                 
+                        query+=",entityName.in(\""+k+"\")" 
-                   
+                        
-             #kr.mergeServices(entities)
+                    params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+                    response = helper.get_request(DTAPIURL, headers, params)        
+                    entities = (response.json())['entities']  
+                    
+                    if len(entities)> 0:
+                       for ent in entities:
+                         
+                         tmp_kr=merge({"displayName":ent["displayName"], "entityId":ent["entityId"],  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"] }, self.config["extendResultObjects"])
+                           
+                         if "isServiceMethodOfService" in ent["fromRelationships"]:
+                           tmp_kr["services"]=ent["fromRelationships"]["isServiceMethodOfService"]
+                        
+                         if options and KROption.RESOLVESERVICES in options and len(tmp_kr["services"])>0:
+                           self.resolveServices(tmp_kr["services"], DTAPIURL, DTAPIToken)
+                        
+                         tmp_KR.append(tmp_kr)
+                          
+                    else:
+                         if k.startswith('SERVICE_METHOD-'):
+                            tmp_kr=merge({"displayName": None,"comparer": "entityId", "entityId":k,  "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
+                         else:
+                            tmp_kr=merge({"displayName":k,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,           
+                        
+                         tmp_KR.append(tmp_kr)
+        
+        
+         return tmp_KR
              
                
     def getKeyRequestsByServices(self, services):
@@ -176,7 +189,7 @@ class KRParser:
             else:
                query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
         
-        params={"entitySelector": query}
+        params={"entitySelector": query,"from":"now-1y"}
         response = helper.get_request(DTAPIURL, headers, params)        
         entities = (response.json())['entities'] 
         
@@ -191,53 +204,39 @@ class KRParser:
                 tmp_methods=self.getKeyRequestsByServices(group["services"])
                 
                 for m in tmp_methods:
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"],  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
+                    group["methods"].append(m["entityId"])
-                    kr.keyRequests.append(tmp)
+                 
+                    # tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"],  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
+                    # kr.keyRequests.append(tmp)
                 
-            for method in group["methods"]:
+            # for method in group["methods"]:
-               if method.startswith('SERVICE_METHOD-'):
+            #    if method.startswith('SERVICE_METHOD-'):
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
+            #         tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
-               else:
+            #    else:
-                    tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
+            #         tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,       
                       
-               kr.keyRequests.append(tmp)
+            #    kr.keyRequests.append(tmp)
             
-            
+        
-               
-            # for service in group["services"]:
-            #     if service.startswith('SERVICE-'):
-            #         tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
-            #     else:
-            #         tmp={"displayName":service,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
-                
-            #     kr.services.append(tmp)
                    
+        kr.keyRequests=self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
                
         if self.options:
-            if KROption.RESOLVEKEYREQUETS in self.options:
-                self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
             
             if KROption.VALIDATE_HASDATA in self.options:
                 self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
                 self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
-        # elif KROption.RESOLVEKEYREQUETS in self.options:
-        #     self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
-        # if KROption.RESOLVESERVICES in self.options:
-        #     self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
             
         return kr
         
 
-    def parseBySLO(self,index,row):
+    def parseBySLO(self,row):
-        #normalize
-        print(index)
         
         try:
             normFilter=self.normalize(row['filter'])
             normExpresseion=self.normalize(row['metricExpression'])
             
-            tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
+            tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"]))
-        
 
             #SLO with Filter
             if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
@@ -248,57 +247,47 @@ class KRParser:
             groups=self.applyPatterns(subject)
         
             tmp_KR.matchedGroups.append(groups)
-            # for g in groups:
+          
-            #     #if g["methods"] != None and len(g["methods"]) > 0:
-            #        tmp_KR.matchedGroups.append(g)
-                
-            #self.process(tmp_KR)
             kr=self.process(tmp_KR)
+            
             with self.lock:
                 self.krs.append(kr)
+                self.pbar.update()
                 
         except Exception as err:
                   print(repr(err)) 
-        #return self.process(tmp_KR)
     
-    def parseBySLO_Threaded(self, slosF):
+    def parse(self, input):
-         self.krs=[]
-         #i=1
-        #  threads = list()
-        #  for index, row in slosF.iterrows():
-        #     logging.info("Main    : create and start thread %d.", index)
-        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
-        #     threads.append(x)
-        #     x.start()
-            
-        #     #krs.append(krp.parseBySLO(row))
         
-        #  for index, thread in enumerate(threads):
+        with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor:
-        #     logging.info("Main    : before joining thread %d.", index)
+             
-        #     thread.join()
+                if type(input) == pd.DataFrame:
-        #     logging.info("Main    : thread %d done", index)
+                    self.pbar = tqdm(total=input["id"].count(),desc=self.name)
-         
+                    
-        #  #resultSlos.extend(krs)    
+                    for index, row in input.iterrows():
-         
+                        executor.submit(self.parseBySLO, row)
-         
+                            
-         with concurrent.futures.ThreadPoolExecutor(10) as executor:   
+                elif type(input)== list:
-            for index, row in slosF.iterrows():
+                    self.pbar = tqdm(total=len(input), desc=self.name)
-                # if i % 25 == 0:
+                    for slo in input:
-                #     time.sleep(0)
+                        executor.submit(self.parseBySLO, slo)
-                #args={index:index, }
+                        
-                executor.submit(self.parseBySLO, index,row)
+                elif type(input) == dict:
-                # print(str(i)+"\n")
+                    #self.pbar = tqdm(total=1, desc=self.name)
-                # i=i+1
+                    #executor.submit(self.parseBySLO, slo)
-        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
+                    self.parseBySLO(input)
-        #     threads.append(x)
+                       
-        #     x.start()
+        
+        return self.krs
         
-         return self.krs
         
     
-    def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ):
+    def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ):
+        self.name=name
         self.DTAPIURL= DTAPIURL
         self.DTAPIToken=DTAPIToken
         self.options=options
-        self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams)
+        self.config=merge({"threads": 3, 
+                           "serviceLookupParams":{"from":"now-2y"},
+                           "extendResultObjects":{}}, config)
         self.krs=[]

KRParser/patterns.py

@@ -22,7 +22,7 @@ class Pattern1:
             for r in result:
                 services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] 
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] 
-                groups.append({"services":services, "methods":methods})
+                groups.append({"services":services, "methods":methods, "_mInSloDef":True})
          #return services, methods
          return groups
 
@@ -38,7 +38,7 @@ class Pattern2:
             for r in result:
                 services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] 
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")] 
-                groups.append({"services":services, "methods":methods})
+                groups.append({"services":services, "methods":methods, "_mInSloDef":True})
                 
          return groups
 
@@ -55,7 +55,7 @@ class Pattern3:
          if result:
             for r in result:
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
-                groups.append({"services":[], "methods":methods})
+                groups.append({"services":[], "methods":methods, "_mInSloDef":True})
            
          return groups
 
@@ -63,15 +63,16 @@ class Pattern4:
    
    def parseServicesAndMethods(self, metricExpression):
          metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-         result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+         #result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
-    
+         result = re.findall(r"type\(\"?service\"?\),[entityName|entityId]*[\.]*[equals|in|contains]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+        
          groups=[]
          if result:
             for r in result:
-               if not r:
+               #if not r:
                 #methods=[s.strip() for s in r.split(",")]
                 services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
-                groups.append({"services":services, "methods":[]})
+                groups.append({"services":services, "methods":[], "_mInSloDef":False})
            
          return groups
 
@@ -81,16 +82,16 @@ class Pattern5:
        
          #Endoce
          metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-         result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)      
+         result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)   
+         #type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)   
          #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
         #services=[]
          #methods=[]
          groups=[]
          if result:
             for r in result:
-                if not r:
+               services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] 
-                  services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] 
                   #methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] 
-                  groups.append({"services":services, "methods":[]})
+               groups.append({"services":services, "methods":[], "_mInSloDef":False})
          #return services, methods
          return groups

KeyRequestParser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: KeyRequestParser
-Version: 0.4
+Version: 0.5
 License: MIT
 
 Parses Keyrequests

KeyRequestParser.egg-info/SOURCES.txt

@@ -7,4 +7,5 @@ KRParser/patterns.py
 KeyRequestParser.egg-info/PKG-INFO
 KeyRequestParser.egg-info/SOURCES.txt
 KeyRequestParser.egg-info/dependency_links.txt
+KeyRequestParser.egg-info/requires.txt
 KeyRequestParser.egg-info/top_level.txt

build/lib/KRParser/keyrequests.py

@@ -58,20 +58,24 @@ class KeyRequestGroup(MutableSequence):
                 query+=",entityId(\""+'","'.join(val["methods"])+"\")"
             else:  
                 query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
-            
+        #  query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)"
+        #  val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
+        #  val['methods'] = list(map(lambda x: x.replace("~","") , val['methods']))
+        #  if len(val["services"]) > 0:
+        #      if val["services"][0].startswith("SERVICE-"):
+        #         query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
+        #      else:
+        #         query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
+                
+        #  if len(val["methods"]) > 0:
+        #      if val["methods"][0].startswith("SERVICE_METHOD-"):
+        #          query+=",entityId(\""+'","'.join(val["methods"])+"\")"
+        #      else:  
+        #          query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
+        
+        #  query+="\")))"
          val["existsQuery"]= query
-         
+               
-    # def createServiceResolveQuery(self, val):
-    #     query="type(SERVICE)"
-    #     val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
-   
-    #     if len(val["services"]) > 0:
-    #         if val["services"][0].startswith("SERVICE-"):
-    #             query+=",entityId(\""+'","'.join(val["services"])+"\")"
-    #         else:    
-    #             query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
-             
-    #     val["resolveServiceQuery"]= query        
 
 
     def insert(self, ii, val):
@@ -127,8 +131,6 @@ class KR:
             return True
             
         return False 
-    
-    
              
 
      def checkKeyRequestsHasData(self):
@@ -140,6 +142,12 @@ class KR:
          for s in listServices:
              if s["entityId"] not in listOfServiceIds:
                  self.services.append(s)
+                 
+    #  def addKeyRequest(self, data): 
+    #     if type(data) == list:
+    #         for m in data:
+    #             self.keyRequests.append({"displayName":m["["displayName"]"], "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""})         
+     
 
      def __init__(self,
       metadata,

build/lib/KRParser/krparser.py

@@ -1,14 +1,12 @@
 import re
 
 from KRParser import patterns, keyrequests, helper
-
-
 from enum import Flag, auto
-import logging
 import threading
 import concurrent.futures
-import time
 from jsonmerge import merge
+import pandas as pd
+from tqdm import *
 
 
 
@@ -19,24 +17,15 @@ class KROption(Flag):
     RESOLVESERVICES = auto()
 
 class KRParser:
-    #threadLimiter = threading.BoundedSemaphore(3)
+    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4(), patterns.Pattern5()]
-    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
     lock = threading.Lock()
     
     def normalize(self,x):
-        #tmp=x.replace("~","")
         tmp=x.replace("\n","")
-        #tmp=tmp.replace("\"/","\"")
-        #tmp=tmp.replace("\"/","") -_>was active
-        #tmp=tmp.replace("/\"","\"")
         tmp=tmp.replace("/\"","")
         tmp=tmp.replace("\"","")
         tmp=tmp.replace("\t","")
-        
         tmp=re.sub("([\s]*)\)", ")", tmp)
-        
-        
-        
         tmp=re.sub("\([\s\n\r]*", "(", tmp)
         tmp=re.sub("\,[\s\n\r]*", ",", tmp)
         tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
@@ -50,7 +39,9 @@ class KRParser:
         for p in self.patterns:        
             groups=p.parseServicesAndMethods(subject)
             
-            if len(groups) > 0:        
+            if len(groups) > 0:
+                for g in groups:
+                    g["pattern"]=str(p.__class__)
                 break
         
         return groups 
@@ -58,40 +49,54 @@ class KRParser:
     
     def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
  
-         DTAPIURL = DTAPIURL + "/api/v2/entities"
+         # DTAPIURL = DTAPIURL + "/api/v2/entities"
 
+        #  headers = {
+        #     'Content-Type': 'application/json',
+        #     'Authorization': 'Api-Token ' + DTAPIToken
+        #  }
+
+        #  for gid, group in enumerate(kr.matchedGroups):
+        #      params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
+        #      response = helper.get_request(DTAPIURL, headers, params)        
+        #      entities = (response.json())['entities']
+             
+        #      if len(entities) > 0:
+        #          y=0
+        #      for method in kr.keyRequests:
+        #          if method["groupId"] == gid:
+        #             found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
+                 
+        #             if len(found) > 0:
+        #                  method["hasData"][tfrom["label"]]=True
+        #             else:
+        #                 method["hasData"][tfrom["label"]]=False
+        
+         DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
+         
          headers = {
             'Content-Type': 'application/json',
             'Authorization': 'Api-Token ' + DTAPIToken
          }
 
          for gid, group in enumerate(kr.matchedGroups):
-             params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
+             params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]}
              response = helper.get_request(DTAPIURL, headers, params)        
-             entities = (response.json())['entities']
+             entities = (response.json())["result"][0]["data"]
              
-             if len(entities) > 0:
-                 y=0
              for method in kr.keyRequests:
                  if method["groupId"] == gid:
-                    found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
+                     
+                    found = [x for x in entities if x["dimensions"][0] == method["entityId"]] 
                  
                     if len(found) > 0:
                          method["hasData"][tfrom["label"]]=True
-                         #method["displayName"]=found[0]["displayName"]
+                         method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None])
-                         #method["entityId"]=found[0]["entityId"]
-                         #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
-                         
-                        #  for idx,o in enumerate(method["services"]):
-                        #     tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
-                        #     if len(tmpS)>0:
-                        #         method["services"][idx]=tmpS[0]
-                             
                     else:
-                        method["hasData"][tfrom["label"]]=False                            
+                        method["hasData"][tfrom["label"]]=False
+                        method["count"][tfrom["label"]]=0        
                             
     def resolveServices(self,services, DTAPIURL, DTAPIToken):
-         #DTAPIURL = DTAPIURL + "/api/v2/entities"
 
          headers = {
             'Content-Type': 'application/json',
@@ -101,14 +106,16 @@ class KRParser:
          for gid, service in enumerate(services):
              query="type(SERVICE),entityId("+service["id"]+")"
             
-             params=merge(self.serviceLookupParams,{"entitySelector": query})
+             params=merge(self.config["serviceLookupParams"],{"entitySelector": query})
-             #params={"entitySelector": query,"from":"now-2y", "fields":"tags"}
+          
              response = helper.get_request(DTAPIURL, headers, params)        
              entities = (response.json())['entities']     
              
              if len(entities)>0:
                  services[gid]=entities[0]
-    
+                 
+                 
+                 
     def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
          DTAPIURL = DTAPIURL + "/api/v2/entities"
 
@@ -117,49 +124,148 @@ class KRParser:
             'Authorization': 'Api-Token ' + DTAPIToken
          }       
          
-         
+         tmp_KR=[]
-         for gid, k in enumerate(kr.keyRequests):
+         for gid, group in enumerate(kr.matchedGroups):
-             try:
+              #try:
-                query="type(service_method)"
+                for k in group["methods"]:
-                group=kr.matchedGroups[k["groupId"]]
+                     
-                
+                     
-                if len(group["services"])> 0:
+                    #tmp_kr={"displayName":None, "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}
-                    if group["services"][0].startswith("SERVICE-"):
-                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
-                    else:    
-                        query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
-                
-                if k["comparer"]=="entityId":
-                    query+=",entityId("+k["entityId"]+")"
-                else:
-                    query+=",entityName.in(\""+k["displayName"]+"\")" 
                     
-                params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+                    query="type(service_method)"
-                response = helper.get_request(DTAPIURL, headers, params)        
-                entities = (response.json())['entities']   
-                
-                #  if len(entities) > 1:
-                #      kr.keyRequests[gid]['foundCount']=len(entities)
-                #      print("Multiple keyrequest found: ")
-                
-                if len(entities)> 0:
-                    kr.keyRequests[gid]["found"]=True
-                    kr.keyRequests[gid]['foundCount']=len(entities)
-                    kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
-                    kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
                     
-                    if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
+                    if len(group["services"])> 0:
-                        kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
+                        
-                
+                        if group["services"][0].startswith("SERVICE-"):
-                if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
+                            query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
-                    self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
+                        else:    
+                            query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
                     
-             except Exception as err:
+                    if k.startswith("SERVICE_METHOD-"):
-                  kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
+                        query+=",entityId("+k+")"
+                    else:
+                        query+=",entityName.in(\""+k+"\")" 
+                        
+                    params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+                    response = helper.get_request(DTAPIURL, headers, params)        
+                    entities = (response.json())['entities']  
+                    
+                    if len(entities)> 0:
+                       for ent in entities:
+                         
+                         tmp_kr=merge({"displayName":ent["displayName"], "entityId":ent["entityId"],  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"] }, self.config["extendResultObjects"])
+                           
+                         if "isServiceMethodOfService" in ent["fromRelationships"]:
+                           tmp_kr["services"]=ent["fromRelationships"]["isServiceMethodOfService"]
+                        
+                         if options and KROption.RESOLVESERVICES in options and len(tmp_kr["services"])>0:
+                           self.resolveServices(tmp_kr["services"], DTAPIURL, DTAPIToken)
+                        
+                         tmp_KR.append(tmp_kr)
+                          
+                    else:
+                         if k.startswith('SERVICE_METHOD-'):
+                            tmp_kr=merge({"displayName": None,"comparer": "entityId", "entityId":k,  "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
+                         else:
+                            tmp_kr=merge({"displayName":k,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,           
+                        
+                         tmp_KR.append(tmp_kr)
+        
+        
+         return tmp_KR
+            #   except Exception as err:
+            #       kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
+                 
+        #  for gid, k in enumerate(kr.keyRequests):
+        #      try:
+        #         query="type(service_method)"
+        #         group=kr.matchedGroups[k["groupId"]]
+                
+        #         if len(group["services"])> 0:
+        #             if group["services"][0].startswith("SERVICE-"):
+        #                 query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
+        #             else:    
+        #                 query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
+                
+        #         if k["comparer"]=="entityId":
+        #             query+=",entityId("+k["entityId"]+")"
+        #         else:
+        #             query+=",entityName.in(\""+k["displayName"]+"\")" 
+                    
+        #         params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+        #         response = helper.get_request(DTAPIURL, headers, params)        
+        #         entities = (response.json())['entities']   
+                
+                
+        #         if len(entities)> 0:
+                    
+        #             kr.keyRequests[gid]["found"]=True
+        #             kr.keyRequests[gid]['foundCount']=len(entities)
+        #             kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
+        #             kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
+                    
+        #             if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
+        #                 kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
+                
+        #         if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
+        #             self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
+                    
+        #      except Exception as err:
+        #           kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
                  
                  
                    
-             #kr.mergeServices(entities)
+             #kr.mergeServices(entities)             
+    
+    # def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
+    #      DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+    #      headers = {
+    #         'Content-Type': 'application/json',
+    #         'Authorization': 'Api-Token ' + DTAPIToken
+    #      }       
+         
+         
+    #      for gid, k in enumerate(kr.keyRequests):
+    #          try:
+    #             query="type(service_method)"
+    #             group=kr.matchedGroups[k["groupId"]]
+                
+    #             if len(group["services"])> 0:
+    #                 if group["services"][0].startswith("SERVICE-"):
+    #                     query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
+    #                 else:    
+    #                     query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
+                
+    #             if k["comparer"]=="entityId":
+    #                 query+=",entityId("+k["entityId"]+")"
+    #             else:
+    #                 query+=",entityName.in(\""+k["displayName"]+"\")" 
+                    
+    #             params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
+    #             response = helper.get_request(DTAPIURL, headers, params)        
+    #             entities = (response.json())['entities']   
+                
+                
+    #             if len(entities)> 0:
+                    
+    #                 kr.keyRequests[gid]["found"]=True
+    #                 kr.keyRequests[gid]['foundCount']=len(entities)
+    #                 kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
+    #                 kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
+                    
+    #                 if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
+    #                     kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
+                
+    #             if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
+    #                 self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
+                    
+    #          except Exception as err:
+    #               kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
+                 
+                 
+                   
+    #          #kr.mergeServices(entities)
              
                
     def getKeyRequestsByServices(self, services):
@@ -176,7 +282,7 @@ class KRParser:
             else:
                query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
         
-        params={"entitySelector": query}
+        params={"entitySelector": query,"from":"now-1y"}
         response = helper.get_request(DTAPIURL, headers, params)        
         entities = (response.json())['entities'] 
         
@@ -191,53 +297,39 @@ class KRParser:
                 tmp_methods=self.getKeyRequestsByServices(group["services"])
                 
                 for m in tmp_methods:
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"],  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
+                    group["methods"].append(m["entityId"])
-                    kr.keyRequests.append(tmp)
+                 
+                    # tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"],  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
+                    # kr.keyRequests.append(tmp)
                 
-            for method in group["methods"]:
+            # for method in group["methods"]:
-               if method.startswith('SERVICE_METHOD-'):
+            #    if method.startswith('SERVICE_METHOD-'):
-                    tmp={"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
+            #         tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method,  "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
-               else:
+            #    else:
-                    tmp={"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
+            #         tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,       
                       
-               kr.keyRequests.append(tmp)
+            #    kr.keyRequests.append(tmp)
             
-            
+        
-               
-            # for service in group["services"]:
-            #     if service.startswith('SERVICE-'):
-            #         tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
-            #     else:
-            #         tmp={"displayName":service,"comparer": "displayName", "entityId":None,  "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,       
-                
-            #     kr.services.append(tmp)
                    
+        kr.keyRequests=self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
                
         if self.options:
-            if KROption.RESOLVEKEYREQUETS in self.options:
-                self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
             
             if KROption.VALIDATE_HASDATA in self.options:
                 self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
                 self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
-        # elif KROption.RESOLVEKEYREQUETS in self.options:
-        #     self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
-        # if KROption.RESOLVESERVICES in self.options:
-        #     self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
             
         return kr
         
 
-    def parseBySLO(self,index,row):
+    def parseBySLO(self,row):
-        #normalize
-        print(index)
         
         try:
             normFilter=self.normalize(row['filter'])
             normExpresseion=self.normalize(row['metricExpression'])
             
-            tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
+            tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"]))
-        
 
             #SLO with Filter
             if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
@@ -248,57 +340,47 @@ class KRParser:
             groups=self.applyPatterns(subject)
         
             tmp_KR.matchedGroups.append(groups)
-            # for g in groups:
+          
-            #     #if g["methods"] != None and len(g["methods"]) > 0:
-            #        tmp_KR.matchedGroups.append(g)
-                
-            #self.process(tmp_KR)
             kr=self.process(tmp_KR)
+            
             with self.lock:
                 self.krs.append(kr)
+                self.pbar.update()
                 
         except Exception as err:
                   print(repr(err)) 
-        #return self.process(tmp_KR)
     
-    def parseBySLO_Threaded(self, slosF):
+    def parse(self, input):
-         self.krs=[]
-         #i=1
-        #  threads = list()
-        #  for index, row in slosF.iterrows():
-        #     logging.info("Main    : create and start thread %d.", index)
-        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
-        #     threads.append(x)
-        #     x.start()
-            
-        #     #krs.append(krp.parseBySLO(row))
         
-        #  for index, thread in enumerate(threads):
+        with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor:
-        #     logging.info("Main    : before joining thread %d.", index)
+             
-        #     thread.join()
+                if type(input) == pd.DataFrame:
-        #     logging.info("Main    : thread %d done", index)
+                    self.pbar = tqdm(total=input["id"].count(),desc=self.name)
-         
+                    
-        #  #resultSlos.extend(krs)    
+                    for index, row in input.iterrows():
-         
+                        executor.submit(self.parseBySLO, row)
-         
+                            
-         with concurrent.futures.ThreadPoolExecutor(10) as executor:   
+                elif type(input)== list:
-            for index, row in slosF.iterrows():
+                    self.pbar = tqdm(total=len(input), desc=self.name)
-                # if i % 25 == 0:
+                    for slo in input:
-                #     time.sleep(0)
+                        executor.submit(self.parseBySLO, slo)
-                #args={index:index, }
+                        
-                executor.submit(self.parseBySLO, index,row)
+                elif type(input) == dict:
-                # print(str(i)+"\n")
+                    #self.pbar = tqdm(total=1, desc=self.name)
-                # i=i+1
+                    #executor.submit(self.parseBySLO, slo)
-        #     x = threading.Thread(target=self.parseBySLO, args=(row,))
+                    self.parseBySLO(input)
-        #     threads.append(x)
+                       
-        #     x.start()
+        
+        return self.krs
         
-         return self.krs
         
     
-    def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ):
+    def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ):
+        self.name=name
         self.DTAPIURL= DTAPIURL
         self.DTAPIToken=DTAPIToken
         self.options=options
-        self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams)
+        self.config=merge({"threads": 3, 
+                           "serviceLookupParams":{"from":"now-2y"},
+                           "extendResultObjects":{}}, config)
         self.krs=[]

build/lib/KRParser/patterns.py

@@ -22,7 +22,7 @@ class Pattern1:
             for r in result:
                 services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] 
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] 
-                groups.append({"services":services, "methods":methods})
+                groups.append({"services":services, "methods":methods, "_mInSloDef":True})
          #return services, methods
          return groups
 
@@ -38,7 +38,7 @@ class Pattern2:
             for r in result:
                 services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] 
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")] 
-                groups.append({"services":services, "methods":methods})
+                groups.append({"services":services, "methods":methods, "_mInSloDef":True})
                 
          return groups
 
@@ -55,7 +55,7 @@ class Pattern3:
          if result:
             for r in result:
                 methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
-                groups.append({"services":[], "methods":methods})
+                groups.append({"services":[], "methods":methods, "_mInSloDef":True})
            
          return groups
 
@@ -63,15 +63,16 @@ class Pattern4:
    
    def parseServicesAndMethods(self, metricExpression):
          metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-         result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+         #result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
-    
+         result = re.findall(r"type\(\"?service\"?\),[entityName|entityId]*[\.]*[equals|in|contains]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+        
          groups=[]
          if result:
             for r in result:
-               if not r:
+               #if not r:
                 #methods=[s.strip() for s in r.split(",")]
                 services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
-                groups.append({"services":services, "methods":[]})
+                groups.append({"services":services, "methods":[], "_mInSloDef":False})
            
          return groups
 
@@ -81,16 +82,16 @@ class Pattern5:
        
          #Endoce
          metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
-         result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)      
+         result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)   
+         #type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)   
          #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
         #services=[]
          #methods=[]
          groups=[]
          if result:
             for r in result:
-                if not r:
+               services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] 
-                  services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] 
                   #methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] 
-                  groups.append({"services":services, "methods":[]})
+               groups.append({"services":services, "methods":[], "_mInSloDef":False})
          #return services, methods
          return groups

setup.py

@@ -1,8 +1,13 @@
 from setuptools import setup, find_packages
 setup(
    name='KeyRequestParser',
-   version='0.4',
+   version='0.6',
    packages=find_packages(include=["KRParser"]),
    license='MIT',
    long_description="Parses Keyrequests",
+   install_requires=[
+        'requests',
+        'jsonmerge',
+        'tqdm'
+    ],
 )

tests/environment.yaml

@@ -0,0 +1,21 @@
+---
+euprod:
+    - name: "EUprod"
+    - env-url: "https://xxu26128.live.dynatrace.com"
+    - env-token-name: "EUPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+naprod:
+    - name: "naprod"
+    - env-url: "https://wgv50241.live.dynatrace.com"
+    - env-token-name: "NAPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+cnprod:
+    - name: "cnprod"
+    - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+    - env-token-name: "CNPROD_TOKEN_VAR"
+    - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
+# #cnpreprod:
+#     - name: "cnpreprod"
+#     - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
+#     - env-token-name: "CNPREPROD_TOKEN_VAR"
+#     - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"

tests/requirements.txt

@@ -0,0 +1,6 @@
+pyyaml
+python-decouple
+requests
+jsonmerge
+pandas
+tqdm

tests/test_patterns.py

@@ -0,0 +1,210 @@
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/..")
+#sys.path.append('..')
+
+import unittest
+import yaml
+#from helper import get_request
+from KRParser.helper import get_request
+from KRParser.krparser import KRParser, KROption
+
+import pandas as pd
+
+
+#from KRParser import helper, krparser
+from decouple import config
+
+
+
+class patterns(unittest.TestCase):
+
+
+    def setUp(self):
+        with open('./tests/environment.yaml') as file:
+          self.env_doc = yaml.safe_load(file)
+    
+    # TP_* SLO definition (Pattern1)
+    # Used SLO: TP_Mobile_Login
+    # https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=2a07c9bf-2c7f-37eb-815b-efda7658bc13
+    def test_TP_Pattern1(self):
+        
+        expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
+        expected_methods=['GET /api/v1/presentation/oauth/config', 'GET /api/v1/presentation/profile-tab', 'POST /api/v1/oauth/token/identifier']
+        expected_pattern="<class 'KRParser.patterns.Pattern1'>"
+        DTURL=self.env_doc['euprod'][1]["env-url"]
+        DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
+        
+        api_url = DTURL+"/api/v2/slo/2a07c9bf-2c7f-37eb-815b-efda7658bc13"   
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTTOKEN
+         }
+        
+        
+        slo_result=get_request(api_url, headers, {})
+        slo=slo_result.json()
+        
+        krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+        result= krp.parse(slo)
+        
+        self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+        self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+        self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
+        
+    # TP_* SLO definition (Pattern4)
+    # Used SLO: TP_Vehicle_Teleservices
+    # https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=18c9bc95-5605-33b4-81e1-10335cf0191b
+    def test_TP_Pattern4(self):
+        
+        expected_services=['ifs-001 (Kinesis) CCM - teleservices - prod', 'ifs-001 (Kinesis) TyreWsIncoming - IN - teleservices - prod', 'ifs-001 (Kinesis) TyreDiIncoming - IN - teleservices - prod', 'ifs-003 (JOYNR) triggerPdmContainerUpdate/notifyPdmContainerStatus - IN/OUT - teleservices - prod', 'ifs-003 (JOYNR) triggerWhitelistUpdate/notifyWhitelistStatus - IN/OUT - wlupdater - prod', 'ifs-003 (JOYNR) triggerStandardOdfUpdate/notifyOdfStatus - IN/OUT - wlupdater - prod', 'ifs-003 (JOYNR) start/stop/notifyTemporaryOdfStatus - INT/OUT - wlupdater - prod', 'ifs-007 (JOYNR) requestPdmResult/notifyPdmResult - IN/OUT - teleservices - prod', 'ifs-002 (JOYNR) request/notifyKeyData - IN/OUT - teleservices - prod', 'ifs-002 (JOYNR) notifyRSUStatus - IN - teleservices - prod', 'VDLM_scallservice - VDLM_scallservice - ifs-002 (JOYNR) notifyMaintenanceStatus - IN - prod', 'VDLM_scallservice - VDLM_scallservice - ifs-002 (JOYNR) requestMaintenanceStatus/confirmMaintenanceStatus - OUT - prod', 'ifs-005 (JOYNR) notify/confirmBatteryStatus - IN/OUT - teleservices - prod', 'ifs-007 (JOYNR) requestDiagResult/notifyDiagResult - IN/OUT - teleservices - prod', 'ifs-005 (SQS) TscsCallTopic - IN - teleservices - prod', 'ifs-006 (SQS) TsddCallTopic - IN - teleservices - prod', 'ifs-010 (SQS) RtcadCallTopic - IN - teleservices - prod', 'VDLM_seam - ifs-004 (SQS) ServiceDemand - IN - teleservices - prod', 'VDLM_seam - ifs-004 (SQS) ServiceDemand - IN - teleservices - prod', 'ifs-003 (MQS) MguItsnIn/Out (ClassificationRequest/Response) - IN/OUT - teleservices - prod', 'ifs-003 (MQS) MguItsnIn/Out (DiagnosisRequest/Response) - IN/OUT - teleservices - prod', 'ifs-005 (MQS) BCallHandlerIstaIn/Out - IN/OUT - teleservices - prod', 'ifs-001 (SQS) BCallIn/Out - IN/OUT - teleservices - prod', 'VDLM_ts-asc - ifs-001 (SQS) AscIn/Out - IN/OUT - prod', 'InternalApiResourceV2 - teleservices - prod', 'VDLM_rtchandler - ifs-010 (SQS) RtcTeleXIn/Out - IN/OUT - prod', 'ckf.bmwgroup.net']
+        expected_methods=['SERVICE_METHOD-0040C1B4C7A410B7', 'SERVICE_METHOD-116706FE1C0B57E9', 'SERVICE_METHOD-49981ADAE87D95D2', 'SERVICE_METHOD-5EB17880C0917DBF', 'SERVICE_METHOD-8226D6EB97776024', 'SERVICE_METHOD-90B715A55C1811EB', 'SERVICE_METHOD-AC1A7EF39680111E', 'SERVICE_METHOD-AEA1756029C53700', 'SERVICE_METHOD-E078C045186CDDA1', 'SERVICE_METHOD-FE7B9FE4485C6B29']
+        expected_pattern="<class 'KRParser.patterns.Pattern4'>"
+        DTURL=self.env_doc['euprod'][1]["env-url"]
+        DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
+        
+        api_url = DTURL+"/api/v2/slo/18c9bc95-5605-33b4-81e1-10335cf0191b"   
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTTOKEN
+         }
+        
+        
+        slo_result=get_request(api_url, headers, {})
+        slo=slo_result.json()
+        
+        krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+        result= krp.parse(slo)
+        
+        self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+        self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+        self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
+        
+    # Simplieifed_* SLO definition (Pattern5)
+    # Used SLO: Simplified_TP_Mobile_Login
+    # https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.slo/ui/slo?gtf=-2h&gf=all&filter=Text%3ATP&slovis=Graph&sloexp=5cc3efd1-fbb9-3504-8075-3cf0e4a2a4fa    
+    def test_Simplified(self):
+        
+        expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
+        expected_methods=['SERVICE_METHOD-0A047D679E974CF0','SERVICE_METHOD-1313A2ED47A33278','SERVICE_METHOD-2D3518701AFE612D','SERVICE_METHOD-45FDB365004EF766','SERVICE_METHOD-51B637F95F50EBE8','SERVICE_METHOD-5A5640BFFC3692D2','SERVICE_METHOD-5B50C30ADC9F8A8B','SERVICE_METHOD-63B88F2A195A44F1','SERVICE_METHOD-6858B9373C1B3C9A','SERVICE_METHOD-72BA0DBBCC1A9AED','SERVICE_METHOD-7B4FDDCAB657C7B3','SERVICE_METHOD-7E6EFAB2B32CBF87','SERVICE_METHOD-83EC6721A281B6F6','SERVICE_METHOD-91B6FB318FC612CD','SERVICE_METHOD-946D89B62E2559C0','SERVICE_METHOD-AD048A48A9FB4B84','SERVICE_METHOD-B20949E0FD947AE5','SERVICE_METHOD-BBC94FF8F8157D42','SERVICE_METHOD-C44DED4B58D56A8E','SERVICE_METHOD-CF980FC5191F17D1','SERVICE_METHOD-E78735CD41F780AC','SERVICE_METHOD-E8483228CE13A2AE','SERVICE_METHOD-E97B7D398DE28B1F','SERVICE_METHOD-F0B28D440F14795A','SERVICE_METHOD-FF77CD9406F022A3']
+        
+        
+        expected_pattern="<class 'KRParser.patterns.Pattern5'>"
+        
+        DTURL=self.env_doc['euprod'][1]["env-url"]
+        DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
+        
+        api_url = DTURL+"/api/v2/slo/5cc3efd1-fbb9-3504-8075-3cf0e4a2a4fa"   #34bb71cf-30bc-3d2e-913f-389517699751"
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTTOKEN
+         }
+        
+        
+        slo_result=get_request(api_url, headers, {})
+        slo=slo_result.json()
+        
+        krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+        result= krp.parse(slo)
+        
+        self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+        self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+        self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)    
+       
+        
+    # entityIds by filter SLO definition (Pattern3)
+    # Used SLO: E2E - Wirkkette "Login"
+    # https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.slo/ui/slo?gtf=-2h&gf=all&filter=Text%3ATP&slovis=Graph&sloexp=5fa56652-5f44-31af-892b-0681ff019b8b    
+    def test_EntityIdsByFilter(self):
+        
+        expected_services=[]
+        expected_methods=['SERVICE_METHOD-088030038E4C48AD', 'SERVICE_METHOD-22DB51030E136454', 'SERVICE_METHOD-93D43FBFAC5DF78F']
+        expected_pattern="<class 'KRParser.patterns.Pattern3'>"
+        
+        DTURL=self.env_doc['euprod'][1]["env-url"]
+        DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
+        
+        api_url = DTURL+"/api/v2/slo/5fa56652-5f44-31af-892b-0681ff019b8b"   #34bb71cf-30bc-3d2e-913f-389517699751"
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTTOKEN
+         }
+        
+        
+        slo_result=get_request(api_url, headers, {})
+        slo=slo_result.json()
+        
+        krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+        result= krp.parse(slo)
+        
+        self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+        self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+        self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)  
+        
+    # def test_contains(self):
+    #     expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
+    #     expected_methods=['SERVICE_METHOD-1313A2ED47A33278', 'SERVICE_METHOD-2D3518701AFE612D', 'SERVICE_METHOD-5A5640BFFC3692D2', 'SERVICE_METHOD-5B50C30ADC9F8A8B', 'SERVICE_METHOD-63B88F2A195A44F1', 'SERVICE_METHOD-6858B9373C1B3C9A', 'SERVICE_METHOD-72BA0DBBCC1A9AED', 'SERVICE_METHOD-7B4FDDCAB657C7B3', 'SERVICE_METHOD-7E6EFAB2B32CBF87', 'SERVICE_METHOD-83EC6721A281B6F6', 'SERVICE_METHOD-91B6FB318FC612CD', 'SERVICE_METHOD-946D89B62E2559C0', 'SERVICE_METHOD-AD048A48A9FB4B84', 'SERVICE_METHOD-B20949E0FD947AE5', 'SERVICE_METHOD-BBC94FF8F8157D42', 'SERVICE_METHOD-C44DED4B58D56A8E', 'SERVICE_METHOD-CF980FC5191F17D1', 'SERVICE_METHOD-E78735CD41F780AC', 'SERVICE_METHOD-E8483228CE13A2AE', 'SERVICE_METHOD-E97B7D398DE28B1F', 'SERVICE_METHOD-F0B28D440F14795A', 'SERVICE_METHOD-FF77CD9406F022A3']
+    #     expected_pattern="<class 'KRParser.patterns.Pattern5'>"
+        
+    #     DTURL=self.env_doc['cnprod'][1]["env-url"]
+    #     DTTOKEN = config(self.env_doc['cnprod'][2].get('env-token-name'))
+        
+    #     api_url = DTURL+"/api/v2/slo/4230f8cb-ffaa-31b3-9e12-091488f330af"   #34bb71cf-30bc-3d2e-913f-389517699751"
+        
+    #     headers = {
+    #         'Content-Type': 'application/json',
+    #         'Authorization': 'Api-Token ' + DTTOKEN
+    #      }
+        
+        
+    #     result_slo=get_request(api_url, headers, {})
+    #     slo=result_slo.json()
+        
+    #     krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+    #     result= krp.parse(slo)
+        
+    #     self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+    #     self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+    #     self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)  
+    # TP_* SLO definition (Pattern1)
+    # Used SLO: TP_Mobile_Login
+    # https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=2a07c9bf-2c7f-37eb-815b-efda7658bc13
+    def test_TP_SmartParking(self):
+        
+        expected_services=['ParkingFinderLite.WebApi']
+        expected_methods=['/sppid7/parkingfinder/v1/EstimatedTimeArrival/getETA', '/parkingfinder/v2.0/getParkingLots']
+        expected_pattern="<class 'KRParser.patterns.Pattern1'>"
+        
+        DTURL=self.env_doc['cnprod'][1]["env-url"]
+        DTTOKEN = config(self.env_doc['cnprod'][2].get('env-token-name'))
+        
+        api_url = DTURL+"/api/v2/slo/91e02599-a243-3201-a87e-91066419e52a"   
+        
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTTOKEN
+         }
+        
+        
+        slo_result=get_request(api_url, headers, {})
+        slo=slo_result.json()
+        
+        krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
+        
+        result= krp.parse(slo)
+        
+        self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
+        # self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
+        # self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)    
+
+if __name__ == '__main__':
+    unittest.main()