diff --git a/createKeyRequestReport.py b/createKeyRequestReport.py index a1aaefa..95eb9c7 100644 --- a/createKeyRequestReport.py +++ b/createKeyRequestReport.py @@ -90,17 +90,24 @@ def main(): DTURL = url.get('env-url') krs=[] - krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN) + #krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN) + krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN) slosF=getSLO(env, DTTOKEN, DTURL) - - for index, row in slosF.iterrows(): - #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": - #if row['id'] == "8fca3fbd-87a8-3005-91c8-727825ed7d99": - #if str.startswith(row["name"],"TP_"): - krs.append(krp.parseBySLO(row)) + #slosF=slosF.loc[slosF['id'] == "a96a4031-e201-3e98-b739-27be7dc85a09"] + krs=krp.parseBySLO_Threaded(slosF) + # for index, row in slosF.iterrows(): + # #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": + # #if row['id'] == "a01cc45e-8369-3623-a156-1b261e6f4c98": + # #if str.startswith(row["name"],"TP_"): + # krs.append(krp.parseBySLO(row)) resultSlos.extend(krs) + + + + # for slo in resultSlos: + # if slo.key ignoerd,noData1M, noData1W= getStats(resultSlos) write_to_excel(ignoerd, noData1M, noData1W) diff --git a/environment.yaml b/environment.yaml index 7e9829a..1cc1e83 100644 --- a/environment.yaml +++ b/environment.yaml @@ -4,16 +4,16 @@ euprod: - env-url: "https://xxu26128.live.dynatrace.com" - env-token-name: "EUPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -naprod: - - name: "naprod" - - env-url: "https://wgv50241.live.dynatrace.com" - - env-token-name: "NAPROD_TOKEN_VAR" - - jenkins: "https://jaws.bmwgroup.net/opapm/" -cnprod: - - name: "cnprod" - - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" - - env-token-name: "CNPROD_TOKEN_VAR" - - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" +# naprod: +# - name: "naprod" +# - env-url: "https://wgv50241.live.dynatrace.com" +# - env-token-name: "NAPROD_TOKEN_VAR" +# - jenkins: "https://jaws.bmwgroup.net/opapm/" +# cnprod: +# - name: "cnprod" +# - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" +# - env-token-name: "CNPROD_TOKEN_VAR" +# - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" # #cnpreprod: # - name: "cnpreprod" # - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35" diff --git a/key_request_parser/helper.py b/key_request_parser/helper.py index 4c311b0..cace378 100644 --- a/key_request_parser/helper.py +++ b/key_request_parser/helper.py @@ -1,6 +1,7 @@ import requests +from requests.adapters import HTTPAdapter, Retry -def get_request(url, headers, params): +def get_requestOld(url, headers, params): #try: response = requests.get(url, headers=headers, params=params, verify=False) response.raise_for_status() @@ -14,6 +15,29 @@ def get_request(url, headers, params): # return "An Unknown Error occurred" + repr(err) return response + +def get_request(url, headers, params): + #try: + session = requests.Session() + retry = Retry(connect=3, backoff_factor=10) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + + #response = requests.get(url, headers=headers, params=params, verify=False) + response = session.get(url,headers=headers, params=params, verify=False) + + response.raise_for_status() + # except requests.exceptions.HTTPError as errh: + # return "An Http Error occurred:" + repr(errh) + # except requests.exceptions.ConnectionError as errc: + # return "An Error Connecting to the API occurred:" + repr(errc) + # except requests.exceptions.Timeout as errt: + # return "A Timeout Error occurred:" + repr(errt) + # except requests.exceptions.RequestException as err: + # return "An Unknown Error occurred" + repr(err) + + return response def contains(list, filter): for x in list: diff --git a/key_request_parser/keyrequests.py b/key_request_parser/keyrequests.py index e23e2fa..a877da3 100644 --- a/key_request_parser/keyrequests.py +++ b/key_request_parser/keyrequests.py @@ -52,12 +52,13 @@ class KeyRequestGroup(MutableSequence): query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" else: query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" - - if val["methods"][0].startswith("SERVICE_METHOD-"): - query+=",entityId(\""+'","'.join(val["methods"])+"\")" - else: - query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" - + + if len(val["methods"]) > 0: + if val["methods"][0].startswith("SERVICE_METHOD-"): + query+=",entityId(\""+'","'.join(val["methods"])+"\")" + else: + query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + val["existsQuery"]= query # def createServiceResolveQuery(self, val): @@ -75,21 +76,23 @@ class KeyRequestGroup(MutableSequence): def insert(self, ii, val): self.createExistsQuery(val) - #self.createServiceResolveQuery(val) + self._list.insert(ii, val) def append(self, val): + for g in val: + if len(self._list) == 0: #self._list.insert(ii, val) - self.insert(len(self._list), val) + self.insert(len(self._list), g) return for group in self._list: - if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0: - self.insert(len(self._list), val) + if len(set(group["services"]) - set(g["services"])) > 0 or len(set(group["methods"]) - set(g["methods"])) > 0: + self.insert(len(self._list), g) diff --git a/key_request_parser/krparser.py b/key_request_parser/krparser.py index 2ac42a0..b370fb7 100644 --- a/key_request_parser/krparser.py +++ b/key_request_parser/krparser.py @@ -3,6 +3,11 @@ import re from key_request_parser import patterns, keyrequests, helper from enum import Flag, auto +import logging +import threading +import concurrent.futures +import time + class KROption(Flag): @@ -12,8 +17,10 @@ class KROption(Flag): RESOLVESERVICES = auto() class KRParser: - patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()] - + #threadLimiter = threading.BoundedSemaphore(3) + patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ] + lock = threading.Lock() + def normalize(self,x): #tmp=x.replace("~","") tmp=x.replace("\n","") @@ -123,7 +130,7 @@ class KRParser: else: query+=",entityName.in(\""+k["displayName"]+"\")" - params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"} + params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -151,11 +158,38 @@ class KRParser: #kr.mergeServices(entities) - + def getKeyRequestsByServices(self, services): + #type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD")) + DTAPIURL = self.DTAPIURL + "/api/v2/entities" + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + self.DTAPIToken + } + + if len(services) > 0: + if services[0].startswith("SERVICE-"): + query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))" + else: + query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))" + + params={"entitySelector": query} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + return entities + def process(self, kr): for gid, group in enumerate(kr.matchedGroups): + + if len(group["services"]) > 0 and len(group["methods"])==0: + tmp_methods=self.getKeyRequestsByServices(group["services"]) + + for m in tmp_methods: + tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + kr.keyRequests.append(tmp) + for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, @@ -163,15 +197,25 @@ class KRParser: tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) + + - - # if self.options and KROption.VALIDATE_EXISTS in self.options: - # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) - self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) - - if KROption.VALIDATE_HASDATA in self.options: - self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) - self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) + # for service in group["services"]: + # if service.startswith('SERVICE-'): + # tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + # else: + # tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + + # kr.services.append(tmp) + + + if self.options: + if KROption.RESOLVEKEYREQUETS in self.options: + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) + + if KROption.VALIDATE_HASDATA in self.options: + self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) # elif KROption.RESOLVEKEYREQUETS in self.options: # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) # if KROption.RESOLVESERVICES in self.options: @@ -180,32 +224,76 @@ class KRParser: return kr - def parseBySLO(self, row): + def parseBySLO(self,index,row): #normalize - normFilter=self.normalize(row['filter']) - normExpresseion=self.normalize(row['metricExpression']) + print(index) - tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) - - - #SLO with Filter - if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"): - subject=normFilter - else: - subject=normExpresseion - - groups=self.applyPatterns(subject) - - for g in groups: - if g["methods"] != None and len(g["methods"]) > 0: - tmp_KR.matchedGroups.append(g) + try: + normFilter=self.normalize(row['filter']) + normExpresseion=self.normalize(row['metricExpression']) - #self.process(tmp_KR) + tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) + - return self.process(tmp_KR) + #SLO with Filter + if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"): + subject=normFilter + else: + subject=normExpresseion + + groups=self.applyPatterns(subject) + + tmp_KR.matchedGroups.append(groups) + # for g in groups: + # #if g["methods"] != None and len(g["methods"]) > 0: + # tmp_KR.matchedGroups.append(g) + + #self.process(tmp_KR) + kr=self.process(tmp_KR) + with self.lock: + self.krs.append(kr) + + except Exception as err: + print(repr(err)) + #return self.process(tmp_KR) + + def parseBySLO_Threaded(self, slosF): + self.krs=[] + #i=1 + # threads = list() + # for index, row in slosF.iterrows(): + # logging.info("Main : create and start thread %d.", index) + # x = threading.Thread(target=self.parseBySLO, args=(row,)) + # threads.append(x) + # x.start() + + # #krs.append(krp.parseBySLO(row)) + + # for index, thread in enumerate(threads): + # logging.info("Main : before joining thread %d.", index) + # thread.join() + # logging.info("Main : thread %d done", index) + + # #resultSlos.extend(krs) + + + with concurrent.futures.ThreadPoolExecutor(10) as executor: + for index, row in slosF.iterrows(): + # if i % 25 == 0: + # time.sleep(0) + #args={index:index, } + executor.submit(self.parseBySLO, index,row) + # print(str(i)+"\n") + # i=i+1 + # x = threading.Thread(target=self.parseBySLO, args=(row,)) + # threads.append(x) + # x.start() + + return self.krs - def __init__(self, options: KROption , DTAPIURL, DTAPIToken ): + def __init__(self, options: KROption=None , DTAPIURL=None, DTAPIToken=None ): self.DTAPIURL= DTAPIURL self.DTAPIToken=DTAPIToken - self.options=options \ No newline at end of file + self.options=options + self.krs=[] \ No newline at end of file diff --git a/key_request_parser/patterns.py b/key_request_parser/patterns.py index c724a73..ebd280b 100644 --- a/key_request_parser/patterns.py +++ b/key_request_parser/patterns.py @@ -47,42 +47,50 @@ class Pattern2: class Pattern3: def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) # services=[] # methods=[] groups=[] if result: for r in result: - methods=[s.strip() for s in r.split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] groups.append({"services":[], "methods":methods}) return groups - - -# class Pattern4: - -# def parseServicesAndMethods(self, metricExpression): -# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) +class Pattern4: -# groups=[] -# methods=[] -# if result: -# for r in result: -# methods.append(r) - -# groups.append({"services":[], "methods":methods}) + def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + groups=[] + if result: + for r in result: + if not r: + #methods=[s.strip() for s in r.split(",")] + services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] + groups.append({"services":services, "methods":[]}) -# return groups + return groups -# class FilterMethodPattern: - -# def parseServicesAndMethods(self, metricExpression): -# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) -# services=[] -# methods=[] -# if result: -# for r in result: -# methods=[s.strip() for s in r.split(",")] - -# return services, methods \ No newline at end of file +class Pattern5: + + def parseServicesAndMethods(self, metricExpression): + + #Endoce + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #services=[] + #methods=[] + groups=[] + if result: + for r in result: + if not r: + services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] + #methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] + groups.append({"services":services, "methods":[]}) + #return services, methods + return groups \ No newline at end of file