diff --git a/createKeyRequestReport.py b/createKeyRequestReport.py index cdfc1de..a1aaefa 100644 --- a/createKeyRequestReport.py +++ b/createKeyRequestReport.py @@ -46,26 +46,26 @@ def getStats(krs): tmpIgnoredList=[] for kr in [a for a in krs if len(a.keyRequests) == 0]: - tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","",kr.metadata["filter"], kr.metadata["metricExpression"]]) + tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","","",kr.metadata["filter"], kr.metadata["metricExpression"]]) noData1M=[] for kr in [s for s in krs if s.hasNoData("1M")==True]: for k in kr.getKeyRequestByHasData("1M"): - noData1M.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) + noData1M.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"],k["services"][0]["displayName"] if len(k["services"]) > 0 else "", kr.metadata["filter"], kr.metadata["metricExpression"]]) noData1W=[] for kr in [s for s in krs if s.hasNoData("1W")==True]: for k in kr.getKeyRequestByHasData("1W"): - noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) + noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], k["services"][0]["displayName"] if len(k["services"]) > 0 else "", kr.metadata["filter"], kr.metadata["metricExpression"]]) # noData1W=[] # for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]: # for k in kr.getNoData1WKeyRequests(): # noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) - columns =['slo', 'env', 'displayName','entityId', 'filter', 'metricExpression'] + columns =['slo', 'env', 'displayName','entityId','service','filter', 'metricExpression'] ignored=pd.DataFrame(tmpIgnoredList, columns=columns, dtype = float) noData_1M=pd.DataFrame(noData1M, columns = columns, dtype = float) noData_1W=pd.DataFrame(noData1W, columns = columns, dtype = float) @@ -90,13 +90,15 @@ def main(): DTURL = url.get('env-url') krs=[] - krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN) + krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES ,DTURL, DTTOKEN) slosF=getSLO(env, DTTOKEN, DTURL) for index, row in slosF.iterrows(): - #if row['id'] == "7a7f9c66-75c1-30b9-a505-801ffa5f9e98": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": - krs.append(krp.parseBySLO(row)) + #if row['id'] == "1ec65bfa-8d66-3215-a094-c289da440f32": #"1de2685e-0f06-370c-8b25-2326426e89c3": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": + #if row['id'] == "8fca3fbd-87a8-3005-91c8-727825ed7d99": + #if str.startswith(row["name"],"TP_"): + krs.append(krp.parseBySLO(row)) resultSlos.extend(krs) diff --git a/key_request_parser/helper.py b/key_request_parser/helper.py index e710997..4c311b0 100644 --- a/key_request_parser/helper.py +++ b/key_request_parser/helper.py @@ -1,19 +1,19 @@ import requests def get_request(url, headers, params): - try: + #try: response = requests.get(url, headers=headers, params=params, verify=False) response.raise_for_status() - except requests.exceptions.HTTPError as errh: - return "An Http Error occurred:" + repr(errh) - except requests.exceptions.ConnectionError as errc: - return "An Error Connecting to the API occurred:" + repr(errc) - except requests.exceptions.Timeout as errt: - return "A Timeout Error occurred:" + repr(errt) - except requests.exceptions.RequestException as err: - return "An Unknown Error occurred" + repr(err) + # except requests.exceptions.HTTPError as errh: + # return "An Http Error occurred:" + repr(errh) + # except requests.exceptions.ConnectionError as errc: + # return "An Error Connecting to the API occurred:" + repr(errc) + # except requests.exceptions.Timeout as errt: + # return "A Timeout Error occurred:" + repr(errt) + # except requests.exceptions.RequestException as err: + # return "An Unknown Error occurred" + repr(err) - return response + return response def contains(list, filter): for x in list: diff --git a/key_request_parser/keyrequests.py b/key_request_parser/keyrequests.py index 6e71a4e..e23e2fa 100644 --- a/key_request_parser/keyrequests.py +++ b/key_request_parser/keyrequests.py @@ -55,17 +55,27 @@ class KeyRequestGroup(MutableSequence): if val["methods"][0].startswith("SERVICE_METHOD-"): query+=",entityId(\""+'","'.join(val["methods"])+"\")" - else: + else: query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" val["existsQuery"]= query - + + # def createServiceResolveQuery(self, val): + # query="type(SERVICE)" + # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + # if len(val["services"]) > 0: + # if val["services"][0].startswith("SERVICE-"): + # query+=",entityId(\""+'","'.join(val["services"])+"\")" + # else: + # query+=",entityName.in(\""+'","'.join(val["services"])+"\")" + + # val["resolveServiceQuery"]= query def insert(self, ii, val): self.createExistsQuery(val) - + #self.createServiceResolveQuery(val) self._list.insert(ii, val) @@ -114,10 +124,19 @@ class KR: return True return False + + def checkKeyRequestsHasData(self): pass + + def mergeServices(self, listServices): + listOfServiceIds=[o["entityId"] for o in self.services] + + for s in listServices: + if s["entityId"] not in listOfServiceIds: + self.services.append(s) def __init__(self, metadata, @@ -130,5 +149,6 @@ class KR: self.matchedGroups = keyRequests_groups self.keyRequests=[] + self.services=[] diff --git a/key_request_parser/krparser.py b/key_request_parser/krparser.py index 34cdede..2ac42a0 100644 --- a/key_request_parser/krparser.py +++ b/key_request_parser/krparser.py @@ -8,9 +8,11 @@ from enum import Flag, auto class KROption(Flag): VALIDATE_EXISTS = auto() VALIDATE_HASDATA = auto() + RESOLVEKEYREQUETS = auto() + RESOLVESERVICES = auto() class KRParser: - patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern4()] + patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()] def normalize(self,x): #tmp=x.replace("~","") @@ -55,7 +57,7 @@ class KRParser: } for gid, group in enumerate(kr.matchedGroups): - params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"]} + params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -66,15 +68,89 @@ class KRParser: found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] if len(found) > 0: - #method["exists"]=True method["hasData"][tfrom["label"]]=True - method["displayName"]=found[0]["displayName"] - method["entityId"]=found[0]["entityId"] + #method["displayName"]=found[0]["displayName"] + #method["entityId"]=found[0]["entityId"] + #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"] + + # for idx,o in enumerate(method["services"]): + # tmpS=[p for p in kr.services if p["entityId"]==o["id"]] + # if len(tmpS)>0: + # method["services"][idx]=tmpS[0] + else: method["hasData"][tfrom["label"]]=False - - + def resolveServices(self,services, DTAPIURL, DTAPIToken): + #DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + for gid, service in enumerate(services): + query="type(SERVICE),entityId("+service["id"]+")" + params={"entitySelector": query,"from":"now-2y"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + if len(entities)>0: + services[gid]=entities[0] + + def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options): + DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + + for gid, k in enumerate(kr.keyRequests): + try: + query="type(service_method)" + group=kr.matchedGroups[k["groupId"]] + + if len(group["services"])> 0: + if group["services"][0].startswith("SERVICE-"): + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))" + else: + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))" + + if k["comparer"]=="entityId": + query+=",entityId("+k["entityId"]+")" + else: + query+=",entityName.in(\""+k["displayName"]+"\")" + + params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + # if len(entities) > 1: + # kr.keyRequests[gid]['foundCount']=len(entities) + # print("Multiple keyrequest found: ") + + if len(entities)> 0: + kr.keyRequests[gid]["found"]=True + kr.keyRequests[gid]['foundCount']=len(entities) + kr.keyRequests[gid]["displayName"]=entities[0]["displayName"] + kr.keyRequests[gid]["entityId"]=entities[0]["entityId"] + + if "isServiceMethodOfService" in entities[0]["fromRelationships"]: + kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] + + if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: + self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) + + except Exception as err: + kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err) + + + + #kr.mergeServices(entities) + + def process(self, kr): @@ -82,20 +158,25 @@ class KRParser: for gid, group in enumerate(kr.matchedGroups): for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): - tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None, + tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, else: - tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None, + tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) # if self.options and KROption.VALIDATE_EXISTS in self.options: # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if KROption.VALIDATE_HASDATA in self.options: self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) - + # elif KROption.RESOLVEKEYREQUETS in self.options: + # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + # if KROption.RESOLVESERVICES in self.options: + # self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken) + return kr diff --git a/key_request_parser/patterns.py b/key_request_parser/patterns.py index 5eeb215..c724a73 100644 --- a/key_request_parser/patterns.py +++ b/key_request_parser/patterns.py @@ -60,20 +60,20 @@ class Pattern3: -class Pattern4: +# class Pattern4: - def parseServicesAndMethods(self, metricExpression): - result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) +# def parseServicesAndMethods(self, metricExpression): +# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) - groups=[] - methods=[] - if result: - for r in result: - methods.append(r) +# groups=[] +# methods=[] +# if result: +# for r in result: +# methods.append(r) - groups.append({"services":[], "methods":methods}) +# groups.append({"services":[], "methods":methods}) - return groups +# return groups # class FilterMethodPattern: