import re from KRParser import patterns, keyrequests, helper from enum import Flag, auto import threading import concurrent.futures from jsonmerge import merge import pandas as pd from tqdm import * class KROption(Flag): VALIDATE_EXISTS = auto() VALIDATE_HASDATA = auto() RESOLVEKEYREQUETS = auto() RESOLVESERVICES = auto() class KRParser: patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ] lock = threading.Lock() def normalize(self,x): tmp=x.replace("\n","") tmp=tmp.replace("/\"","") tmp=tmp.replace("\"","") tmp=tmp.replace("\t","") tmp=re.sub("([\s]*)\)", ")", tmp) tmp=re.sub("\([\s\n\r]*", "(", tmp) tmp=re.sub("\,[\s\n\r]*", ",", tmp) tmp=re.sub("\)[\s\n\r]*,", "),", tmp) tmp=re.sub("in[\s\n\r]*\(", "in(", tmp) return tmp def applyPatterns(self,subject): groups=None for p in self.patterns: groups=p.parseServicesAndMethods(subject) if len(groups) > 0: break return groups def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): # DTAPIURL = DTAPIURL + "/api/v2/entities" # headers = { # 'Content-Type': 'application/json', # 'Authorization': 'Api-Token ' + DTAPIToken # } # for gid, group in enumerate(kr.matchedGroups): # params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} # response = helper.get_request(DTAPIURL, headers, params) # entities = (response.json())['entities'] # if len(entities) > 0: # y=0 # for method in kr.keyRequests: # if method["groupId"] == gid: # found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] # if len(found) > 0: # method["hasData"][tfrom["label"]]=True # else: # method["hasData"][tfrom["label"]]=False DTAPIURL = DTAPIURL + "/api/v2/metrics/query" headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } for gid, group in enumerate(kr.matchedGroups): params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())["result"][0]["data"] for method in kr.keyRequests: if method["groupId"] == gid: found = [x for x in entities if x["dimensions"][0] == method["entityId"]] if len(found) > 0: method["hasData"][tfrom["label"]]=True method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None]) else: method["hasData"][tfrom["label"]]=False method["count"][tfrom["label"]]=0 def resolveServices(self,services, DTAPIURL, DTAPIToken): headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } for gid, service in enumerate(services): query="type(SERVICE),entityId("+service["id"]+")" params=merge(self.config["serviceLookupParams"],{"entitySelector": query}) response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] if len(entities)>0: services[gid]=entities[0] def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options): DTAPIURL = DTAPIURL + "/api/v2/entities" headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } for gid, k in enumerate(kr.keyRequests): try: query="type(service_method)" group=kr.matchedGroups[k["groupId"]] if len(group["services"])> 0: if group["services"][0].startswith("SERVICE-"): query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))" else: query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))" if k["comparer"]=="entityId": query+=",entityId("+k["entityId"]+")" else: query+=",entityName.in(\""+k["displayName"]+"\")" params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] if len(entities)> 0: kr.keyRequests[gid]["found"]=True kr.keyRequests[gid]['foundCount']=len(entities) kr.keyRequests[gid]["displayName"]=entities[0]["displayName"] kr.keyRequests[gid]["entityId"]=entities[0]["entityId"] if "isServiceMethodOfService" in entities[0]["fromRelationships"]: kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) except Exception as err: kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err) #kr.mergeServices(entities) def getKeyRequestsByServices(self, services): #type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD")) DTAPIURL = self.DTAPIURL + "/api/v2/entities" headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + self.DTAPIToken } if len(services) > 0: if services[0].startswith("SERVICE-"): query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))" else: query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))" params={"entitySelector": query} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] return entities def process(self, kr): for gid, group in enumerate(kr.matchedGroups): if len(group["services"]) > 0 and len(group["methods"])==0: tmp_methods=self.getKeyRequestsByServices(group["services"]) for m in tmp_methods: tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, else: tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if self.options: if KROption.VALIDATE_HASDATA in self.options: self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) return kr def parseBySLO(self,row): try: normFilter=self.normalize(row['filter']) normExpresseion=self.normalize(row['metricExpression']) tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"])) #SLO with Filter if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"): subject=normFilter else: subject=normExpresseion groups=self.applyPatterns(subject) tmp_KR.matchedGroups.append(groups) kr=self.process(tmp_KR) with self.lock: self.krs.append(kr) self.pbar.update() except Exception as err: print(repr(err)) def parse(self, input): with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor: if type(input) == pd.DataFrame: self.pbar = tqdm(total=input["id"].count(),desc=self.name) for index, row in input.iterrows(): executor.submit(self.parseBySLO, row) elif type(input)== list: self.pbar = tqdm(total=len(input), desc=self.name) for slo in input: executor.submit(self.parseBySLO, slo) elif type(input) == dict: self.pbar = tqdm(total=1, desc=self.name) executor.submit(self.parseBySLO, row) return self.krs def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ): self.name=name self.DTAPIURL= DTAPIURL self.DTAPIToken=DTAPIToken self.options=options self.config=merge({"threads": 3, "serviceLookupParams":{"from":"now-2y"}, "extendResultObjects":{}}, config) self.krs=[]