277 lines
12 KiB
Python
277 lines
12 KiB
Python
import re
|
|
|
|
from KRParser import patterns, keyrequests, helper
|
|
from enum import Flag, auto
|
|
import threading
|
|
import concurrent.futures
|
|
from jsonmerge import merge
|
|
import pandas as pd
|
|
from tqdm import *
|
|
|
|
|
|
|
|
class KROption(Flag):
|
|
VALIDATE_EXISTS = auto()
|
|
VALIDATE_HASDATA = auto()
|
|
RESOLVEKEYREQUETS = auto()
|
|
RESOLVESERVICES = auto()
|
|
|
|
class KRParser:
|
|
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
|
|
lock = threading.Lock()
|
|
|
|
def normalize(self,x):
|
|
tmp=x.replace("\n","")
|
|
tmp=tmp.replace("/\"","")
|
|
tmp=tmp.replace("\"","")
|
|
tmp=tmp.replace("\t","")
|
|
tmp=re.sub("([\s]*)\)", ")", tmp)
|
|
tmp=re.sub("\([\s\n\r]*", "(", tmp)
|
|
tmp=re.sub("\,[\s\n\r]*", ",", tmp)
|
|
tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
|
|
tmp=re.sub("in[\s\n\r]*\(", "in(", tmp)
|
|
|
|
return tmp
|
|
|
|
|
|
def applyPatterns(self,subject):
|
|
groups=None
|
|
for p in self.patterns:
|
|
groups=p.parseServicesAndMethods(subject)
|
|
|
|
if len(groups) > 0:
|
|
break
|
|
|
|
return groups
|
|
|
|
|
|
def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
|
|
|
|
# DTAPIURL = DTAPIURL + "/api/v2/entities"
|
|
|
|
# headers = {
|
|
# 'Content-Type': 'application/json',
|
|
# 'Authorization': 'Api-Token ' + DTAPIToken
|
|
# }
|
|
|
|
# for gid, group in enumerate(kr.matchedGroups):
|
|
# params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
|
# response = helper.get_request(DTAPIURL, headers, params)
|
|
# entities = (response.json())['entities']
|
|
|
|
# if len(entities) > 0:
|
|
# y=0
|
|
# for method in kr.keyRequests:
|
|
# if method["groupId"] == gid:
|
|
# found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
|
|
|
# if len(found) > 0:
|
|
# method["hasData"][tfrom["label"]]=True
|
|
# else:
|
|
# method["hasData"][tfrom["label"]]=False
|
|
|
|
DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
|
|
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Api-Token ' + DTAPIToken
|
|
}
|
|
|
|
for gid, group in enumerate(kr.matchedGroups):
|
|
params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]}
|
|
response = helper.get_request(DTAPIURL, headers, params)
|
|
entities = (response.json())["result"][0]["data"]
|
|
|
|
for method in kr.keyRequests:
|
|
if method["groupId"] == gid:
|
|
|
|
found = [x for x in entities if x["dimensions"][0] == method["entityId"]]
|
|
|
|
if len(found) > 0:
|
|
method["hasData"][tfrom["label"]]=True
|
|
method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None])
|
|
else:
|
|
method["hasData"][tfrom["label"]]=False
|
|
method["count"][tfrom["label"]]=0
|
|
|
|
def resolveServices(self,services, DTAPIURL, DTAPIToken):
|
|
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Api-Token ' + DTAPIToken
|
|
}
|
|
|
|
for gid, service in enumerate(services):
|
|
query="type(SERVICE),entityId("+service["id"]+")"
|
|
|
|
params=merge(self.config["serviceLookupParams"],{"entitySelector": query})
|
|
|
|
response = helper.get_request(DTAPIURL, headers, params)
|
|
entities = (response.json())['entities']
|
|
|
|
if len(entities)>0:
|
|
services[gid]=entities[0]
|
|
|
|
def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
|
|
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
|
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Api-Token ' + DTAPIToken
|
|
}
|
|
|
|
|
|
for gid, k in enumerate(kr.keyRequests):
|
|
try:
|
|
query="type(service_method)"
|
|
group=kr.matchedGroups[k["groupId"]]
|
|
|
|
if len(group["services"])> 0:
|
|
if group["services"][0].startswith("SERVICE-"):
|
|
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
|
else:
|
|
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
|
|
|
if k["comparer"]=="entityId":
|
|
query+=",entityId("+k["entityId"]+")"
|
|
else:
|
|
query+=",entityName.in(\""+k["displayName"]+"\")"
|
|
|
|
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
|
response = helper.get_request(DTAPIURL, headers, params)
|
|
entities = (response.json())['entities']
|
|
|
|
|
|
if len(entities)> 0:
|
|
kr.keyRequests[gid]["found"]=True
|
|
kr.keyRequests[gid]['foundCount']=len(entities)
|
|
kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
|
kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
|
|
|
if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
|
kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
|
|
|
if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
|
self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
|
|
|
except Exception as err:
|
|
kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
|
|
|
|
|
|
|
#kr.mergeServices(entities)
|
|
|
|
|
|
def getKeyRequestsByServices(self, services):
|
|
#type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD"))
|
|
DTAPIURL = self.DTAPIURL + "/api/v2/entities"
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': 'Api-Token ' + self.DTAPIToken
|
|
}
|
|
|
|
if len(services) > 0:
|
|
if services[0].startswith("SERVICE-"):
|
|
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))"
|
|
else:
|
|
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
|
|
|
|
params={"entitySelector": query}
|
|
response = helper.get_request(DTAPIURL, headers, params)
|
|
entities = (response.json())['entities']
|
|
|
|
return entities
|
|
|
|
|
|
def process(self, kr):
|
|
|
|
for gid, group in enumerate(kr.matchedGroups):
|
|
|
|
if len(group["services"]) > 0 and len(group["methods"])==0:
|
|
tmp_methods=self.getKeyRequestsByServices(group["services"])
|
|
|
|
for m in tmp_methods:
|
|
tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
|
kr.keyRequests.append(tmp)
|
|
|
|
for method in group["methods"]:
|
|
if method.startswith('SERVICE_METHOD-'):
|
|
tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
|
else:
|
|
tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
|
|
|
kr.keyRequests.append(tmp)
|
|
|
|
|
|
|
|
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
|
|
|
if self.options:
|
|
|
|
if KROption.VALIDATE_HASDATA in self.options:
|
|
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
|
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
|
|
|
return kr
|
|
|
|
|
|
def parseBySLO(self,row):
|
|
|
|
try:
|
|
normFilter=self.normalize(row['filter'])
|
|
normExpresseion=self.normalize(row['metricExpression'])
|
|
|
|
tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"]))
|
|
|
|
#SLO with Filter
|
|
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
|
|
subject=normFilter
|
|
else:
|
|
subject=normExpresseion
|
|
|
|
groups=self.applyPatterns(subject)
|
|
|
|
tmp_KR.matchedGroups.append(groups)
|
|
|
|
kr=self.process(tmp_KR)
|
|
|
|
with self.lock:
|
|
self.krs.append(kr)
|
|
self.pbar.update()
|
|
|
|
except Exception as err:
|
|
print(repr(err))
|
|
|
|
def parse(self, input):
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor:
|
|
|
|
if type(input) == pd.DataFrame:
|
|
self.pbar = tqdm(total=input["id"].count(),desc=self.name)
|
|
|
|
for index, row in input.iterrows():
|
|
executor.submit(self.parseBySLO, row)
|
|
|
|
elif type(input)== list:
|
|
self.pbar = tqdm(total=len(input), desc=self.name)
|
|
for slo in input:
|
|
executor.submit(self.parseBySLO, slo)
|
|
|
|
elif type(input) == dict:
|
|
self.pbar = tqdm(total=1, desc=self.name)
|
|
executor.submit(self.parseBySLO, row)
|
|
|
|
|
|
return self.krs
|
|
|
|
|
|
|
|
def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ):
|
|
self.name=name
|
|
self.DTAPIURL= DTAPIURL
|
|
self.DTAPIToken=DTAPIToken
|
|
self.options=options
|
|
self.config=merge({"threads": 3,
|
|
"serviceLookupParams":{"from":"now-2y"},
|
|
"extendResultObjects":{}}, config)
|
|
self.krs=[] |