diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..fca413f --- /dev/null +++ b/Readme.md @@ -0,0 +1,45 @@ +# Report Host & ProcessGroups for SLOs +This repository holds the code to generate the relevant Host and Processgroup Report. + +Ouput is an excel file of all Host and Processgroups for all relevant SLOs (starting with (TP_)) + + +# Prerequisites +## +Python Version >= 3.10.x + +## Python packages +Before executing scripts, python requirements have to be satisfied. To do so, execute following command: + pip install -r requirements.txt + +## .env file + +To provide authentication for API calls, create ".env" file in the script directory with following definition: + + = + is name of environment variable. This name should be passed to "environment.yaml" file as "env-token-name" parameter + Example: + environment.yaml file: "- env-token-name: "GLOBAL_CONFIG_TOKEN" + .env file: "GLOBAL_CONFIG_TOKEN=XXXXXXXXXXX" + +# Usage + + python create_report.py + +# Files + +## create_report.py + +This scripts generates the report. + +## environment.yaml +File containing environments to execute --auto-upload + + Environment name: + name: string #name ov environment + env-url: str #url of environment + env-token-name: str #name of environment variable containing API token + +## requirements.txt + +File containing required python packages \ No newline at end of file diff --git a/_BAK/__init__.py b/_BAK/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/_BAK/helper.py b/_BAK/helper.py new file mode 100644 index 0000000..4c311b0 --- /dev/null +++ b/_BAK/helper.py @@ -0,0 +1,22 @@ +import requests + +def get_request(url, headers, params): + #try: + response = requests.get(url, headers=headers, params=params, verify=False) + response.raise_for_status() + # except requests.exceptions.HTTPError as errh: + # return "An Http Error occurred:" + repr(errh) + # except requests.exceptions.ConnectionError as errc: + # return "An Error Connecting to the API occurred:" + repr(errc) + # except requests.exceptions.Timeout as errt: + # return "A Timeout Error occurred:" + repr(errt) + # except requests.exceptions.RequestException as err: + # return "An Unknown Error occurred" + repr(err) + + return response + +def contains(list, filter): + for x in list: + if filter(x): + return True + return False \ No newline at end of file diff --git a/_BAK/keyrequests.py b/_BAK/keyrequests.py new file mode 100644 index 0000000..e23e2fa --- /dev/null +++ b/_BAK/keyrequests.py @@ -0,0 +1,154 @@ +try: + # Python 3 + from collections.abc import MutableSequence +except ImportError: + # Python 2.7 + from collections import MutableSequence + +class KeyRequestGroup(MutableSequence): + """A container for manipulating lists of hosts""" + def __init__(self, data=None): + """Initialize the class""" + super(KeyRequestGroup, self).__init__() + if (data is not None): + self._list = list(data) + else: + self._list = list() + + def __repr__(self): + return "<{0} {1}>".format(self.__class__.__name__, self._list) + + def __len__(self): + """List length""" + return len(self._list) + + def __getitem__(self, ii): + """Get a list item""" + if isinstance(ii, slice): + return self.__class__(self._list[ii]) + else: + return self._list[ii] + + def __delitem__(self, ii): + """Delete an item""" + del self._list[ii] + + def __setitem__(self, ii, val): + # optional: self._acl_check(val) + self._list[ii] = val + + def __str__(self): + return str(self._list) + + def createExistsQuery(self, val): + + query="type(service_method)" + + val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + val['methods'] = list(map(lambda x: x.replace("~","") , val['methods'])) + #case Service Names exists + if len(val["services"]) > 0: + if val["services"][0].startswith("SERVICE-"): + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" + else: + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" + + if val["methods"][0].startswith("SERVICE_METHOD-"): + query+=",entityId(\""+'","'.join(val["methods"])+"\")" + else: + query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + + val["existsQuery"]= query + + # def createServiceResolveQuery(self, val): + # query="type(SERVICE)" + # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + + # if len(val["services"]) > 0: + # if val["services"][0].startswith("SERVICE-"): + # query+=",entityId(\""+'","'.join(val["services"])+"\")" + # else: + # query+=",entityName.in(\""+'","'.join(val["services"])+"\")" + + # val["resolveServiceQuery"]= query + + + def insert(self, ii, val): + self.createExistsQuery(val) + #self.createServiceResolveQuery(val) + self._list.insert(ii, val) + + + + def append(self, val): + + if len(self._list) == 0: + #self._list.insert(ii, val) + self.insert(len(self._list), val) + return + + for group in self._list: + if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0: + self.insert(len(self._list), val) + + + +class KR: + + # def getNotExistingKeyRequests(self): + # return [k for k in self.keyRequests if k['exists']==False] + + # def hasNotExistingKeyRequests(self): + # for k in self.keyRequests: + # if k['exists']==False: + # return True + + # return False + + # def getNoData1WKeyRequests(self): + # return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True] + + # def hasNoData1WKeyRequests(self): + # for k in self.keyRequests: + # if k['hasData_1W']==False and k['exists'] == True: + # return True + + # return False + + def getKeyRequestByHasData(self,label): + return [k for k in self.keyRequests if k['hasData'][label]==False] + + def hasNoData(self,label): + for k in self.keyRequests: + if k['hasData'][label]==False: + return True + + return False + + + + + def checkKeyRequestsHasData(self): + pass + + def mergeServices(self, listServices): + listOfServiceIds=[o["entityId"] for o in self.services] + + for s in listServices: + if s["entityId"] not in listOfServiceIds: + self.services.append(s) + + def __init__(self, + metadata, + matchedGroups: KeyRequestGroup = None): + self.metadata=metadata + + if matchedGroups == None: + self.matchedGroups = KeyRequestGroup() + else: + self.matchedGroups = keyRequests_groups + + self.keyRequests=[] + self.services=[] + + diff --git a/_BAK/krparser.py b/_BAK/krparser.py new file mode 100644 index 0000000..2ac42a0 --- /dev/null +++ b/_BAK/krparser.py @@ -0,0 +1,211 @@ +import re + +from key_request_parser import patterns, keyrequests, helper + +from enum import Flag, auto + + +class KROption(Flag): + VALIDATE_EXISTS = auto() + VALIDATE_HASDATA = auto() + RESOLVEKEYREQUETS = auto() + RESOLVESERVICES = auto() + +class KRParser: + patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()] + + def normalize(self,x): + #tmp=x.replace("~","") + tmp=x.replace("\n","") + #tmp=tmp.replace("\"/","\"") + #tmp=tmp.replace("\"/","") -_>was active + #tmp=tmp.replace("/\"","\"") + tmp=tmp.replace("/\"","") + tmp=tmp.replace("\"","") + tmp=tmp.replace("\t","") + + tmp=re.sub("([\s]*)\)", ")", tmp) + + + + tmp=re.sub("\([\s\n\r]*", "(", tmp) + tmp=re.sub("\,[\s\n\r]*", ",", tmp) + tmp=re.sub("\)[\s\n\r]*,", "),", tmp) + tmp=re.sub("in[\s\n\r]*\(", "in(", tmp) + + return tmp + + + def applyPatterns(self,subject): + groups=None + for p in self.patterns: + groups=p.parseServicesAndMethods(subject) + + if len(groups) > 0: + break + + return groups + + + def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): + + DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + for gid, group in enumerate(kr.matchedGroups): + params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + if len(entities) > 0: + y=0 + for method in kr.keyRequests: + if method["groupId"] == gid: + found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + if len(found) > 0: + method["hasData"][tfrom["label"]]=True + #method["displayName"]=found[0]["displayName"] + #method["entityId"]=found[0]["entityId"] + #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"] + + # for idx,o in enumerate(method["services"]): + # tmpS=[p for p in kr.services if p["entityId"]==o["id"]] + # if len(tmpS)>0: + # method["services"][idx]=tmpS[0] + + else: + method["hasData"][tfrom["label"]]=False + + def resolveServices(self,services, DTAPIURL, DTAPIToken): + #DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + for gid, service in enumerate(services): + query="type(SERVICE),entityId("+service["id"]+")" + params={"entitySelector": query,"from":"now-2y"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + if len(entities)>0: + services[gid]=entities[0] + + def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options): + DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + + for gid, k in enumerate(kr.keyRequests): + try: + query="type(service_method)" + group=kr.matchedGroups[k["groupId"]] + + if len(group["services"])> 0: + if group["services"][0].startswith("SERVICE-"): + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))" + else: + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))" + + if k["comparer"]=="entityId": + query+=",entityId("+k["entityId"]+")" + else: + query+=",entityName.in(\""+k["displayName"]+"\")" + + params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + # if len(entities) > 1: + # kr.keyRequests[gid]['foundCount']=len(entities) + # print("Multiple keyrequest found: ") + + if len(entities)> 0: + kr.keyRequests[gid]["found"]=True + kr.keyRequests[gid]['foundCount']=len(entities) + kr.keyRequests[gid]["displayName"]=entities[0]["displayName"] + kr.keyRequests[gid]["entityId"]=entities[0]["entityId"] + + if "isServiceMethodOfService" in entities[0]["fromRelationships"]: + kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] + + if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: + self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) + + except Exception as err: + kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err) + + + + #kr.mergeServices(entities) + + + + + def process(self, kr): + + for gid, group in enumerate(kr.matchedGroups): + for method in group["methods"]: + if method.startswith('SERVICE_METHOD-'): + tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + else: + tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + + kr.keyRequests.append(tmp) + + + # if self.options and KROption.VALIDATE_EXISTS in self.options: + # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) + + if KROption.VALIDATE_HASDATA in self.options: + self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) + # elif KROption.RESOLVEKEYREQUETS in self.options: + # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + # if KROption.RESOLVESERVICES in self.options: + # self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken) + + return kr + + + def parseBySLO(self, row): + #normalize + normFilter=self.normalize(row['filter']) + normExpresseion=self.normalize(row['metricExpression']) + + tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) + + + #SLO with Filter + if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"): + subject=normFilter + else: + subject=normExpresseion + + groups=self.applyPatterns(subject) + + for g in groups: + if g["methods"] != None and len(g["methods"]) > 0: + tmp_KR.matchedGroups.append(g) + + #self.process(tmp_KR) + + return self.process(tmp_KR) + + + def __init__(self, options: KROption , DTAPIURL, DTAPIToken ): + self.DTAPIURL= DTAPIURL + self.DTAPIToken=DTAPIToken + self.options=options \ No newline at end of file diff --git a/_BAK/patterns.py b/_BAK/patterns.py new file mode 100644 index 0000000..c724a73 --- /dev/null +++ b/_BAK/patterns.py @@ -0,0 +1,88 @@ +import re +import urllib + +class Pattern1: + + + def parseServicesAndMethods(self, metricExpression): + + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*[\)]*", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*]*[\"]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*\")[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*)\)\"", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + #Endoce + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)\,entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #services=[] + #methods=[] + groups=[] + if result: + for r in result: + services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] + groups.append({"services":services, "methods":methods}) + #return services, methods + return groups + +class Pattern2: + + def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\([~]*service[~]*\),entityName[\.]*[in]*\(([^\)]*)\),tag\(([^\)]*)\)\),entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + # services=[] + # methods=[] + groups=[] + if result: + for r in result: + services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")] + groups.append({"services":services, "methods":methods}) + + return groups + + + +class Pattern3: + + def parseServicesAndMethods(self, metricExpression): + result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + # services=[] + # methods=[] + groups=[] + if result: + for r in result: + methods=[s.strip() for s in r.split(",")] + groups.append({"services":[], "methods":methods}) + + return groups + + + +# class Pattern4: + +# def parseServicesAndMethods(self, metricExpression): +# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + +# groups=[] +# methods=[] +# if result: +# for r in result: +# methods.append(r) + +# groups.append({"services":[], "methods":methods}) + +# return groups + +# class FilterMethodPattern: + +# def parseServicesAndMethods(self, metricExpression): +# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) +# services=[] +# methods=[] +# if result: +# for r in result: +# methods=[s.strip() for s in r.split(",")] + +# return services, methods \ No newline at end of file diff --git a/create_report.py b/create_report.py index 841b38a..c659c55 100644 --- a/create_report.py +++ b/create_report.py @@ -393,22 +393,24 @@ def main() -> None: # krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN) slosF = get_slo(env, DTTOKEN, DTURL) - slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"] - #slosF = slosF[slosF["name"].str.startswith("TP_")] + #slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"] + slosF = slosF[slosF["name"].str.startswith("TP_")] # parse the metric Expression to get Services and Requests krs = [] - krp = krparser.KRParser( - krparser.KROption.VALIDATE_EXISTS - # | krparser.KROption.VALIDATE_HASDATA - | krparser.KROption.RESOLVESERVICES, - DTURL, - DTTOKEN, - ) + # krp = krparser.KRParser( + # krparser.KROption.VALIDATE_EXISTS + # # | krparser.KROption.VALIDATE_HASDATA + # | krparser.KROption.RESOLVESERVICES, + # DTURL, + # DTTOKEN, + # ) + krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN) - for index, row in slosF.iterrows(): - krs.append(krp.parseBySLO(row)) + krs=krp.parseBySLO_Threaded(slosF) + # for index, row in slosF.iterrows(): + # krs.append(krp.parseBySLO(row)) # x = 0 diff --git a/key_request_parser/helper.py b/key_request_parser/helper.py index 4c311b0..cace378 100644 --- a/key_request_parser/helper.py +++ b/key_request_parser/helper.py @@ -1,6 +1,7 @@ import requests +from requests.adapters import HTTPAdapter, Retry -def get_request(url, headers, params): +def get_requestOld(url, headers, params): #try: response = requests.get(url, headers=headers, params=params, verify=False) response.raise_for_status() @@ -14,6 +15,29 @@ def get_request(url, headers, params): # return "An Unknown Error occurred" + repr(err) return response + +def get_request(url, headers, params): + #try: + session = requests.Session() + retry = Retry(connect=3, backoff_factor=10) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + + #response = requests.get(url, headers=headers, params=params, verify=False) + response = session.get(url,headers=headers, params=params, verify=False) + + response.raise_for_status() + # except requests.exceptions.HTTPError as errh: + # return "An Http Error occurred:" + repr(errh) + # except requests.exceptions.ConnectionError as errc: + # return "An Error Connecting to the API occurred:" + repr(errc) + # except requests.exceptions.Timeout as errt: + # return "A Timeout Error occurred:" + repr(errt) + # except requests.exceptions.RequestException as err: + # return "An Unknown Error occurred" + repr(err) + + return response def contains(list, filter): for x in list: diff --git a/key_request_parser/keyrequests.py b/key_request_parser/keyrequests.py index e23e2fa..a877da3 100644 --- a/key_request_parser/keyrequests.py +++ b/key_request_parser/keyrequests.py @@ -52,12 +52,13 @@ class KeyRequestGroup(MutableSequence): query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" else: query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" - - if val["methods"][0].startswith("SERVICE_METHOD-"): - query+=",entityId(\""+'","'.join(val["methods"])+"\")" - else: - query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" - + + if len(val["methods"]) > 0: + if val["methods"][0].startswith("SERVICE_METHOD-"): + query+=",entityId(\""+'","'.join(val["methods"])+"\")" + else: + query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + val["existsQuery"]= query # def createServiceResolveQuery(self, val): @@ -75,21 +76,23 @@ class KeyRequestGroup(MutableSequence): def insert(self, ii, val): self.createExistsQuery(val) - #self.createServiceResolveQuery(val) + self._list.insert(ii, val) def append(self, val): + for g in val: + if len(self._list) == 0: #self._list.insert(ii, val) - self.insert(len(self._list), val) + self.insert(len(self._list), g) return for group in self._list: - if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0: - self.insert(len(self._list), val) + if len(set(group["services"]) - set(g["services"])) > 0 or len(set(group["methods"]) - set(g["methods"])) > 0: + self.insert(len(self._list), g) diff --git a/key_request_parser/krparser.py b/key_request_parser/krparser.py index 2ac42a0..b370fb7 100644 --- a/key_request_parser/krparser.py +++ b/key_request_parser/krparser.py @@ -3,6 +3,11 @@ import re from key_request_parser import patterns, keyrequests, helper from enum import Flag, auto +import logging +import threading +import concurrent.futures +import time + class KROption(Flag): @@ -12,8 +17,10 @@ class KROption(Flag): RESOLVESERVICES = auto() class KRParser: - patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()] - + #threadLimiter = threading.BoundedSemaphore(3) + patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ] + lock = threading.Lock() + def normalize(self,x): #tmp=x.replace("~","") tmp=x.replace("\n","") @@ -123,7 +130,7 @@ class KRParser: else: query+=",entityName.in(\""+k["displayName"]+"\")" - params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"} + params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -151,11 +158,38 @@ class KRParser: #kr.mergeServices(entities) - + def getKeyRequestsByServices(self, services): + #type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD")) + DTAPIURL = self.DTAPIURL + "/api/v2/entities" + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + self.DTAPIToken + } + + if len(services) > 0: + if services[0].startswith("SERVICE-"): + query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))" + else: + query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))" + + params={"entitySelector": query} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + return entities + def process(self, kr): for gid, group in enumerate(kr.matchedGroups): + + if len(group["services"]) > 0 and len(group["methods"])==0: + tmp_methods=self.getKeyRequestsByServices(group["services"]) + + for m in tmp_methods: + tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + kr.keyRequests.append(tmp) + for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, @@ -163,15 +197,25 @@ class KRParser: tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) + + - - # if self.options and KROption.VALIDATE_EXISTS in self.options: - # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) - self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) - - if KROption.VALIDATE_HASDATA in self.options: - self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) - self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) + # for service in group["services"]: + # if service.startswith('SERVICE-'): + # tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + # else: + # tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + + # kr.services.append(tmp) + + + if self.options: + if KROption.RESOLVEKEYREQUETS in self.options: + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) + + if KROption.VALIDATE_HASDATA in self.options: + self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) # elif KROption.RESOLVEKEYREQUETS in self.options: # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) # if KROption.RESOLVESERVICES in self.options: @@ -180,32 +224,76 @@ class KRParser: return kr - def parseBySLO(self, row): + def parseBySLO(self,index,row): #normalize - normFilter=self.normalize(row['filter']) - normExpresseion=self.normalize(row['metricExpression']) + print(index) - tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) - - - #SLO with Filter - if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"): - subject=normFilter - else: - subject=normExpresseion - - groups=self.applyPatterns(subject) - - for g in groups: - if g["methods"] != None and len(g["methods"]) > 0: - tmp_KR.matchedGroups.append(g) + try: + normFilter=self.normalize(row['filter']) + normExpresseion=self.normalize(row['metricExpression']) - #self.process(tmp_KR) + tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) + - return self.process(tmp_KR) + #SLO with Filter + if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"): + subject=normFilter + else: + subject=normExpresseion + + groups=self.applyPatterns(subject) + + tmp_KR.matchedGroups.append(groups) + # for g in groups: + # #if g["methods"] != None and len(g["methods"]) > 0: + # tmp_KR.matchedGroups.append(g) + + #self.process(tmp_KR) + kr=self.process(tmp_KR) + with self.lock: + self.krs.append(kr) + + except Exception as err: + print(repr(err)) + #return self.process(tmp_KR) + + def parseBySLO_Threaded(self, slosF): + self.krs=[] + #i=1 + # threads = list() + # for index, row in slosF.iterrows(): + # logging.info("Main : create and start thread %d.", index) + # x = threading.Thread(target=self.parseBySLO, args=(row,)) + # threads.append(x) + # x.start() + + # #krs.append(krp.parseBySLO(row)) + + # for index, thread in enumerate(threads): + # logging.info("Main : before joining thread %d.", index) + # thread.join() + # logging.info("Main : thread %d done", index) + + # #resultSlos.extend(krs) + + + with concurrent.futures.ThreadPoolExecutor(10) as executor: + for index, row in slosF.iterrows(): + # if i % 25 == 0: + # time.sleep(0) + #args={index:index, } + executor.submit(self.parseBySLO, index,row) + # print(str(i)+"\n") + # i=i+1 + # x = threading.Thread(target=self.parseBySLO, args=(row,)) + # threads.append(x) + # x.start() + + return self.krs - def __init__(self, options: KROption , DTAPIURL, DTAPIToken ): + def __init__(self, options: KROption=None , DTAPIURL=None, DTAPIToken=None ): self.DTAPIURL= DTAPIURL self.DTAPIToken=DTAPIToken - self.options=options \ No newline at end of file + self.options=options + self.krs=[] \ No newline at end of file diff --git a/key_request_parser/patterns.py b/key_request_parser/patterns.py index c724a73..ebd280b 100644 --- a/key_request_parser/patterns.py +++ b/key_request_parser/patterns.py @@ -47,42 +47,50 @@ class Pattern2: class Pattern3: def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) # services=[] # methods=[] groups=[] if result: for r in result: - methods=[s.strip() for s in r.split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] groups.append({"services":[], "methods":methods}) return groups - - -# class Pattern4: - -# def parseServicesAndMethods(self, metricExpression): -# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) +class Pattern4: -# groups=[] -# methods=[] -# if result: -# for r in result: -# methods.append(r) - -# groups.append({"services":[], "methods":methods}) + def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + groups=[] + if result: + for r in result: + if not r: + #methods=[s.strip() for s in r.split(",")] + services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] + groups.append({"services":services, "methods":[]}) -# return groups + return groups -# class FilterMethodPattern: - -# def parseServicesAndMethods(self, metricExpression): -# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) -# services=[] -# methods=[] -# if result: -# for r in result: -# methods=[s.strip() for s in r.split(",")] - -# return services, methods \ No newline at end of file +class Pattern5: + + def parseServicesAndMethods(self, metricExpression): + + #Endoce + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #services=[] + #methods=[] + groups=[] + if result: + for r in result: + if not r: + services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")] + #methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] + groups.append({"services":services, "methods":[]}) + #return services, methods + return groups \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a496a0d..af8e829 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ python-decouple requests pyyaml pandas -xlsxwriter \ No newline at end of file +xlsxwriter +jsonmerge \ No newline at end of file