finalizations & Readme.md
ermisw
parent
85d6002bc3
commit
e265dd1338
|
|
@ -0,0 +1,45 @@
|
|||
# Report Host & ProcessGroups for SLOs
|
||||
This repository holds the code to generate the relevant Host and Processgroup Report.
|
||||
|
||||
Ouput is an excel file of all Host and Processgroups for all relevant SLOs (starting with (TP_))
|
||||
|
||||
|
||||
# Prerequisites
|
||||
##
|
||||
Python Version >= 3.10.x
|
||||
|
||||
## Python packages
|
||||
Before executing scripts, python requirements have to be satisfied. To do so, execute following command:
|
||||
pip install -r requirements.txt
|
||||
|
||||
## .env file
|
||||
|
||||
To provide authentication for API calls, create ".env" file in the script directory with following definition:
|
||||
|
||||
<ENV NAME>=<ENV TOKEN>
|
||||
<ENV NAME> is name of environment variable. This name should be passed to "environment.yaml" file as "env-token-name" parameter
|
||||
Example:
|
||||
environment.yaml file: "- env-token-name: "GLOBAL_CONFIG_TOKEN"
|
||||
.env file: "GLOBAL_CONFIG_TOKEN=XXXXXXXXXXX"
|
||||
|
||||
# Usage
|
||||
|
||||
python create_report.py
|
||||
|
||||
# Files
|
||||
|
||||
## create_report.py
|
||||
|
||||
This scripts generates the report.
|
||||
|
||||
## environment.yaml
|
||||
File containing environments to execute --auto-upload
|
||||
|
||||
Environment name:
|
||||
name: string #name ov environment
|
||||
env-url: str #url of environment
|
||||
env-token-name: str #name of environment variable containing API token
|
||||
|
||||
## requirements.txt
|
||||
|
||||
File containing required python packages
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import requests
|
||||
|
||||
def get_request(url, headers, params):
|
||||
#try:
|
||||
response = requests.get(url, headers=headers, params=params, verify=False)
|
||||
response.raise_for_status()
|
||||
# except requests.exceptions.HTTPError as errh:
|
||||
# return "An Http Error occurred:" + repr(errh)
|
||||
# except requests.exceptions.ConnectionError as errc:
|
||||
# return "An Error Connecting to the API occurred:" + repr(errc)
|
||||
# except requests.exceptions.Timeout as errt:
|
||||
# return "A Timeout Error occurred:" + repr(errt)
|
||||
# except requests.exceptions.RequestException as err:
|
||||
# return "An Unknown Error occurred" + repr(err)
|
||||
|
||||
return response
|
||||
|
||||
def contains(list, filter):
|
||||
for x in list:
|
||||
if filter(x):
|
||||
return True
|
||||
return False
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
try:
|
||||
# Python 3
|
||||
from collections.abc import MutableSequence
|
||||
except ImportError:
|
||||
# Python 2.7
|
||||
from collections import MutableSequence
|
||||
|
||||
class KeyRequestGroup(MutableSequence):
|
||||
"""A container for manipulating lists of hosts"""
|
||||
def __init__(self, data=None):
|
||||
"""Initialize the class"""
|
||||
super(KeyRequestGroup, self).__init__()
|
||||
if (data is not None):
|
||||
self._list = list(data)
|
||||
else:
|
||||
self._list = list()
|
||||
|
||||
def __repr__(self):
|
||||
return "<{0} {1}>".format(self.__class__.__name__, self._list)
|
||||
|
||||
def __len__(self):
|
||||
"""List length"""
|
||||
return len(self._list)
|
||||
|
||||
def __getitem__(self, ii):
|
||||
"""Get a list item"""
|
||||
if isinstance(ii, slice):
|
||||
return self.__class__(self._list[ii])
|
||||
else:
|
||||
return self._list[ii]
|
||||
|
||||
def __delitem__(self, ii):
|
||||
"""Delete an item"""
|
||||
del self._list[ii]
|
||||
|
||||
def __setitem__(self, ii, val):
|
||||
# optional: self._acl_check(val)
|
||||
self._list[ii] = val
|
||||
|
||||
def __str__(self):
|
||||
return str(self._list)
|
||||
|
||||
def createExistsQuery(self, val):
|
||||
|
||||
query="type(service_method)"
|
||||
|
||||
val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
val['methods'] = list(map(lambda x: x.replace("~","") , val['methods']))
|
||||
#case Service Names exists
|
||||
if len(val["services"]) > 0:
|
||||
if val["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
|
||||
|
||||
if val["methods"][0].startswith("SERVICE_METHOD-"):
|
||||
query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
else:
|
||||
query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
|
||||
val["existsQuery"]= query
|
||||
|
||||
# def createServiceResolveQuery(self, val):
|
||||
# query="type(SERVICE)"
|
||||
# val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
|
||||
# if len(val["services"]) > 0:
|
||||
# if val["services"][0].startswith("SERVICE-"):
|
||||
# query+=",entityId(\""+'","'.join(val["services"])+"\")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
|
||||
|
||||
# val["resolveServiceQuery"]= query
|
||||
|
||||
|
||||
def insert(self, ii, val):
|
||||
self.createExistsQuery(val)
|
||||
#self.createServiceResolveQuery(val)
|
||||
self._list.insert(ii, val)
|
||||
|
||||
|
||||
|
||||
def append(self, val):
|
||||
|
||||
if len(self._list) == 0:
|
||||
#self._list.insert(ii, val)
|
||||
self.insert(len(self._list), val)
|
||||
return
|
||||
|
||||
for group in self._list:
|
||||
if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0:
|
||||
self.insert(len(self._list), val)
|
||||
|
||||
|
||||
|
||||
class KR:
|
||||
|
||||
# def getNotExistingKeyRequests(self):
|
||||
# return [k for k in self.keyRequests if k['exists']==False]
|
||||
|
||||
# def hasNotExistingKeyRequests(self):
|
||||
# for k in self.keyRequests:
|
||||
# if k['exists']==False:
|
||||
# return True
|
||||
|
||||
# return False
|
||||
|
||||
# def getNoData1WKeyRequests(self):
|
||||
# return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True]
|
||||
|
||||
# def hasNoData1WKeyRequests(self):
|
||||
# for k in self.keyRequests:
|
||||
# if k['hasData_1W']==False and k['exists'] == True:
|
||||
# return True
|
||||
|
||||
# return False
|
||||
|
||||
def getKeyRequestByHasData(self,label):
|
||||
return [k for k in self.keyRequests if k['hasData'][label]==False]
|
||||
|
||||
def hasNoData(self,label):
|
||||
for k in self.keyRequests:
|
||||
if k['hasData'][label]==False:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def checkKeyRequestsHasData(self):
|
||||
pass
|
||||
|
||||
def mergeServices(self, listServices):
|
||||
listOfServiceIds=[o["entityId"] for o in self.services]
|
||||
|
||||
for s in listServices:
|
||||
if s["entityId"] not in listOfServiceIds:
|
||||
self.services.append(s)
|
||||
|
||||
def __init__(self,
|
||||
metadata,
|
||||
matchedGroups: KeyRequestGroup = None):
|
||||
self.metadata=metadata
|
||||
|
||||
if matchedGroups == None:
|
||||
self.matchedGroups = KeyRequestGroup()
|
||||
else:
|
||||
self.matchedGroups = keyRequests_groups
|
||||
|
||||
self.keyRequests=[]
|
||||
self.services=[]
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
import re
|
||||
|
||||
from key_request_parser import patterns, keyrequests, helper
|
||||
|
||||
from enum import Flag, auto
|
||||
|
||||
|
||||
class KROption(Flag):
|
||||
VALIDATE_EXISTS = auto()
|
||||
VALIDATE_HASDATA = auto()
|
||||
RESOLVEKEYREQUETS = auto()
|
||||
RESOLVESERVICES = auto()
|
||||
|
||||
class KRParser:
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()]
|
||||
|
||||
def normalize(self,x):
|
||||
#tmp=x.replace("~","")
|
||||
tmp=x.replace("\n","")
|
||||
#tmp=tmp.replace("\"/","\"")
|
||||
#tmp=tmp.replace("\"/","") -_>was active
|
||||
#tmp=tmp.replace("/\"","\"")
|
||||
tmp=tmp.replace("/\"","")
|
||||
tmp=tmp.replace("\"","")
|
||||
tmp=tmp.replace("\t","")
|
||||
|
||||
tmp=re.sub("([\s]*)\)", ")", tmp)
|
||||
|
||||
|
||||
|
||||
tmp=re.sub("\([\s\n\r]*", "(", tmp)
|
||||
tmp=re.sub("\,[\s\n\r]*", ",", tmp)
|
||||
tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
|
||||
tmp=re.sub("in[\s\n\r]*\(", "in(", tmp)
|
||||
|
||||
return tmp
|
||||
|
||||
|
||||
def applyPatterns(self,subject):
|
||||
groups=None
|
||||
for p in self.patterns:
|
||||
groups=p.parseServicesAndMethods(subject)
|
||||
|
||||
if len(groups) > 0:
|
||||
break
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
|
||||
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities) > 0:
|
||||
y=0
|
||||
for method in kr.keyRequests:
|
||||
if method["groupId"] == gid:
|
||||
found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
||||
|
||||
if len(found) > 0:
|
||||
method["hasData"][tfrom["label"]]=True
|
||||
#method["displayName"]=found[0]["displayName"]
|
||||
#method["entityId"]=found[0]["entityId"]
|
||||
#method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
# for idx,o in enumerate(method["services"]):
|
||||
# tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
|
||||
# if len(tmpS)>0:
|
||||
# method["services"][idx]=tmpS[0]
|
||||
|
||||
else:
|
||||
method["hasData"][tfrom["label"]]=False
|
||||
|
||||
def resolveServices(self,services, DTAPIURL, DTAPIToken):
|
||||
#DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
for gid, service in enumerate(services):
|
||||
query="type(SERVICE),entityId("+service["id"]+")"
|
||||
params={"entitySelector": query,"from":"now-2y"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities)>0:
|
||||
services[gid]=entities[0]
|
||||
|
||||
def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
|
||||
for gid, k in enumerate(kr.keyRequests):
|
||||
try:
|
||||
query="type(service_method)"
|
||||
group=kr.matchedGroups[k["groupId"]]
|
||||
|
||||
if len(group["services"])> 0:
|
||||
if group["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
|
||||
if k["comparer"]=="entityId":
|
||||
query+=",entityId("+k["entityId"]+")"
|
||||
else:
|
||||
query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
|
||||
params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
# if len(entities) > 1:
|
||||
# kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
# print("Multiple keyrequest found: ")
|
||||
|
||||
if len(entities)> 0:
|
||||
kr.keyRequests[gid]["found"]=True
|
||||
kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
||||
kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
||||
|
||||
if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
||||
kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
||||
self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
except Exception as err:
|
||||
kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
|
||||
|
||||
|
||||
#kr.mergeServices(entities)
|
||||
|
||||
|
||||
|
||||
|
||||
def process(self, kr):
|
||||
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
for method in group["methods"]:
|
||||
if method.startswith('SERVICE_METHOD-'):
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
else:
|
||||
tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
|
||||
# if self.options and KROption.VALIDATE_EXISTS in self.options:
|
||||
# self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
|
||||
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if KROption.VALIDATE_HASDATA in self.options:
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
||||
# elif KROption.RESOLVEKEYREQUETS in self.options:
|
||||
# self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
# if KROption.RESOLVESERVICES in self.options:
|
||||
# self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
|
||||
|
||||
return kr
|
||||
|
||||
|
||||
def parseBySLO(self, row):
|
||||
#normalize
|
||||
normFilter=self.normalize(row['filter'])
|
||||
normExpresseion=self.normalize(row['metricExpression'])
|
||||
|
||||
tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
|
||||
|
||||
|
||||
#SLO with Filter
|
||||
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"):
|
||||
subject=normFilter
|
||||
else:
|
||||
subject=normExpresseion
|
||||
|
||||
groups=self.applyPatterns(subject)
|
||||
|
||||
for g in groups:
|
||||
if g["methods"] != None and len(g["methods"]) > 0:
|
||||
tmp_KR.matchedGroups.append(g)
|
||||
|
||||
#self.process(tmp_KR)
|
||||
|
||||
return self.process(tmp_KR)
|
||||
|
||||
|
||||
def __init__(self, options: KROption , DTAPIURL, DTAPIToken ):
|
||||
self.DTAPIURL= DTAPIURL
|
||||
self.DTAPIToken=DTAPIToken
|
||||
self.options=options
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import re
|
||||
import urllib
|
||||
|
||||
class Pattern1:
|
||||
|
||||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*[\)]*", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*]*[\"]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*\")[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*)\)\"", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
#Endoce
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)\,entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#services=[]
|
||||
#methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":methods})
|
||||
#return services, methods
|
||||
return groups
|
||||
|
||||
class Pattern2:
|
||||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\([~]*service[~]*\),entityName[\.]*[in]*\(([^\)]*)\),tag\(([^\)]*)\)\),entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
# services=[]
|
||||
# methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")]
|
||||
groups.append({"services":services, "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
|
||||
class Pattern3:
|
||||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
# services=[]
|
||||
# methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
methods=[s.strip() for s in r.split(",")]
|
||||
groups.append({"services":[], "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
|
||||
# class Pattern4:
|
||||
|
||||
# def parseServicesAndMethods(self, metricExpression):
|
||||
# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
# groups=[]
|
||||
# methods=[]
|
||||
# if result:
|
||||
# for r in result:
|
||||
# methods.append(r)
|
||||
|
||||
# groups.append({"services":[], "methods":methods})
|
||||
|
||||
# return groups
|
||||
|
||||
# class FilterMethodPattern:
|
||||
|
||||
# def parseServicesAndMethods(self, metricExpression):
|
||||
# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
# services=[]
|
||||
# methods=[]
|
||||
# if result:
|
||||
# for r in result:
|
||||
# methods=[s.strip() for s in r.split(",")]
|
||||
|
||||
# return services, methods
|
||||
|
|
@ -393,22 +393,24 @@ def main() -> None:
|
|||
# krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN)
|
||||
|
||||
slosF = get_slo(env, DTTOKEN, DTURL)
|
||||
slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"]
|
||||
#slosF = slosF[slosF["name"].str.startswith("TP_")]
|
||||
#slosF = slosF[slosF["id"]=="9c5b0581-acc2-3e70-97d3-531700f78b65"]
|
||||
slosF = slosF[slosF["name"].str.startswith("TP_")]
|
||||
|
||||
# parse the metric Expression to get Services and Requests
|
||||
|
||||
krs = []
|
||||
krp = krparser.KRParser(
|
||||
krparser.KROption.VALIDATE_EXISTS
|
||||
# | krparser.KROption.VALIDATE_HASDATA
|
||||
| krparser.KROption.RESOLVESERVICES,
|
||||
DTURL,
|
||||
DTTOKEN,
|
||||
)
|
||||
# krp = krparser.KRParser(
|
||||
# krparser.KROption.VALIDATE_EXISTS
|
||||
# # | krparser.KROption.VALIDATE_HASDATA
|
||||
# | krparser.KROption.RESOLVESERVICES,
|
||||
# DTURL,
|
||||
# DTTOKEN,
|
||||
# )
|
||||
krp = krparser.KRParser(options=krparser.KROption.RESOLVEKEYREQUETS | krparser.KROption.RESOLVESERVICES, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
for index, row in slosF.iterrows():
|
||||
krs.append(krp.parseBySLO(row))
|
||||
krs=krp.parseBySLO_Threaded(slosF)
|
||||
# for index, row in slosF.iterrows():
|
||||
# krs.append(krp.parseBySLO(row))
|
||||
|
||||
# x = 0
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import requests
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
|
||||
def get_request(url, headers, params):
|
||||
def get_requestOld(url, headers, params):
|
||||
#try:
|
||||
response = requests.get(url, headers=headers, params=params, verify=False)
|
||||
response.raise_for_status()
|
||||
|
|
@ -14,6 +15,29 @@ def get_request(url, headers, params):
|
|||
# return "An Unknown Error occurred" + repr(err)
|
||||
|
||||
return response
|
||||
|
||||
def get_request(url, headers, params):
|
||||
#try:
|
||||
session = requests.Session()
|
||||
retry = Retry(connect=3, backoff_factor=10)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
|
||||
#response = requests.get(url, headers=headers, params=params, verify=False)
|
||||
response = session.get(url,headers=headers, params=params, verify=False)
|
||||
|
||||
response.raise_for_status()
|
||||
# except requests.exceptions.HTTPError as errh:
|
||||
# return "An Http Error occurred:" + repr(errh)
|
||||
# except requests.exceptions.ConnectionError as errc:
|
||||
# return "An Error Connecting to the API occurred:" + repr(errc)
|
||||
# except requests.exceptions.Timeout as errt:
|
||||
# return "A Timeout Error occurred:" + repr(errt)
|
||||
# except requests.exceptions.RequestException as err:
|
||||
# return "An Unknown Error occurred" + repr(err)
|
||||
|
||||
return response
|
||||
|
||||
def contains(list, filter):
|
||||
for x in list:
|
||||
|
|
|
|||
|
|
@ -52,12 +52,13 @@ class KeyRequestGroup(MutableSequence):
|
|||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
|
||||
|
||||
if val["methods"][0].startswith("SERVICE_METHOD-"):
|
||||
query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
else:
|
||||
query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
|
||||
|
||||
if len(val["methods"]) > 0:
|
||||
if val["methods"][0].startswith("SERVICE_METHOD-"):
|
||||
query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
else:
|
||||
query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
|
||||
val["existsQuery"]= query
|
||||
|
||||
# def createServiceResolveQuery(self, val):
|
||||
|
|
@ -75,21 +76,23 @@ class KeyRequestGroup(MutableSequence):
|
|||
|
||||
def insert(self, ii, val):
|
||||
self.createExistsQuery(val)
|
||||
#self.createServiceResolveQuery(val)
|
||||
|
||||
self._list.insert(ii, val)
|
||||
|
||||
|
||||
|
||||
def append(self, val):
|
||||
|
||||
for g in val:
|
||||
|
||||
if len(self._list) == 0:
|
||||
#self._list.insert(ii, val)
|
||||
self.insert(len(self._list), val)
|
||||
self.insert(len(self._list), g)
|
||||
return
|
||||
|
||||
for group in self._list:
|
||||
if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0:
|
||||
self.insert(len(self._list), val)
|
||||
if len(set(group["services"]) - set(g["services"])) > 0 or len(set(group["methods"]) - set(g["methods"])) > 0:
|
||||
self.insert(len(self._list), g)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ import re
|
|||
from key_request_parser import patterns, keyrequests, helper
|
||||
|
||||
from enum import Flag, auto
|
||||
import logging
|
||||
import threading
|
||||
import concurrent.futures
|
||||
import time
|
||||
|
||||
|
||||
|
||||
class KROption(Flag):
|
||||
|
|
@ -12,8 +17,10 @@ class KROption(Flag):
|
|||
RESOLVESERVICES = auto()
|
||||
|
||||
class KRParser:
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()]
|
||||
|
||||
#threadLimiter = threading.BoundedSemaphore(3)
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
|
||||
lock = threading.Lock()
|
||||
|
||||
def normalize(self,x):
|
||||
#tmp=x.replace("~","")
|
||||
tmp=x.replace("\n","")
|
||||
|
|
@ -123,7 +130,7 @@ class KRParser:
|
|||
else:
|
||||
query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
|
||||
params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"}
|
||||
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
|
|
@ -151,11 +158,38 @@ class KRParser:
|
|||
#kr.mergeServices(entities)
|
||||
|
||||
|
||||
|
||||
def getKeyRequestsByServices(self, services):
|
||||
#type(SERVICE_METHOD),fromRelationship.isServiceMethodOfService(type("SERVICE"),entityName.in("btc-user-composite-service - PROD"))
|
||||
DTAPIURL = self.DTAPIURL + "/api/v2/entities"
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + self.DTAPIToken
|
||||
}
|
||||
|
||||
if len(services) > 0:
|
||||
if services[0].startswith("SERVICE-"):
|
||||
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(services)+"\"))"
|
||||
else:
|
||||
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
|
||||
|
||||
params={"entitySelector": query}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
return entities
|
||||
|
||||
|
||||
def process(self, kr):
|
||||
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
|
||||
if len(group["services"]) > 0 and len(group["methods"])==0:
|
||||
tmp_methods=self.getKeyRequestsByServices(group["services"])
|
||||
|
||||
for m in tmp_methods:
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
for method in group["methods"]:
|
||||
if method.startswith('SERVICE_METHOD-'):
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
|
@ -163,15 +197,25 @@ class KRParser:
|
|||
tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
|
||||
|
||||
|
||||
# if self.options and KROption.VALIDATE_EXISTS in self.options:
|
||||
# self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
|
||||
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if KROption.VALIDATE_HASDATA in self.options:
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
||||
# for service in group["services"]:
|
||||
# if service.startswith('SERVICE-'):
|
||||
# tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
# else:
|
||||
# tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
# kr.services.append(tmp)
|
||||
|
||||
|
||||
if self.options:
|
||||
if KROption.RESOLVEKEYREQUETS in self.options:
|
||||
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if KROption.VALIDATE_HASDATA in self.options:
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
||||
# elif KROption.RESOLVEKEYREQUETS in self.options:
|
||||
# self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
# if KROption.RESOLVESERVICES in self.options:
|
||||
|
|
@ -180,32 +224,76 @@ class KRParser:
|
|||
return kr
|
||||
|
||||
|
||||
def parseBySLO(self, row):
|
||||
def parseBySLO(self,index,row):
|
||||
#normalize
|
||||
normFilter=self.normalize(row['filter'])
|
||||
normExpresseion=self.normalize(row['metricExpression'])
|
||||
print(index)
|
||||
|
||||
tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
|
||||
|
||||
|
||||
#SLO with Filter
|
||||
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"):
|
||||
subject=normFilter
|
||||
else:
|
||||
subject=normExpresseion
|
||||
|
||||
groups=self.applyPatterns(subject)
|
||||
|
||||
for g in groups:
|
||||
if g["methods"] != None and len(g["methods"]) > 0:
|
||||
tmp_KR.matchedGroups.append(g)
|
||||
try:
|
||||
normFilter=self.normalize(row['filter'])
|
||||
normExpresseion=self.normalize(row['metricExpression'])
|
||||
|
||||
#self.process(tmp_KR)
|
||||
tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
|
||||
|
||||
|
||||
return self.process(tmp_KR)
|
||||
#SLO with Filter
|
||||
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
|
||||
subject=normFilter
|
||||
else:
|
||||
subject=normExpresseion
|
||||
|
||||
groups=self.applyPatterns(subject)
|
||||
|
||||
tmp_KR.matchedGroups.append(groups)
|
||||
# for g in groups:
|
||||
# #if g["methods"] != None and len(g["methods"]) > 0:
|
||||
# tmp_KR.matchedGroups.append(g)
|
||||
|
||||
#self.process(tmp_KR)
|
||||
kr=self.process(tmp_KR)
|
||||
with self.lock:
|
||||
self.krs.append(kr)
|
||||
|
||||
except Exception as err:
|
||||
print(repr(err))
|
||||
#return self.process(tmp_KR)
|
||||
|
||||
def parseBySLO_Threaded(self, slosF):
|
||||
self.krs=[]
|
||||
#i=1
|
||||
# threads = list()
|
||||
# for index, row in slosF.iterrows():
|
||||
# logging.info("Main : create and start thread %d.", index)
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
# #krs.append(krp.parseBySLO(row))
|
||||
|
||||
# for index, thread in enumerate(threads):
|
||||
# logging.info("Main : before joining thread %d.", index)
|
||||
# thread.join()
|
||||
# logging.info("Main : thread %d done", index)
|
||||
|
||||
# #resultSlos.extend(krs)
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(10) as executor:
|
||||
for index, row in slosF.iterrows():
|
||||
# if i % 25 == 0:
|
||||
# time.sleep(0)
|
||||
#args={index:index, }
|
||||
executor.submit(self.parseBySLO, index,row)
|
||||
# print(str(i)+"\n")
|
||||
# i=i+1
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
return self.krs
|
||||
|
||||
|
||||
def __init__(self, options: KROption , DTAPIURL, DTAPIToken ):
|
||||
def __init__(self, options: KROption=None , DTAPIURL=None, DTAPIToken=None ):
|
||||
self.DTAPIURL= DTAPIURL
|
||||
self.DTAPIToken=DTAPIToken
|
||||
self.options=options
|
||||
self.options=options
|
||||
self.krs=[]
|
||||
|
|
@ -47,42 +47,50 @@ class Pattern2:
|
|||
class Pattern3:
|
||||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
# services=[]
|
||||
# methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
methods=[s.strip() for s in r.split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":[], "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
||||
|
||||
# class Pattern4:
|
||||
|
||||
# def parseServicesAndMethods(self, metricExpression):
|
||||
# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
class Pattern4:
|
||||
|
||||
# groups=[]
|
||||
# methods=[]
|
||||
# if result:
|
||||
# for r in result:
|
||||
# methods.append(r)
|
||||
|
||||
# groups.append({"services":[], "methods":methods})
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
if not r:
|
||||
#methods=[s.strip() for s in r.split(",")]
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":services, "methods":[]})
|
||||
|
||||
# return groups
|
||||
return groups
|
||||
|
||||
# class FilterMethodPattern:
|
||||
|
||||
# def parseServicesAndMethods(self, metricExpression):
|
||||
# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
# services=[]
|
||||
# methods=[]
|
||||
# if result:
|
||||
# for r in result:
|
||||
# methods=[s.strip() for s in r.split(",")]
|
||||
|
||||
# return services, methods
|
||||
class Pattern5:
|
||||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
|
||||
#Endoce
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#services=[]
|
||||
#methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
if not r:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
#methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":[]})
|
||||
#return services, methods
|
||||
return groups
|
||||
|
|
@ -3,4 +3,5 @@ python-decouple
|
|||
requests
|
||||
pyyaml
|
||||
pandas
|
||||
xlsxwriter
|
||||
xlsxwriter
|
||||
jsonmerge
|
||||
Loading…
Reference in New Issue