Compare commits
No commits in common. "99dc302d5d489ee45c4dab2950315aa3badf8bac" and "dec4e7f136f423fbc741d5734abcb3581a922e6e" have entirely different histories.
99dc302d5d
...
dec4e7f136
|
|
@ -1,138 +0,0 @@
|
|||
.vscode
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
### Terraform stuff
|
||||
**/.terraform/*
|
||||
crash.log
|
||||
*.tfvars
|
||||
|
||||
#excel reports
|
||||
*.xlsx
|
||||
|
|
@ -58,24 +58,20 @@ class KeyRequestGroup(MutableSequence):
|
|||
query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
else:
|
||||
query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
# query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)"
|
||||
# val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
# val['methods'] = list(map(lambda x: x.replace("~","") , val['methods']))
|
||||
# if len(val["services"]) > 0:
|
||||
# if val["services"][0].startswith("SERVICE-"):
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
|
||||
# else:
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
|
||||
|
||||
# if len(val["methods"]) > 0:
|
||||
# if val["methods"][0].startswith("SERVICE_METHOD-"):
|
||||
# query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
|
||||
# query+="\")))"
|
||||
|
||||
val["existsQuery"]= query
|
||||
|
||||
|
||||
# def createServiceResolveQuery(self, val):
|
||||
# query="type(SERVICE)"
|
||||
# val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
|
||||
# if len(val["services"]) > 0:
|
||||
# if val["services"][0].startswith("SERVICE-"):
|
||||
# query+=",entityId(\""+'","'.join(val["services"])+"\")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
|
||||
|
||||
# val["resolveServiceQuery"]= query
|
||||
|
||||
|
||||
def insert(self, ii, val):
|
||||
|
|
@ -131,6 +127,8 @@ class KR:
|
|||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def checkKeyRequestsHasData(self):
|
||||
|
|
@ -142,12 +140,6 @@ class KR:
|
|||
for s in listServices:
|
||||
if s["entityId"] not in listOfServiceIds:
|
||||
self.services.append(s)
|
||||
|
||||
# def addKeyRequest(self, data):
|
||||
# if type(data) == list:
|
||||
# for m in data:
|
||||
# self.keyRequests.append({"displayName":m["["displayName"]"], "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""})
|
||||
|
||||
|
||||
def __init__(self,
|
||||
metadata,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import re
|
||||
|
||||
from KRParser import patterns, keyrequests, helper
|
||||
|
||||
|
||||
from enum import Flag, auto
|
||||
import logging
|
||||
import threading
|
||||
import concurrent.futures
|
||||
import time
|
||||
from jsonmerge import merge
|
||||
import pandas as pd
|
||||
from tqdm import *
|
||||
|
||||
|
||||
|
||||
|
|
@ -17,15 +19,24 @@ class KROption(Flag):
|
|||
RESOLVESERVICES = auto()
|
||||
|
||||
class KRParser:
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4(), patterns.Pattern5()]
|
||||
#threadLimiter = threading.BoundedSemaphore(3)
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
|
||||
lock = threading.Lock()
|
||||
|
||||
def normalize(self,x):
|
||||
#tmp=x.replace("~","")
|
||||
tmp=x.replace("\n","")
|
||||
#tmp=tmp.replace("\"/","\"")
|
||||
#tmp=tmp.replace("\"/","") -_>was active
|
||||
#tmp=tmp.replace("/\"","\"")
|
||||
tmp=tmp.replace("/\"","")
|
||||
tmp=tmp.replace("\"","")
|
||||
tmp=tmp.replace("\t","")
|
||||
|
||||
tmp=re.sub("([\s]*)\)", ")", tmp)
|
||||
|
||||
|
||||
|
||||
tmp=re.sub("\([\s\n\r]*", "(", tmp)
|
||||
tmp=re.sub("\,[\s\n\r]*", ",", tmp)
|
||||
tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
|
||||
|
|
@ -39,9 +50,7 @@ class KRParser:
|
|||
for p in self.patterns:
|
||||
groups=p.parseServicesAndMethods(subject)
|
||||
|
||||
if len(groups) > 0:
|
||||
for g in groups:
|
||||
g["pattern"]=str(p.__class__)
|
||||
if len(groups) > 0:
|
||||
break
|
||||
|
||||
return groups
|
||||
|
|
@ -49,54 +58,40 @@ class KRParser:
|
|||
|
||||
def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
|
||||
|
||||
# DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
# headers = {
|
||||
# 'Content-Type': 'application/json',
|
||||
# 'Authorization': 'Api-Token ' + DTAPIToken
|
||||
# }
|
||||
|
||||
# for gid, group in enumerate(kr.matchedGroups):
|
||||
# params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
||||
# response = helper.get_request(DTAPIURL, headers, params)
|
||||
# entities = (response.json())['entities']
|
||||
|
||||
# if len(entities) > 0:
|
||||
# y=0
|
||||
# for method in kr.keyRequests:
|
||||
# if method["groupId"] == gid:
|
||||
# found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
||||
|
||||
# if len(found) > 0:
|
||||
# method["hasData"][tfrom["label"]]=True
|
||||
# else:
|
||||
# method["hasData"][tfrom["label"]]=False
|
||||
|
||||
DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]}
|
||||
params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())["result"][0]["data"]
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities) > 0:
|
||||
y=0
|
||||
for method in kr.keyRequests:
|
||||
if method["groupId"] == gid:
|
||||
|
||||
found = [x for x in entities if x["dimensions"][0] == method["entityId"]]
|
||||
found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
||||
|
||||
if len(found) > 0:
|
||||
method["hasData"][tfrom["label"]]=True
|
||||
method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None])
|
||||
#method["displayName"]=found[0]["displayName"]
|
||||
#method["entityId"]=found[0]["entityId"]
|
||||
#method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
# for idx,o in enumerate(method["services"]):
|
||||
# tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
|
||||
# if len(tmpS)>0:
|
||||
# method["services"][idx]=tmpS[0]
|
||||
|
||||
else:
|
||||
method["hasData"][tfrom["label"]]=False
|
||||
method["count"][tfrom["label"]]=0
|
||||
method["hasData"][tfrom["label"]]=False
|
||||
|
||||
def resolveServices(self,services, DTAPIURL, DTAPIToken):
|
||||
#DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -106,16 +101,14 @@ class KRParser:
|
|||
for gid, service in enumerate(services):
|
||||
query="type(SERVICE),entityId("+service["id"]+")"
|
||||
|
||||
params=merge(self.config["serviceLookupParams"],{"entitySelector": query})
|
||||
|
||||
params=merge(self.serviceLookupParams,{"entitySelector": query})
|
||||
#params={"entitySelector": query,"from":"now-2y", "fields":"tags"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities)>0:
|
||||
services[gid]=entities[0]
|
||||
|
||||
|
||||
|
||||
|
||||
def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
|
|
@ -124,55 +117,49 @@ class KRParser:
|
|||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
tmp_KR=[]
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
#try:
|
||||
for k in group["methods"]:
|
||||
|
||||
|
||||
#tmp_kr={"displayName":None, "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}
|
||||
|
||||
for gid, k in enumerate(kr.keyRequests):
|
||||
try:
|
||||
query="type(service_method)"
|
||||
group=kr.matchedGroups[k["groupId"]]
|
||||
|
||||
if len(group["services"])> 0:
|
||||
if group["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
|
||||
if k["comparer"]=="entityId":
|
||||
query+=",entityId("+k["entityId"]+")"
|
||||
else:
|
||||
query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
|
||||
query="type(service_method)"
|
||||
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
# if len(entities) > 1:
|
||||
# kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
# print("Multiple keyrequest found: ")
|
||||
|
||||
if len(entities)> 0:
|
||||
kr.keyRequests[gid]["found"]=True
|
||||
kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
||||
kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
||||
|
||||
if len(group["services"])> 0:
|
||||
|
||||
if group["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
||||
kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
||||
self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
if k.startswith("SERVICE_METHOD-"):
|
||||
query+=",entityId("+k+")"
|
||||
else:
|
||||
query+=",entityName.in(\""+k+"\")"
|
||||
|
||||
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities)> 0:
|
||||
for ent in entities:
|
||||
|
||||
tmp_kr=merge({"displayName":ent["displayName"], "entityId":ent["entityId"], "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"] }, self.config["extendResultObjects"])
|
||||
|
||||
if "isServiceMethodOfService" in ent["fromRelationships"]:
|
||||
tmp_kr["services"]=ent["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
if options and KROption.RESOLVESERVICES in options and len(tmp_kr["services"])>0:
|
||||
self.resolveServices(tmp_kr["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
tmp_KR.append(tmp_kr)
|
||||
|
||||
else:
|
||||
if k.startswith('SERVICE_METHOD-'):
|
||||
tmp_kr=merge({"displayName": None,"comparer": "entityId", "entityId":k, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
else:
|
||||
tmp_kr=merge({"displayName":k,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
|
||||
tmp_KR.append(tmp_kr)
|
||||
|
||||
|
||||
return tmp_KR
|
||||
except Exception as err:
|
||||
kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
|
||||
|
||||
|
||||
#kr.mergeServices(entities)
|
||||
|
||||
|
||||
def getKeyRequestsByServices(self, services):
|
||||
|
|
@ -189,7 +176,7 @@ class KRParser:
|
|||
else:
|
||||
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
|
||||
|
||||
params={"entitySelector": query,"from":"now-1y"}
|
||||
params={"entitySelector": query}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
|
|
@ -204,39 +191,53 @@ class KRParser:
|
|||
tmp_methods=self.getKeyRequestsByServices(group["services"])
|
||||
|
||||
for m in tmp_methods:
|
||||
group["methods"].append(m["entityId"])
|
||||
|
||||
# tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
# kr.keyRequests.append(tmp)
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
# for method in group["methods"]:
|
||||
# if method.startswith('SERVICE_METHOD-'):
|
||||
# tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
# else:
|
||||
# tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
for method in group["methods"]:
|
||||
if method.startswith('SERVICE_METHOD-'):
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
else:
|
||||
tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
# kr.keyRequests.append(tmp)
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
|
||||
|
||||
|
||||
# for service in group["services"]:
|
||||
# if service.startswith('SERVICE-'):
|
||||
# tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
# else:
|
||||
# tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
# kr.services.append(tmp)
|
||||
|
||||
kr.keyRequests=self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if self.options:
|
||||
if KROption.RESOLVEKEYREQUETS in self.options:
|
||||
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if KROption.VALIDATE_HASDATA in self.options:
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
||||
# elif KROption.RESOLVEKEYREQUETS in self.options:
|
||||
# self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
# if KROption.RESOLVESERVICES in self.options:
|
||||
# self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
|
||||
|
||||
return kr
|
||||
|
||||
|
||||
def parseBySLO(self,row):
|
||||
def parseBySLO(self,index,row):
|
||||
#normalize
|
||||
print(index)
|
||||
|
||||
try:
|
||||
normFilter=self.normalize(row['filter'])
|
||||
normExpresseion=self.normalize(row['metricExpression'])
|
||||
|
||||
tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"]))
|
||||
tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
|
||||
|
||||
|
||||
#SLO with Filter
|
||||
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
|
||||
|
|
@ -247,47 +248,57 @@ class KRParser:
|
|||
groups=self.applyPatterns(subject)
|
||||
|
||||
tmp_KR.matchedGroups.append(groups)
|
||||
|
||||
# for g in groups:
|
||||
# #if g["methods"] != None and len(g["methods"]) > 0:
|
||||
# tmp_KR.matchedGroups.append(g)
|
||||
|
||||
#self.process(tmp_KR)
|
||||
kr=self.process(tmp_KR)
|
||||
|
||||
with self.lock:
|
||||
self.krs.append(kr)
|
||||
self.pbar.update()
|
||||
|
||||
except Exception as err:
|
||||
print(repr(err))
|
||||
#return self.process(tmp_KR)
|
||||
|
||||
def parse(self, input):
|
||||
def parseBySLO_Threaded(self, slosF):
|
||||
self.krs=[]
|
||||
#i=1
|
||||
# threads = list()
|
||||
# for index, row in slosF.iterrows():
|
||||
# logging.info("Main : create and start thread %d.", index)
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
# #krs.append(krp.parseBySLO(row))
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor:
|
||||
|
||||
if type(input) == pd.DataFrame:
|
||||
self.pbar = tqdm(total=input["id"].count(),desc=self.name)
|
||||
|
||||
for index, row in input.iterrows():
|
||||
executor.submit(self.parseBySLO, row)
|
||||
|
||||
elif type(input)== list:
|
||||
self.pbar = tqdm(total=len(input), desc=self.name)
|
||||
for slo in input:
|
||||
executor.submit(self.parseBySLO, slo)
|
||||
|
||||
elif type(input) == dict:
|
||||
#self.pbar = tqdm(total=1, desc=self.name)
|
||||
#executor.submit(self.parseBySLO, slo)
|
||||
self.parseBySLO(input)
|
||||
|
||||
|
||||
return self.krs
|
||||
# for index, thread in enumerate(threads):
|
||||
# logging.info("Main : before joining thread %d.", index)
|
||||
# thread.join()
|
||||
# logging.info("Main : thread %d done", index)
|
||||
|
||||
# #resultSlos.extend(krs)
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(10) as executor:
|
||||
for index, row in slosF.iterrows():
|
||||
# if i % 25 == 0:
|
||||
# time.sleep(0)
|
||||
#args={index:index, }
|
||||
executor.submit(self.parseBySLO, index,row)
|
||||
# print(str(i)+"\n")
|
||||
# i=i+1
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
return self.krs
|
||||
|
||||
|
||||
def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ):
|
||||
self.name=name
|
||||
def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ):
|
||||
self.DTAPIURL= DTAPIURL
|
||||
self.DTAPIToken=DTAPIToken
|
||||
self.options=options
|
||||
self.config=merge({"threads": 3,
|
||||
"serviceLookupParams":{"from":"now-2y"},
|
||||
"extendResultObjects":{}}, config)
|
||||
self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams)
|
||||
self.krs=[]
|
||||
|
|
@ -22,7 +22,7 @@ class Pattern1:
|
|||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":services, "methods":methods})
|
||||
#return services, methods
|
||||
return groups
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ class Pattern2:
|
|||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")]
|
||||
groups.append({"services":services, "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":services, "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ class Pattern3:
|
|||
if result:
|
||||
for r in result:
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":[], "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":[], "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -63,16 +63,15 @@ class Pattern4:
|
|||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service\"?\),[entityName|entityId]*[\.]*[equals|in|contains]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
#if not r:
|
||||
if not r:
|
||||
#methods=[s.strip() for s in r.split(",")]
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":services, "methods":[], "_mInSloDef":False})
|
||||
groups.append({"services":services, "methods":[]})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -82,16 +81,16 @@ class Pattern5:
|
|||
|
||||
#Endoce
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#services=[]
|
||||
#methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
if not r:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
#methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":[], "_mInSloDef":False})
|
||||
groups.append({"services":services, "methods":[]})
|
||||
#return services, methods
|
||||
return groups
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
Metadata-Version: 2.1
|
||||
Name: KeyRequestParser
|
||||
Version: 0.5
|
||||
Version: 0.4
|
||||
License: MIT
|
||||
|
||||
Parses Keyrequests
|
||||
|
|
|
|||
|
|
@ -7,5 +7,4 @@ KRParser/patterns.py
|
|||
KeyRequestParser.egg-info/PKG-INFO
|
||||
KeyRequestParser.egg-info/SOURCES.txt
|
||||
KeyRequestParser.egg-info/dependency_links.txt
|
||||
KeyRequestParser.egg-info/requires.txt
|
||||
KeyRequestParser.egg-info/top_level.txt
|
||||
|
|
@ -58,24 +58,20 @@ class KeyRequestGroup(MutableSequence):
|
|||
query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
else:
|
||||
query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
# query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)"
|
||||
# val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
# val['methods'] = list(map(lambda x: x.replace("~","") , val['methods']))
|
||||
# if len(val["services"]) > 0:
|
||||
# if val["services"][0].startswith("SERVICE-"):
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
|
||||
# else:
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
|
||||
|
||||
# if len(val["methods"]) > 0:
|
||||
# if val["methods"][0].startswith("SERVICE_METHOD-"):
|
||||
# query+=",entityId(\""+'","'.join(val["methods"])+"\")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
|
||||
|
||||
# query+="\")))"
|
||||
|
||||
val["existsQuery"]= query
|
||||
|
||||
|
||||
# def createServiceResolveQuery(self, val):
|
||||
# query="type(SERVICE)"
|
||||
# val['services'] = list(map(lambda x: x.replace("~","") , val['services']))
|
||||
|
||||
# if len(val["services"]) > 0:
|
||||
# if val["services"][0].startswith("SERVICE-"):
|
||||
# query+=",entityId(\""+'","'.join(val["services"])+"\")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+'","'.join(val["services"])+"\")"
|
||||
|
||||
# val["resolveServiceQuery"]= query
|
||||
|
||||
|
||||
def insert(self, ii, val):
|
||||
|
|
@ -131,6 +127,8 @@ class KR:
|
|||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
|
||||
|
||||
def checkKeyRequestsHasData(self):
|
||||
|
|
@ -142,12 +140,6 @@ class KR:
|
|||
for s in listServices:
|
||||
if s["entityId"] not in listOfServiceIds:
|
||||
self.services.append(s)
|
||||
|
||||
# def addKeyRequest(self, data):
|
||||
# if type(data) == list:
|
||||
# for m in data:
|
||||
# self.keyRequests.append({"displayName":m["["displayName"]"], "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""})
|
||||
|
||||
|
||||
def __init__(self,
|
||||
metadata,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import re
|
||||
|
||||
from KRParser import patterns, keyrequests, helper
|
||||
|
||||
|
||||
from enum import Flag, auto
|
||||
import logging
|
||||
import threading
|
||||
import concurrent.futures
|
||||
import time
|
||||
from jsonmerge import merge
|
||||
import pandas as pd
|
||||
from tqdm import *
|
||||
|
||||
|
||||
|
||||
|
|
@ -17,15 +19,24 @@ class KROption(Flag):
|
|||
RESOLVESERVICES = auto()
|
||||
|
||||
class KRParser:
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4(), patterns.Pattern5()]
|
||||
#threadLimiter = threading.BoundedSemaphore(3)
|
||||
patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ]
|
||||
lock = threading.Lock()
|
||||
|
||||
def normalize(self,x):
|
||||
#tmp=x.replace("~","")
|
||||
tmp=x.replace("\n","")
|
||||
#tmp=tmp.replace("\"/","\"")
|
||||
#tmp=tmp.replace("\"/","") -_>was active
|
||||
#tmp=tmp.replace("/\"","\"")
|
||||
tmp=tmp.replace("/\"","")
|
||||
tmp=tmp.replace("\"","")
|
||||
tmp=tmp.replace("\t","")
|
||||
|
||||
tmp=re.sub("([\s]*)\)", ")", tmp)
|
||||
|
||||
|
||||
|
||||
tmp=re.sub("\([\s\n\r]*", "(", tmp)
|
||||
tmp=re.sub("\,[\s\n\r]*", ",", tmp)
|
||||
tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
|
||||
|
|
@ -39,9 +50,7 @@ class KRParser:
|
|||
for p in self.patterns:
|
||||
groups=p.parseServicesAndMethods(subject)
|
||||
|
||||
if len(groups) > 0:
|
||||
for g in groups:
|
||||
g["pattern"]=str(p.__class__)
|
||||
if len(groups) > 0:
|
||||
break
|
||||
|
||||
return groups
|
||||
|
|
@ -49,54 +58,40 @@ class KRParser:
|
|||
|
||||
def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken):
|
||||
|
||||
# DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
# headers = {
|
||||
# 'Content-Type': 'application/json',
|
||||
# 'Authorization': 'Api-Token ' + DTAPIToken
|
||||
# }
|
||||
|
||||
# for gid, group in enumerate(kr.matchedGroups):
|
||||
# params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
||||
# response = helper.get_request(DTAPIURL, headers, params)
|
||||
# entities = (response.json())['entities']
|
||||
|
||||
# if len(entities) > 0:
|
||||
# y=0
|
||||
# for method in kr.keyRequests:
|
||||
# if method["groupId"] == gid:
|
||||
# found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
||||
|
||||
# if len(found) > 0:
|
||||
# method["hasData"][tfrom["label"]]=True
|
||||
# else:
|
||||
# method["hasData"][tfrom["label"]]=False
|
||||
|
||||
DTAPIURL = DTAPIURL + "/api/v2/metrics/query"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]}
|
||||
params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())["result"][0]["data"]
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities) > 0:
|
||||
y=0
|
||||
for method in kr.keyRequests:
|
||||
if method["groupId"] == gid:
|
||||
|
||||
found = [x for x in entities if x["dimensions"][0] == method["entityId"]]
|
||||
found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
|
||||
|
||||
if len(found) > 0:
|
||||
method["hasData"][tfrom["label"]]=True
|
||||
method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None])
|
||||
#method["displayName"]=found[0]["displayName"]
|
||||
#method["entityId"]=found[0]["entityId"]
|
||||
#method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
# for idx,o in enumerate(method["services"]):
|
||||
# tmpS=[p for p in kr.services if p["entityId"]==o["id"]]
|
||||
# if len(tmpS)>0:
|
||||
# method["services"][idx]=tmpS[0]
|
||||
|
||||
else:
|
||||
method["hasData"][tfrom["label"]]=False
|
||||
method["count"][tfrom["label"]]=0
|
||||
method["hasData"][tfrom["label"]]=False
|
||||
|
||||
def resolveServices(self,services, DTAPIURL, DTAPIToken):
|
||||
#DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
|
|
@ -106,16 +101,14 @@ class KRParser:
|
|||
for gid, service in enumerate(services):
|
||||
query="type(SERVICE),entityId("+service["id"]+")"
|
||||
|
||||
params=merge(self.config["serviceLookupParams"],{"entitySelector": query})
|
||||
|
||||
params=merge(self.serviceLookupParams,{"entitySelector": query})
|
||||
#params={"entitySelector": query,"from":"now-2y", "fields":"tags"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities)>0:
|
||||
services[gid]=entities[0]
|
||||
|
||||
|
||||
|
||||
|
||||
def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
|
||||
DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
|
|
@ -124,148 +117,49 @@ class KRParser:
|
|||
'Authorization': 'Api-Token ' + DTAPIToken
|
||||
}
|
||||
|
||||
tmp_KR=[]
|
||||
for gid, group in enumerate(kr.matchedGroups):
|
||||
#try:
|
||||
for k in group["methods"]:
|
||||
|
||||
|
||||
#tmp_kr={"displayName":None, "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}
|
||||
|
||||
query="type(service_method)"
|
||||
|
||||
if len(group["services"])> 0:
|
||||
|
||||
if group["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
|
||||
if k.startswith("SERVICE_METHOD-"):
|
||||
query+=",entityId("+k+")"
|
||||
else:
|
||||
query+=",entityName.in(\""+k+"\")"
|
||||
|
||||
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
if len(entities)> 0:
|
||||
for ent in entities:
|
||||
|
||||
tmp_kr=merge({"displayName":ent["displayName"], "entityId":ent["entityId"], "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"] }, self.config["extendResultObjects"])
|
||||
|
||||
if "isServiceMethodOfService" in ent["fromRelationships"]:
|
||||
tmp_kr["services"]=ent["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
if options and KROption.RESOLVESERVICES in options and len(tmp_kr["services"])>0:
|
||||
self.resolveServices(tmp_kr["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
tmp_KR.append(tmp_kr)
|
||||
|
||||
else:
|
||||
if k.startswith('SERVICE_METHOD-'):
|
||||
tmp_kr=merge({"displayName": None,"comparer": "entityId", "entityId":k, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
else:
|
||||
tmp_kr=merge({"displayName":k,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":"", "_mInSloDef": group["_mInSloDef"]}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
|
||||
tmp_KR.append(tmp_kr)
|
||||
|
||||
|
||||
return tmp_KR
|
||||
# except Exception as err:
|
||||
# kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
|
||||
# for gid, k in enumerate(kr.keyRequests):
|
||||
# try:
|
||||
# query="type(service_method)"
|
||||
# group=kr.matchedGroups[k["groupId"]]
|
||||
|
||||
for gid, k in enumerate(kr.keyRequests):
|
||||
try:
|
||||
query="type(service_method)"
|
||||
group=kr.matchedGroups[k["groupId"]]
|
||||
|
||||
# if len(group["services"])> 0:
|
||||
# if group["services"][0].startswith("SERVICE-"):
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
# else:
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
if len(group["services"])> 0:
|
||||
if group["services"][0].startswith("SERVICE-"):
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
else:
|
||||
query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
|
||||
# if k["comparer"]=="entityId":
|
||||
# query+=",entityId("+k["entityId"]+")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
if k["comparer"]=="entityId":
|
||||
query+=",entityId("+k["entityId"]+")"
|
||||
else:
|
||||
query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
|
||||
# params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
# response = helper.get_request(DTAPIURL, headers, params)
|
||||
# entities = (response.json())['entities']
|
||||
params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
# if len(entities) > 1:
|
||||
# kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
# print("Multiple keyrequest found: ")
|
||||
|
||||
# if len(entities)> 0:
|
||||
if len(entities)> 0:
|
||||
kr.keyRequests[gid]["found"]=True
|
||||
kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
||||
kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
||||
|
||||
# kr.keyRequests[gid]["found"]=True
|
||||
# kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
# kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
||||
# kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
||||
|
||||
# if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
||||
# kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
||||
kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
# if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
||||
# self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
||||
if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
||||
self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
# except Exception as err:
|
||||
# kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
except Exception as err:
|
||||
kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
|
||||
|
||||
|
||||
#kr.mergeServices(entities)
|
||||
|
||||
# def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options):
|
||||
# DTAPIURL = DTAPIURL + "/api/v2/entities"
|
||||
|
||||
# headers = {
|
||||
# 'Content-Type': 'application/json',
|
||||
# 'Authorization': 'Api-Token ' + DTAPIToken
|
||||
# }
|
||||
|
||||
|
||||
# for gid, k in enumerate(kr.keyRequests):
|
||||
# try:
|
||||
# query="type(service_method)"
|
||||
# group=kr.matchedGroups[k["groupId"]]
|
||||
|
||||
# if len(group["services"])> 0:
|
||||
# if group["services"][0].startswith("SERVICE-"):
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))"
|
||||
# else:
|
||||
# query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))"
|
||||
|
||||
# if k["comparer"]=="entityId":
|
||||
# query+=",entityId("+k["entityId"]+")"
|
||||
# else:
|
||||
# query+=",entityName.in(\""+k["displayName"]+"\")"
|
||||
|
||||
# params={"entitySelector": query, "from":"now-1y","fields": "fromRelationships"}
|
||||
# response = helper.get_request(DTAPIURL, headers, params)
|
||||
# entities = (response.json())['entities']
|
||||
|
||||
|
||||
# if len(entities)> 0:
|
||||
|
||||
# kr.keyRequests[gid]["found"]=True
|
||||
# kr.keyRequests[gid]['foundCount']=len(entities)
|
||||
# kr.keyRequests[gid]["displayName"]=entities[0]["displayName"]
|
||||
# kr.keyRequests[gid]["entityId"]=entities[0]["entityId"]
|
||||
|
||||
# if "isServiceMethodOfService" in entities[0]["fromRelationships"]:
|
||||
# kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"]
|
||||
|
||||
# if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0:
|
||||
# self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken)
|
||||
|
||||
# except Exception as err:
|
||||
# kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err)
|
||||
|
||||
|
||||
|
||||
# #kr.mergeServices(entities)
|
||||
#kr.mergeServices(entities)
|
||||
|
||||
|
||||
def getKeyRequestsByServices(self, services):
|
||||
|
|
@ -282,7 +176,7 @@ class KRParser:
|
|||
else:
|
||||
query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\"))"
|
||||
|
||||
params={"entitySelector": query,"from":"now-1y"}
|
||||
params={"entitySelector": query}
|
||||
response = helper.get_request(DTAPIURL, headers, params)
|
||||
entities = (response.json())['entities']
|
||||
|
||||
|
|
@ -297,39 +191,53 @@ class KRParser:
|
|||
tmp_methods=self.getKeyRequestsByServices(group["services"])
|
||||
|
||||
for m in tmp_methods:
|
||||
group["methods"].append(m["entityId"])
|
||||
|
||||
# tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
# kr.keyRequests.append(tmp)
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
# for method in group["methods"]:
|
||||
# if method.startswith('SERVICE_METHOD-'):
|
||||
# tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
# else:
|
||||
# tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None,
|
||||
for method in group["methods"]:
|
||||
if method.startswith('SERVICE_METHOD-'):
|
||||
tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
else:
|
||||
tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
# kr.keyRequests.append(tmp)
|
||||
kr.keyRequests.append(tmp)
|
||||
|
||||
|
||||
|
||||
|
||||
# for service in group["services"]:
|
||||
# if service.startswith('SERVICE-'):
|
||||
# tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
# else:
|
||||
# tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None,
|
||||
|
||||
# kr.services.append(tmp)
|
||||
|
||||
kr.keyRequests=self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if self.options:
|
||||
if KROption.RESOLVEKEYREQUETS in self.options:
|
||||
self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options)
|
||||
|
||||
if KROption.VALIDATE_HASDATA in self.options:
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken)
|
||||
# elif KROption.RESOLVEKEYREQUETS in self.options:
|
||||
# self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken)
|
||||
# if KROption.RESOLVESERVICES in self.options:
|
||||
# self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken)
|
||||
|
||||
return kr
|
||||
|
||||
|
||||
def parseBySLO(self,row):
|
||||
def parseBySLO(self,index,row):
|
||||
#normalize
|
||||
print(index)
|
||||
|
||||
try:
|
||||
normFilter=self.normalize(row['filter'])
|
||||
normExpresseion=self.normalize(row['metricExpression'])
|
||||
|
||||
tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"]))
|
||||
tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
|
||||
|
||||
|
||||
#SLO with Filter
|
||||
if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"):
|
||||
|
|
@ -340,47 +248,57 @@ class KRParser:
|
|||
groups=self.applyPatterns(subject)
|
||||
|
||||
tmp_KR.matchedGroups.append(groups)
|
||||
|
||||
# for g in groups:
|
||||
# #if g["methods"] != None and len(g["methods"]) > 0:
|
||||
# tmp_KR.matchedGroups.append(g)
|
||||
|
||||
#self.process(tmp_KR)
|
||||
kr=self.process(tmp_KR)
|
||||
|
||||
with self.lock:
|
||||
self.krs.append(kr)
|
||||
self.pbar.update()
|
||||
|
||||
except Exception as err:
|
||||
print(repr(err))
|
||||
#return self.process(tmp_KR)
|
||||
|
||||
def parse(self, input):
|
||||
def parseBySLO_Threaded(self, slosF):
|
||||
self.krs=[]
|
||||
#i=1
|
||||
# threads = list()
|
||||
# for index, row in slosF.iterrows():
|
||||
# logging.info("Main : create and start thread %d.", index)
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
# #krs.append(krp.parseBySLO(row))
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor:
|
||||
|
||||
if type(input) == pd.DataFrame:
|
||||
self.pbar = tqdm(total=input["id"].count(),desc=self.name)
|
||||
|
||||
for index, row in input.iterrows():
|
||||
executor.submit(self.parseBySLO, row)
|
||||
|
||||
elif type(input)== list:
|
||||
self.pbar = tqdm(total=len(input), desc=self.name)
|
||||
for slo in input:
|
||||
executor.submit(self.parseBySLO, slo)
|
||||
|
||||
elif type(input) == dict:
|
||||
#self.pbar = tqdm(total=1, desc=self.name)
|
||||
#executor.submit(self.parseBySLO, slo)
|
||||
self.parseBySLO(input)
|
||||
|
||||
|
||||
return self.krs
|
||||
# for index, thread in enumerate(threads):
|
||||
# logging.info("Main : before joining thread %d.", index)
|
||||
# thread.join()
|
||||
# logging.info("Main : thread %d done", index)
|
||||
|
||||
# #resultSlos.extend(krs)
|
||||
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(10) as executor:
|
||||
for index, row in slosF.iterrows():
|
||||
# if i % 25 == 0:
|
||||
# time.sleep(0)
|
||||
#args={index:index, }
|
||||
executor.submit(self.parseBySLO, index,row)
|
||||
# print(str(i)+"\n")
|
||||
# i=i+1
|
||||
# x = threading.Thread(target=self.parseBySLO, args=(row,))
|
||||
# threads.append(x)
|
||||
# x.start()
|
||||
|
||||
return self.krs
|
||||
|
||||
|
||||
def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ):
|
||||
self.name=name
|
||||
def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ):
|
||||
self.DTAPIURL= DTAPIURL
|
||||
self.DTAPIToken=DTAPIToken
|
||||
self.options=options
|
||||
self.config=merge({"threads": 3,
|
||||
"serviceLookupParams":{"from":"now-2y"},
|
||||
"extendResultObjects":{}}, config)
|
||||
self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams)
|
||||
self.krs=[]
|
||||
|
|
@ -22,7 +22,7 @@ class Pattern1:
|
|||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":services, "methods":methods})
|
||||
#return services, methods
|
||||
return groups
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ class Pattern2:
|
|||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")]
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")]
|
||||
groups.append({"services":services, "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":services, "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ class Pattern3:
|
|||
if result:
|
||||
for r in result:
|
||||
methods=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":[], "methods":methods, "_mInSloDef":True})
|
||||
groups.append({"services":[], "methods":methods})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -63,16 +63,15 @@ class Pattern4:
|
|||
|
||||
def parseServicesAndMethods(self, metricExpression):
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service\"?\),[entityName|entityId]*[\.]*[equals|in|contains]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
result = re.findall(r"type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*[entityName|entityId].*[equals|in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
#if not r:
|
||||
if not r:
|
||||
#methods=[s.strip() for s in r.split(",")]
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
groups.append({"services":services, "methods":[], "_mInSloDef":False})
|
||||
groups.append({"services":services, "methods":[]})
|
||||
|
||||
return groups
|
||||
|
||||
|
|
@ -82,16 +81,16 @@ class Pattern5:
|
|||
|
||||
#Endoce
|
||||
metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)
|
||||
result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
|
||||
#services=[]
|
||||
#methods=[]
|
||||
groups=[]
|
||||
if result:
|
||||
for r in result:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
if not r:
|
||||
services=[s.strip() for s in urllib.parse.unquote_plus(r).split(",")]
|
||||
#methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")]
|
||||
groups.append({"services":services, "methods":[], "_mInSloDef":False})
|
||||
groups.append({"services":services, "methods":[]})
|
||||
#return services, methods
|
||||
return groups
|
||||
Binary file not shown.
7
setup.py
7
setup.py
|
|
@ -1,13 +1,8 @@
|
|||
from setuptools import setup, find_packages
|
||||
setup(
|
||||
name='KeyRequestParser',
|
||||
version='0.6',
|
||||
version='0.4',
|
||||
packages=find_packages(include=["KRParser"]),
|
||||
license='MIT',
|
||||
long_description="Parses Keyrequests",
|
||||
install_requires=[
|
||||
'requests',
|
||||
'jsonmerge',
|
||||
'tqdm'
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
---
|
||||
euprod:
|
||||
- name: "EUprod"
|
||||
- env-url: "https://xxu26128.live.dynatrace.com"
|
||||
- env-token-name: "EUPROD_TOKEN_VAR"
|
||||
- jenkins: "https://jaws.bmwgroup.net/opapm/"
|
||||
naprod:
|
||||
- name: "naprod"
|
||||
- env-url: "https://wgv50241.live.dynatrace.com"
|
||||
- env-token-name: "NAPROD_TOKEN_VAR"
|
||||
- jenkins: "https://jaws.bmwgroup.net/opapm/"
|
||||
cnprod:
|
||||
- name: "cnprod"
|
||||
- env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
|
||||
- env-token-name: "CNPROD_TOKEN_VAR"
|
||||
- jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
|
||||
# #cnpreprod:
|
||||
# - name: "cnpreprod"
|
||||
# - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
|
||||
# - env-token-name: "CNPREPROD_TOKEN_VAR"
|
||||
# - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
pyyaml
|
||||
python-decouple
|
||||
requests
|
||||
jsonmerge
|
||||
pandas
|
||||
tqdm
|
||||
|
|
@ -1,210 +0,0 @@
|
|||
import sys
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/..")
|
||||
#sys.path.append('..')
|
||||
|
||||
import unittest
|
||||
import yaml
|
||||
#from helper import get_request
|
||||
from KRParser.helper import get_request
|
||||
from KRParser.krparser import KRParser, KROption
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
#from KRParser import helper, krparser
|
||||
from decouple import config
|
||||
|
||||
|
||||
|
||||
class patterns(unittest.TestCase):
|
||||
|
||||
|
||||
def setUp(self):
|
||||
with open('./tests/environment.yaml') as file:
|
||||
self.env_doc = yaml.safe_load(file)
|
||||
|
||||
# TP_* SLO definition (Pattern1)
|
||||
# Used SLO: TP_Mobile_Login
|
||||
# https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=2a07c9bf-2c7f-37eb-815b-efda7658bc13
|
||||
def test_TP_Pattern1(self):
|
||||
|
||||
expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
|
||||
expected_methods=['GET /api/v1/presentation/oauth/config', 'GET /api/v1/presentation/profile-tab', 'POST /api/v1/oauth/token/identifier']
|
||||
expected_pattern="<class 'KRParser.patterns.Pattern1'>"
|
||||
DTURL=self.env_doc['euprod'][1]["env-url"]
|
||||
DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
|
||||
|
||||
api_url = DTURL+"/api/v2/slo/2a07c9bf-2c7f-37eb-815b-efda7658bc13"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTTOKEN
|
||||
}
|
||||
|
||||
|
||||
slo_result=get_request(api_url, headers, {})
|
||||
slo=slo_result.json()
|
||||
|
||||
krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
result= krp.parse(slo)
|
||||
|
||||
self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
|
||||
# TP_* SLO definition (Pattern4)
|
||||
# Used SLO: TP_Vehicle_Teleservices
|
||||
# https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=18c9bc95-5605-33b4-81e1-10335cf0191b
|
||||
def test_TP_Pattern4(self):
|
||||
|
||||
expected_services=['ifs-001 (Kinesis) CCM - teleservices - prod', 'ifs-001 (Kinesis) TyreWsIncoming - IN - teleservices - prod', 'ifs-001 (Kinesis) TyreDiIncoming - IN - teleservices - prod', 'ifs-003 (JOYNR) triggerPdmContainerUpdate/notifyPdmContainerStatus - IN/OUT - teleservices - prod', 'ifs-003 (JOYNR) triggerWhitelistUpdate/notifyWhitelistStatus - IN/OUT - wlupdater - prod', 'ifs-003 (JOYNR) triggerStandardOdfUpdate/notifyOdfStatus - IN/OUT - wlupdater - prod', 'ifs-003 (JOYNR) start/stop/notifyTemporaryOdfStatus - INT/OUT - wlupdater - prod', 'ifs-007 (JOYNR) requestPdmResult/notifyPdmResult - IN/OUT - teleservices - prod', 'ifs-002 (JOYNR) request/notifyKeyData - IN/OUT - teleservices - prod', 'ifs-002 (JOYNR) notifyRSUStatus - IN - teleservices - prod', 'VDLM_scallservice - VDLM_scallservice - ifs-002 (JOYNR) notifyMaintenanceStatus - IN - prod', 'VDLM_scallservice - VDLM_scallservice - ifs-002 (JOYNR) requestMaintenanceStatus/confirmMaintenanceStatus - OUT - prod', 'ifs-005 (JOYNR) notify/confirmBatteryStatus - IN/OUT - teleservices - prod', 'ifs-007 (JOYNR) requestDiagResult/notifyDiagResult - IN/OUT - teleservices - prod', 'ifs-005 (SQS) TscsCallTopic - IN - teleservices - prod', 'ifs-006 (SQS) TsddCallTopic - IN - teleservices - prod', 'ifs-010 (SQS) RtcadCallTopic - IN - teleservices - prod', 'VDLM_seam - ifs-004 (SQS) ServiceDemand - IN - teleservices - prod', 'VDLM_seam - ifs-004 (SQS) ServiceDemand - IN - teleservices - prod', 'ifs-003 (MQS) MguItsnIn/Out (ClassificationRequest/Response) - IN/OUT - teleservices - prod', 'ifs-003 (MQS) MguItsnIn/Out (DiagnosisRequest/Response) - IN/OUT - teleservices - prod', 'ifs-005 (MQS) BCallHandlerIstaIn/Out - IN/OUT - teleservices - prod', 'ifs-001 (SQS) BCallIn/Out - IN/OUT - teleservices - prod', 'VDLM_ts-asc - ifs-001 (SQS) AscIn/Out - IN/OUT - prod', 'InternalApiResourceV2 - teleservices - prod', 'VDLM_rtchandler - ifs-010 (SQS) RtcTeleXIn/Out - IN/OUT - prod', 'ckf.bmwgroup.net']
|
||||
expected_methods=['SERVICE_METHOD-0040C1B4C7A410B7', 'SERVICE_METHOD-116706FE1C0B57E9', 'SERVICE_METHOD-49981ADAE87D95D2', 'SERVICE_METHOD-5EB17880C0917DBF', 'SERVICE_METHOD-8226D6EB97776024', 'SERVICE_METHOD-90B715A55C1811EB', 'SERVICE_METHOD-AC1A7EF39680111E', 'SERVICE_METHOD-AEA1756029C53700', 'SERVICE_METHOD-E078C045186CDDA1', 'SERVICE_METHOD-FE7B9FE4485C6B29']
|
||||
expected_pattern="<class 'KRParser.patterns.Pattern4'>"
|
||||
DTURL=self.env_doc['euprod'][1]["env-url"]
|
||||
DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
|
||||
|
||||
api_url = DTURL+"/api/v2/slo/18c9bc95-5605-33b4-81e1-10335cf0191b"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTTOKEN
|
||||
}
|
||||
|
||||
|
||||
slo_result=get_request(api_url, headers, {})
|
||||
slo=slo_result.json()
|
||||
|
||||
krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
result= krp.parse(slo)
|
||||
|
||||
self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
|
||||
# Simplieifed_* SLO definition (Pattern5)
|
||||
# Used SLO: Simplified_TP_Mobile_Login
|
||||
# https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.slo/ui/slo?gtf=-2h&gf=all&filter=Text%3ATP&slovis=Graph&sloexp=5cc3efd1-fbb9-3504-8075-3cf0e4a2a4fa
|
||||
def test_Simplified(self):
|
||||
|
||||
expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
|
||||
expected_methods=['SERVICE_METHOD-0A047D679E974CF0','SERVICE_METHOD-1313A2ED47A33278','SERVICE_METHOD-2D3518701AFE612D','SERVICE_METHOD-45FDB365004EF766','SERVICE_METHOD-51B637F95F50EBE8','SERVICE_METHOD-5A5640BFFC3692D2','SERVICE_METHOD-5B50C30ADC9F8A8B','SERVICE_METHOD-63B88F2A195A44F1','SERVICE_METHOD-6858B9373C1B3C9A','SERVICE_METHOD-72BA0DBBCC1A9AED','SERVICE_METHOD-7B4FDDCAB657C7B3','SERVICE_METHOD-7E6EFAB2B32CBF87','SERVICE_METHOD-83EC6721A281B6F6','SERVICE_METHOD-91B6FB318FC612CD','SERVICE_METHOD-946D89B62E2559C0','SERVICE_METHOD-AD048A48A9FB4B84','SERVICE_METHOD-B20949E0FD947AE5','SERVICE_METHOD-BBC94FF8F8157D42','SERVICE_METHOD-C44DED4B58D56A8E','SERVICE_METHOD-CF980FC5191F17D1','SERVICE_METHOD-E78735CD41F780AC','SERVICE_METHOD-E8483228CE13A2AE','SERVICE_METHOD-E97B7D398DE28B1F','SERVICE_METHOD-F0B28D440F14795A','SERVICE_METHOD-FF77CD9406F022A3']
|
||||
|
||||
|
||||
expected_pattern="<class 'KRParser.patterns.Pattern5'>"
|
||||
|
||||
DTURL=self.env_doc['euprod'][1]["env-url"]
|
||||
DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
|
||||
|
||||
api_url = DTURL+"/api/v2/slo/5cc3efd1-fbb9-3504-8075-3cf0e4a2a4fa" #34bb71cf-30bc-3d2e-913f-389517699751"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTTOKEN
|
||||
}
|
||||
|
||||
|
||||
slo_result=get_request(api_url, headers, {})
|
||||
slo=slo_result.json()
|
||||
|
||||
krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
result= krp.parse(slo)
|
||||
|
||||
self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
|
||||
|
||||
# entityIds by filter SLO definition (Pattern3)
|
||||
# Used SLO: E2E - Wirkkette "Login"
|
||||
# https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.slo/ui/slo?gtf=-2h&gf=all&filter=Text%3ATP&slovis=Graph&sloexp=5fa56652-5f44-31af-892b-0681ff019b8b
|
||||
def test_EntityIdsByFilter(self):
|
||||
|
||||
expected_services=[]
|
||||
expected_methods=['SERVICE_METHOD-088030038E4C48AD', 'SERVICE_METHOD-22DB51030E136454', 'SERVICE_METHOD-93D43FBFAC5DF78F']
|
||||
expected_pattern="<class 'KRParser.patterns.Pattern3'>"
|
||||
|
||||
DTURL=self.env_doc['euprod'][1]["env-url"]
|
||||
DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name'))
|
||||
|
||||
api_url = DTURL+"/api/v2/slo/5fa56652-5f44-31af-892b-0681ff019b8b" #34bb71cf-30bc-3d2e-913f-389517699751"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTTOKEN
|
||||
}
|
||||
|
||||
|
||||
slo_result=get_request(api_url, headers, {})
|
||||
slo=slo_result.json()
|
||||
|
||||
krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
result= krp.parse(slo)
|
||||
|
||||
self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
|
||||
# def test_contains(self):
|
||||
# expected_services=['btc-user-composite-service - PROD', 'btc-connected-oauth-service - PROD']
|
||||
# expected_methods=['SERVICE_METHOD-1313A2ED47A33278', 'SERVICE_METHOD-2D3518701AFE612D', 'SERVICE_METHOD-5A5640BFFC3692D2', 'SERVICE_METHOD-5B50C30ADC9F8A8B', 'SERVICE_METHOD-63B88F2A195A44F1', 'SERVICE_METHOD-6858B9373C1B3C9A', 'SERVICE_METHOD-72BA0DBBCC1A9AED', 'SERVICE_METHOD-7B4FDDCAB657C7B3', 'SERVICE_METHOD-7E6EFAB2B32CBF87', 'SERVICE_METHOD-83EC6721A281B6F6', 'SERVICE_METHOD-91B6FB318FC612CD', 'SERVICE_METHOD-946D89B62E2559C0', 'SERVICE_METHOD-AD048A48A9FB4B84', 'SERVICE_METHOD-B20949E0FD947AE5', 'SERVICE_METHOD-BBC94FF8F8157D42', 'SERVICE_METHOD-C44DED4B58D56A8E', 'SERVICE_METHOD-CF980FC5191F17D1', 'SERVICE_METHOD-E78735CD41F780AC', 'SERVICE_METHOD-E8483228CE13A2AE', 'SERVICE_METHOD-E97B7D398DE28B1F', 'SERVICE_METHOD-F0B28D440F14795A', 'SERVICE_METHOD-FF77CD9406F022A3']
|
||||
# expected_pattern="<class 'KRParser.patterns.Pattern5'>"
|
||||
|
||||
# DTURL=self.env_doc['cnprod'][1]["env-url"]
|
||||
# DTTOKEN = config(self.env_doc['cnprod'][2].get('env-token-name'))
|
||||
|
||||
# api_url = DTURL+"/api/v2/slo/4230f8cb-ffaa-31b3-9e12-091488f330af" #34bb71cf-30bc-3d2e-913f-389517699751"
|
||||
|
||||
# headers = {
|
||||
# 'Content-Type': 'application/json',
|
||||
# 'Authorization': 'Api-Token ' + DTTOKEN
|
||||
# }
|
||||
|
||||
|
||||
# result_slo=get_request(api_url, headers, {})
|
||||
# slo=result_slo.json()
|
||||
|
||||
# krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
# result= krp.parse(slo)
|
||||
|
||||
# self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
# self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
# self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
# TP_* SLO definition (Pattern1)
|
||||
# Used SLO: TP_Mobile_Login
|
||||
# https://xxu26128.apps.dynatrace.com/ui/apps/dynatrace.classic.settings/ui/settings/builtin:monitoring.slo?gtf=-2h&gf=all&id=2a07c9bf-2c7f-37eb-815b-efda7658bc13
|
||||
def test_TP_SmartParking(self):
|
||||
|
||||
expected_services=['ParkingFinderLite.WebApi']
|
||||
expected_methods=['/sppid7/parkingfinder/v1/EstimatedTimeArrival/getETA', '/parkingfinder/v2.0/getParkingLots']
|
||||
expected_pattern="<class 'KRParser.patterns.Pattern1'>"
|
||||
|
||||
DTURL=self.env_doc['cnprod'][1]["env-url"]
|
||||
DTTOKEN = config(self.env_doc['cnprod'][2].get('env-token-name'))
|
||||
|
||||
api_url = DTURL+"/api/v2/slo/91e02599-a243-3201-a87e-91066419e52a"
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Api-Token ' + DTTOKEN
|
||||
}
|
||||
|
||||
|
||||
slo_result=get_request(api_url, headers, {})
|
||||
slo=slo_result.json()
|
||||
|
||||
krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":1,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN)
|
||||
|
||||
result= krp.parse(slo)
|
||||
|
||||
self.assertListEqual(result[0].matchedGroups[0]["services"], expected_services)
|
||||
# self.assertListEqual(result[0].matchedGroups[0]["methods"], expected_methods)
|
||||
# self.assertEqual(result[0].matchedGroups[0]["pattern"], expected_pattern)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue