From 031fc0aa26a2cc8f1d149c3230ce862a6238602f Mon Sep 17 00:00:00 2001 From: ermisw Date: Mon, 8 May 2023 19:06:50 +0200 Subject: [PATCH] added request count & some fixes --- .gitignore | 138 +++++++++++++++++++++ KRParser/keyrequests.py | 30 +++-- KRParser/krparser.py | 169 +++++++++++--------------- KeyRequestParser.egg-info/PKG-INFO | 2 +- KeyRequestParser.egg-info/SOURCES.txt | 1 + build/lib/KRParser/keyrequests.py | 30 +++-- build/lib/KRParser/krparser.py | 169 +++++++++++--------------- dist/KeyRequestParser-0.4-py3.10.egg | Bin 15308 -> 15294 bytes setup.py | 7 +- tests/TestStringMethods.py | 66 ++++++++++ tests/__init__.py | 0 tests/environment.yaml | 21 ++++ tests/requirements.txt | 6 + 13 files changed, 415 insertions(+), 224 deletions(-) create mode 100644 .gitignore create mode 100644 tests/TestStringMethods.py create mode 100644 tests/__init__.py create mode 100644 tests/environment.yaml create mode 100644 tests/requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0738441 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +.vscode +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Terraform stuff +**/.terraform/* +crash.log +*.tfvars + +#excel reports +*.xlsx \ No newline at end of file diff --git a/KRParser/keyrequests.py b/KRParser/keyrequests.py index a877da3..432ba16 100644 --- a/KRParser/keyrequests.py +++ b/KRParser/keyrequests.py @@ -58,20 +58,24 @@ class KeyRequestGroup(MutableSequence): query+=",entityId(\""+'","'.join(val["methods"])+"\")" else: query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" - + # query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)" + # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + # val['methods'] = list(map(lambda x: x.replace("~","") , val['methods'])) + # if len(val["services"]) > 0: + # if val["services"][0].startswith("SERVICE-"): + # query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" + # else: + # query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" + + # if len(val["methods"]) > 0: + # if val["methods"][0].startswith("SERVICE_METHOD-"): + # query+=",entityId(\""+'","'.join(val["methods"])+"\")" + # else: + # query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + + # query+="\")))" val["existsQuery"]= query - - # def createServiceResolveQuery(self, val): - # query="type(SERVICE)" - # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) - - # if len(val["services"]) > 0: - # if val["services"][0].startswith("SERVICE-"): - # query+=",entityId(\""+'","'.join(val["services"])+"\")" - # else: - # query+=",entityName.in(\""+'","'.join(val["services"])+"\")" - - # val["resolveServiceQuery"]= query + def insert(self, ii, val): diff --git a/KRParser/krparser.py b/KRParser/krparser.py index 7305ba0..b634a6d 100644 --- a/KRParser/krparser.py +++ b/KRParser/krparser.py @@ -1,14 +1,12 @@ import re from KRParser import patterns, keyrequests, helper - - from enum import Flag, auto -import logging import threading import concurrent.futures -import time from jsonmerge import merge +import pandas as pd +from tqdm import * @@ -19,24 +17,15 @@ class KROption(Flag): RESOLVESERVICES = auto() class KRParser: - #threadLimiter = threading.BoundedSemaphore(3) patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ] lock = threading.Lock() def normalize(self,x): - #tmp=x.replace("~","") tmp=x.replace("\n","") - #tmp=tmp.replace("\"/","\"") - #tmp=tmp.replace("\"/","") -_>was active - #tmp=tmp.replace("/\"","\"") tmp=tmp.replace("/\"","") tmp=tmp.replace("\"","") tmp=tmp.replace("\t","") - tmp=re.sub("([\s]*)\)", ")", tmp) - - - tmp=re.sub("\([\s\n\r]*", "(", tmp) tmp=re.sub("\,[\s\n\r]*", ",", tmp) tmp=re.sub("\)[\s\n\r]*,", "),", tmp) @@ -58,40 +47,54 @@ class KRParser: def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): - DTAPIURL = DTAPIURL + "/api/v2/entities" + # DTAPIURL = DTAPIURL + "/api/v2/entities" + # headers = { + # 'Content-Type': 'application/json', + # 'Authorization': 'Api-Token ' + DTAPIToken + # } + + # for gid, group in enumerate(kr.matchedGroups): + # params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + # response = helper.get_request(DTAPIURL, headers, params) + # entities = (response.json())['entities'] + + # if len(entities) > 0: + # y=0 + # for method in kr.keyRequests: + # if method["groupId"] == gid: + # found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + # if len(found) > 0: + # method["hasData"][tfrom["label"]]=True + # else: + # method["hasData"][tfrom["label"]]=False + + DTAPIURL = DTAPIURL + "/api/v2/metrics/query" + headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } for gid, group in enumerate(kr.matchedGroups): - params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]} response = helper.get_request(DTAPIURL, headers, params) - entities = (response.json())['entities'] + entities = (response.json())["result"][0]["data"] - if len(entities) > 0: - y=0 for method in kr.keyRequests: if method["groupId"] == gid: - found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + found = [x for x in entities if x["dimensions"][0] == method["entityId"]] if len(found) > 0: method["hasData"][tfrom["label"]]=True - #method["displayName"]=found[0]["displayName"] - #method["entityId"]=found[0]["entityId"] - #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"] - - # for idx,o in enumerate(method["services"]): - # tmpS=[p for p in kr.services if p["entityId"]==o["id"]] - # if len(tmpS)>0: - # method["services"][idx]=tmpS[0] - + method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None]) else: - method["hasData"][tfrom["label"]]=False + method["hasData"][tfrom["label"]]=False + method["count"][tfrom["label"]]=0 def resolveServices(self,services, DTAPIURL, DTAPIToken): - #DTAPIURL = DTAPIURL + "/api/v2/entities" headers = { 'Content-Type': 'application/json', @@ -101,8 +104,8 @@ class KRParser: for gid, service in enumerate(services): query="type(SERVICE),entityId("+service["id"]+")" - params=merge(self.serviceLookupParams,{"entitySelector": query}) - #params={"entitySelector": query,"from":"now-2y", "fields":"tags"} + params=merge(self.config["serviceLookupParams"],{"entitySelector": query}) + response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -138,9 +141,6 @@ class KRParser: response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] - # if len(entities) > 1: - # kr.keyRequests[gid]['foundCount']=len(entities) - # print("Multiple keyrequest found: ") if len(entities)> 0: kr.keyRequests[gid]["found"]=True @@ -151,7 +151,7 @@ class KRParser: if "isServiceMethodOfService" in entities[0]["fromRelationships"]: kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] - if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: + if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) except Exception as err: @@ -191,53 +191,37 @@ class KRParser: tmp_methods=self.getKeyRequestsByServices(group["services"]) for m in tmp_methods: - tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): - tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, else: - tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) - - - # for service in group["services"]: - # if service.startswith('SERVICE-'): - # tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, - # else: - # tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, - - # kr.services.append(tmp) + + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if self.options: - if KROption.RESOLVEKEYREQUETS in self.options: - self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if KROption.VALIDATE_HASDATA in self.options: self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) - # elif KROption.RESOLVEKEYREQUETS in self.options: - # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) - # if KROption.RESOLVESERVICES in self.options: - # self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken) return kr - def parseBySLO(self,index,row): - #normalize - print(index) + def parseBySLO(self,row): try: normFilter=self.normalize(row['filter']) normExpresseion=self.normalize(row['metricExpression']) - tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) - + tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"])) #SLO with Filter if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"): @@ -248,57 +232,46 @@ class KRParser: groups=self.applyPatterns(subject) tmp_KR.matchedGroups.append(groups) - # for g in groups: - # #if g["methods"] != None and len(g["methods"]) > 0: - # tmp_KR.matchedGroups.append(g) - - #self.process(tmp_KR) + kr=self.process(tmp_KR) + with self.lock: self.krs.append(kr) + self.pbar.update() except Exception as err: print(repr(err)) - #return self.process(tmp_KR) - def parseBySLO_Threaded(self, slosF): - self.krs=[] - #i=1 - # threads = list() - # for index, row in slosF.iterrows(): - # logging.info("Main : create and start thread %d.", index) - # x = threading.Thread(target=self.parseBySLO, args=(row,)) - # threads.append(x) - # x.start() - - # #krs.append(krp.parseBySLO(row)) + def parse(self, input): - # for index, thread in enumerate(threads): - # logging.info("Main : before joining thread %d.", index) - # thread.join() - # logging.info("Main : thread %d done", index) - - # #resultSlos.extend(krs) - - - with concurrent.futures.ThreadPoolExecutor(10) as executor: - for index, row in slosF.iterrows(): - # if i % 25 == 0: - # time.sleep(0) - #args={index:index, } - executor.submit(self.parseBySLO, index,row) - # print(str(i)+"\n") - # i=i+1 - # x = threading.Thread(target=self.parseBySLO, args=(row,)) - # threads.append(x) - # x.start() + with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor: + + if type(input) == pd.DataFrame: + self.pbar = tqdm(total=input["id"].count(),desc=self.name) + + for index, row in input.iterrows(): + executor.submit(self.parseBySLO, row) + + elif type(input)== list: + self.pbar = tqdm(total=len(input), desc=self.name) + for slo in input: + executor.submit(self.parseBySLO, slo) + + elif type(input) == dict: + self.pbar = tqdm(total=1, desc=self.name) + executor.submit(self.parseBySLO, row) + + + return self.krs - return self.krs - def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ): + def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ): + self.name=name self.DTAPIURL= DTAPIURL self.DTAPIToken=DTAPIToken self.options=options - self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams) + self.config=merge({"threads": 3, + "serviceLookupParams":{"from":"now-2y"}, + "extendResultObjects":{}}, config) self.krs=[] \ No newline at end of file diff --git a/KeyRequestParser.egg-info/PKG-INFO b/KeyRequestParser.egg-info/PKG-INFO index eb80030..7886544 100644 --- a/KeyRequestParser.egg-info/PKG-INFO +++ b/KeyRequestParser.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: KeyRequestParser -Version: 0.4 +Version: 0.5 License: MIT Parses Keyrequests diff --git a/KeyRequestParser.egg-info/SOURCES.txt b/KeyRequestParser.egg-info/SOURCES.txt index edb4624..5e4aeb8 100644 --- a/KeyRequestParser.egg-info/SOURCES.txt +++ b/KeyRequestParser.egg-info/SOURCES.txt @@ -7,4 +7,5 @@ KRParser/patterns.py KeyRequestParser.egg-info/PKG-INFO KeyRequestParser.egg-info/SOURCES.txt KeyRequestParser.egg-info/dependency_links.txt +KeyRequestParser.egg-info/requires.txt KeyRequestParser.egg-info/top_level.txt \ No newline at end of file diff --git a/build/lib/KRParser/keyrequests.py b/build/lib/KRParser/keyrequests.py index a877da3..432ba16 100644 --- a/build/lib/KRParser/keyrequests.py +++ b/build/lib/KRParser/keyrequests.py @@ -58,20 +58,24 @@ class KeyRequestGroup(MutableSequence): query+=",entityId(\""+'","'.join(val["methods"])+"\")" else: query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" - + # query="builtin:service.keyRequest.count.total:filter(in(\"dt.entity.service_method\",entitySelector(\"type(service_method)" + # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + # val['methods'] = list(map(lambda x: x.replace("~","") , val['methods'])) + # if len(val["services"]) > 0: + # if val["services"][0].startswith("SERVICE-"): + # query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" + # else: + # query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" + + # if len(val["methods"]) > 0: + # if val["methods"][0].startswith("SERVICE_METHOD-"): + # query+=",entityId(\""+'","'.join(val["methods"])+"\")" + # else: + # query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + + # query+="\")))" val["existsQuery"]= query - - # def createServiceResolveQuery(self, val): - # query="type(SERVICE)" - # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) - - # if len(val["services"]) > 0: - # if val["services"][0].startswith("SERVICE-"): - # query+=",entityId(\""+'","'.join(val["services"])+"\")" - # else: - # query+=",entityName.in(\""+'","'.join(val["services"])+"\")" - - # val["resolveServiceQuery"]= query + def insert(self, ii, val): diff --git a/build/lib/KRParser/krparser.py b/build/lib/KRParser/krparser.py index 7305ba0..b634a6d 100644 --- a/build/lib/KRParser/krparser.py +++ b/build/lib/KRParser/krparser.py @@ -1,14 +1,12 @@ import re from KRParser import patterns, keyrequests, helper - - from enum import Flag, auto -import logging import threading import concurrent.futures -import time from jsonmerge import merge +import pandas as pd +from tqdm import * @@ -19,24 +17,15 @@ class KROption(Flag): RESOLVESERVICES = auto() class KRParser: - #threadLimiter = threading.BoundedSemaphore(3) patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern5(), patterns.Pattern4() ] lock = threading.Lock() def normalize(self,x): - #tmp=x.replace("~","") tmp=x.replace("\n","") - #tmp=tmp.replace("\"/","\"") - #tmp=tmp.replace("\"/","") -_>was active - #tmp=tmp.replace("/\"","\"") tmp=tmp.replace("/\"","") tmp=tmp.replace("\"","") tmp=tmp.replace("\t","") - tmp=re.sub("([\s]*)\)", ")", tmp) - - - tmp=re.sub("\([\s\n\r]*", "(", tmp) tmp=re.sub("\,[\s\n\r]*", ",", tmp) tmp=re.sub("\)[\s\n\r]*,", "),", tmp) @@ -58,40 +47,54 @@ class KRParser: def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): - DTAPIURL = DTAPIURL + "/api/v2/entities" + # DTAPIURL = DTAPIURL + "/api/v2/entities" + # headers = { + # 'Content-Type': 'application/json', + # 'Authorization': 'Api-Token ' + DTAPIToken + # } + + # for gid, group in enumerate(kr.matchedGroups): + # params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + # response = helper.get_request(DTAPIURL, headers, params) + # entities = (response.json())['entities'] + + # if len(entities) > 0: + # y=0 + # for method in kr.keyRequests: + # if method["groupId"] == gid: + # found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + # if len(found) > 0: + # method["hasData"][tfrom["label"]]=True + # else: + # method["hasData"][tfrom["label"]]=False + + DTAPIURL = DTAPIURL + "/api/v2/metrics/query" + headers = { 'Content-Type': 'application/json', 'Authorization': 'Api-Token ' + DTAPIToken } for gid, group in enumerate(kr.matchedGroups): - params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + params={"entitySelector": group["existsQuery"], "resolution":"1d", "metricSelector": "builtin:service.keyRequest.count.total", "from":tfrom["tfrom"]} response = helper.get_request(DTAPIURL, headers, params) - entities = (response.json())['entities'] + entities = (response.json())["result"][0]["data"] - if len(entities) > 0: - y=0 for method in kr.keyRequests: if method["groupId"] == gid: - found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + found = [x for x in entities if x["dimensions"][0] == method["entityId"]] if len(found) > 0: method["hasData"][tfrom["label"]]=True - #method["displayName"]=found[0]["displayName"] - #method["entityId"]=found[0]["entityId"] - #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"] - - # for idx,o in enumerate(method["services"]): - # tmpS=[p for p in kr.services if p["entityId"]==o["id"]] - # if len(tmpS)>0: - # method["services"][idx]=tmpS[0] - + method["count"][tfrom["label"]]=sum([x for x in found[0]['values'] if x != None]) else: - method["hasData"][tfrom["label"]]=False + method["hasData"][tfrom["label"]]=False + method["count"][tfrom["label"]]=0 def resolveServices(self,services, DTAPIURL, DTAPIToken): - #DTAPIURL = DTAPIURL + "/api/v2/entities" headers = { 'Content-Type': 'application/json', @@ -101,8 +104,8 @@ class KRParser: for gid, service in enumerate(services): query="type(SERVICE),entityId("+service["id"]+")" - params=merge(self.serviceLookupParams,{"entitySelector": query}) - #params={"entitySelector": query,"from":"now-2y", "fields":"tags"} + params=merge(self.config["serviceLookupParams"],{"entitySelector": query}) + response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -138,9 +141,6 @@ class KRParser: response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] - # if len(entities) > 1: - # kr.keyRequests[gid]['foundCount']=len(entities) - # print("Multiple keyrequest found: ") if len(entities)> 0: kr.keyRequests[gid]["found"]=True @@ -151,7 +151,7 @@ class KRParser: if "isServiceMethodOfService" in entities[0]["fromRelationships"]: kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] - if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: + if options and KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) except Exception as err: @@ -191,53 +191,37 @@ class KRParser: tmp_methods=self.getKeyRequestsByServices(group["services"]) for m in tmp_methods: - tmp={"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName": None,"comparer": "entityId", "entityId":m["entityId"], "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""},self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): - tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{},"count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, else: - tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + tmp=merge({"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "count":{},"services":[], "found":False, "foundCount":0, "exception":""}, self.config["extendResultObjects"]) #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) - - - # for service in group["services"]: - # if service.startswith('SERVICE-'): - # tmp={"displayName": None,"comparer": "entityId", "entityId":service, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, - # else: - # tmp={"displayName":service,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "keyReuqests":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, - - # kr.services.append(tmp) + + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if self.options: - if KROption.RESOLVEKEYREQUETS in self.options: - self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) if KROption.VALIDATE_HASDATA in self.options: self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) - # elif KROption.RESOLVEKEYREQUETS in self.options: - # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) - # if KROption.RESOLVESERVICES in self.options: - # self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken) return kr - def parseBySLO(self,index,row): - #normalize - print(index) + def parseBySLO(self,row): try: normFilter=self.normalize(row['filter']) normExpresseion=self.normalize(row['metricExpression']) - tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) - + tmp_KR = keyrequests.KR(merge({"sloName":row["name"], "sloId":row["id"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}, self.config["extendResultObjects"])) #SLO with Filter if normFilter.upper().startswith("TYPE(SERVICE_METHOD),") or normFilter.upper().startswith("TYPE(SERVICE),"): @@ -248,57 +232,46 @@ class KRParser: groups=self.applyPatterns(subject) tmp_KR.matchedGroups.append(groups) - # for g in groups: - # #if g["methods"] != None and len(g["methods"]) > 0: - # tmp_KR.matchedGroups.append(g) - - #self.process(tmp_KR) + kr=self.process(tmp_KR) + with self.lock: self.krs.append(kr) + self.pbar.update() except Exception as err: print(repr(err)) - #return self.process(tmp_KR) - def parseBySLO_Threaded(self, slosF): - self.krs=[] - #i=1 - # threads = list() - # for index, row in slosF.iterrows(): - # logging.info("Main : create and start thread %d.", index) - # x = threading.Thread(target=self.parseBySLO, args=(row,)) - # threads.append(x) - # x.start() - - # #krs.append(krp.parseBySLO(row)) + def parse(self, input): - # for index, thread in enumerate(threads): - # logging.info("Main : before joining thread %d.", index) - # thread.join() - # logging.info("Main : thread %d done", index) - - # #resultSlos.extend(krs) - - - with concurrent.futures.ThreadPoolExecutor(10) as executor: - for index, row in slosF.iterrows(): - # if i % 25 == 0: - # time.sleep(0) - #args={index:index, } - executor.submit(self.parseBySLO, index,row) - # print(str(i)+"\n") - # i=i+1 - # x = threading.Thread(target=self.parseBySLO, args=(row,)) - # threads.append(x) - # x.start() + with concurrent.futures.ThreadPoolExecutor(self.config["threads"]) as executor: + + if type(input) == pd.DataFrame: + self.pbar = tqdm(total=input["id"].count(),desc=self.name) + + for index, row in input.iterrows(): + executor.submit(self.parseBySLO, row) + + elif type(input)== list: + self.pbar = tqdm(total=len(input), desc=self.name) + for slo in input: + executor.submit(self.parseBySLO, slo) + + elif type(input) == dict: + self.pbar = tqdm(total=1, desc=self.name) + executor.submit(self.parseBySLO, row) + + + return self.krs - return self.krs - def __init__(self, options: KROption=None ,serviceLookupParams={}, DTAPIURL=None, DTAPIToken=None ): + def __init__(self, name="Default Parser", options: KROption=None ,config={}, DTAPIURL=None, DTAPIToken=None ): + self.name=name self.DTAPIURL= DTAPIURL self.DTAPIToken=DTAPIToken self.options=options - self.serviceLookupParams=merge({"from":"now-2y"},serviceLookupParams) + self.config=merge({"threads": 3, + "serviceLookupParams":{"from":"now-2y"}, + "extendResultObjects":{}}, config) self.krs=[] \ No newline at end of file diff --git a/dist/KeyRequestParser-0.4-py3.10.egg b/dist/KeyRequestParser-0.4-py3.10.egg index 615747838f35cfe917c99d63173b131136fa6995..2c585a4e31a4fe2aa49ad5ee45cde5411ec31f63 100644 GIT binary patch delta 6704 zcmZ9RWmFVUyM>38?(UHs8tD|IyE~-2rNaSfq`M@gVSu5#g#o0y8B#i=1mu37Yuz9B zop2^<8Wcg4g~6%X(>V%8=NRq>iXH=O1wU|Fdfj%rg285B~P& zAcGGXv+L_4FY;99Ze!}w$Wj<=`y=z<3GiAtD#18AI8ZFScO?B+b^j&&bYt7=a%5q9 zX$zI#UHCnfNw=s!|FoTQez+y0AT{vSunkSBx4vgu{-WHPd}`ELfkY~xKm-0v7sF!w zK!3Z~NyH_ea8h)O8G!P3zVC;DCm2=h1o0i3k0246N z&(yYR<&a);YJnF@7~;QxJT7}1q8f}R^~>DgV`5QJuT6{(fW>v^W{ggI@b$HJQjvL1 z6z+FyI6y=1N`T$b6gnK+1syO^R^&47@vZ0Wfc@C&6y&_avwa2keJ8djET)Y7#ne-@ zdqK`x9yw-kO8&L`ZrHV9dDo`x9ElrXydZkIi_>XB)*^E(u3sVs8 zXb|0;$$g(_4*T51D`O1*%@z|;DN7uF*l{Jtblpv%g@Oh3=^n5McPGtyxPFL1WARrWD~ch<2JnN_BsqFKbtB3lvVfHj_*lv< zFB$xH;SPMRAd&pI1d3;BK2>DeK-Je`Li9@buwm8VW8yon+qmI@Qgf}{0WsptYdvqW zKcm%?%l=Cml*+vk_TzAtN6JkAOHz@Y)&jDdN=`$+WhcA}pB1|!#^)$29t)oBUQYHCOUWldxlly*dU5Jt;6M|> z7-XNIORWK;lOQJss|9-rLE_xP6odq{p^>Fj>!`9!B}Jjd=ZD;6w(mS+EBrm!z%8>5 z`NwD%79{P($3 zl~W$yb??pPGZK3We4?TVHm`GVu3uY~ohfb2*p8a6i(4!_J6a6nw&Z3N2=Q^N zMI*(#SgxwmckLnRY#C^#v@^K~^qTxjcmOS9oWFD9*C7Wh<=nl#^>Ii~Z5%ruG2r{4 zQ{6_dI*_!eiKrmP3-%%+wpVAM=@CI^H?E?6sy!}+7R^^)1 zN3r^nZ9o*&&X^M>+p2IT_I? zVpUxGWyEzdr!nLOGE3(?D>ckke&zxWcY2q`nHlJTUUvd5 z`f8BkC%$d4Br1`0w1(sU{}yS$bRIpchw|o?M4<89%#3I}>~Z_g*|3xE()SuqPY%Z2`$sP@y4SrSt16V6 z`KgL_RX$Unv|h%R&&Jo>b1D+o>YwLjwU@nN zvUA#Ztg?tFR?p!g*UG5ppAIzi;rUcAo5a#4D3L>7qFy0gdgTtDWz=E~w^&Ug(>tzJ z-^Lc}_h()YA1WDsxztlG&0f>*tiDp3=%U$OE$UUKbAOz+=|ogp4zRR*S2n5wkX*)VHfzr~u! zg~$?#EuLGtz$^#T- zpr)9sJa4jW@XODj#i!Q|XYK0XL!COq=GBaiozSA5}69(Y+KuA%lHwV z#S*w$KW4XxY= Rj|m62YO1n&Uf!?Z#XW%^sx0nEDZF8Tx={1UKDEktcxLf z$yvPyr2zpb-@<(|8+`-z#o+8pGNU&39uw+we(u3Z(x>t9I-n0S5bm+w9iEuIw4}Vy z&bpwDt9bFGN{ z>t#Foc#p~jjGdW8w#v=M-D+wYU;ozHiPc><EroY`r5|9ds(kLlJt=bGA%v3a%NYicCx@W$v#(qzmoq`>Bjs!F;0m6`~f-_ zi8e+xf_4rs#!gH{`+|sj6sMMGY7rSa;o&4@y7_b-apeNvz5$qT|0)3`!6Gg$zcHhi zT-b__N-Z3*y4S{uj()Iw6H?TWdB^#C2S_#@q1lW4k9TSAdRMZ_o<*=<_Yf-D^RR`t2z}1K& zQSfiLypQ6vPY(k>IcH(-S2V?M(rbcZ=aGqdpFet9JpQR{vs)@5wK$*eYOhgqj*E6r z*`G&Wxw3J7qC+Ei+}WHFY%b^HysdcSs%5JEqYH-IC03wfU8p^5t?6@(xv8+hI9JE< zSc9PB{hceI8Ja-k-oZuYZRa2QqearZ{u|}Z32`0QtDdb(O)RECV)}`uxR@tFFgfV z(T!?^7*b_cR&KZBjt|moV&qNunV~qA(iN27(Hvwx4@8_7r!8$viu7j zSwk`}HCY*eB$5dfLt(d^`L2D19&P|<$J-vWc`>74$~ zXjhFTR}&6YiZz6|6K}RFG!6%SpRvSM|3tWnlD}iUkb&w;-h@O`>hJDuiXiDxeL*fME?&$dm~c)Z`8-@S%s)d}75)Q6?j z=Pq~-<7C%M54(tmh%7W$D+7b6Sj}qNG08MT7*v)hi}fHPJ>qe2@(_k`sAGn$L;n7P z*l@wvb5DXjb1Fh$c+HPQqU+ogRP%A`^gQN!3>QX?7JJE75Eu;wxdHv4vn*CRtx#i; zA4DO*HvN}fcDrjB7&Gs0Eat`9B5HcLa9#n|8NUh-l;OZ`Vef$XQV3l zoy(b9-<RQqVG>kLy!|8A?Si-#D3b!i%!*bH8<1AgWFiMD@T@SyHCGSDpcshm$O@ z9mN(Zf_&uv1H78^2FP_PQ33{@d^i*iEW#ivx=XFZ1Lwusrn%#vTQ=}wjb`3y@sgbu(DIcG)!?!7vqT?>BWP60(( zRZ@@&RgTva&loa}5kke?JXhD?t1FVXP(et%zL0PMkRd;W7r|^CX!riu0|)c#-&en0 zf)igt9#|bSp}Y-HMwpmP3IYi(QN1Y2shVtOs-7T%UU{=O1XW~}y6MEc6qJYv4zVB=jU_!M ztvdXB-(mUX@Xv-SR@XcPcC4=0&E8G#J!p(=Qx@)LtZ*wjHG{M8&cun%Ecn=ooAswL zn)ZZk_yWBv%fx)(PDOW$kQNa$tiEEJfJONuI_t7w&a0(3tUYL>o=LYlF`6*iJN zB1SHvXCSGZ@&o|P14+kIn6;>?MJIo=3_jQ=$=p>c%du8ZFol!stGHHiSjge)PFa%_ zirMHGUTArND=MqTg0T?3*N0EdxNwN@Hb6@U6?c_FI z{#8U>X71_VTi#cm{i-_4ejCI9p%wc;LvXd=8k6XC ztI{v>OP5!pXlkk6Qma7W>LkS3rsKqdcC6(C%mr$u5tF&`fXR49#rT;E3>!yjS4}dR zeccBg`M&86IzJTlVtDv0LyPY~H)1=Sh;R8+V=ZZ<7&oOgRG%Rfai$Lz$!M?|(&O8i zio3WcPfegK;fO_ra19Uh2;bO?c{PnA%KN6xc&M??H^I@aBmOq|=9RQC`|ZdcPpU z)DOd=2qdBYB6!Jxa>-2Zm1VO+M;)RQKq-!!Z;SUCML;F7aZm>woMKlp?>Eq+{kl!N zF=b(Ta*FPx34pEapfixo`rwlrmhz;7@RkviJ+M`r-WDGDwh%#(gu7$$Y5gczE19va z7DV!V@0R+>KX|if6t|R?fKVwkVrh>2is6QZM3=_qM(s$d$~g&HvfCTNqoaSBGZ-Md zZw1|xpI$R%eQLpMR~Z>I`+ImzaBUS=3n*#L^d|<3lth98zgyX%vo~uNXtf=H%PYB& zl9#ewEY%-b2ub6P{vwi3SwJb&kz#!Uy5KD;u0GIO=$vR_Bj>9=;wx!C8V+$vDY@jY zU_(9lJHiVjCpJx7A=S!I4Ei@~6lQtI6#B`-)X||=N3me}MLUNOJX78NCj=spA#J>E zm}VWYu*J(2AmcW6RY^Ls*tq>N`y6Haf;Z`I&Cl5LJ8psRpzp-FFEa-r(uCX(YS3A$ zg(&Xnk;qjfPopwCWcA9s_$#IxoU+TR3b&rN3Lng8uVds zFr*gF_5~9L(GAD?tB~MLit`XCFWHiCn!SGylqX%OT;q{{56$w!4J?R0t{c8xs>zQw zAXFp17ZIa8b`OUF-7`YEQ>Q^RZ-?PDKZv5yJsL2F>_jI(K*Ixu?DggqkdVS9fFq=HVaTZClG~kv~}+>(GzyjS(Qkg ze?BDFzlg?|ccRDMUxXF2MEVdUD4AH|3+d{K`0h;>zPa=fg871;wlN+QxLqhfz0Ho=H}G5>!e5OSB*mc46S5-{u~kN} z4?cu};G4n791oq+BF*ncAIXu09hMa*>GAfy3-0&8rG9mr zX?-lpNL$ETooX;Sc4#xbUBd~XUM9R+A58c=of|}trX%D5i5SaqA9ksE#tOBOvwQQi z^3#(?Mf8u7;uw8l(mmH$@vOvq%UyCa`BBqs!n$p&5_F~u8i0tl*#cnSwEM3-!B+mj zRpQRV%(yc$4go7~KTmE40)Jg_rha58{oRwVX68hF6jyT=FXQET>XFySw={V;ec<*l zH_vc;%5}~UZ^$?>v8n*fP!4N)YLklCtJ8oUO1z^!Mmn<#R2dzoh7~vbLwAu+dp}s? z14qzft44zL%UFA@Wyl#woxA4pd*M zQS#rFuu0SRG>N9~sWG?nDxRM)-0NlCeXq&;<5^*bQi9JWw9+tCpOj*AM2Uaboe-mV ziy^q<<#v&u5+7rxfKyZ-#h4-wjOEVavA2%XEZOo~6S7tztBZS9xC^rnXycW0RWJ(j zdMbhMCrRGIdTGOhJ4f^4LO~+O(2~RXQ%j1AsJ$(rx8KONZi0(=h+op->;YQ-JdQ~S zdKy^6A%OD2yI-#e|EEN09OcPE`Iq^u1poCWsl|ft|GoDF75fq^Xl4q^ZbcWGSLpbpH*E6Xf}v28aMa6A}Qx_MhtJe@{zj z4B5ZgIFYyir_tfd`%X&_Ac#L_VE6c&b;{pHxWWa(?VLZA&+`kfA@vlIIVFdoFFO1;7TH6Za zj)eZxr%;n-m~>kB9RRQd6-Lp5a3w!px7~P=FFVE4a7iT!LhfIdnH7MCx5peggt>IM z2G(&0Ap>dQ`(_HmgVe(=k4>~HvUK0658^$Wm0i~*x--Lr%(Da`G~e@f&sn}+L|wdO zB^Vbq%0oR{3KILgF5J^t>4!(Uw}Pa(NPkipVtGs9viu=G^!#J@0YL{5!lYl|mD-1q z-+Lwk3ETAY@O8g_dh})^d^s`m6p|1wo^)nxcl;K9>7#$t@d1r>VjW!(38Ihg|7x%N zo#i<^xZH!b3enq928vZ)P(=AC@(!O|8h-%)`tUW8X@@WkeV9V}0!KE3lnT2L1yZo? zowB_*+AP}M!c|D4* zSP%n-7}mnXD#RNI;t#M9h|6b6uajif+Noa zA((JF9_KDD4yMp{8e|SPo|q|qVd0J;;6~IJy=n=+6pIT(!eqthG70yR=J)~b$@X!= z3$b83nBFaaW{7l6t%bBJ$~1m!k@wr;GRAlg@Or4!N^@$1z+cT=DhY>-^C|ZFDbgRq zl2dQ3w9y6=6z#Vj}MD^{&*Co<=%&2JV(_eC%v5{L13HD6<@X1?a0HIogJG&Ib9kNH;h0#{0%+e(bXEOCG#eQL$ ze*Jh3@-1MHeUvPWYz?TxJsK@RXf4*U%&`FvF!5-!V2xJX3`sc5jmYJ&8m&q$Dae|5 zGm|fSHw?@#AdolJ{8^fIDFt-*&4AmAWfIX>S}qBG*p)CT@re^@pr&P5xJr*=l+=0C zNf=8xqQ1Ci%8i3wsSo=q7lYidQL?@!@hJ*J&N{VCnPs;Ho1QJU^X$*VHm!{IWn(C( zZkM!zqwnL)c)MA#N+kg{_WfoW^Z<5__BRs*rK_x;Md!qf;nFI^kWw!Os`$S4(~iOc7#YF zKJ#?HY@?>F9#A)Q|I>&ixl>m6y2U8&6>k23cu ze2-Hr+KrBwAc2LagP}Z>wB9s_2!R)fO3J}N)uz%q$una0XFhuLowe?RQ98`o2&>lW zyh9iiJsRGpbPiGO76PKZ)SVvt{$WebwTM+Fa9v5WMujOtsreORn}UW?zM4a&?X7BE zDU;3BtmbvSi!ILKF9w$KUnQ@%hHl@9_@LHEs5vhdProD_U0-`=$8AiiVRXeIrPWF2 zLpHKA#ORNhmjOX&?`13ImhGj-;h8tRd%_%o*(mJ%&Q%grk&IW7SKU)`({*c4gA0#B zOVTwp(RI_zgTBop|FO1TFdfnagKuR`U4)zbeXdr1anqHj>c1O)^b`JN1iS7Bl!Y>l z4R@4<-%sz&_uC>5QM2p%k5*Vh#1qPkUi}CiWabsED{tpRUzmMl*$U`r+@;@iuh*&) zkbz9vuJk@EH*VH;Lx0jym^XOL#B_CU_GZ#5W>b?*gJ1_VpE(PCq^)7GRo0Ca-Fz4} zge_g$EoSdsB9><20!VI|~0Yig^c zC^e2K-7)r(5a1*tDef-o9{zlqlo1SxLam)NH?&*0BhB6{-^>-+mckIUKDn_MxEqGD zuZfsxM*$WFA5q+PWf0F@x0SVevWQ>T@f236w1qCysIox&)>#Iz`^};nHE!0x1K5_% zlk{$@&Qlb%qO z`5#e2D_5$m8+Bq_*ODK%&Db@S@XGNr+~Z{o{HrI;zZs?;5zV+ljtO-dwG=#r;I4~o z-y_7-w`%JpEIIAL4EZKXY5V!!3UH?DZ&W;mWb<5_Ob^~8Pue0+dWbCrYTRhyEiH$s z-|mIXF4yT)r%iajc7AuTX}-uf29+nT{uB_5gU>RExaIk3X|^}L3spG?kW&K_Clg0>e^_L4nP0WTc5A6$h4a?P6`s`fW7{y7#g71TT0xII>mr> z6K!5rtfeet#S8K&)|#N4$dhQ&M+6S~^2P`(A}#l=Ys=wLdhVj;zMy?`rfP*huq!h} z9*Vo)U8q+>CIts;Pf5T^jGH7*`N&;n&N;_$JU)FJ6|>KWz_ua>{l?&2PHgFo=WTz% zj+0!w-<8iUi>w%FZ?hYyH*DV2Y&Ex-{i)+ox=B%$xs09BN>N|RuU#W3aJnlqIx#Kc zTiNrIW!9EsG*wRuVx$$50Ucj<1SD3zla!P<58K2x*g zqB@h#SGtA~a%f}BKjX;FX?$Y>we{ys8Jn9hNHlXqYt>kLr1V>C#CehmBL#Z+E<1?B zTgRI6c{oppJgR3Cl}fzxQ`K>Xm}vd|4bvsKfj6)U-??$yvk zf!l2GLnjqAAxln2G_p*l_$uK?-d?HLPS(=u41b2><-0@4Q~_>VmRcMl@}5jH zklUvQKf+ZE_&`dLHG!w1g~ZQ==XYK5Wu~&(ylDYR2hd0`m?4d6M$x~UsI=AWsh&C; zFM#3Otz?Dfe*Bj#hj*XXbqsTA(6O-Wo?pBN2u?;fWUFG-oncODT!fB?2a@$*XnkKG zfZB9i3An!jdJF&O-vFIVkOlj9!%LKljQvlf*39bRJ1Rs607yRo0L%br7e4`%M*tsE zucPOoe<&UZv|_ANm?U&5P5MEGY2Uu@blwX( z5ZC~5+U(r`10kI%BD&(Qf@#DJgksrBJSh@xiP@zQqRo|X69tbBJtpke>q)~Bs^q!?2&Xq7)^Up%50Y6OPa2*$+_NcpRI$Ej_q1i7BRl_V~ z+(rKO=dmP0A&&*ZjVsAx1>bwWae#g280Hti4gheg&hxryFzI?hY6KNNw z;(e=FptcyZ@=GZb!D*yBKS{OHkj=F;c_*1QOoRNwke5q2WKQFg`S&nzc#yIvmY#9P zNg3ZK6jXC@Xr9Rp?AfFy{Dht`Lww6&M61L2mBsg$sGSXKXI9l?#aYdM0)Ib5_on^1 z7ilBB3dPmvLsOzLt<|%q{rZrPm!pC|v|x^b7>f-O}9d9B9Uz&iL$-PX?)@g;ZO@CL6?< za)BQLFgUrUlbgIUz|2PPn6cs!DELP!9Do+HCY6!dP$iVf9>FJmoa~pcKa~4c#3}10 zpv)`d-A(!&=lraq8+rP*;Eb`j7_G;y?(`UJTC&G($nRe>M>eZMeh>Y#inAVmi&yS) zbO?=_k`@B(`}OIq3wEoSx;7O8<4~h&vIoWTBk_yU!tn;2JxhTc;w{9V5YWjQ685$5 zA>S9ynHRii8wt2n7CyJ--qxLlcGkoavc*+>@NrIGa)kP_hLLk7ztzg2Ukv=`#CZOW zA=^472sM++FjkovY=QdlJor=3kLx}f7rygVij#AC z^)|l*c-wjX=||Qg^(ZS0+t21$m(qs7=_F2a5Dn74<&q~hB8XOyZjOQEvc_N)RV=fr z3gvde4t?pM8`qw&`7b`oM2Ltm!X{QOAb#X# zh?wrkPo&eyW04@74a$!6mEu#oJUbZGOhlg}`}hgUs^WSa+l| zer@Z4Vo_-{Pr;oTW<((JA4ao(k}laJyMy$*xk7|=W0|T@psu;JIJzATBlAEyev{-C zF%5FdW(lSRTl}NB4@cOYr0{$8l<4=T5S~z!C@zcUubw0~gt&Wg__b?$nzrq;9W-}j zNi53gUl4KT>N8{JZK+;`u7EtbM!?$CwwS&&8C^jvb*DQfY!rnL0);WTeWp z49$PEz7nP$O}$2%!UKhcR$dO%lM0J&cLr#l~^x$#o>H`qux2L!V`+{T*&t6Tvr$ z@Nh3OBBc9`qAu2N#TGdm=(pFuv-hXtH=jSYOBDoWq~eQ6M37V`m{?PzK!h3Ddsnv6 z8jOy01}fvVr`6eby{0VXTMdI6_&zkXB;eBSj_nLjT?xM<-q0AJWv@Bge0}ro@ru|g zCZGF&&nhc-%8aA!!zk;6DT^c}p%XDi&Ijd&A!>wcUvA-bvrrTK=97J@bC}?%)j~Af zav{2%^388i3sH%R$J2Te1(Km(+Ts@8>5MHn@|sRLN#jml^qrU8D}RtSe($%&md}I! z8%9%N@~Yqvzb6pOY6RB6&}`z4=Bqj~7lM3r2-Sp1f{RL`^|0{zM|w%Hf&KvOOgGHu zC|sv5Bx~!-E5G`987Gclnk>oiO>|#5l$T)w5_1sGr*QJ5_0OyU#Sp?kqsW*U=qZ5) zDee*<_s43-c@cOe;@#>FxBJj~x`5l$O8_>S^>f76AJZWtxX5;aLrsW9+<2+Vv<0nZa&OVaf)mi6!?9fqZB?1N#1g+hpsNjx>w{hi|osonunldj$yr0rM4Ho zZH%BZf)GtVfTvIG!LslNPxcSfA8{lukf%Jx?Kh-a<86GYR24H{HNCclwD7&{sN~-box% z(W9U)Xn=hb1i?-!A$h(A>c!oUr20d<&kSWtUp`K|nWOP`ef~zx1)P@on{MmShX<$1 z8BKpj2a#f}aQ~AJDtc$zANBwUgYSZLe zCl{q3XcJB7SGWyrzQ~E3_S3LaUY#^D1j0j#u$t&RkT5a6+Cw@a#4Y00`EVD}SPL3oDmODEKOs>YP0_2z9GO@8eL6z`rs%D7 zuqJb2qtxmLmx6~+x+4JM$*?Y8g@`d23-~20Z-91v{TNS(9Wq}Dw|GWBKwCRIsoDZP zh&5stLXXInQF6Jey8S`{5^wwdkb?l>m9El61c?Na4Tfay`oW6DHd8+&h4AbvDV*XT zur<}hcnM}M_(@4nPlY?#x<0IKiVGg$T!mq->D*LSh!Ge%l4qkJB?aT#Y`bI9cgGhJ zE$-9Y2Tx?7p}x-F7Z^m;H8_3m;0vV0i)@#Jyo+CeF^(kAnk+ahH&&IzDskG@|CZ>s z75T0D+7vz+FPns`aX9lio)hracL)u%OU%W&>Or>rSXxadVv&njG$oVQOMT~Zq%aUnAYGi|ADZA+h z`LsRj7y9VL4>&-bxT&RW*>r!;-(>|dRz~29x2GgJ+We&P)sh$WH9!=J1?Ea$glt>) z`#2<+FMKwQkyq1}+m?j9OF%L$DRu+qNFrFy13n|Ws$Tc5`B|}Ir%__rNi2*^m35|l z;wK3nmXVHexrl(Ta$4xlT zc=P4MG-LBczk=R{Dci84XFA@6>Grw3h=VsoWQ$~}QKhs=VX;a*t9UpR>`P(*$pfMz z&M4e_FH40+n%3lZsag1%TT!LBTh6RSTc6vd`+&yVanm{ylUDU(=0=UVwlNglDY%&m z%m|+bLj1}tzo@qN$bo)sbQXh6zW@>_=}uK5cby|B&@8?Z=$Nh%a2WE$5~vN#nu^4| zo+c2S#l#aO07*Cglz4)ul>M$1E|64;)^!a+CxIkP=;vEt{=ibJ5tEKprxBWU;BcpB z-v^G49a~CMnm^zBLAT}CEK6j)KMJ~}MvY>V=fqw`{#UC~+rplT@RtKMc>fkD&_rI5 zzvl&B>Ay305SH3ZT^JNZS({y%vBj{^W`2ppySZw&p< WlllTgu&wNW?~fCpLsaMe*ZW^QhvaAg diff --git a/setup.py b/setup.py index 1732ce4..367c67c 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,13 @@ from setuptools import setup, find_packages setup( name='KeyRequestParser', - version='0.4', + version='0.5', packages=find_packages(include=["KRParser"]), license='MIT', long_description="Parses Keyrequests", + install_requires=[ + 'requests', + 'jsonmerge', + 'tqdm' + ], ) diff --git a/tests/TestStringMethods.py b/tests/TestStringMethods.py new file mode 100644 index 0000000..6fbfeac --- /dev/null +++ b/tests/TestStringMethods.py @@ -0,0 +1,66 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/..") +#sys.path.append('..') + +import unittest +import yaml +#from helper import get_request +from KRParser.helper import get_request +from KRParser.krparser import KRParser, KROption + +import pandas as pd + + +#from KRParser import helper, krparser +from decouple import config + + + +class TestStringMethods(unittest.TestCase): + + + def setUp(self): + with open('./tests/environment.yaml') as file: + self.env_doc = yaml.safe_load(file) + + + def test_upper(self): + self.assertEqual('foo'.upper(), 'FOO') + + def test_isupper(self): + self.assertTrue('FOO'.isupper()) + self.assertFalse('Foo'.isupper()) + + def test_simplifySLOs(self): + DTURL=self.env_doc['euprod'][1]["env-url"] + DTTOKEN = config(self.env_doc['euprod'][2].get('env-token-name')) + + api_url = DTURL+"/api/v2/slo/15c29ec3-71a7-3298-9e99-aad2e5bf347c" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTTOKEN + } + + + result=get_request(api_url, headers, {}) + + krp = KRParser(options=KROption.RESOLVEKEYREQUETS | KROption.VALIDATE_HASDATA | KROption.RESOLVESERVICES, config={"threads":10,"serviceLookupParams":{"fields":"tags"},"extendResultObjects":{"env":"emea"}}, DTAPIURL=DTURL, DTAPIToken=DTTOKEN) + + #df = pd.DataFrame.from_dict(result.json().items()) + #for index, row in pd.DataFrame(result.json().items()).iterrows(): + slo=result.json() + slo["env"]="emea" + + listSlo=[slo, slo] + krp.parse(listSlo) + + #krp.parseBySLO(0,slo) + #krp.parseBySLO_Threaded(0, ) + + #self.assertEqual(s.split(), ['hello', 'world']) + # check that s.split fails when the separator is not a strin + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/environment.yaml b/tests/environment.yaml new file mode 100644 index 0000000..7e9829a --- /dev/null +++ b/tests/environment.yaml @@ -0,0 +1,21 @@ +--- +euprod: + - name: "EUprod" + - env-url: "https://xxu26128.live.dynatrace.com" + - env-token-name: "EUPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" +naprod: + - name: "naprod" + - env-url: "https://wgv50241.live.dynatrace.com" + - env-token-name: "NAPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" +cnprod: + - name: "cnprod" + - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" + - env-token-name: "CNPROD_TOKEN_VAR" + - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" +# #cnpreprod: +# - name: "cnpreprod" +# - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35" +# - env-token-name: "CNPREPROD_TOKEN_VAR" +# - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..4cdb679 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +pyyaml +python-decouple +requests +jsonmerge +pandas +tqdm \ No newline at end of file