diff --git a/createKeyRequestReport.py b/createKeyRequestReport.py index 857c735..f35db06 100644 --- a/createKeyRequestReport.py +++ b/createKeyRequestReport.py @@ -56,12 +56,12 @@ def getSLO(ENV, DTAPIToken, DTENV): return df -def write_to_excel(ignored, notExists, noData1W): +def write_to_excel(ignored, noData1M, noData1W): fileName = "./KeyRequest_Report_"+ str(datetime.date.today()) +".xlsx" writer = pd.ExcelWriter(fileName) - notExists.to_excel(writer, sheet_name='notExists') + noData1M.to_excel(writer, sheet_name='noData1M') noData1W.to_excel(writer, sheet_name='noData1W') ignored.to_excel(writer, sheet_name='ignored') @@ -76,22 +76,29 @@ def getStats(krs): for kr in [a for a in krs if len(a.keyRequests) == 0]: tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","",kr.metadata["filter"], kr.metadata["metricExpression"]]) - keyReuqestsNotExists=[] - for kr in [s for s in krs if s.hasNotExistingKeyRequests()==True]: - for k in kr.getNotExistingKeyRequests(): - keyReuqestsNotExists.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) - + + + noData1M=[] + for kr in [s for s in krs if s.hasNoData("1M")==True]: + for k in kr.getKeyRequestByHasData("1M"): + noData1M.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) + noData1W=[] - for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]: - for k in kr.getNoData1WKeyRequests(): - noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) + for kr in [s for s in krs if s.hasNoData("1W")==True]: + for k in kr.getKeyRequestByHasData("1W"): + noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) + + # noData1W=[] + # for kr in [s for s in krs if s.hasNoData1WKeyRequests()==True]: + # for k in kr.getNoData1WKeyRequests(): + # noData1W.append([kr.metadata["sloName"], kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]]) columns =['slo', 'env', 'displayName','entityId', 'filter', 'metricExpression'] ignored=pd.DataFrame(tmpIgnoredList, columns=columns, dtype = float) - notExists=pd.DataFrame(keyReuqestsNotExists, columns = columns, dtype = float) + noData_1M=pd.DataFrame(noData1M, columns = columns, dtype = float) noData_1W=pd.DataFrame(noData1W, columns = columns, dtype = float) - return ignored, notExists, noData_1W + return ignored, noData_1M, noData_1W def main(slo_path): @@ -116,13 +123,13 @@ def main(slo_path): slosF=getSLO(env, DTTOKEN, DTURL) for index, row in slosF.iterrows(): - #if row['id'] == "69c57b3f-725f-36fb-88ef-fb64b03eb0ce": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": + if row['id'] == "7a7f9c66-75c1-30b9-a505-801ffa5f9e98": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52": krs.append(krp.parseBySLO(row)) resultSlos.extend(krs) - ignoerd,notExists, noData1W= getStats(resultSlos) - write_to_excel(ignoerd, notExists, noData1W) + ignoerd,noData1M, noData1W= getStats(resultSlos) + write_to_excel(ignoerd, noData1M, noData1W) if __name__ == "__main__": main('./slo_parameter.yaml') \ No newline at end of file diff --git a/environment.yaml b/environment.yaml index ba3f0e8..310703f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -4,21 +4,12 @@ euprod: - env-url: "https://xxu26128.live.dynatrace.com" - env-token-name: "EUPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -#eupreprod: - - name: "eupreprod" - - env-url: "https://qqk70169.live.dynatrace.com" - - env-token-name: "EUPREPROD_TOKEN_VAR" - - jenkins: "https://jaws.bmwgroup.net/opapm/" -#napreprod: - - name: "napreprod" - - env-url: "https://onb44935.live.dynatrace.com" - - env-token-name: "NAPREPROD_TOKEN_VAR" - - jenkins: "https://jaws.bmwgroup.net/opapm/" - naprod: - - name: "naprod" - - env-url: "https://wgv50241.live.dynatrace.com" - - env-token-name: "NAPROD_TOKEN_VAR" - - jenkins: "https://jaws.bmwgroup.net/opapm/" + +# naprod: +# - name: "naprod" +# - env-url: "https://wgv50241.live.dynatrace.com" +# - env-token-name: "NAPROD_TOKEN_VAR" +# - jenkins: "https://jaws.bmwgroup.net/opapm/" # cnprod: # - name: "cnprod" # - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" diff --git a/key_request_parser/keyrequests.py b/key_request_parser/keyrequests.py index c6a1363..6e71a4e 100644 --- a/key_request_parser/keyrequests.py +++ b/key_request_parser/keyrequests.py @@ -85,23 +85,33 @@ class KeyRequestGroup(MutableSequence): class KR: - def getNotExistingKeyRequests(self): - return [k for k in self.keyRequests if k['exists']==False] + # def getNotExistingKeyRequests(self): + # return [k for k in self.keyRequests if k['exists']==False] - def hasNotExistingKeyRequests(self): - for k in self.keyRequests: - if k['exists']==False: - return True + # def hasNotExistingKeyRequests(self): + # for k in self.keyRequests: + # if k['exists']==False: + # return True - return False + # return False - def getNoData1WKeyRequests(self): - return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True] + # def getNoData1WKeyRequests(self): + # return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True] - def hasNoData1WKeyRequests(self): + # def hasNoData1WKeyRequests(self): + # for k in self.keyRequests: + # if k['hasData_1W']==False and k['exists'] == True: + # return True + + # return False + + def getKeyRequestByHasData(self,label): + return [k for k in self.keyRequests if k['hasData'][label]==False] + + def hasNoData(self,label): for k in self.keyRequests: - if k['hasData_1W']==False and k['exists'] == True: - return True + if k['hasData'][label]==False: + return True return False diff --git a/key_request_parser/krparser.py b/key_request_parser/krparser.py index d789d9d..34cdede 100644 --- a/key_request_parser/krparser.py +++ b/key_request_parser/krparser.py @@ -45,42 +45,7 @@ class KRParser: return groups - # def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken): - - # DTAPIURL = DTAPIURL + "/api/v2/entities" - - # headers = { - # 'Content-Type': 'application/json', - # 'Authorization': 'Api-Token ' + DTAPIToken - # } - - # for group in kr.matchedGroups: - # params={"entitySelector": group["existsQuery"]} - # response = helper.get_request(DTAPIURL, headers, params) - # entities = (response.json())['entities'] - - # for method in group["methods"]: - # comparer=None - # if method.startswith('SERVICE_METHOD-'): - # comparer="entityId" - # else: - # comparer="displayName" - - # found = [x for x in entities if x[comparer] == method] - - # if len(found) > 0: - # #Keyrequest exists - # tmp=found[0] - # tmp["exists"]=True - # else: - # #Keyrequest not exists - # tmp={"displayName":method,"type": "SERVICE_METHOD", "entityId":method, "exists":False} - - # kr.keyRequests.append(tmp) - - # return kr - - def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken): + def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): DTAPIURL = DTAPIURL + "/api/v2/entities" @@ -90,7 +55,7 @@ class KRParser: } for gid, group in enumerate(kr.matchedGroups): - params={"entitySelector": group["existsQuery"]} + params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"]} response = helper.get_request(DTAPIURL, headers, params) entities = (response.json())['entities'] @@ -101,41 +66,14 @@ class KRParser: found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] if len(found) > 0: - method["exists"]=True + #method["exists"]=True + method["hasData"][tfrom["label"]]=True method["displayName"]=found[0]["displayName"] method["entityId"]=found[0]["entityId"] else: - method["exists"]=False - - def checkKeyRequetsHasData(self,kr, DTAPIURL, DTAPIToken): - - DTAPIURL = DTAPIURL + "/api/v2/metrics/query" - - headers = { - 'Content-Type': 'application/json', - 'Authorization': 'Api-Token ' + DTAPIToken - } - - existingKRs=[k for k in kr.keyRequests if k["exists"]==True] - - result=[] - if len(existingKRs) > 0: - query='builtin:service.keyRequest.count.total:filter(and(or(in("dt.entity.service_method",entitySelector("type(service_method),entityId(~"'+"~\",~\"".join([s["entityId"] for s in existingKRs])+'~")"))))):splitBy("dt.entity.service_method"):sort(value(auto,descending)):limit(20)' - params={"metricSelector": query, "resolution":"1d", "from":"now-1w"} - response = helper.get_request(DTAPIURL, headers, params) - result = (response.json())['result'][0]['data'] - - for k in kr.keyRequests: - if k["exists"]==False: - k['hasData_1W']=None + method["hasData"][tfrom["label"]]=False + - if len(result)> 0: - if len([r for r in result if k["entityId"] in r['dimensions'] and len(r['values']) > 0]) > 0: - k['hasData_1W']=True - else: - k['hasData_1W']=False - - @@ -144,18 +82,19 @@ class KRParser: for gid, group in enumerate(kr.matchedGroups): for method in group["methods"]: if method.startswith('SERVICE_METHOD-'): - tmp={"displayName": None,"comparer": "entityId", "entityId":method, "exists":None, "groupId":gid} + tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None, else: - tmp={"displayName":method,"comparer": "displayName", "entityId":None, "exists":None, "groupId":gid} + tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}} #"exists":None, 'hasData_1W':None, kr.keyRequests.append(tmp) - if self.options and KROption.VALIDATE_EXISTS in self.options: - self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) + # if self.options and KROption.VALIDATE_EXISTS in self.options: + # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) - if KROption.VALIDATE_HASDATA in self.options: - self.checkKeyRequetsHasData(kr,self.DTAPIURL, self.DTAPIToken) + if KROption.VALIDATE_HASDATA in self.options: + self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) return kr