restructured

createKeyRequestReport.py

@@ -1,4 +1,4 @@
-from tracemalloc import start
+#from tracemalloc import start
 from decouple import config
 import sys
 import yaml
@@ -10,37 +10,38 @@ import pandas as pd
 import argparse
 import warnings 
 import os 
-import re
+#import re
 #import glob
 
 import dynatraceAPI
 from pagination import Pagionation
 
 import types
-import SLO
+#import SLO
-from patterns.Pattern1 import Pattern1, Pattern2, Pattern3, Pattern4
+#from patterns.Pattern1 import Pattern1, Pattern2, Pattern3, Pattern4
-import urllib.parse
+#import urllib.parse
 
+from key_request_parser import krparser
 
 warnings.filterwarnings("ignore")
 
 
-patterns=[Pattern1(), Pattern2(), Pattern3(), Pattern4()]
+#patterns=[Pattern1(), Pattern2(), Pattern3(), Pattern4()]
 
-def get_request(url, headers):
+# def get_request(url, headers):
-    try:
+#     try:
-        response = requests.get(url, headers=headers)
+#         response = requests.get(url, headers=headers)
-        response.raise_for_status()
+#         response.raise_for_status()
-    except requests.exceptions.HTTPError as errh:
+#     except requests.exceptions.HTTPError as errh:
-        return "An Http Error occurred:" + repr(errh)
+#         return "An Http Error occurred:" + repr(errh)
-    except requests.exceptions.ConnectionError as errc:
+#     except requests.exceptions.ConnectionError as errc:
-        return "An Error Connecting to the API occurred:" + repr(errc)
+#         return "An Error Connecting to the API occurred:" + repr(errc)
-    except requests.exceptions.Timeout as errt:
+#     except requests.exceptions.Timeout as errt:
-        return "A Timeout Error occurred:" + repr(errt)
+#         return "A Timeout Error occurred:" + repr(errt)
-    except requests.exceptions.RequestException as err:
+#     except requests.exceptions.RequestException as err:
-        return "An Unknown Error occurred" + repr(err)
+#         return "An Unknown Error occurred" + repr(err)
     
-    return response
+#     return response
 
 def getSLO(ENV, DTAPIToken, DTENV):
     # DTENV = base url
@@ -50,179 +51,10 @@ def getSLO(ENV, DTAPIToken, DTENV):
     # gets all slos and filter later
     api_url_report = "/api/v2/slo"
     pages = dtclient.returnPageination(api_url_report, my_params_report, "slo")
-    #only_wanted = [x for x in pages.elements if str.lower(selector) in str.lower(x['description'])]
     df = pd.DataFrame(pages.elements)
     df["env"]=ENV
     return df
 
-
-
-
-# def init_argparse():
-#     parser = argparse.ArgumentParser(
-#         usage="%(prog)s [--fromDate] [toDate] or [preSelect]",
-#         description="gather SLO in daily slices for given Timeframe"
-#     )
-#     parser.add_argument(
-#         "-f","--fromDate",
-#         help = "YYYY-mm-dd e.g. 2022-01-01"
-#     )
-#     parser.add_argument(
-#         "-t","--toDate",
-#         help = "YYYY-mm-dd e.g. 2022-01-31"
-#     )
-#     parser.add_argument(
-#         "-p","--preSelect",
-#         help = "day | week | month - gathers the data for the last full day, week or month"
-#     )
-#     parser.add_argument(
-#         "-s","--slices",
-#         help = "h | d | t | y - writes the slices hourly, daily, total or year to date into ecxel. given in any order"
-#     )
-    
-
-#     return parser
-
-# def check_inputs(args):
-#     '''
-#     This functions is the single point of true for arguments. If new arguments are added they need to be added in here. Returns from and to date.
-#     '''
-#     if args.preSelect and (args.fromDate or args.toDate):
-#         print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate")
-#         sys.exit()
-    
-#     elif args.fromDate and not args.toDate:
-#         print("--fromDate only in conjunction with --toDate")
-#         sys.exit()
-        
-#     elif args.toDate and not args.fromDate:
-#         print("--toDate only in conjunction with --fromDate")
-#         sys.exit()
-    
-#     elif  args.toDate and args.fromDate and not args.preSelect:
-#         try:
-#             #fromDate = datetime.date.fromisoformat(args.fromDate)
-#             fromDate = datetime.datetime.strptime(args.fromDate, "%Y-%m-%d")
-            
-#             #toDate = datetime.date.fromisoformat(args.toDate)
-#             toDate = datetime.datetime.strptime(args.toDate, "%Y-%m-%d")
-#         except Exception as e:
-#             print("Progam closed: " + str(e))
-#             sys.exit()
-        
-#         if toDate < fromDate:
-#             print("--toDate can't be older than --fromDate")
-#             sys.exit()
-        
-#         if toDate > datetime.date.today() or fromDate > datetime.date.today():
-#             print("--toDate or --fromDate can't be in the future")
-#             sys.exit()
-        
-#     elif args.preSelect and not args.fromDate and not args.toDate:
-        
-#         date = datetime.date.today()
-
-#         if args.preSelect == "week":
-#             fromDate, toDate = previous_week_range(date)
-#         elif args.preSelect == "month":
-#             fromDate, toDate = previous_month_range(date)
-#         elif args.preSelect == "day":
-#             fromDate, toDate = previous_day_range(date)        
-#         else:
-#             print("--preSelect must be week or month")
-#             sys.exit()
-#     else:
-#         print("Invalid arguments, please use --help")
-#         sys.exit()
-#     if args.slices == None:
-#         print("-s or --slices must not be null and needs at least one letter of h d t or y, lower- or uppercase.")
-#         sys.exit()
-#     elif sum([1 if one_inp in str.lower(args.slices) else 0 for one_inp in ['h','d','t','y'] ]) == 0:
-#         print("-s or --slices must has at least one letter of h d t or y, lower- or uppercase.")
-#         sys.exit()
-#     return fromDate, toDate
-
-
-
-
-
-
-
-def applyPatterns(subject):
-    groups=None
-    for p in patterns:        
-        groups=p.parseServicesAndMethods(subject)
-        
-        if len(groups) > 0:        
-            break
-     
-    return groups 
-    
-
-def parseAndCreateSLOObject(row):
-    
-    #normalize
-    normFilter=normalize(row['filter'])
-    normExpresseion=normalize(row['metricExpression'])
-    tmp_SLO=SLO.SLO(row["name"],row["env"],normExpresseion,normFilter, None)
-
-    #SLO with Filter
-    if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"):
-       subject=normFilter     
-    else:
-       subject=normExpresseion
-
-    groups=applyPatterns(subject)
-    # for p in patterns:
-    #     #services, methods=p.parseServicesAndMethods(subject)
-        
-    #     groups=p.parseServicesAndMethods(subject)
-    []
-    for g in groups:
-        if g["methods"] != None and len(g["methods"]) > 0:
-                #tmp_SLO.keyRequestGroup.append({"services":services,"methods":methods})
-            tmp_SLO.keyRequestGroup.append(g)
-        
-        # if len(groups) > 0:        
-        #     break
-    
-
-    return tmp_SLO
-    
-
-
-def normalize(x):
-    #tmp=x.replace("~","")
-    tmp=x.replace("\n","")
-    #tmp=tmp.replace("\"/","\"")
-    #tmp=tmp.replace("\"/","") -_>was active
-    #tmp=tmp.replace("/\"","\"")
-    tmp=tmp.replace("/\"","")
-    tmp=tmp.replace("\"","")
-    tmp=tmp.replace("\t","")
-    
-    tmp=re.sub("([\s]*)\)", ")", tmp)
-    
-    
-    
-    tmp=re.sub("\([\s\n\r]*", "(", tmp)
-    tmp=re.sub("\,[\s\n\r]*", ",", tmp)
-    tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
-    tmp=re.sub("in[\s\n\r]*\(", "in(", tmp)
-    
-    
-    
-    #tmp=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: "xxx", tmp,flags=re.IGNORECASE|re.X|re.MULTILINE)
-    
-    return tmp
-
-def getParsedSLOs(ENV, DTTOKEN, DTURL):
-     slos=[]
-     slosF=getSLO(ENV, DTTOKEN, DTURL)
-     for index, row in slosF.iterrows():
-            #if row['id'] == "06292149-0f7f-34f6-b226-dfd9f680486d": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
-                slos.append(parseAndCreateSLOObject(row))
-     return slos
  
 def write_to_excel(ignored, notExists):
     
@@ -236,35 +68,31 @@ def write_to_excel(ignored, notExists):
     writer.close()
 
     
-def getStats(slos):
+def getStats(krs):
     
     
     tmpIgnoredList=[]
-    for slo in [a for a in slos if len(a.keyRequests) == 0]:
+    for kr in [a for a in krs if len(a.keyRequests) == 0]:
-        tmpIgnoredList.append([slo.sloName, slo.env,"",slo.filter, slo.metricExpression])
+        tmpIgnoredList.append([kr.metadata["sloName"], kr.metadata["env"],"","",kr.metadata["filter"], kr.metadata["metricExpression"]])
     
     keyReuqestsNotExists=[]
-    for slo in [s for s in slos if s.hasNotExistingKeyRequests()==True]:
+    for kr in [s for s in krs if s.hasNotExistingKeyRequests()==True]:
-         for k in slo.getNotExistingKeyRequests():
+         for k in kr.getNotExistingKeyRequests():
-             keyReuqestsNotExists.append([slo.sloName, slo.env, k["displayName"], slo.filter, slo.metricExpression])
+             keyReuqestsNotExists.append([kr.metadata["sloName"],  kr.metadata["env"], k["displayName"], k["entityId"], kr.metadata["filter"], kr.metadata["metricExpression"]])
-    #keyrequestsNotExists=pd.DataFrame(flatternList([a for a in slos if len(a.keyRequests) == 0]), columns =['slo', 'env', 'keyRequest', 'filter', 'metricExpression'], dtype = float)  
     
-    ignored=pd.DataFrame(tmpIgnoredList, columns =['slo', 'env', 'keyRequest', 'filter', 'metricExpression'], dtype = float)
+    columns =['slo', 'env', 'displayName','entityId', 'filter', 'metricExpression']
-    notExists=pd.DataFrame(keyReuqestsNotExists, columns =['slo', 'env', 'keyRequest', 'filter', 'metricExpression'], dtype = float)
+    ignored=pd.DataFrame(tmpIgnoredList, columns=columns, dtype = float)
+    notExists=pd.DataFrame(keyReuqestsNotExists, columns = columns, dtype = float)
 
     return ignored, notExists
 
 def main(slo_path):
     
-    
     resultSlos=[]
     
-    
     with open('./environment.yaml') as file:
         env_doc = yaml.safe_load(file)
 
-    
-
     #iterate through all environments
     for env, doc in env_doc.items():    
         token = dict(doc[2])
@@ -275,23 +103,16 @@ def main(slo_path):
                 DTTOKEN = config(token.get('env-token-name'))
                 DTURL = url.get('env-url')
 
-                #getParsedSLOs(DTTOKEN, DTURL)
+                krs=[]
-                slos=getParsedSLOs(env,DTTOKEN, DTURL)
+                krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN)
                 
-                for slo in slos:
+                slosF=getSLO(env, DTTOKEN, DTURL)
-                    slo.checkKeyRequetsExists(DTURL, DTTOKEN)
+                
-
+                for index, row in slosF.iterrows():
-
+                  #if row['id'] == "69c57b3f-725f-36fb-88ef-fb64b03eb0ce": #or row['id'] == "ab1bf34a-10fc-3446-9cc7-79d257498a52":
-
+                   krs.append(krp.parseBySLO(row))
-
+              
-                # count = sum(map(lambda x : len(x.keyRequests) > 0, slos))
+                resultSlos.extend(krs)
-                # count_ignored=sum(map(lambda x : len(x.keyRequests) == 0, slos))
-                # ignored=[a for a in slos if len(a.keyRequests) == 0]
-                # for i in ignored:
-                #     if i.metricExpression:
-                #      print(i.metricExpression+"                "+i.sloName+"\n\n")
-                # x=0
-                resultSlos.extend(slos)
                 
     ignoerd,notExists= getStats(resultSlos)            
     write_to_excel(ignoerd,notExists)

key_request_parser/keyrequests.py

@@ -78,10 +78,9 @@ class KeyRequestGroup(MutableSequence):
             if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0:
                self.insert(len(self._list), val)
 
-
+         
-
+         
-from helper import get_request,contains
+class KR:
-class SLO:
     
      def getNotExistingKeyRequests(self):
          return [k for k in self.keyRequests if k['exists']==False]
@@ -91,59 +90,21 @@ class SLO:
             if k['exists']==False:
                 return True
             
-        return False
+        return False 
-
-     def checkKeyRequetsExists(self, DTAPIURL, DTAPIToken):
- 
-         DTAPIURL = DTAPIURL + "/api/v2/entities"
-
-         headers = {
-            'Content-Type': 'application/json',
-            'Authorization': 'Api-Token ' + DTAPIToken
-         }
-
-         for group in self.keyRequestGroup:
-             params={"entitySelector": group["existsQuery"]}
-             response = get_request(DTAPIURL, headers, params)        
-             entities = (response.json())['entities']
-             
-             for method in group["methods"]:
-               comparer=None
-               if method.startswith('SERVICE_METHOD-'):
-                  comparer="entityId"
-               else:
-                  comparer="displayName"
-
-               found = [x for x in entities if x[comparer] == method]   
-               if len(found) > 0:
-                  #Keyrequest exists 
-                  tmp=found[0]
-                  tmp["exists"]=True
-               else:
-                  #Keyrequest not exists
-                  tmp={"displayName":method,"type": "SERVICE_METHOD", "entityId":method, "exists":False}
-                    
-               self.keyRequests.append(tmp)     
              
 
      def checkKeyRequestsHasData(self):
          pass
 
      def __init__(self,
-      sloName,
+      metadata,
-      env,
+      matchedGroups: KeyRequestGroup = None):
-      metricExpression,
+         self.metadata=metadata
-      filter,
+          
-      keyRequests_groups: KeyRequestGroup = None):
+         if matchedGroups == None:
-         self.sloName=sloName
+            self.matchedGroups = KeyRequestGroup()
-         self.env=env
-         self.metricExpression=metricExpression
-         self.filter=filter
-         
-         if keyRequests_groups == None:
-            self.keyRequestGroup = KeyRequestGroup()
          else:
-            self.keyRequestGroup = keyRequests_groups
+            self.matchedGroups = keyRequests_groups
 
          self.keyRequests=[]
       

key_request_parser/krparser.py

@@ -0,0 +1,161 @@
+import re
+
+from key_request_parser import patterns, keyrequests, helper
+
+from enum import Flag, auto
+
+
+class KROption(Flag):
+    VALIDATE_EXISTS = auto()
+    VALIDATE_HASDATA = auto()
+
+class KRParser:
+    patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3(), patterns.Pattern4()]
+
+    def normalize(self,x):
+        #tmp=x.replace("~","")
+        tmp=x.replace("\n","")
+        #tmp=tmp.replace("\"/","\"")
+        #tmp=tmp.replace("\"/","") -_>was active
+        #tmp=tmp.replace("/\"","\"")
+        tmp=tmp.replace("/\"","")
+        tmp=tmp.replace("\"","")
+        tmp=tmp.replace("\t","")
+        
+        tmp=re.sub("([\s]*)\)", ")", tmp)
+        
+        
+        
+        tmp=re.sub("\([\s\n\r]*", "(", tmp)
+        tmp=re.sub("\,[\s\n\r]*", ",", tmp)
+        tmp=re.sub("\)[\s\n\r]*,", "),", tmp)
+        tmp=re.sub("in[\s\n\r]*\(", "in(", tmp)
+        
+        return tmp
+        
+
+    def applyPatterns(self,subject):
+        groups=None
+        for p in self.patterns:        
+            groups=p.parseServicesAndMethods(subject)
+            
+            if len(groups) > 0:        
+                break
+        
+        return groups 
+  
+    
+    # def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken):
+ 
+    #      DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+    #      headers = {
+    #         'Content-Type': 'application/json',
+    #         'Authorization': 'Api-Token ' + DTAPIToken
+    #      }
+
+    #      for group in kr.matchedGroups:
+    #          params={"entitySelector": group["existsQuery"]}
+    #          response = helper.get_request(DTAPIURL, headers, params)        
+    #          entities = (response.json())['entities']
+             
+    #          for method in group["methods"]:
+    #            comparer=None
+    #            if method.startswith('SERVICE_METHOD-'):
+    #               comparer="entityId"
+    #            else:
+    #               comparer="displayName"
+
+    #            found = [x for x in entities if x[comparer] == method]
+                  
+    #            if len(found) > 0:
+    #               #Keyrequest exists 
+    #               tmp=found[0]
+    #               tmp["exists"]=True
+    #            else:
+    #               #Keyrequest not exists
+    #               tmp={"displayName":method,"type": "SERVICE_METHOD", "entityId":method, "exists":False}
+                    
+    #            kr.keyRequests.append(tmp)
+        
+    #      return kr
+     
+    def checkKeyRequetsExists(self,kr, DTAPIURL, DTAPIToken):
+ 
+         DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+         headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTAPIToken
+         }
+
+         for gid, group in enumerate(kr.matchedGroups):
+             params={"entitySelector": group["existsQuery"]}
+             response = helper.get_request(DTAPIURL, headers, params)        
+             entities = (response.json())['entities']
+             
+             if len(entities) > 0:
+                 y=0
+             for method in kr.keyRequests:
+                 if method["groupId"] == gid:
+                    found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]]
+                 
+                    if len(found) > 0:
+                         method["exists"]=True
+                         method["displayName"]=found[0]["displayName"]
+                         method["entityId"]=found[0]["entityId"]
+                    else:
+                        method["exists"]=False    
+                     
+                    
+    
+    def process(self, kr):
+        
+        for gid, group in enumerate(kr.matchedGroups):
+            for method in group["methods"]:
+               if method.startswith('SERVICE_METHOD-'):
+                    tmp={"displayName": None,"comparer": "entityId", "entityId":method, "exists":None, "groupId":gid}
+               else:
+                    tmp={"displayName":method,"comparer": "displayName", "entityId":None, "exists":None, "groupId":gid}        
+                      
+               kr.keyRequests.append(tmp)
+               
+        
+        if KROption.VALIDATE_EXISTS in self.options:
+          self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken)
+          
+          if KROption.VALIDATE_HASDATA in self.options:
+              x=0
+        
+        return kr
+        
+
+    def parseBySLO(self, row):
+        #normalize
+        normFilter=self.normalize(row['filter'])
+        normExpresseion=self.normalize(row['metricExpression'])
+        
+        tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None})
+    
+
+        #SLO with Filter
+        if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"):
+         subject=normFilter     
+        else:
+         subject=normExpresseion
+
+        groups=self.applyPatterns(subject)
+    
+        for g in groups:
+            if g["methods"] != None and len(g["methods"]) > 0:
+                tmp_KR.matchedGroups.append(g)
+            
+        #self.process(tmp_KR)
+
+        return self.process(tmp_KR)
+        
+    
+    def __init__(self, options: KROption , DTAPIURL, DTAPIToken ):
+        self.DTAPIURL= DTAPIURL
+        self.DTAPIToken=DTAPIToken
+        self.options=options