init

.gitignore

@@ -0,0 +1,138 @@
+.vscode
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Terraform stuff
+**/.terraform/*
+crash.log
+*.tfvars
+
+#excel reports
+*.xlsx

SLO.py

@@ -0,0 +1,133 @@
+try:
+    # Python 3
+    from collections.abc import MutableSequence
+except ImportError:
+    # Python 2.7
+    from collections import MutableSequence
+
+class KeyRequestGroup(MutableSequence):
+    """A container for manipulating lists of hosts"""
+    def __init__(self, data=None):
+        """Initialize the class"""
+        super(KeyRequestGroup, self).__init__()
+        if (data is not None):
+            self._list = list(data)
+        else:
+            self._list = list()
+
+    def __repr__(self):
+        return "<{0} {1}>".format(self.__class__.__name__, self._list)
+
+    def __len__(self):
+        """List length"""
+        return len(self._list)
+
+    def __getitem__(self, ii):
+        """Get a list item"""
+        if isinstance(ii, slice):
+            return self.__class__(self._list[ii])
+        else:
+            return self._list[ii]
+
+    def __delitem__(self, ii):
+        """Delete an item"""
+        del self._list[ii]
+
+    def __setitem__(self, ii, val):
+        # optional: self._acl_check(val)
+        self._list[ii] = val
+
+    def __str__(self):
+        return str(self._list)
+
+    def createExistsQuery(self, val):
+      
+         query="type(service_method)"
+         
+         #case Service Names exists
+         if len(val["services"]) > 0:
+            if val["services"][0].startswith("SERVICE-"):
+                query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))"
+            else:    
+                query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))"
+     
+         if val["methods"][0].startswith("SERVICE_METHOD-"):
+            query+=",entityId(\""+'","'.join(val["methods"])+"\")"
+         else:   
+            query+=",entityName.in(\""+'","'.join(val["methods"])+"\")"
+         
+         val["existsQuery"]= query
+
+
+    def insert(self, ii, val):
+       self.createExistsQuery(val)
+       self._list.insert(ii, val)
+
+           
+
+    def append(self, val):
+
+         if len(self._list) == 0:
+            #self._list.insert(ii, val)
+            self.insert(len(self._list), val)
+            return
+
+         for group in self._list:
+            if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0:
+               self.insert(len(self._list), val)
+
+
+
+from helper import get_request,contains
+class SLO:
+
+     def checkKeyRequetsExists(self, DTAPIURL, DTAPIToken):
+
+           
+         DTAPIURL = DTAPIURL + "/api/v2/entities"
+
+         headers = {
+            'Content-Type': 'application/json',
+            'Authorization': 'Api-Token ' + DTAPIToken
+         }
+
+         for group in self.keyRequestGroup:
+             params={"entitySelector": group["existsQuery"]}
+             response = get_request(DTAPIURL, headers, params)        
+             entities = (response.json())['entities']
+             
+             for method in group["methods"]:
+               comparer=None
+               if method.startswith('SERVICE_METHOD-'):
+                  comparer="entityId"
+               else:
+                  comparer="displayName"
+
+               found = [x for x in entities if x[comparer] == method]   
+               if len(found) > 0:
+                  #Keyrequest exists 
+                  tmp=found[0]
+                  tmp["exists"]=True
+               else:
+                  #Keyrequest not exists
+                  tmp={"displayName":method,"type": "SERVICE_METHOD", "entityId":method, "exists":False}
+                    
+               self.keyRequests.append(tmp)     
+             
+
+     def checkKeyRequestsHasData(self):
+         pass
+
+     def __init__(self,
+      sloName,
+      metricExpression,
+      keyRequests_groups: KeyRequestGroup = None):
+         self.sloName=sloName
+         self.metricExpression=metricExpression
+         if keyRequests_groups == None:
+            self.keyRequestGroup = KeyRequestGroup()
+         else:
+            self.keyRequestGroup = keyRequests_groups
+
+         self.keyRequests=[]   
+         

createKeyRequestReport.py

@@ -0,0 +1,346 @@
+from tracemalloc import start
+from decouple import config
+import sys
+import yaml
+import datetime
+import time
+import pandas as pd
+#import requests
+#import openpyxl
+import argparse
+import warnings 
+import os 
+import re
+#import glob
+
+import dynatraceAPI
+from pagination import Pagionation
+
+import types
+import SLO
+from patterns.Pattern1 import Pattern1, Pattern2, Pattern3
+
+# import importlib
+
+# from dynamic_import import *
+
+# from patterns import a
+
+warnings.filterwarnings("ignore")
+
+
+# module = __import__("patterns.a")
+# my_class = getattr(module, "a.APattern")
+# instance = my_class()
+# mod, modCl = dynamic_imp("patterns", "APattern")
+# modCl.APattern("asdsad")
+# path = os.path.dirname(os.path.abspath(__file__))+"/patterns"
+
+# for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']:
+#     mod = __import__('.'.join([__name__, py]), fromlist=[py])
+#     classes = [getattr(mod, x) for x in dir(mod) if isinstance(getattr(mod, x), type)]
+#     for cls in classes:
+#         setattr(sys.modules[__name__], cls.__name__, cls)
+
+# module = importlib.import_module('patterns.a')
+# my_class = getattr(module, 'MyClass')
+# my_instance = my_class()
+#warning, there are warnings which are ignored!
+
+# patterns=[
+#     {"regex": "type\(\"?service_method\"?\),.*fromRelationship\.isServiceMethodOfService\(.*type\(\"?service\"?\),entityName\.in\(([^\)]*).*,entityName\.in\(([^\)]*)"}
+# ];
+
+patterns=[Pattern1(), Pattern2(), Pattern3()]
+
+def get_request(url, headers):
+    try:
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as errh:
+        return "An Http Error occurred:" + repr(errh)
+    except requests.exceptions.ConnectionError as errc:
+        return "An Error Connecting to the API occurred:" + repr(errc)
+    except requests.exceptions.Timeout as errt:
+        return "A Timeout Error occurred:" + repr(errt)
+    except requests.exceptions.RequestException as err:
+        return "An Unknown Error occurred" + repr(err)
+    
+    return response
+
+def getSLO(DTAPIToken, DTENV):
+    # DTENV = base url
+    # DTAPIToken = sec token
+    dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken)
+    my_params_report = {'pageSize': 25}
+    # gets all slos and filter later
+    api_url_report = "/api/v2/slo"
+    pages = dtclient.returnPageination(api_url_report, my_params_report, "slo")
+    #only_wanted = [x for x in pages.elements if str.lower(selector) in str.lower(x['description'])]
+    df = pd.DataFrame(pages.elements)
+    return df
+
+
+
+
+def init_argparse():
+    parser = argparse.ArgumentParser(
+        usage="%(prog)s [--fromDate] [toDate] or [preSelect]",
+        description="gather SLO in daily slices for given Timeframe"
+    )
+    parser.add_argument(
+        "-f","--fromDate",
+        help = "YYYY-mm-dd e.g. 2022-01-01"
+    )
+    parser.add_argument(
+        "-t","--toDate",
+        help = "YYYY-mm-dd e.g. 2022-01-31"
+    )
+    parser.add_argument(
+        "-p","--preSelect",
+        help = "day | week | month - gathers the data for the last full day, week or month"
+    )
+    parser.add_argument(
+        "-s","--slices",
+        help = "h | d | t | y - writes the slices hourly, daily, total or year to date into ecxel. given in any order"
+    )
+    
+
+    return parser
+
+def check_inputs(args):
+    '''
+    This functions is the single point of true for arguments. If new arguments are added they need to be added in here. Returns from and to date.
+    '''
+    if args.preSelect and (args.fromDate or args.toDate):
+        print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate")
+        sys.exit()
+    
+    elif args.fromDate and not args.toDate:
+        print("--fromDate only in conjunction with --toDate")
+        sys.exit()
+        
+    elif args.toDate and not args.fromDate:
+        print("--toDate only in conjunction with --fromDate")
+        sys.exit()
+    
+    elif  args.toDate and args.fromDate and not args.preSelect:
+        try:
+            #fromDate = datetime.date.fromisoformat(args.fromDate)
+            fromDate = datetime.datetime.strptime(args.fromDate, "%Y-%m-%d")
+            
+            #toDate = datetime.date.fromisoformat(args.toDate)
+            toDate = datetime.datetime.strptime(args.toDate, "%Y-%m-%d")
+        except Exception as e:
+            print("Progam closed: " + str(e))
+            sys.exit()
+        
+        if toDate < fromDate:
+            print("--toDate can't be older than --fromDate")
+            sys.exit()
+        
+        if toDate > datetime.date.today() or fromDate > datetime.date.today():
+            print("--toDate or --fromDate can't be in the future")
+            sys.exit()
+        
+    elif args.preSelect and not args.fromDate and not args.toDate:
+        
+        date = datetime.date.today()
+
+        if args.preSelect == "week":
+            fromDate, toDate = previous_week_range(date)
+        elif args.preSelect == "month":
+            fromDate, toDate = previous_month_range(date)
+        elif args.preSelect == "day":
+            fromDate, toDate = previous_day_range(date)        
+        else:
+            print("--preSelect must be week or month")
+            sys.exit()
+    else:
+        print("Invalid arguments, please use --help")
+        sys.exit()
+    if args.slices == None:
+        print("-s or --slices must not be null and needs at least one letter of h d t or y, lower- or uppercase.")
+        sys.exit()
+    elif sum([1 if one_inp in str.lower(args.slices) else 0 for one_inp in ['h','d','t','y'] ]) == 0:
+        print("-s or --slices must has at least one letter of h d t or y, lower- or uppercase.")
+        sys.exit()
+    return fromDate, toDate
+
+
+
+
+def write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd):
+    touchpoints = ['Vehicle' , 'Mobile']
+    if args.preSelect == 'day':
+        today = datetime.date.today()
+        yesterday = today - datetime.timedelta(days = 1)
+        fileName = "./QM_Report_"+ str(yesterday)  +".xlsx"
+    else:
+        fileName = "./QM_Report_" + str(fromDate.isocalendar()[1]) + ".xlsx"
+
+    writer = pd.ExcelWriter(fileName)
+
+    if not totalall.empty and 't' in str.lower(args.slices):
+        totalall = totalall[totalall['Touchpoint'].isin(touchpoints)]
+        totalall.to_excel(writer, sheet_name='total')
+
+    if not dailyall.empty and 'd' in str.lower(args.slices):
+        dailyall = dailyall[dailyall['Touchpoint'].isin(touchpoints)]
+        dailyall.to_excel(writer, sheet_name='daily')
+    
+    if not hourlyall.empty and 'h' in str.lower(args.slices):
+        hourlyall = hourlyall[hourlyall['Touchpoint'].isin(touchpoints)]
+        hourlyall.to_excel(writer, sheet_name='hourly')
+    
+    if not ytd.empty and 'y' in str.lower(args.slices):
+        ytd = ytd[ytd['Touchpoint'].isin(touchpoints)]
+        ytd.to_excel(writer, sheet_name='YTD') 
+
+    writer.save()
+    writer.close()
+
+
+def testProcess(matches):
+    services=[s.strip() for s in matches.group(1).split(",")] 
+    methods=[s.strip() for s in  matches.group(2).split(",")]
+    
+    query="type(service_method),fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(services)+"\")),entityName.in(\""+'","'.join(methods)+"\")"
+
+#     env = DTENV
+#     DTAPIToken = DTAPIToken
+     
+
+#     DTAPIURL = env + "/api/v2/apiTokens"
+
+#     headers = {
+#     'Content-Type': 'application/json',
+#     'Authorization': 'Api-Token ' + DTAPIToken
+#      }
+
+   
+#    data = '{"name":"' + tokenname + '","scopes":["InstallerDownload","ReadConfig","WriteConfig","DataExport"]}'
+#   #data = '{"name":"' + tokenname + '","scopes":["ReadConfig","DataExport","entities.read"]}'
+
+#    r = post_request(DTAPIURL,headers,data, proxies)
+
+    return 
+
+
+def parseAndCreateSLOObject(row):
+    
+
+    normalizedMetric=normalize(row['metricExpression'])
+    tmp_SLO=SLO.SLO(row["name"], normalizedMetric, None)
+    
+    for p in patterns:
+        services, methods=p.parseServicesAndMethods(normalizedMetric)
+    
+        if methods != None and len(methods) > 0:
+            tmp_SLO.keyRequestGroup.append({"services":services,"methods":methods})
+            break
+    
+
+    return tmp_SLO
+    
+
+
+def normalize(x):
+    tmp=x.replace("~","")
+    tmp=tmp.replace("\n","")
+    #tmp=tmp.replace("\"/","\"")
+    tmp=tmp.replace("\"/","")
+    #tmp=tmp.replace("/\"","\"")
+    tmp=tmp.replace("/\"","")
+    tmp=tmp.replace("\"","")
+    return tmp
+
+
+# def start(dfSLOS):
+#     #dfSLOS['normalizedMetricExpression']= dfSLOS.apply(lambda row : normalize(row['metricExpression'], row['filter']))
+    
+#     for index, row in dfSLOS.iterrows():
+#         #if row["id"]=="3a2d280d-83bb-3e45-bafc-e79c1429c79b":
+#              findKeyrequests(normalize(row['metricExpression']))
+
+def main(slo_path):
+    
+    with open('./environment.yaml') as file:
+        env_doc = yaml.safe_load(file)
+
+    slos=[]
+
+    #iterate through all environments
+    for item, doc in env_doc.items():    
+        token = dict(doc[2])
+        url = dict(doc[1])
+
+        if(config(token.get('env-token-name')) != ""):
+                print("Gather data, hold on a minute")
+                DTTOKEN = config(token.get('env-token-name'))
+                DTURL = url.get('env-url')
+
+                slosF=getSLO(DTTOKEN, DTURL)
+                for index, row in slosF.iterrows():
+                    #if row['id'] == "75165058-75c6-385e-a78e-b6ea3457f87d":
+                        slos.append(parseAndCreateSLOObject(row))
+                        print("huhu")
+                
+                for slo in slos:
+                    slo.checkKeyRequetsExists(DTURL, DTTOKEN)
+
+                x=0
+
+
+    # slo_configs = load_slo_parameter(slo_path)
+    
+    # for one_slo_config in slo_configs:
+    #     hub, selector_type, selector_var, yearstart, header_name = one_slo_config
+    #     print(f"For the slo config was '{slo_path}' used with the config '{header_name}'.")
+    #     for item, doc in env_doc.items():
+    #         if not item in hub:
+    #             print(f"{item} will be skipped since it is not in {hub}, which was selected in {slo_path}")
+    #             continue
+    #         token = dict(doc[2])
+    #         url = dict(doc[1])
+    #         print("Crawling through: " + item)
+    #         print("Check if token exists in environment...")
+    #         if(config(token.get('env-token-name')) != ""):
+    #             print("Gather data, hold on a minute")
+    #             DTTOKEN = config(token.get('env-token-name'))
+    #             DTURL = url.get('env-url')
+                
+    #             ###Calc daily SLO
+    #             if 'd' in str.lower(args.slices):
+    #                 dailyall = get_one_slice(item, DTTOKEN, DTURL, days, dailyall, selector_var, selector_type)
+    #             #Calc hourly SLO
+    #             if 'h' in str.lower(args.slices):
+    #                 hourlyall = get_one_slice(item, DTTOKEN, DTURL, hours, hourlyall, selector_var, selector_type)
+    #         ###Calc Overall YTD SLO
+    #             if 'y' in str.lower(args.slices):
+    #                 ytd = get_slice_ytd_total(DTTOKEN,DTURL,item, yearstart, days['endTime'].max(), 'Date', fromDate.year, ytd, selector_var, selector_type)
+    #         ###Calc Overall SLO
+    #             if 't' in str.lower(args.slices):
+    #                 totalall = get_slice_ytd_total(DTTOKEN,DTURL,item, days['startTime'].min(), days['endTime'].max(), 'Date', fromDate.isocalendar()[1], totalall, selector_var, selector_type)
+    #         else: 
+    #             print("token not found, skipping " + item)
+    #         write_slo_to_excel(args, fromDate, hourlyall, dailyall, totalall, ytd)
+    # print("It took {} seconds to run this script".format(time.time()-start_timer))
+
+if __name__ == "__main__":
+
+    # for file in glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "*.py")):
+    #    name = os.path.splitext(os.path.basename(file))[0]
+    #    if name == "Animal" or name == "main":  # avoid main.py and Animal.py
+    #        continue
+    #    # add package prefix to name, if required
+    #    module = __import__(name)
+    #    try:
+    #        # get the class
+    #        cls = getattr(module, name)
+    #        # instantiate the class and call the method
+    #        cls().feed()
+    #    except Exception as e:
+    #         print(e)
+
+    main('./slo_parameter.yaml')

dynatraceAPI.py

@@ -0,0 +1,40 @@
+import logging
+from typing import Dict
+import os
+import sys
+
+file_dir = os.path.dirname(__file__)
+sys.path.append(file_dir)
+
+from more_utils.httpClient import HttpClient
+import pagination
+
+
+class Dynatrace:
+    def __init__(
+        self,
+        base_url: str,
+        token: str,
+        log: logging.Logger = None,
+        proxies: Dict = None,
+        too_many_requests_strategy=None,
+        retries: int = 0,
+        retry_delay_ms: int = 0,
+        
+    ):
+        self.__http_client = HttpClient(
+            base_url, token, log, proxies, too_many_requests_strategy, retries, retry_delay_ms
+        )
+
+    def returnPageination(self,path,params,list_item):
+        page = pagination.Pagionation(self.__http_client,path,params,list_item=list_item)
+        return page
+    
+    def returnSingle(self,path):
+        response = self.__http_client.make_request(path)
+        json_response = response.json()
+        return json_response
+
+
+
+

environment.yaml

@@ -0,0 +1,31 @@
+---
+euprod:
+    - name: "EUprod"
+    - env-url: "https://xxu26128.live.dynatrace.com"
+    - env-token-name: "EUPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+#eupreprod:
+    - name: "eupreprod"
+    - env-url: "https://qqk70169.live.dynatrace.com"
+    - env-token-name: "EUPREPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+#napreprod:
+    - name: "napreprod"
+    - env-url: "https://onb44935.live.dynatrace.com"
+    - env-token-name: "NAPREPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+naprod:
+    - name: "naprod"
+    - env-url: "https://wgv50241.live.dynatrace.com"
+    - env-token-name: "NAPROD_TOKEN_VAR"
+    - jenkins: "https://jaws.bmwgroup.net/opapm/"
+cnprod:
+    - name: "cnprod"
+    - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b"
+    - env-token-name: "CNPROD_TOKEN_VAR"
+    - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"
+#cnpreprod:
+    - name: "cnpreprod"
+    - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35"
+    - env-token-name: "CNPREPROD_TOKEN_VAR"
+    - jenkins: "https://jaws-china.bmwgroup.net/opmaas/"

helper.py

@@ -0,0 +1,22 @@
+import requests
+
+def get_request(url, headers, params):
+      try:
+         response = requests.get(url, headers=headers, params=params)
+         response.raise_for_status()
+      except requests.exceptions.HTTPError as errh:
+         return "An Http Error occurred:" + repr(errh)
+      except requests.exceptions.ConnectionError as errc:
+         return "An Error Connecting to the API occurred:" + repr(errc)
+      except requests.exceptions.Timeout as errt:
+         return "A Timeout Error occurred:" + repr(errt)
+      except requests.exceptions.RequestException as err:
+         return "An Unknown Error occurred" + repr(err)
+      
+      return response
+
+def contains(list, filter):
+            for x in list:
+               if filter(x):
+                  return True
+            return False

more_utils/httpClient.py

@@ -0,0 +1,116 @@
+import logging
+from typing import Dict, Optional, Any
+import time
+
+import requests
+import urllib3
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+TOO_MANY_REQUESTS_WAIT = "wait"
+
+
+##Not sure where/why this is here
+class DynatraceRetry(Retry):
+    def get_backoff_time(self):
+        return self.backoff_factor
+
+
+class HttpClient:
+    def __init__(
+        self,
+        base_url: str,
+        token: str,
+        log: logging.Logger = None,
+        proxies: Dict = None,
+        too_many_requests_strategy=None,
+        retries: int = 0,
+        retry_delay_ms: int = 0,
+        #mc_jsession_id: Optional[str] = None,
+        #mc_b925d32c: Optional[str] = None,
+        #mc_sso_csrf_cookie: Optional[str] = None,
+    ):
+        while base_url.endswith("/"):
+            base_url = base_url[:-1]
+        self.base_url = base_url
+        
+        if proxies is None:
+            proxies = {}
+        self.proxies = proxies
+
+        self.auth_header = {"Authorization": f"Api-Token {token}"}
+        self.log = log
+        if self.log is None:
+            self.log = logging.getLogger(__name__)
+            self.log.setLevel(logging.WARNING)
+            st = logging.StreamHandler()
+            fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(thread)d - %(filename)s:%(lineno)d - %(message)s")
+            st.setFormatter(fmt)
+            self.log.addHandler(st)
+
+        self.too_many_requests_strategy = too_many_requests_strategy
+        retry_delay_s = retry_delay_ms / 1000
+
+        try:
+            self.retries = Retry(
+                total=retries,
+                backoff_factor=retry_delay_s,
+                status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504],
+                allowed_methods=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"],
+                raise_on_status=False,
+            )
+        except TypeError:  # Older version of urllib3?
+            self.retries = Retry(
+                total=retries,
+                backoff_factor=retry_delay_s,
+                status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504],
+                method_whitelist=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"],
+                raise_on_status=False,
+            )
+
+        # This is for internal dynatrace usage
+        #self.mc_jsession_id = mc_jsession_id
+        #self.mc_b925d32c = mc_b925d32c
+        #self.mc_sso_csrf_cookie = mc_sso_csrf_cookie
+
+    def make_request(
+        self, path: str, params: Optional[Any] = None, headers: Optional[Dict] = None, method="GET", data=None, files=None, query_params=None
+    ) -> requests.Response:
+        url = f"{self.base_url}{path}"
+
+        body = None
+        if method in ["POST", "PUT"]:
+            body = params
+            params = query_params
+
+        if headers is None:
+            headers = {}
+        if files is None and "content-type" not in [key.lower() for key in headers.keys()]:
+            headers.update({"content-type": "application/json"})
+        headers.update(self.auth_header)
+
+        cookies = None
+        #if self.mc_b925d32c and self.mc_sso_csrf_cookie and self.mc_jsession_id:
+        #    headers.update({"Cookie": f"JSESSIONID={self.mc_jsession_id}; ssoCSRFCookie={self.mc_sso_csrf_cookie}; b925d32c={self.mc_b925d32c}"})
+        #    cookies = {"JSESSIONID": self.mc_jsession_id, "ssoCSRFCookie": self.mc_sso_csrf_cookie, "b925d32c": self.mc_b925d32c}
+
+        s = requests.Session()
+        s.mount("https://", HTTPAdapter(max_retries=self.retries))
+
+        self.log.debug(f"Making {method} request to '{url}' with params {params} and body: {body}")
+        r = s.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies, data=data, cookies=cookies, files=files)
+        self.log.debug(f"Received response '{r}'")
+
+        while r.status_code == 429 and self.too_many_requests_strategy == TOO_MANY_REQUESTS_WAIT:
+            sleep_amount = int(r.headers.get("retry-after", 5))
+            self.log.warning(f"Sleeping for {sleep_amount}s because we have received an HTTP 429")
+            time.sleep(sleep_amount)
+            r = requests.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies)
+
+        if r.status_code >= 400:
+            raise Exception(f"Error making request to {url}: {r}. Response: {r.text}")
+
+        return r

pagination.py

@@ -0,0 +1,68 @@
+from more_utils.httpClient import HttpClient
+
+
+
+class Pagionation():
+    def __init__(self, http_client, target_url, target_params=None, headers=None, list_item="result"):
+        #self.__target_class = target_class
+        self.__http_client: HttpClient = http_client
+        self.__target_url = target_url
+        self.__target_params = target_params
+        self.__headers = headers
+        self.__list_item = list_item
+        self._has_next_page = True
+        self.__total_count = None
+        self.__page_size = None
+        self.elements = self._get_next_page()
+
+        
+
+    def __iter__(self):# -> Iterator[T]:
+        for element in self.__elements:
+            yield element
+
+        while self._has_next_page:
+            new_elements = self._get_next_page()
+            for element in new_elements:
+                yield element
+
+
+    def __len__(self):
+        return self.__total_count or len(self.__elements)
+
+    def _get_next_page(self):
+        response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers)
+        json_response = response.json()
+        data = []
+        if json_response.get("nextPageKey", None):
+            self._has_next_page = True
+            self.__target_params = {"nextPageKey": json_response["nextPageKey"]}
+        else:
+            self._has_next_page = False
+
+        if self.__list_item in json_response:
+            elements = json_response[self.__list_item]
+            self.__total_count = json_response.get("totalCount") or len(elements)
+
+        while self._has_next_page == True:
+            self.__target_url = self.__target_url.split("?")[0]
+            elements += self._get_response()
+            
+        return elements
+
+    def _get_response(self):
+        response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers)
+        json_response = response.json()
+        data = []
+        if json_response.get("nextPageKey", None):
+            self._has_next_page = True
+            self.__target_params = {"nextPageKey": json_response["nextPageKey"]}
+        else:
+            self._has_next_page = False
+
+        if self.__list_item in json_response:
+            elements = json_response[self.__list_item]
+            self.__total_count = json_response.get("totalCount") or len(elements)
+
+        
+        return elements

patterns/Pattern1.py

@@ -0,0 +1,41 @@
+import re
+
+class Pattern1:
+
+   def parseServicesAndMethods(self, metricExpression):
+         result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*\.[\s\n\r]*in[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[\s\n\r]*in\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+         services=[]
+         methods=[]
+         if result:
+            for r in result:
+                services=[s.strip() for s in r[0].split(",")] 
+                methods=[s.strip() for s in r[1].split(",")] 
+           
+         return services, methods
+
+class Pattern2:
+
+   def parseServicesAndMethods(self, metricExpression):
+         result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*\.[\s\n\r]*in[\s\n\r]*\([\s\n\r]*([^\)]*)\),[\s\n\r]*tag\([^\)]*[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[\s\n\r]*in\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+         services=[]
+         methods=[]
+         if result:
+            for r in result:
+                services=[s.strip() for s in r[0].split(",")] 
+                methods=[s.strip() for s in r[1].split(",")] 
+           
+         return services, methods
+
+
+
+class Pattern3:
+
+   def parseServicesAndMethods(self, metricExpression):
+         result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE)
+         services=[]
+         methods=[]
+         if result:
+            for r in result:
+                methods=[s.strip() for s in r.split(",")] 
+           
+         return services, methods

requirements.txt

@@ -0,0 +1,7 @@
+python-decouple
+pyyaml
+pandas
+requests
+datetime
+argparse
+openpyxl