commit 2c7d9912585889b3071cb0d11ffa751ba70ec7b4 Author: Selimovic Amsal Date: Fri Apr 21 12:31:32 2023 +0200 Initial Commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0738441 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +.vscode +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Terraform stuff +**/.terraform/* +crash.log +*.tfvars + +#excel reports +*.xlsx \ No newline at end of file diff --git a/create_report.py b/create_report.py new file mode 100644 index 0000000..efd1603 --- /dev/null +++ b/create_report.py @@ -0,0 +1,153 @@ +import yaml +from decouple import config +import dynatraceAPI +import pandas as pd +from pagination import Pagionation +from key_request_parser import krparser + +from datetime import datetime, timedelta +import datetime +import re + +def getSLO(ENV, DTAPIToken, DTENV): + # DTENV = base url + # DTAPIToken = sec token + dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken) + my_params_report = {'pageSize': 25} + # gets all slos and filter later + api_url_report = "/api/v2/slo" + pages = dtclient.returnPageination(api_url_report, my_params_report, "slo") + df = pd.DataFrame(pages.elements) + df["env"]=ENV + return df + +def previous_week_range(date: datetime): + start_date = date + timedelta(-date.weekday(), weeks=-1) + end_date = date + timedelta(-date.weekday() -1) + return start_date, end_date + + +def main(): +#Get All SLOs + + reportItem={} + with open('./environment.yaml') as file: + env_doc = yaml.safe_load(file) + + for env, doc in env_doc.items(): + token = dict(doc[2]) + url = dict(doc[1]) + + if(config(token.get('env-token-name')) != ""): + print("Gather data, hold on a minute") + DTTOKEN = config(token.get('env-token-name')) + DTURL = url.get('env-url') + + #krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA ,DTURL, DTTOKEN) + + slosF = getSLO(env, DTTOKEN, DTURL) + slosF = slosF[slosF['name'].str.startswith('TP_')] + + + +#parse the metric Expression to get Services and Requests + + krs=[] + krp = krparser.KRParser(krparser.KROption.VALIDATE_EXISTS | krparser.KROption.VALIDATE_HASDATA | krparser.KROption.RESOLVESERVICES, DTURL, DTTOKEN) + + for index, row in slosF.iterrows(): + krs.append(krp.parseBySLO(row)) + + x=0 + + #SLO Name | SERVICE | PROCESS GRUPPE | TAGS + {"sloname": { + "sloname":$sloname$, + "services":[{ + "serviceName": "$servicename$" + }] + }, + "sloname": { + "sloname":$sloname$, + "services":[{ + "serviceName": "$servicename$" + }] + # { + "SLO 1": { + "sloname": "SLO 1", + "services": [ + { + "serviceName": "Service 1", + "processGroup": "Process Group 1", + "tags": ["tag1", "tag2"] + }, + { + "serviceName": "Service 2", + "processGroup": "Process Group 2", + "tags": ["tag3", "tag4"] + } + ] + }, + "SLO 2": { + "sloname": "SLO 2", + "services": [ + { + "serviceName": "Service 3", + "processGroup": "Process Group 3", + "tags": ["tag5", "tag6"] + } + ] + } +} +# + }} + for kr in krs: + reportItem[kr.metadata.name]= + + # DEBUG + # print(krs) + for x in krs: + print(x.metadata) + + + # regex ~ XYZ ~ + + #filtering the SLO dataframe based on those dates and the service name + + +# import datetime + + +# import re + +#extract all the service names that start with "TP_" +metric_expression = 'sum(duration), filter(entity.service.name, beginsWith("TP_"))' +# Define regex pattern to match service names +regex_pattern = r'beginsWith\("TP_(.*?)"\)' +# Find all matches of pattern in metric expression +matches = re.findall(regex_pattern, metric_expression) + +print(matches) + + +#get detail info service names +import requests +import json +environment_id = "ENVIRONMENT_ID" +api_token = "dt0c01.MBX344ELRJNDGSWGOSK3JBE5.KHJ6BLKLZ6UB6NKJQRXYINLDCKN36HGOK6ECYAOZ3REF2KHWM2NIBIKOITVXCWUT" +headers = {"Authorization": "Api-Token {}".format(api_token)} + +# url = "https://xxu26128.live.dynatrace.com/api/v1/entity/services".format(environment_id) +params = {"filter": "displayName LIKE 'TP_%'"} + +# response = requests.get(url, headers=headers, params=params) +# response_json = json.loads(response.text) + +# for service in response_json["entities"]: +# display_name = service["displayName"] +# print("Service Name: {0}".format(display_name)) + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dynatraceAPI.py b/dynatraceAPI.py new file mode 100644 index 0000000..9e59f9a --- /dev/null +++ b/dynatraceAPI.py @@ -0,0 +1,40 @@ +import logging +from typing import Dict +import os +import sys + +file_dir = os.path.dirname(__file__) +sys.path.append(file_dir) + +from more_utils.httpClient import HttpClient +import pagination + + +class Dynatrace: + def __init__( + self, + base_url: str, + token: str, + log: logging.Logger = None, + proxies: Dict = None, + too_many_requests_strategy=None, + retries: int = 0, + retry_delay_ms: int = 0, + + ): + self.__http_client = HttpClient( + base_url, token, log, proxies, too_many_requests_strategy, retries, retry_delay_ms + ) + + def returnPageination(self,path,params,list_item): + page = pagination.Pagionation(self.__http_client,path,params,list_item=list_item) + return page + + def returnSingle(self,path): + response = self.__http_client.make_request(path) + json_response = response.json() + return json_response + + + + diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..9d2b15c --- /dev/null +++ b/environment.yaml @@ -0,0 +1,16 @@ +--- +euprod: + - name: "emeaprod" + - env-url: "https://xxu26128.live.dynatrace.com" + - env-token-name: "EUPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" +naprod: + - name: "naprod" + - env-url: "https://wgv50241.live.dynatrace.com" + - env-token-name: "NAPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" +cnprod: + - name: "cnprod" + - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" + - env-token-name: "CNPROD_TOKEN_VAR" + - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" \ No newline at end of file diff --git a/key_request_parser/__init__.py b/key_request_parser/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/key_request_parser/helper.py b/key_request_parser/helper.py new file mode 100644 index 0000000..4c311b0 --- /dev/null +++ b/key_request_parser/helper.py @@ -0,0 +1,22 @@ +import requests + +def get_request(url, headers, params): + #try: + response = requests.get(url, headers=headers, params=params, verify=False) + response.raise_for_status() + # except requests.exceptions.HTTPError as errh: + # return "An Http Error occurred:" + repr(errh) + # except requests.exceptions.ConnectionError as errc: + # return "An Error Connecting to the API occurred:" + repr(errc) + # except requests.exceptions.Timeout as errt: + # return "A Timeout Error occurred:" + repr(errt) + # except requests.exceptions.RequestException as err: + # return "An Unknown Error occurred" + repr(err) + + return response + +def contains(list, filter): + for x in list: + if filter(x): + return True + return False \ No newline at end of file diff --git a/key_request_parser/keyrequests.py b/key_request_parser/keyrequests.py new file mode 100644 index 0000000..e23e2fa --- /dev/null +++ b/key_request_parser/keyrequests.py @@ -0,0 +1,154 @@ +try: + # Python 3 + from collections.abc import MutableSequence +except ImportError: + # Python 2.7 + from collections import MutableSequence + +class KeyRequestGroup(MutableSequence): + """A container for manipulating lists of hosts""" + def __init__(self, data=None): + """Initialize the class""" + super(KeyRequestGroup, self).__init__() + if (data is not None): + self._list = list(data) + else: + self._list = list() + + def __repr__(self): + return "<{0} {1}>".format(self.__class__.__name__, self._list) + + def __len__(self): + """List length""" + return len(self._list) + + def __getitem__(self, ii): + """Get a list item""" + if isinstance(ii, slice): + return self.__class__(self._list[ii]) + else: + return self._list[ii] + + def __delitem__(self, ii): + """Delete an item""" + del self._list[ii] + + def __setitem__(self, ii, val): + # optional: self._acl_check(val) + self._list[ii] = val + + def __str__(self): + return str(self._list) + + def createExistsQuery(self, val): + + query="type(service_method)" + + val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + val['methods'] = list(map(lambda x: x.replace("~","") , val['methods'])) + #case Service Names exists + if len(val["services"]) > 0: + if val["services"][0].startswith("SERVICE-"): + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(val["services"])+"\"))" + else: + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(val["services"])+"\"))" + + if val["methods"][0].startswith("SERVICE_METHOD-"): + query+=",entityId(\""+'","'.join(val["methods"])+"\")" + else: + query+=",entityName.in(\""+'","'.join(val["methods"])+"\")" + + val["existsQuery"]= query + + # def createServiceResolveQuery(self, val): + # query="type(SERVICE)" + # val['services'] = list(map(lambda x: x.replace("~","") , val['services'])) + + # if len(val["services"]) > 0: + # if val["services"][0].startswith("SERVICE-"): + # query+=",entityId(\""+'","'.join(val["services"])+"\")" + # else: + # query+=",entityName.in(\""+'","'.join(val["services"])+"\")" + + # val["resolveServiceQuery"]= query + + + def insert(self, ii, val): + self.createExistsQuery(val) + #self.createServiceResolveQuery(val) + self._list.insert(ii, val) + + + + def append(self, val): + + if len(self._list) == 0: + #self._list.insert(ii, val) + self.insert(len(self._list), val) + return + + for group in self._list: + if len(set(group["services"]) - set(val["services"])) > 0 or len(set(group["methods"]) - set(val["methods"])) > 0: + self.insert(len(self._list), val) + + + +class KR: + + # def getNotExistingKeyRequests(self): + # return [k for k in self.keyRequests if k['exists']==False] + + # def hasNotExistingKeyRequests(self): + # for k in self.keyRequests: + # if k['exists']==False: + # return True + + # return False + + # def getNoData1WKeyRequests(self): + # return [k for k in self.keyRequests if k['hasData_1W']==False and k['exists']==True] + + # def hasNoData1WKeyRequests(self): + # for k in self.keyRequests: + # if k['hasData_1W']==False and k['exists'] == True: + # return True + + # return False + + def getKeyRequestByHasData(self,label): + return [k for k in self.keyRequests if k['hasData'][label]==False] + + def hasNoData(self,label): + for k in self.keyRequests: + if k['hasData'][label]==False: + return True + + return False + + + + + def checkKeyRequestsHasData(self): + pass + + def mergeServices(self, listServices): + listOfServiceIds=[o["entityId"] for o in self.services] + + for s in listServices: + if s["entityId"] not in listOfServiceIds: + self.services.append(s) + + def __init__(self, + metadata, + matchedGroups: KeyRequestGroup = None): + self.metadata=metadata + + if matchedGroups == None: + self.matchedGroups = KeyRequestGroup() + else: + self.matchedGroups = keyRequests_groups + + self.keyRequests=[] + self.services=[] + + diff --git a/key_request_parser/krparser.py b/key_request_parser/krparser.py new file mode 100644 index 0000000..2ac42a0 --- /dev/null +++ b/key_request_parser/krparser.py @@ -0,0 +1,211 @@ +import re + +from key_request_parser import patterns, keyrequests, helper + +from enum import Flag, auto + + +class KROption(Flag): + VALIDATE_EXISTS = auto() + VALIDATE_HASDATA = auto() + RESOLVEKEYREQUETS = auto() + RESOLVESERVICES = auto() + +class KRParser: + patterns=[patterns.Pattern1(), patterns.Pattern2(), patterns.Pattern3()] + + def normalize(self,x): + #tmp=x.replace("~","") + tmp=x.replace("\n","") + #tmp=tmp.replace("\"/","\"") + #tmp=tmp.replace("\"/","") -_>was active + #tmp=tmp.replace("/\"","\"") + tmp=tmp.replace("/\"","") + tmp=tmp.replace("\"","") + tmp=tmp.replace("\t","") + + tmp=re.sub("([\s]*)\)", ")", tmp) + + + + tmp=re.sub("\([\s\n\r]*", "(", tmp) + tmp=re.sub("\,[\s\n\r]*", ",", tmp) + tmp=re.sub("\)[\s\n\r]*,", "),", tmp) + tmp=re.sub("in[\s\n\r]*\(", "in(", tmp) + + return tmp + + + def applyPatterns(self,subject): + groups=None + for p in self.patterns: + groups=p.parseServicesAndMethods(subject) + + if len(groups) > 0: + break + + return groups + + + def checkKeyRequetsHasData(self,kr, tfrom, DTAPIURL, DTAPIToken): + + DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + for gid, group in enumerate(kr.matchedGroups): + params={"entitySelector": group["existsQuery"], "from":tfrom["tfrom"], "fields": "fromRelationships"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + if len(entities) > 0: + y=0 + for method in kr.keyRequests: + if method["groupId"] == gid: + found = [x for x in entities if x[method["comparer"]] == method[method["comparer"]]] + + if len(found) > 0: + method["hasData"][tfrom["label"]]=True + #method["displayName"]=found[0]["displayName"] + #method["entityId"]=found[0]["entityId"] + #method["services"]=found[0]["fromRelationships"]["isServiceMethodOfService"] + + # for idx,o in enumerate(method["services"]): + # tmpS=[p for p in kr.services if p["entityId"]==o["id"]] + # if len(tmpS)>0: + # method["services"][idx]=tmpS[0] + + else: + method["hasData"][tfrom["label"]]=False + + def resolveServices(self,services, DTAPIURL, DTAPIToken): + #DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + for gid, service in enumerate(services): + query="type(SERVICE),entityId("+service["id"]+")" + params={"entitySelector": query,"from":"now-2y"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + if len(entities)>0: + services[gid]=entities[0] + + def resolveKeyRequests(self,kr, DTAPIURL, DTAPIToken, options): + DTAPIURL = DTAPIURL + "/api/v2/entities" + + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + + for gid, k in enumerate(kr.keyRequests): + try: + query="type(service_method)" + group=kr.matchedGroups[k["groupId"]] + + if len(group["services"])> 0: + if group["services"][0].startswith("SERVICE-"): + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityId(\""+'","'.join(group["services"])+"\"))" + else: + query+=",fromRelationship.isServiceMethodOfService(type(\"SERVICE\"),entityName.in(\""+'","'.join(group["services"])+"\"))" + + if k["comparer"]=="entityId": + query+=",entityId("+k["entityId"]+")" + else: + query+=",entityName.in(\""+k["displayName"]+"\")" + + params={"entitySelector": query, "from":"now-2y","fields": "fromRelationships"} + response = helper.get_request(DTAPIURL, headers, params) + entities = (response.json())['entities'] + + # if len(entities) > 1: + # kr.keyRequests[gid]['foundCount']=len(entities) + # print("Multiple keyrequest found: ") + + if len(entities)> 0: + kr.keyRequests[gid]["found"]=True + kr.keyRequests[gid]['foundCount']=len(entities) + kr.keyRequests[gid]["displayName"]=entities[0]["displayName"] + kr.keyRequests[gid]["entityId"]=entities[0]["entityId"] + + if "isServiceMethodOfService" in entities[0]["fromRelationships"]: + kr.keyRequests[gid]["services"]=entities[0]["fromRelationships"]["isServiceMethodOfService"] + + if KROption.RESOLVESERVICES in options and len( kr.keyRequests[gid]["services"])>0: + self.resolveServices(kr.keyRequests[gid]["services"], DTAPIURL, DTAPIToken) + + except Exception as err: + kr.keyRequests[gid]["exception"]="resolveKeyRequests failed: "+repr(err) + + + + #kr.mergeServices(entities) + + + + + def process(self, kr): + + for gid, group in enumerate(kr.matchedGroups): + for method in group["methods"]: + if method.startswith('SERVICE_METHOD-'): + tmp={"displayName": None,"comparer": "entityId", "entityId":method, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + else: + tmp={"displayName":method,"comparer": "displayName", "entityId":None, "groupId":gid, "hasData":{}, "services":[], "found":False, "foundCount":0, "exception":""} #"exists":None, 'hasData_1W':None, + + kr.keyRequests.append(tmp) + + + # if self.options and KROption.VALIDATE_EXISTS in self.options: + # self.checkKeyRequetsExists(kr,self.DTAPIURL, self.DTAPIToken) + self.resolveKeyRequests(kr,self.DTAPIURL, self.DTAPIToken, self.options) + + if KROption.VALIDATE_HASDATA in self.options: + self.checkKeyRequetsHasData(kr,{"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + self.checkKeyRequetsHasData(kr,{"label":"1M", "tfrom":"now-1M"},self.DTAPIURL, self.DTAPIToken) + # elif KROption.RESOLVEKEYREQUETS in self.options: + # self.checkKeyRequetsHasData(kr, {"label":"1W", "tfrom":"now-1w"},self.DTAPIURL, self.DTAPIToken) + # if KROption.RESOLVESERVICES in self.options: + # self.resolveServices(kr,self.DTAPIURL, self.DTAPIToken) + + return kr + + + def parseBySLO(self, row): + #normalize + normFilter=self.normalize(row['filter']) + normExpresseion=self.normalize(row['metricExpression']) + + tmp_KR = keyrequests.KR({"sloName":row["name"], "env":row["env"], "metricExpression": normExpresseion, "filter": normFilter, "matchedGroups": None}) + + + #SLO with Filter + if normFilter.upper().startswith("TYPE(SERVICE_METHOD),"): + subject=normFilter + else: + subject=normExpresseion + + groups=self.applyPatterns(subject) + + for g in groups: + if g["methods"] != None and len(g["methods"]) > 0: + tmp_KR.matchedGroups.append(g) + + #self.process(tmp_KR) + + return self.process(tmp_KR) + + + def __init__(self, options: KROption , DTAPIURL, DTAPIToken ): + self.DTAPIURL= DTAPIURL + self.DTAPIToken=DTAPIToken + self.options=options \ No newline at end of file diff --git a/key_request_parser/patterns.py b/key_request_parser/patterns.py new file mode 100644 index 0000000..c724a73 --- /dev/null +++ b/key_request_parser/patterns.py @@ -0,0 +1,88 @@ +import re +import urllib + +class Pattern1: + + + def parseServicesAndMethods(self, metricExpression): + + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*[\)]*", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*]*[\"]*)[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*([\"]*[^.*\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*\")[\s\n\r]*\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*\.[in]*[\s\n\r]*\([\s\n\r]*(\"[^.*]*)\)\"", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + + #Endoce + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\(\"?service\"?\),entityName[\.]*[in]*\(([^\)]*)\)\)\,entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*fromRelationship[\s\n\r]*\.[\s\n\r]*isServiceMethodOfService[\s\n\r]*\([\s\n\r]*type\(\"?service\"?\)[\s\n\r]*,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*[\s\n\r]*\([\s\n\r]*([^\)]*)\)[\s\n\r]*\)[\s\n\r]*\,[\s\n\r]*entityName[\s\n\r]*[\.]*[\s\n\r]*[in]*\([\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + #services=[] + #methods=[] + groups=[] + if result: + for r in result: + services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r[1]).split(",")] + groups.append({"services":services, "methods":methods}) + #return services, methods + return groups + +class Pattern2: + + def parseServicesAndMethods(self, metricExpression): + metricExpression=re.sub(r'~([A-Z0-9\:\<\>\_\$\.\s\-\,\(\),\[\]\\\\/*]*)~', lambda m: str(urllib.parse.quote_plus(m.group(1))), metricExpression, flags=re.IGNORECASE|re.X|re.MULTILINE) + result = re.findall(r"type\(\"?service_method\"?\),fromRelationship\.isServiceMethodOfService\(type\([~]*service[~]*\),entityName[\.]*[in]*\(([^\)]*)\),tag\(([^\)]*)\)\),entityName[\.]*[in]*\(([^\)]*)\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + # services=[] + # methods=[] + groups=[] + if result: + for r in result: + services=[s.strip() for s in urllib.parse.unquote_plus(r[0]).split(",")] + methods=[s.strip() for s in urllib.parse.unquote_plus(r[2]).split(",")] + groups.append({"services":services, "methods":methods}) + + return groups + + + +class Pattern3: + + def parseServicesAndMethods(self, metricExpression): + result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + # services=[] + # methods=[] + groups=[] + if result: + for r in result: + methods=[s.strip() for s in r.split(",")] + groups.append({"services":[], "methods":methods}) + + return groups + + + +# class Pattern4: + +# def parseServicesAndMethods(self, metricExpression): +# result = re.findall(r"service_method,([^\)]*)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) + +# groups=[] +# methods=[] +# if result: +# for r in result: +# methods.append(r) + +# groups.append({"services":[], "methods":methods}) + +# return groups + +# class FilterMethodPattern: + +# def parseServicesAndMethods(self, metricExpression): +# result = re.findall(r"type\(\"?service_method\"?\)[\s\n\r]*,[\s\n\r]*entityId[\s\n\r]*[\s\n\r]*\([\s\n\r]*[\s\n\r]*([^\)]*)[\s\n\r]*\)", metricExpression,flags=re.IGNORECASE|re.X|re.MULTILINE) +# services=[] +# methods=[] +# if result: +# for r in result: +# methods=[s.strip() for s in r.split(",")] + +# return services, methods \ No newline at end of file diff --git a/more_utils/__init__.py b/more_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/more_utils/httpClient.py b/more_utils/httpClient.py new file mode 100644 index 0000000..2c72a6a --- /dev/null +++ b/more_utils/httpClient.py @@ -0,0 +1,116 @@ +import logging +from typing import Dict, Optional, Any +import time + +import requests +import urllib3 +from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +TOO_MANY_REQUESTS_WAIT = "wait" + + +##Not sure where/why this is here +class DynatraceRetry(Retry): + def get_backoff_time(self): + return self.backoff_factor + + +class HttpClient: + def __init__( + self, + base_url: str, + token: str, + log: logging.Logger = None, + proxies: Dict = None, + too_many_requests_strategy=None, + retries: int = 0, + retry_delay_ms: int = 0, + #mc_jsession_id: Optional[str] = None, + #mc_b925d32c: Optional[str] = None, + #mc_sso_csrf_cookie: Optional[str] = None, + ): + while base_url.endswith("/"): + base_url = base_url[:-1] + self.base_url = base_url + + if proxies is None: + proxies = {} + self.proxies = proxies + + self.auth_header = {"Authorization": f"Api-Token {token}"} + self.log = log + if self.log is None: + self.log = logging.getLogger(__name__) + self.log.setLevel(logging.WARNING) + st = logging.StreamHandler() + fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(thread)d - %(filename)s:%(lineno)d - %(message)s") + st.setFormatter(fmt) + self.log.addHandler(st) + + self.too_many_requests_strategy = too_many_requests_strategy + retry_delay_s = retry_delay_ms / 1000 + + try: + self.retries = Retry( + total=retries, + backoff_factor=retry_delay_s, + status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], + allowed_methods=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], + raise_on_status=False, + ) + except TypeError: # Older version of urllib3? + self.retries = Retry( + total=retries, + backoff_factor=retry_delay_s, + status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], + method_whitelist=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], + raise_on_status=False, + ) + + # This is for internal dynatrace usage + #self.mc_jsession_id = mc_jsession_id + #self.mc_b925d32c = mc_b925d32c + #self.mc_sso_csrf_cookie = mc_sso_csrf_cookie + + def make_request( + self, path: str, params: Optional[Any] = None, headers: Optional[Dict] = None, method="GET", data=None, files=None, query_params=None + ) -> requests.Response: + url = f"{self.base_url}{path}" + + body = None + if method in ["POST", "PUT"]: + body = params + params = query_params + + if headers is None: + headers = {} + if files is None and "content-type" not in [key.lower() for key in headers.keys()]: + headers.update({"content-type": "application/json"}) + headers.update(self.auth_header) + + cookies = None + #if self.mc_b925d32c and self.mc_sso_csrf_cookie and self.mc_jsession_id: + # headers.update({"Cookie": f"JSESSIONID={self.mc_jsession_id}; ssoCSRFCookie={self.mc_sso_csrf_cookie}; b925d32c={self.mc_b925d32c}"}) + # cookies = {"JSESSIONID": self.mc_jsession_id, "ssoCSRFCookie": self.mc_sso_csrf_cookie, "b925d32c": self.mc_b925d32c} + + s = requests.Session() + s.mount("https://", HTTPAdapter(max_retries=self.retries)) + + self.log.debug(f"Making {method} request to '{url}' with params {params} and body: {body}") + r = s.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies, data=data, cookies=cookies, files=files) + self.log.debug(f"Received response '{r}'") + + while r.status_code == 429 and self.too_many_requests_strategy == TOO_MANY_REQUESTS_WAIT: + sleep_amount = int(r.headers.get("retry-after", 5)) + self.log.warning(f"Sleeping for {sleep_amount}s because we have received an HTTP 429") + time.sleep(sleep_amount) + r = requests.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies) + + if r.status_code >= 400: + raise Exception(f"Error making request to {url}: {r}. Response: {r.text}") + + return r \ No newline at end of file diff --git a/pagination.py b/pagination.py new file mode 100644 index 0000000..e072567 --- /dev/null +++ b/pagination.py @@ -0,0 +1,68 @@ +from more_utils.httpClient import HttpClient + + + +class Pagionation(): + def __init__(self, http_client, target_url, target_params=None, headers=None, list_item="result"): + #self.__target_class = target_class + self.__http_client: HttpClient = http_client + self.__target_url = target_url + self.__target_params = target_params + self.__headers = headers + self.__list_item = list_item + self._has_next_page = True + self.__total_count = None + self.__page_size = None + self.elements = self._get_next_page() + + + + def __iter__(self):# -> Iterator[T]: + for element in self.__elements: + yield element + + while self._has_next_page: + new_elements = self._get_next_page() + for element in new_elements: + yield element + + + def __len__(self): + return self.__total_count or len(self.__elements) + + def _get_next_page(self): + response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) + json_response = response.json() + data = [] + if json_response.get("nextPageKey", None): + self._has_next_page = True + self.__target_params = {"nextPageKey": json_response["nextPageKey"]} + else: + self._has_next_page = False + + if self.__list_item in json_response: + elements = json_response[self.__list_item] + self.__total_count = json_response.get("totalCount") or len(elements) + + while self._has_next_page == True: + self.__target_url = self.__target_url.split("?")[0] + elements += self._get_response() + + return elements + + def _get_response(self): + response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) + json_response = response.json() + data = [] + if json_response.get("nextPageKey", None): + self._has_next_page = True + self.__target_params = {"nextPageKey": json_response["nextPageKey"]} + else: + self._has_next_page = False + + if self.__list_item in json_response: + elements = json_response[self.__list_item] + self.__total_count = json_response.get("totalCount") or len(elements) + + + return elements \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c38313e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# Package mgmt +python-decouple +requests +pyyaml +pandas \ No newline at end of file