commit 273372db7a45e1145800320445de91eafed69b5c Author: SLW\ARNAUA Date: Tue Jul 4 13:03:42 2023 +0200 initial diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b7464da --- /dev/null +++ b/.gitignore @@ -0,0 +1,151 @@ +.vscode +.idea +# Byte-compiled / optimized / DLL files +__pycache__/ +more_utils/__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Terraform stuff +**/.terraform/* +crash.log +*.tfvars + +#excel reports +*.xlsx +*.csv + +# for dev +slo_parameter.yaml +metricexpressions.json +*.bak +*.json +failed_requests.txt + +# other +*.txt \ No newline at end of file diff --git a/dynatraceAPI.py b/dynatraceAPI.py new file mode 100644 index 0000000..9e59f9a --- /dev/null +++ b/dynatraceAPI.py @@ -0,0 +1,40 @@ +import logging +from typing import Dict +import os +import sys + +file_dir = os.path.dirname(__file__) +sys.path.append(file_dir) + +from more_utils.httpClient import HttpClient +import pagination + + +class Dynatrace: + def __init__( + self, + base_url: str, + token: str, + log: logging.Logger = None, + proxies: Dict = None, + too_many_requests_strategy=None, + retries: int = 0, + retry_delay_ms: int = 0, + + ): + self.__http_client = HttpClient( + base_url, token, log, proxies, too_many_requests_strategy, retries, retry_delay_ms + ) + + def returnPageination(self,path,params,list_item): + page = pagination.Pagionation(self.__http_client,path,params,list_item=list_item) + return page + + def returnSingle(self,path): + response = self.__http_client.make_request(path) + json_response = response.json() + return json_response + + + + diff --git a/env-config.yaml b/env-config.yaml new file mode 100644 index 0000000..edae2ed --- /dev/null +++ b/env-config.yaml @@ -0,0 +1,43 @@ +--- +euprod-coco: + - name: "euprod" + - env-url: "https://xxu26128.live.dynatrace.com" + - env-token-name: "EUPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" + - type: "coco" +euprod-gcdm: + - name: "euprod" + - env-url: "https://moh22956.live.dynatrace.com" + - env-token-name: "EUPRODSAAS_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" + - type: "gcdm" +eupreprod-coco: + - name: "eupreprod" + - env-url: "https://qqk70169.live.dynatrace.com" + - env-token-name: "EUPREPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" + - type: "coco" +naprod-coco: + - name: "naprod" + - env-url: "https://wgv50241.live.dynatrace.com" + - env-token-name: "NAPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" + - type: "coco" +napreprod-coco: + - name: "napreprod" + - env-url: "https://onb44935.live.dynatrace.com" + - env-token-name: "NAPREPROD_TOKEN_VAR" + - jenkins: "https://jaws.bmwgroup.net/opapm/" + - type: "coco" +cnprod-coco: + - name: "cnprod" + - env-url: "https://dyna-synth-cn.bmwgroup.com.cn/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" + - env-token-name: "CNPROD_TOKEN_VAR" + - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" + - type: "coco" +cnpreprod-coco: + - name: "cnpreprod" + - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35" + - env-token-name: "CNPREPROD_TOKEN_VAR" + - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" + - type: "coco" diff --git a/main.py b/main.py new file mode 100644 index 0000000..b0febf0 --- /dev/null +++ b/main.py @@ -0,0 +1,75 @@ +import os +import dynatraceAPI +import logging +from decouple import config +import yaml +import pandas as pd +from dynatrace import Dynatrace +from dynatrace import TOO_MANY_REQUESTS_WAIT +from dynatrace.environment_v2.tokens_api import SCOPE_METRICS_READ, SCOPE_METRICS_INGEST + +def getD(DTURL, DTTOKEN, metricSelector, resolution, fromDate, toDate): + # Create a Dynatrace client + dt = Dynatrace(DTURL, DTTOKEN) + for metric in dt.metrics.query(metricSelector, resolution, fromDate, toDate): + with open('metrics.txt', 'w') as f: + f.write(str(metric)) + + +def getDashboardsWithViewCount(DTAPIToken, DTENV, metricSelector, resolution, + fromDate, toDate): + + dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken, + logging.Logger("ERROR"), None, None, 0, + 2 * 1000) + params = { + "metricSelector": f"{metricSelector}", + "resolution": f"{resolution}", + "fromDate": f"{fromDate}", + "toDate": f"{toDate}" + } + + api_url_report = "/api/v2/metrics/query" + pages = dtclient.returnPageination(api_url_report, params, "data") + df = pd.DataFrame(pages.elements) + return df + + +def getDashboards(DTAPIToken, DTENV): + + dtclient = dynatraceAPI.Dynatrace(DTENV, DTAPIToken, + logging.Logger("ERROR"), None, None, 0, + 2 * 1000) + my_params = { + "owner": "", + "tags": [], + } + + api_url_report = "/api/config/v1/dashboards" + pages = dtclient.returnPageination(api_url_report, my_params, "dashboards") + df = pd.DataFrame(pages.elements) + return df + +if __name__ == "__main__": + + metricSelector = "builtin:dashboards.viewCount:splitBy(id):sort(value(auto,ascending))" + resolution = "1M" + fromDate = "now-6M" + toDate = "now" + + with open(os.path.basename("./env-config.yaml")) as env_cfg: + env_config = yaml.safe_load(env_cfg) + + for item, doc in env_config.items(): + token = dict(doc[2]) + url = dict(doc[1]) + print(item, " crawling through ...") + print(item, " checking token ...") + if config(token.get("env-token-name"), default='') != "": + print(item, " fetching all dashboards ...") + DTTOKEN = config(token.get("env-token-name"), default='') + DTURL = url.get("env-url") + getDashboards(DTTOKEN, DTURL) + getDashboardsWithViewCount(DTTOKEN, DTURL, metricSelector, + resolution, fromDate, toDate) + getD(DTURL, DTTOKEN, metricSelector, resolution, fromDate, toDate) \ No newline at end of file diff --git a/more_utils/__init__.py b/more_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/more_utils/httpClient.py b/more_utils/httpClient.py new file mode 100644 index 0000000..2c72a6a --- /dev/null +++ b/more_utils/httpClient.py @@ -0,0 +1,116 @@ +import logging +from typing import Dict, Optional, Any +import time + +import requests +import urllib3 +from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +TOO_MANY_REQUESTS_WAIT = "wait" + + +##Not sure where/why this is here +class DynatraceRetry(Retry): + def get_backoff_time(self): + return self.backoff_factor + + +class HttpClient: + def __init__( + self, + base_url: str, + token: str, + log: logging.Logger = None, + proxies: Dict = None, + too_many_requests_strategy=None, + retries: int = 0, + retry_delay_ms: int = 0, + #mc_jsession_id: Optional[str] = None, + #mc_b925d32c: Optional[str] = None, + #mc_sso_csrf_cookie: Optional[str] = None, + ): + while base_url.endswith("/"): + base_url = base_url[:-1] + self.base_url = base_url + + if proxies is None: + proxies = {} + self.proxies = proxies + + self.auth_header = {"Authorization": f"Api-Token {token}"} + self.log = log + if self.log is None: + self.log = logging.getLogger(__name__) + self.log.setLevel(logging.WARNING) + st = logging.StreamHandler() + fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(thread)d - %(filename)s:%(lineno)d - %(message)s") + st.setFormatter(fmt) + self.log.addHandler(st) + + self.too_many_requests_strategy = too_many_requests_strategy + retry_delay_s = retry_delay_ms / 1000 + + try: + self.retries = Retry( + total=retries, + backoff_factor=retry_delay_s, + status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], + allowed_methods=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], + raise_on_status=False, + ) + except TypeError: # Older version of urllib3? + self.retries = Retry( + total=retries, + backoff_factor=retry_delay_s, + status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], + method_whitelist=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], + raise_on_status=False, + ) + + # This is for internal dynatrace usage + #self.mc_jsession_id = mc_jsession_id + #self.mc_b925d32c = mc_b925d32c + #self.mc_sso_csrf_cookie = mc_sso_csrf_cookie + + def make_request( + self, path: str, params: Optional[Any] = None, headers: Optional[Dict] = None, method="GET", data=None, files=None, query_params=None + ) -> requests.Response: + url = f"{self.base_url}{path}" + + body = None + if method in ["POST", "PUT"]: + body = params + params = query_params + + if headers is None: + headers = {} + if files is None and "content-type" not in [key.lower() for key in headers.keys()]: + headers.update({"content-type": "application/json"}) + headers.update(self.auth_header) + + cookies = None + #if self.mc_b925d32c and self.mc_sso_csrf_cookie and self.mc_jsession_id: + # headers.update({"Cookie": f"JSESSIONID={self.mc_jsession_id}; ssoCSRFCookie={self.mc_sso_csrf_cookie}; b925d32c={self.mc_b925d32c}"}) + # cookies = {"JSESSIONID": self.mc_jsession_id, "ssoCSRFCookie": self.mc_sso_csrf_cookie, "b925d32c": self.mc_b925d32c} + + s = requests.Session() + s.mount("https://", HTTPAdapter(max_retries=self.retries)) + + self.log.debug(f"Making {method} request to '{url}' with params {params} and body: {body}") + r = s.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies, data=data, cookies=cookies, files=files) + self.log.debug(f"Received response '{r}'") + + while r.status_code == 429 and self.too_many_requests_strategy == TOO_MANY_REQUESTS_WAIT: + sleep_amount = int(r.headers.get("retry-after", 5)) + self.log.warning(f"Sleeping for {sleep_amount}s because we have received an HTTP 429") + time.sleep(sleep_amount) + r = requests.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies) + + if r.status_code >= 400: + raise Exception(f"Error making request to {url}: {r}. Response: {r.text}") + + return r \ No newline at end of file diff --git a/pagination.py b/pagination.py new file mode 100644 index 0000000..e072567 --- /dev/null +++ b/pagination.py @@ -0,0 +1,68 @@ +from more_utils.httpClient import HttpClient + + + +class Pagionation(): + def __init__(self, http_client, target_url, target_params=None, headers=None, list_item="result"): + #self.__target_class = target_class + self.__http_client: HttpClient = http_client + self.__target_url = target_url + self.__target_params = target_params + self.__headers = headers + self.__list_item = list_item + self._has_next_page = True + self.__total_count = None + self.__page_size = None + self.elements = self._get_next_page() + + + + def __iter__(self):# -> Iterator[T]: + for element in self.__elements: + yield element + + while self._has_next_page: + new_elements = self._get_next_page() + for element in new_elements: + yield element + + + def __len__(self): + return self.__total_count or len(self.__elements) + + def _get_next_page(self): + response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) + json_response = response.json() + data = [] + if json_response.get("nextPageKey", None): + self._has_next_page = True + self.__target_params = {"nextPageKey": json_response["nextPageKey"]} + else: + self._has_next_page = False + + if self.__list_item in json_response: + elements = json_response[self.__list_item] + self.__total_count = json_response.get("totalCount") or len(elements) + + while self._has_next_page == True: + self.__target_url = self.__target_url.split("?")[0] + elements += self._get_response() + + return elements + + def _get_response(self): + response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) + json_response = response.json() + data = [] + if json_response.get("nextPageKey", None): + self._has_next_page = True + self.__target_params = {"nextPageKey": json_response["nextPageKey"]} + else: + self._has_next_page = False + + if self.__list_item in json_response: + elements = json_response[self.__list_item] + self.__total_count = json_response.get("totalCount") or len(elements) + + + return elements \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..7363ed3 --- /dev/null +++ b/test.py @@ -0,0 +1,88 @@ +import time +import typing +import urllib +import requests +import yaml +from decouple import config +import os +import pandas as pd + +def build_params(params: typing.Dict) -> str: + """ + Builds the parameter dictionary to a formatted string + Args: + params (typing.Dict): Parameters as dictionary as stated on dynatrace documentation + + Returns: + str: Returns the query string + """ + query_string = "&".join( + f"{key}={urllib.parse.quote(value)}" for key, value in params.items() + ) + return query_string + +def get_data_from_dynatrace( + throttling_rate: float | int, + token: str, + env_url: str, + params: typing.Dict | str, + route: str, +) -> typing.Dict: + """ + Sends out GET request to dynatrace + Args: + throttling (float | int ): If needed set timeout for throttling + token (str): Token for dynatrace API + env_url (str): Url for the respective environment + params (typing.Dict | str): Parameters as dictionary as stated on dynatrace documentation + route (str): Route for the request + Returns: + typing.Dict: Returns the response as + """ + time.sleep(throttling_rate) + if type(params) is dict: + params_string = f"?{build_params(params)}" + elif type(params) is str: + params_string = f"/{params}" + + headers = {"Authorization": f"Api-Token {token}"} + host_response = requests.get( + f"{env_url}/api/v2/{route}{params_string}", + headers=headers, + verify=False, + ) + + if host_response.status_code == 200: + return host_response.json() + else: + # TODO: proper error handling + print(f"ERROR - {host_response.status_code}") + +if __name__ == "__main__": + + metricSelector = "builtin:dashboards.viewCount:splitBy(id):sort(value(auto,ascending))" + resolution = "1M" + fromDate = "now-6M" + toDate = "now" + + my_params = { + # "nextPageKey": 300, + "metricSelector": f"{metricSelector}", + "resolution": f"{resolution}", + "fromDate" : f"{fromDate}", + "toDate" : f"{toDate}" + } + + with open(os.path.basename("./env-config.yaml")) as env_cfg: + env_config = yaml.safe_load(env_cfg) + + for item, doc in env_config.items(): + token = dict(doc[2]) + url = dict(doc[1]) + print(item, " crawling through ...") + print(item, " checking token ...") + if config(token.get("env-token-name"), default='') != "": + print(item, " fetching all dashboards ...") + DTTOKEN = config(token.get("env-token-name"), default='') + DTURL = url.get("env-url") + get_data_from_dynatrace(0, DTTOKEN, DTURL, my_params, "metrics/query") \ No newline at end of file