diff --git a/dynatraceAPI.py b/dynatraceAPI.py deleted file mode 100644 index 9e59f9a..0000000 --- a/dynatraceAPI.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging -from typing import Dict -import os -import sys - -file_dir = os.path.dirname(__file__) -sys.path.append(file_dir) - -from more_utils.httpClient import HttpClient -import pagination - - -class Dynatrace: - def __init__( - self, - base_url: str, - token: str, - log: logging.Logger = None, - proxies: Dict = None, - too_many_requests_strategy=None, - retries: int = 0, - retry_delay_ms: int = 0, - - ): - self.__http_client = HttpClient( - base_url, token, log, proxies, too_many_requests_strategy, retries, retry_delay_ms - ) - - def returnPageination(self,path,params,list_item): - page = pagination.Pagionation(self.__http_client,path,params,list_item=list_item) - return page - - def returnSingle(self,path): - response = self.__http_client.make_request(path) - json_response = response.json() - return json_response - - - - diff --git a/main.py b/main.py index 571335e..4776eb2 100644 --- a/main.py +++ b/main.py @@ -1,13 +1,16 @@ import os -import dynatraceAPI import logging from decouple import config import yaml -import pandas as pd from dynatrace import Dynatrace import logging +def format_block(string, max): + string_length = len(string) + string = (f'{string}{" "*(max-string_length)}') + return string + def calculateDifference(dashboards, viewCounts): ids = [] for stub in getattr(dashboards, "_PaginatedList__elements"): @@ -17,11 +20,16 @@ def calculateDifference(dashboards, viewCounts): for metricSeries in getattr(viewCounts, "_PaginatedList__elements"): for metric in getattr(metricSeries, "data"): viewIds.append(getattr(metric, "dimension_map")["id"]) - + obsolete = [] - for value in ids: - if value not in viewIds: - obsolete.append(value) + # for value in ids: + # if value not in viewIds: + # obsolete.append(value) + + for stub in getattr(dashboards, "_PaginatedList__elements"): + if getattr(stub, "id") not in viewIds: + obsolete.append(stub) + return obsolete @@ -29,12 +37,14 @@ def getDashboardsWithViewCount(DT_CLIENT, METRIC_SELECTOR, RESOLUTION, FROM_DATE, TO_DATE): metrics = DT_CLIENT.metrics.query(METRIC_SELECTOR, RESOLUTION, FROM_DATE, TO_DATE) - return metrics + count = getattr(metrics, "_PaginatedList__total_count") + return count, metrics def getDashboards(DT_CLIENT): dashboards = DT_CLIENT.dashboards.list(owner=None, tags=None) - return dashboards + n_dashboards = getattr(dashboards, "_PaginatedList__total_count") + return n_dashboards, dashboards if __name__ == "__main__": @@ -45,8 +55,8 @@ if __name__ == "__main__": environment = yaml.safe_load(env_cfg) for item, doc in environment.items(): - logging.info("%s checking token...", str(item)) - + logging.debug("%s checking token...", str(item)) + if config(dict(doc[2]).get("env-token-name"), default='') != "": DT_URL = dict(doc[1]).get("env-url") DT_TOKEN = config(dict(doc[2]).get("env-token-name"), default='') @@ -54,21 +64,44 @@ if __name__ == "__main__": RESOLUTION = dict(doc[6]).get("resolution") FROM_DATE= dict(doc[7]).get("fromDate") TO_DATE= dict(doc[8]).get("toDate") - - logging.info("%s init Dynatrace client...", str(item)) + + logging.debug("%s init Dynatrace client...", str(item)) DT_CLIENT = Dynatrace(DT_URL, DT_TOKEN, logging.Logger("ERROR"), None, None, 0, 10*1000) - logging.info("%s get all dashboards...", str(item)) - dashboards = getDashboards(DT_CLIENT) + logging.debug("%s get all dashboards...", str(item)) + n_dashboards, dashboards = getDashboards(DT_CLIENT) + logging.info("%s %s total dashboards", str(item), str(n_dashboards)) - logging.info("%s get viewcount of dashboards older than 6 months..." - , str(item)) - viewCounts = getDashboardsWithViewCount(DT_CLIENT, METRIC_SELECTOR, - RESOLUTION, FROM_DATE, - TO_DATE) + logging.debug("%s get dashboards with viewCount, resolution %s ...", + str(item), RESOLUTION) + count, viewCounts = getDashboardsWithViewCount(DT_CLIENT, + METRIC_SELECTOR, + RESOLUTION, + FROM_DATE, + TO_DATE) + logging.info("%s %s dashboards with viewCount and older than 6 " + "Months", str(item), str(count)) - logging.info("%s calculate difference...", str(item)) - result = calculateDifference(dashboards, viewCounts) - - logging.info("%s calculated!", str(item)) \ No newline at end of file + logging.debug("%s store ids of obsolete dashboards...", str(item)) + results = calculateDifference(dashboards, viewCounts) + logging.info("%s %s dashboards with 0 viewCount!", + str(item), n_dashboards - count) + for result in results: + id = getattr(result, "id") + name = getattr(result, "name") + owner = getattr(result, "owner") + filename = os.path.join(".\log", item+"-log.txt") + os.makedirs(os.path.dirname(filename), exist_ok=True) + + with open(filename, "a", encoding="utf-8") as f: + f.write(f"{'id: ' + id + ' '} " + + f"{'name: ' + name + ' '} " + + f"{'owner: ' + owner}" + + "\n") + # logging.info(f"{'id: %s '}" + # f"{'name: %s'}" + # f"{'owner: %s'}", + # format_block(id, 50), + # format_block(name, 70), owner) + print("finished") diff --git a/more_utils/__init__.py b/more_utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/more_utils/httpClient.py b/more_utils/httpClient.py deleted file mode 100644 index 2c72a6a..0000000 --- a/more_utils/httpClient.py +++ /dev/null @@ -1,116 +0,0 @@ -import logging -from typing import Dict, Optional, Any -import time - -import requests -import urllib3 -from urllib3.util.retry import Retry -from requests.adapters import HTTPAdapter - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - - -TOO_MANY_REQUESTS_WAIT = "wait" - - -##Not sure where/why this is here -class DynatraceRetry(Retry): - def get_backoff_time(self): - return self.backoff_factor - - -class HttpClient: - def __init__( - self, - base_url: str, - token: str, - log: logging.Logger = None, - proxies: Dict = None, - too_many_requests_strategy=None, - retries: int = 0, - retry_delay_ms: int = 0, - #mc_jsession_id: Optional[str] = None, - #mc_b925d32c: Optional[str] = None, - #mc_sso_csrf_cookie: Optional[str] = None, - ): - while base_url.endswith("/"): - base_url = base_url[:-1] - self.base_url = base_url - - if proxies is None: - proxies = {} - self.proxies = proxies - - self.auth_header = {"Authorization": f"Api-Token {token}"} - self.log = log - if self.log is None: - self.log = logging.getLogger(__name__) - self.log.setLevel(logging.WARNING) - st = logging.StreamHandler() - fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(thread)d - %(filename)s:%(lineno)d - %(message)s") - st.setFormatter(fmt) - self.log.addHandler(st) - - self.too_many_requests_strategy = too_many_requests_strategy - retry_delay_s = retry_delay_ms / 1000 - - try: - self.retries = Retry( - total=retries, - backoff_factor=retry_delay_s, - status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], - allowed_methods=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], - raise_on_status=False, - ) - except TypeError: # Older version of urllib3? - self.retries = Retry( - total=retries, - backoff_factor=retry_delay_s, - status_forcelist=[400, 401, 403, 404, 413, 429, 500, 502, 503, 504], - method_whitelist=["TRACE", "PUT", "DELETE", "OPTIONS", "HEAD", "GET", "POST"], - raise_on_status=False, - ) - - # This is for internal dynatrace usage - #self.mc_jsession_id = mc_jsession_id - #self.mc_b925d32c = mc_b925d32c - #self.mc_sso_csrf_cookie = mc_sso_csrf_cookie - - def make_request( - self, path: str, params: Optional[Any] = None, headers: Optional[Dict] = None, method="GET", data=None, files=None, query_params=None - ) -> requests.Response: - url = f"{self.base_url}{path}" - - body = None - if method in ["POST", "PUT"]: - body = params - params = query_params - - if headers is None: - headers = {} - if files is None and "content-type" not in [key.lower() for key in headers.keys()]: - headers.update({"content-type": "application/json"}) - headers.update(self.auth_header) - - cookies = None - #if self.mc_b925d32c and self.mc_sso_csrf_cookie and self.mc_jsession_id: - # headers.update({"Cookie": f"JSESSIONID={self.mc_jsession_id}; ssoCSRFCookie={self.mc_sso_csrf_cookie}; b925d32c={self.mc_b925d32c}"}) - # cookies = {"JSESSIONID": self.mc_jsession_id, "ssoCSRFCookie": self.mc_sso_csrf_cookie, "b925d32c": self.mc_b925d32c} - - s = requests.Session() - s.mount("https://", HTTPAdapter(max_retries=self.retries)) - - self.log.debug(f"Making {method} request to '{url}' with params {params} and body: {body}") - r = s.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies, data=data, cookies=cookies, files=files) - self.log.debug(f"Received response '{r}'") - - while r.status_code == 429 and self.too_many_requests_strategy == TOO_MANY_REQUESTS_WAIT: - sleep_amount = int(r.headers.get("retry-after", 5)) - self.log.warning(f"Sleeping for {sleep_amount}s because we have received an HTTP 429") - time.sleep(sleep_amount) - r = requests.request(method, url, headers=headers, params=params, json=body, verify=False, proxies=self.proxies) - - if r.status_code >= 400: - raise Exception(f"Error making request to {url}: {r}. Response: {r.text}") - - return r \ No newline at end of file diff --git a/pagination.py b/pagination.py deleted file mode 100644 index e072567..0000000 --- a/pagination.py +++ /dev/null @@ -1,68 +0,0 @@ -from more_utils.httpClient import HttpClient - - - -class Pagionation(): - def __init__(self, http_client, target_url, target_params=None, headers=None, list_item="result"): - #self.__target_class = target_class - self.__http_client: HttpClient = http_client - self.__target_url = target_url - self.__target_params = target_params - self.__headers = headers - self.__list_item = list_item - self._has_next_page = True - self.__total_count = None - self.__page_size = None - self.elements = self._get_next_page() - - - - def __iter__(self):# -> Iterator[T]: - for element in self.__elements: - yield element - - while self._has_next_page: - new_elements = self._get_next_page() - for element in new_elements: - yield element - - - def __len__(self): - return self.__total_count or len(self.__elements) - - def _get_next_page(self): - response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) - json_response = response.json() - data = [] - if json_response.get("nextPageKey", None): - self._has_next_page = True - self.__target_params = {"nextPageKey": json_response["nextPageKey"]} - else: - self._has_next_page = False - - if self.__list_item in json_response: - elements = json_response[self.__list_item] - self.__total_count = json_response.get("totalCount") or len(elements) - - while self._has_next_page == True: - self.__target_url = self.__target_url.split("?")[0] - elements += self._get_response() - - return elements - - def _get_response(self): - response = self.__http_client.make_request(self.__target_url, params=self.__target_params, headers=self.__headers) - json_response = response.json() - data = [] - if json_response.get("nextPageKey", None): - self._has_next_page = True - self.__target_params = {"nextPageKey": json_response["nextPageKey"]} - else: - self._has_next_page = False - - if self.__list_item in json_response: - elements = json_response[self.__list_item] - self.__total_count = json_response.get("totalCount") or len(elements) - - - return elements \ No newline at end of file diff --git a/test.py b/test.py index 7363ed3..81b6aeb 100644 --- a/test.py +++ b/test.py @@ -1,88 +1,6 @@ -import time -import typing -import urllib -import requests -import yaml -from decouple import config -import os -import pandas as pd -def build_params(params: typing.Dict) -> str: - """ - Builds the parameter dictionary to a formatted string - Args: - params (typing.Dict): Parameters as dictionary as stated on dynatrace documentation - - Returns: - str: Returns the query string - """ - query_string = "&".join( - f"{key}={urllib.parse.quote(value)}" for key, value in params.items() - ) - return query_string - -def get_data_from_dynatrace( - throttling_rate: float | int, - token: str, - env_url: str, - params: typing.Dict | str, - route: str, -) -> typing.Dict: - """ - Sends out GET request to dynatrace - Args: - throttling (float | int ): If needed set timeout for throttling - token (str): Token for dynatrace API - env_url (str): Url for the respective environment - params (typing.Dict | str): Parameters as dictionary as stated on dynatrace documentation - route (str): Route for the request - Returns: - typing.Dict: Returns the response as - """ - time.sleep(throttling_rate) - if type(params) is dict: - params_string = f"?{build_params(params)}" - elif type(params) is str: - params_string = f"/{params}" - - headers = {"Authorization": f"Api-Token {token}"} - host_response = requests.get( - f"{env_url}/api/v2/{route}{params_string}", - headers=headers, - verify=False, - ) - - if host_response.status_code == 200: - return host_response.json() - else: - # TODO: proper error handling - print(f"ERROR - {host_response.status_code}") - -if __name__ == "__main__": - - metricSelector = "builtin:dashboards.viewCount:splitBy(id):sort(value(auto,ascending))" - resolution = "1M" - fromDate = "now-6M" - toDate = "now" - - my_params = { - # "nextPageKey": 300, - "metricSelector": f"{metricSelector}", - "resolution": f"{resolution}", - "fromDate" : f"{fromDate}", - "toDate" : f"{toDate}" - } - - with open(os.path.basename("./env-config.yaml")) as env_cfg: - env_config = yaml.safe_load(env_cfg) - - for item, doc in env_config.items(): - token = dict(doc[2]) - url = dict(doc[1]) - print(item, " crawling through ...") - print(item, " checking token ...") - if config(token.get("env-token-name"), default='') != "": - print(item, " fetching all dashboards ...") - DTTOKEN = config(token.get("env-token-name"), default='') - DTURL = url.get("env-url") - get_data_from_dynatrace(0, DTTOKEN, DTURL, my_params, "metrics/query") \ No newline at end of file +format_block("SCHSHSHSHSHSHSHSCHSHSHSHSHSHSH") +format_block("SCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSH") +format_block("SCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSH") +format_block("SCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSH") +format_block("SCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSHSCHSHSHSHSHSHSH")