diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1216e03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,135 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Terraform stuff +**/.terraform/* +crash.log +*.tfvars + diff --git a/environment.yaml b/environment.yaml index 572b277..bdb214e 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,27 +3,27 @@ euprod: - env-url: "https://xxu26128.live.dynatrace.com" - env-token-name: "EUPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -eupreprod: +#eupreprod: - name: "eupreprod" - env-url: "https://qqk70169.live.dynatrace.com" - env-token-name: "EUPREPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -napreprod: +#napreprod: - name: "napreprod" - env-url: "https://onb44935.live.dynatrace.com" - env-token-name: "NAPREPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -naprod: +#naprod: - name: "naprod" - env-url: "https://wgv50241.live.dynatrace.com" - env-token-name: "NAPROD_TOKEN_VAR" - jenkins: "https://jaws.bmwgroup.net/opapm/" -cnprod: +#cnprod: - name: "cnprod" - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/b921f1b9-c00e-4031-b9d1-f5a0d530757b" - env-token-name: "CNPROD_TOKEN_VAR" - jenkins: "https://jaws-china.bmwgroup.net/opmaas/" -cnpreprod: +#cnpreprod: - name: "cnpreprod" - env-url: "https://dynatracemgd-tsp.bmwgroup.net/e/ab88c03b-b7fc-45f0-9115-9e9ecc0ced35" - env-token-name: "CNPREPROD_TOKEN_VAR" diff --git a/euprod.xlsx b/euprod.xlsx new file mode 100644 index 0000000..e226288 Binary files /dev/null and b/euprod.xlsx differ diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..dfb0407 --- /dev/null +++ b/utils.py @@ -0,0 +1,199 @@ +from tracemalloc import start +from decouple import config +import sys +import yaml +import datetime +import time +import pandas as pd +import requests +import openpyxl +import argparse + +def init_argparse() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + usage="%(prog)s [--fromDate] [toDate] or [preSelect]", + description="gather SLO in daily slices for given Timeframe" + ) + parser.add_argument( + "-f","--fromDate", + help = "YYYY-mm-dd e.g. 2022-01-01" + ) + parser.add_argument( + "-t","--toDate", + help = "YYYY-mm-dd e.g. 2022-01-31" + ) + parser.add_argument( + "-p","--preSelect", + help = "week | month - gathers the data for the last full week or month" + ) + + return parser + +def make_request(url, headers,verify,parameters): + try: + response = requests.get(url, headers=headers,verify=verify,params=parameters) + response.raise_for_status() + except requests.exceptions.HTTPError as errh: + return "An Http Error occurred:" + repr(errh) + except requests.exceptions.ConnectionError as errc: + return "An Error Connecting to the API occurred:" + repr(errc) + except requests.exceptions.Timeout as errt: + return "A Timeout Error occurred:" + repr(errt) + except requests.exceptions.RequestException as err: + return "An Unknown Error occurred" + repr(err) + + return response + +def previous_week_range(date): + start_date = date + datetime.timedelta(-date.weekday(), weeks=-1) + end_date = date + datetime.timedelta(-date.weekday() - 1) + return start_date, end_date + +def previous_month_range(date): + end_date = date.replace(day=1) - datetime.timedelta(days=1) + start_date = end_date.replace(day=1) + return start_date, end_date + +def getSLO(DTAPIToken, DTENV, fromDate, toDate): + + env = DTENV + DTAPIToken = DTAPIToken + + if (DTENV.find('dynatracemgd') != -1): + verify=False + else: + verify=True + + DTAPIURL= env + "/api/v2/slo" + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Api-Token ' + DTAPIToken + } + + parameters = { + "pageSize": 25, + "from": int(fromDate), + "to": int(toDate), + "timeFrame": "GTF", + "evaluate": True, + "sloSelector": "text(\"CoCo-QM-Report\")" + } + r = make_request(DTAPIURL,headers=headers,parameters=parameters,verify=verify) + + df = pd.json_normalize(r.json()['slo']) + + return df + +def get_daily_slice(start_date, end_date): + tempstart = start_date + days = pd.DataFrame() + + #Add the first day + tempend = tempstart + datetime.timedelta(hours=24) + startms = time.mktime(tempstart.timetuple()) * 1000 + endms = time.mktime(tempend.timetuple()) * 1000 + + row = {'Date':tempstart,'startTime':startms, 'endTime':endms} + days = days.append(row,ignore_index=True) + + while tempstart < end_date: + tempstart = tempstart + datetime.timedelta(hours=24) + tempend = tempstart + datetime.timedelta(hours=24) + startms = time.mktime(tempstart.timetuple()) * 1000 + endms = time.mktime(tempend.timetuple()) * 1000 + + row = {'Date':tempstart,'startTime':startms, 'endTime':endms} + days = days.append(row,ignore_index=True) + + return days + +def main() -> None: + + parser = init_argparse() + args = parser.parse_args() + + if args.preSelect and (args.fromDate or args.toDate): + print("--preSelect must not be used in conjuntion with --fromDate and/or --toDate") + sys.exit() + + elif args.fromDate and not args.toDate: + print("--fromDate only in conjunction with --toDate") + sys.exit() + + elif args.toDate and not args.fromDate: + print("--toDate only in conjunction with --fromDate") + sys.exit() + + elif args.toDate and args.fromDate and not args.preSelect: + try: + fromDate = datetime.date.fromisoformat(args.fromDate) + toDate = datetime.date.fromisoformat(args.toDate) + except Exception as e: + print("Progam closed: " + str(e)) + sys.exit() + + if toDate < fromDate: + print("--toDate can't be older than --fromDate") + sys.exit() + + if toDate > datetime.date.today() or fromDate > datetime.date.today(): + print("--toDate or --fromDate can't be in the future") + sys.exit() + + elif args.preSelect and not args.fromDate and not args.toDate: + + date = datetime.date.today() + + if args.preSelect == "week": + fromDate, toDate = previous_week_range(date) + elif args.preSelect == "month": + fromDate, toDate = previous_month_range(date) + else: + print("--preSelect must be week or month") + sys.exit() + + else: + print("Invalid arguments, please use --help") + sys.exit() + print("fromDate: " + str(fromDate)) + print("toDate: " + str(toDate)) + + days = get_daily_slice(fromDate,toDate) + with open('./environment.yaml') as file: + doc = yaml.safe_load(file) + + for item, doc in doc.items(): + token = dict(doc[2]) + url = dict(doc[1]) + print("Crawling through: " + item) + print("Check if token exists in environment...") + if(config(token.get('env-token-name')) != ""): + print("Gather data, hold on a minute") + DTTOKEN = config(token.get('env-token-name')) + DTURL = url.get('env-url') + + + + df = pd.DataFrame() + for index, row in days.iterrows(): + temp_df = getSLO(DTTOKEN,DTURL,row['startTime'],row['endTime']) + temp_df['Date'] = row['Date'] + df = pd.concat([df,temp_df],ignore_index=True) + + #sort columns in a try block - if API is returning columns which are non exist, this will not fail the script + + try: + df = df[['Date', 'id', 'enabled', 'name', 'description', 'evaluatedPercentage', 'errorBudget', 'status', 'error', 'target','warning', 'evaluationType', 'timeframe', 'metricExpression', 'filter']] + except Exception as e: + print("Could not rearrange columns: " + e) + + + writer = pd.ExcelWriter("./"+ item +'.xlsx') + df.to_excel(writer, sheet_name=str(item).split(" ")[0]) + writer.save() + + else: + print("token not found, skipping " + item) + +if __name__ == "__main__": + main() \ No newline at end of file