Compare commits
No commits in common. "d7b23f220c26defd58d6fd06872b22eb5155f57f" and "6fbc96c2c054c6c2b0adf6913a705b60144a0163" have entirely different histories.
d7b23f220c
...
6fbc96c2c0
|
|
@ -142,6 +142,4 @@ crash.log
|
|||
# for dev
|
||||
slo_parameter.yaml
|
||||
metricexpressions.json
|
||||
*.bak
|
||||
*.json
|
||||
failed_requests.txt
|
||||
*.bak
|
||||
159
kpi_extension.py
159
kpi_extension.py
|
|
@ -65,10 +65,6 @@ class ReportReader:
|
|||
columns = df.columns
|
||||
if "Unnamed: 0" in columns:
|
||||
df = df.drop("Unnamed: 0", axis=1)
|
||||
if sheet_name != "total":
|
||||
df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d")
|
||||
if DEBUG:
|
||||
Helper.console_output(f"dtypes of {sheet_name}\n{df.dtypes}")
|
||||
self.qm_report_df[sheet_name] = df
|
||||
|
||||
def get_sheet_names(self) -> typing.List:
|
||||
|
|
@ -120,7 +116,13 @@ class QmReportWriter:
|
|||
for slo_id in self.kpis[sheet][hub].keys():
|
||||
for query in self.kpis[sheet][hub][slo_id]:
|
||||
if len(query) > 0:
|
||||
if query["result"] != "None" and len(query["result"]) > 0:
|
||||
if (
|
||||
query["result"] != "None"
|
||||
and len(query["result"]) > 0
|
||||
# and len(query["result"][0]["data"]) > 0
|
||||
# and len(query["result"][0]["data"][0]["values"]) > 0
|
||||
):
|
||||
# values = query["result"][0]["data"][0]["values"][0]
|
||||
values = query["api_result"]
|
||||
mask = (
|
||||
(self.report_dfs[sheet]["HUB"] == hub.split("-")[0])
|
||||
|
|
@ -143,31 +145,27 @@ class QmReportWriter:
|
|||
existing_value = self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
]
|
||||
try:
|
||||
if not existing_value.empty:
|
||||
existing_value = existing_value.iloc[0]
|
||||
if existing_value is not None:
|
||||
self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
] = (existing_value + values)
|
||||
else:
|
||||
self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
] = values
|
||||
|
||||
if not existing_value.empty:
|
||||
existing_value = existing_value.iloc[0]
|
||||
if existing_value is not None:
|
||||
self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
] = (existing_value + values)
|
||||
else:
|
||||
self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
] = values
|
||||
except Exception as e:
|
||||
print(f"_combine_dataset EXCEPTION: {e}")
|
||||
else:
|
||||
self.report_dfs[sheet].loc[
|
||||
mask, query["kpi_name"]
|
||||
] = values
|
||||
|
||||
# self.report_dfs[sheet].loc[
|
||||
# mask, query["kpi_name"]
|
||||
# ] = values
|
||||
|
||||
self._write_report_to_xlsx()
|
||||
if DEBUG:
|
||||
self._write_to_csv()
|
||||
|
||||
def _write_report_to_xlsx(self):
|
||||
Helper.console_output("Writing XLSX")
|
||||
|
|
@ -184,19 +182,8 @@ class QmReportWriter:
|
|||
worksheet = writer.sheets[sheet_name]
|
||||
worksheet.autofilter(0, 0, dataframe.shape[0], dataframe.shape[1] - 1)
|
||||
|
||||
# format date
|
||||
if sheet_name != "total":
|
||||
fmt = workbook.add_format({"num_format": "yyyy-mm-dd"})
|
||||
for row, date_time in enumerate(dataframe["Date"], start=1):
|
||||
worksheet.write_datetime(row, 0, date_time, fmt)
|
||||
|
||||
writer.close()
|
||||
|
||||
def _write_to_csv(self):
|
||||
Helper.console_output("Writing CSV")
|
||||
for sheet_name, dataframe in self.report_dfs.items():
|
||||
dataframe.to_csv(f"test_{sheet_name}.csv", index=False)
|
||||
|
||||
|
||||
class DynatraceDataGetter:
|
||||
def __init__(self) -> None:
|
||||
|
|
@ -274,7 +261,6 @@ class KPIGetter:
|
|||
report_reader.run()
|
||||
|
||||
# Get SLO IDs from first sheet and build metric expression queries.
|
||||
|
||||
for i, sheet in enumerate(report_reader.qm_report_ids.keys()):
|
||||
if i == 0:
|
||||
for hub in report_reader.qm_report_ids[sheet].keys():
|
||||
|
|
@ -427,17 +413,23 @@ class KPIGetter:
|
|||
f'{row["HUB"]}-{row["type"]}'
|
||||
][row["id"]]["services_transformed"],
|
||||
)
|
||||
# resolution 1d in daily
|
||||
self.metric_expressions[sheet][f'{row["HUB"]}-{row["type"]}'][
|
||||
row["id"]
|
||||
].append(
|
||||
# self._template_metric_expression(
|
||||
# "kpi_1",
|
||||
# metric_kpi1,
|
||||
# from_timestamp_ms,
|
||||
# to_timestamp_ms,
|
||||
# timeframe,
|
||||
# row["timeframe"],
|
||||
# )
|
||||
{
|
||||
"kpi_name": "kpi_1",
|
||||
"metric": metric_kpi1,
|
||||
"from_date": from_timestamp_ms,
|
||||
"to_date": to_timestamp_ms,
|
||||
# "resolution": timeframe,
|
||||
"resolution": self._get_resolution_for_kpi_data(),
|
||||
"resolution": timeframe,
|
||||
"timeframe": row["timeframe"],
|
||||
}
|
||||
)
|
||||
|
|
@ -463,14 +455,7 @@ class KPIGetter:
|
|||
# )
|
||||
# )
|
||||
|
||||
if (
|
||||
REPORT_TYPE == "day"
|
||||
or REPORT_TYPE == "month"
|
||||
and sheet != "total"
|
||||
or REPORT_TYPE == "week"
|
||||
and sheet != "total"
|
||||
):
|
||||
# if REPORT_TYPE == "day":
|
||||
if REPORT_TYPE == "day":
|
||||
metric_count_shifted = self._build_kpi_metric_for_query(
|
||||
"count_shifted",
|
||||
timeframe,
|
||||
|
|
@ -665,14 +650,6 @@ class KPIGetter:
|
|||
)
|
||||
|
||||
else:
|
||||
if DEBUG:
|
||||
Helper.console_output(
|
||||
f"Nothing received for: {hub} - {slo} - {result} - {self.metric_expressions[sheet][hub][slo][index]['kpi_name']}"
|
||||
)
|
||||
with open("./failed_requests.txt", "a") as f:
|
||||
f.write(
|
||||
f"Nothing received for: {hub} - {slo}\n{json.dumps(result, indent=4)}\n{self.metric_expressions[sheet][hub][slo][index]['kpi_name']}\n{'-'*80}\n"
|
||||
)
|
||||
self.metric_expressions[sheet][hub][slo][index][
|
||||
"result"
|
||||
] = "None"
|
||||
|
|
@ -694,48 +671,24 @@ class KPIGetter:
|
|||
for data in result[0]["data"]:
|
||||
result_values.append(data["values"][0])
|
||||
return sum(result_values) / len(result_values)
|
||||
# elif result_type == "count-7d":
|
||||
# result_values = []
|
||||
# option 2
|
||||
# if any(elem is None for elem in result[0]["data"][0]["values"]):
|
||||
# for value in result[0]["data"][0]["values"]:
|
||||
# if value is not None:
|
||||
# return value
|
||||
# option 2 end
|
||||
elif result_type == "kpi_1" or result_type == "count-7d":
|
||||
# 2nd value + none check
|
||||
if len(result[0]["data"][0]["values"]) > 0:
|
||||
if len(result[0]["data"][0]["values"]) == 2:
|
||||
if (
|
||||
result[0]["data"][0]["values"][1] != "None"
|
||||
or result[0]["data"][0]["values"][1] != None
|
||||
):
|
||||
return result[0]["data"][0]["values"][1]
|
||||
elif (
|
||||
result[0]["data"][0]["values"][0] != "None"
|
||||
or result[0]["data"][0]["values"][0] != None
|
||||
):
|
||||
return result[0]["data"][0]["values"][1]
|
||||
else:
|
||||
return "None"
|
||||
else:
|
||||
return result[0]["data"][0]["values"][0]
|
||||
else:
|
||||
if DEBUG:
|
||||
Helper.console_output(
|
||||
f"Extraction No Result: {result_type}\n{result}"
|
||||
)
|
||||
return "None"
|
||||
# if len(result[0]["data"][0]["values"]) > 0:
|
||||
# result_values = []
|
||||
elif result_type == "count-7d":
|
||||
result_values = []
|
||||
# option 2
|
||||
# if any(elem is None for elem in result[0]["data"][0]["values"]):
|
||||
# for value in result[0]["data"][0]["values"]:
|
||||
# if value == None:
|
||||
# result_values.append(0)
|
||||
# else:
|
||||
# result_values.append(value)
|
||||
# return sum(result_values) / len(result_values)
|
||||
# else:
|
||||
# return result[0]["data"][0]["values"][0]
|
||||
# if value is not None:
|
||||
# return value
|
||||
# option 2 end
|
||||
for value in result[0]["data"][0]["values"]:
|
||||
if value == None:
|
||||
result_values.append(0)
|
||||
else:
|
||||
result_values.append(value)
|
||||
return int(sum(result_values) / len(result_values))
|
||||
# elif result_type == "kpi_1":
|
||||
# result_values = []
|
||||
# for value in result[0]["data"][0]["values"]:
|
||||
# return sum(result_values) / len(result_values)
|
||||
# elif result_type == "count":
|
||||
# # DEBUG
|
||||
# Helper.console_output(result[0]["data"])
|
||||
|
|
@ -845,14 +798,6 @@ class KPIGetter:
|
|||
if REPORT_TYPE == "month":
|
||||
return "1M"
|
||||
|
||||
def _get_resolution_for_kpi_data(self) -> str:
|
||||
if REPORT_TYPE == "day":
|
||||
return "1d"
|
||||
if REPORT_TYPE == "week":
|
||||
return "1w"
|
||||
if REPORT_TYPE == "month":
|
||||
return "1M"
|
||||
|
||||
def _build_kpi_metric_for_query(
|
||||
self, kpi_type: str, timeframe: str, service: str = None, request: str = None
|
||||
) -> typing.Union[str, bool]:
|
||||
|
|
@ -1109,14 +1054,7 @@ class Helper:
|
|||
Cleans up files created in debugging mode.
|
||||
"""
|
||||
Helper.console_output("Cleaning up debug files")
|
||||
files = [
|
||||
"./metricexpressions.json",
|
||||
"./slo_results.txt",
|
||||
"./test.xlsx",
|
||||
"./test_total.csv",
|
||||
"./test_daily.csv",
|
||||
"./failed_requests.txt",
|
||||
]
|
||||
files = ["./metricexpressions.json", "./slo_results.txt", "./test.xlsx"]
|
||||
for file in files:
|
||||
if os.path.exists(file):
|
||||
os.remove(file)
|
||||
|
|
@ -1138,7 +1076,4 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception as e:
|
||||
print(f"main EXCEPTION: {e}")
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -7,4 +7,4 @@ argparse
|
|||
openpyxl
|
||||
# git+https://atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git
|
||||
XlsxWriter==3.0.9
|
||||
xlrd==2.0.1
|
||||
xlrd==2.0.1
|
||||
Loading…
Reference in New Issue