diff --git a/convert.py b/convert.py index bb75e7b..a6391c5 100644 --- a/convert.py +++ b/convert.py @@ -1,49 +1,104 @@ -import yaml +import sys +import itertools import jinja2 -import sys, os, shutil, pathlib -from glob import glob -from yaml.loader import SafeLoader - +import re +import yaml +import os +from nested_lookup import nested_lookup # pre-initialization get current working directory cwd = os.getcwd() # defines +SLOS_FOLDER_BASENAME = os.path.basename("slos") +OUTPUT_FOLDER_BASENAME = os.path.basename("output") TEMPLATE_FOLDER_BASENAME = os.path.basename("templates") -TEMPLATE_FILE_BASENAME = os.path.basename("terraform-template.j2") -YAML_FILE_BASENAME = os.path.basename("TP_FTS.yaml") +TEMPLATE_FILES = { + "terraform-template.j2":["builtin:service.keyRequest.errors.fivexx.rate"], + "terraform-template-2.j2":["builtin:service.keyRequest.errors.server.successCount", + "builtin:service.keyRequest.count.server"], + "terraform-template-3.j2":["builtin:service.keyRequest.errors.fivexx.count", + "builtin:service.keyRequest.count.server"], + "terraform-template-4.j2":["builtin:service.successes.server.rate"], + "terraform-template-5.j2":["calc:service.vehicleservice_oes_responsecode_998_requestcount", + "calc:service.vehicleservice_oes_responsecode_999_requestcount", + "calc:service.vehicleservice_oes_fivexx_count", + "calc:service.vehicleservice_oes_request_count_total"] +} + +SLO_FILE_BASENAME = os.path.basename("TP_FTS-example.yaml") # define entries slo_id = None slo_name = None +module_name = None displayname = None department = None +description = None doc_url = None - slo_definition_tresholds_warning = None slo_definition_tresholds_failure = None +environments = [] +metrics = {} +services = {} +keyRequests = {} -emea_prod_metric = None -emea_prod_filter_service = [] -emea_prod_filter_keyRequests = [] - -na_prod_metric = None -na_prod_filter_service = [] -na_prod_filter_keyRequests = [] - -cn_prod_metric = None -cn_prod_filter_service = [] -cn_prod_filter_keyRequests = [] +# methods +def logic(metrics,environment,TEMPLATE_FILES): + for template,metric in TEMPLATE_FILES.items(): + print(set(metric)) + print(set(metrics[environment])) + if set(metrics[environment]) == set(metric): + return template,metric + else: + print("Lists do not match.") # read yaml file -with open(YAML_FILE_BASENAME) as f: - data = list(yaml.safe_load_all(f)) +with open(os.path.join(SLOS_FOLDER_BASENAME,SLO_FILE_BASENAME)) as f: + slos = list(yaml.safe_load_all(f)) + +# fill values +for i,slo in enumerate(slos): + slo_id = slo["slo_id"] + slo_name = slo["slo_name"] + module_name = re.sub(r'[^a-zA-Z ]+', '', slo_name).replace(' ', '_').replace('__', '_') + displayname = slo["displayname"] + description = slo["description"] + department = slo["department"] + doc_url = slo["doc_url"] + slo_definition_tresholds_warning = slo["slo_definition"]["tresholds"]["warning"] + slo_definition_tresholds_failure = slo["slo_definition"]["tresholds"]["failure"] + slo["slo_definition"].pop("tresholds") + for j,environment in enumerate(slo["slo_definition"]): + environments.append(environment) + metrics[environment] = nested_lookup('metric',slo["slo_definition"][environment]) + services[environment] = '~",\n\t\t\t\t~"'.join(nested_lookup('service',slo["slo_definition"][environment])) + keyRequests[environment] = nested_lookup('keyRequests',slo["slo_definition"][environment]) + keyRequests[environment] = '~",\n\t\t\t\t~"'.join(itertools.chain.from_iterable(keyRequests[environment])) + environment = None # read template file -env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_FOLDER_BASENAME), - trim_blocks=True, - lstrip_blocks=True) -template = env.get_template(TEMPLATE_FILE_BASENAME) - - +jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_FOLDER_BASENAME), + trim_blocks=True, + lstrip_blocks=True) +# create template file +for e,environment in enumerate(environments): + folder_path = os.path.join(cwd,OUTPUT_FOLDER_BASENAME,environment.replace('-','_'),"slo") + if not os.path.exists(folder_path): + os.makedirs(folder_path) + with open(os.path.join(folder_path,module_name+".yaml"),"w+") as file: + t,m = logic(metrics,environment,TEMPLATE_FILES) + jinja_template = jinja_environment.get_template(t) + file.write(jinja_template.render(module=module_name, + slo_name=slo_name, + metric=m[0], + metricA=m[0], + metricB=m[1], + metricC=m[2], + metricD=m[3], + description=description, + service=services[environment], + keyRequest=keyRequests[environment], + target=slo_definition_tresholds_failure, + warning=slo_definition_tresholds_warning)) diff --git a/output/EMEA_Prod/slo/Wirkkette_FTS_Service_Reliability_of_key_requests.yaml b/output/EMEA_Prod/slo/Wirkkette_FTS_Service_Reliability_of_key_requests.yaml new file mode 100644 index 0000000..e69de29 diff --git a/slos/TP_Example.yaml b/slos/TP_Example.yaml new file mode 100644 index 0000000..478b285 --- /dev/null +++ b/slos/TP_Example.yaml @@ -0,0 +1,42 @@ +--- +slo_id: "" +slo_name: 'Wirkkette "FTS Service" - Reliability of key requests' +displayname: "Free Text Seach" +department: "DE-322" +description: "CoCo-QM-Report_Draft" +doc_url: "https://atc.bmwgroup.net/confluence/x/YCOqdQ" +slo_definition: + tresholds: + warning: 99 #traffic light orange + failure: 98 #traffic light red + EMEA-Prod: + - metric: "emea_m1" + filter: + - service: "emea_m1s1" + keyRequests: + - "emea_m1s1k1" + - service: "emea_m1s2" + keyRequests: + - "emea_m1s2k1" + - metric: "emea_m2" + filter: + - service: "emea_m2s1" + keyRequests: + - "emea_m2s1k1" + - "emea_m2s1k2" + - service: "emea_m2s2" + keyRequests: + - "emea_m2s2k1" + - "emea_m2s2k2" + - "emea_m2s2k3" + NA-Prod: + - metric: "na_m1" + filter: + - service: "na_m1s1" + keyRequests: + - "na_m1s1k1" + - "na_m1s1k2" + - "na_m1s1k3" + - service: "na_m1s2" + keyRequests: + - "na_m1s2k1" diff --git a/slos/TP_FTS-example.yaml b/slos/TP_FTS-example.yaml new file mode 100644 index 0000000..a8ec051 --- /dev/null +++ b/slos/TP_FTS-example.yaml @@ -0,0 +1,43 @@ +--- +slo_id: "" +slo_name: 'Wirkkette "FTS Service" - Reliability of key requests' +displayname: "Free Text Seach" +department: "DE-322" +description: "CoCo-QM-Report_Draft" +doc_url: "https://atc.bmwgroup.net/confluence/x/YCOqdQ" +slo_definition: + tresholds: + warning: 99 #traffic light orange + failure: 98 #traffic light red + EMEA-Prod: + - metric: "builtin:service.keyRequest.errors.fivexx.count" + filter: + - service: "SimplePOIs" + keyRequests: + - "getSimplePOIs (FTS Calls)" + - service: "POIs" + keyRequests: + - "getPOIs (FTS Calls)" + - metric: "builtin:service.keyRequest.count.server" + filter: + - service: "SimplePOIs" + keyRequests: + - "getSimplePOIs (FTS Calls)" + - service: "POIs" + keyRequests: + - "getPOIs (FTS Calls)" + NA-Prod: + - metric: "builtin:service.keyRequest.errors.fivexx.rate" + filter: + - service: "SimplePOIs" + keyRequests: + - "getSimplePOIs (FTS Calls)" + - service: "POIs" + keyRequests: + - "getPOIs (FTS Calls)" + CN-Prod: + - metric: "builtin:service.keyRequest.errors.fivexx.rate" + filter: + - service: "POIs" + keyRequests: + - "getPOIs (FTS Calls)" diff --git a/TP_FTS.yaml b/slos/TP_FTS.yaml similarity index 96% rename from TP_FTS.yaml rename to slos/TP_FTS.yaml index 184ec51..7719c2f 100644 --- a/TP_FTS.yaml +++ b/slos/TP_FTS.yaml @@ -3,6 +3,7 @@ slo_id: "" slo_name: 'Wirkkette "FTS Service" - Reliability of key requests' displayname: "Free Text Seach" department: "DE-322" +description: "CoCo-QM-Report_Draft" doc_url: "https://atc.bmwgroup.net/confluence/x/YCOqdQ" slo_definition: tresholds: diff --git a/slos/TP_RTTI.yaml b/slos/TP_RTTI.yaml new file mode 100644 index 0000000..05841b2 --- /dev/null +++ b/slos/TP_RTTI.yaml @@ -0,0 +1,40 @@ +--- +slo_id: "WK23" +slo_name: 'Wirkkette "RTTI-Service" - Reliability of key requests' +displayname: "Real Time Traffic Info" +department: "DE-322" +description: "CoCo-QM-Report_Vehicle" +doc_url: "https://atc.bmwgroup.net/confluence/x/zMLwdQ" +slo_definition: + tresholds: + warning: 99 #traffic light orange + failure: 98 #traffic light red + EMEA-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "Netty on 0.0.0.0:8080 - rtti - prod" + keyRequests: + - "/api/default" + - "/api/tw" + - "/api/ru" + - "/api/au" + - "/api/sea" + - "/api/br" + - "/api/mx" + - "/api/za" + - "/api/tw-mgu21" + - "/api/nz" + - "/api/ae" + - "/api/kw" + NA-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "Netty on 0.0.0.0:8080 - rtti - prod" + keyRequests: + - "/api/na" + CN-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "Netty on 0.0.0.0:8080 - rtti - prod" + keyRequests: + - "/api/cn" diff --git a/slos/TP_SendToCar_MGU.yaml b/slos/TP_SendToCar_MGU.yaml new file mode 100644 index 0000000..6135a7e --- /dev/null +++ b/slos/TP_SendToCar_MGU.yaml @@ -0,0 +1,32 @@ +--- +slo_id: "" +slo_name: 'Wirkkette "Send to Car (MGU)" - Reliability of key requests' +displayname: "Send to Vehicle (MGU)" +department: "DE-320" +description: "CoCo-QM-Report_Mobile" +doc_url: "https://atc.bmwgroup.net/confluence/x/qFSqdQ" +slo_definition: + tresholds: + warning: 99 #traffic light orange + failure: 98 #traffic light red + EMEA-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "TripService - PROD" + keyRequests: + - "/v2/motorist/[UUID]/trips" + - "/v3/motorist/[UUID]/trips" + NA-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "TripService - PROD" + keyRequests: + - "/v2/motorist/[UUID]/trips" + - "/v3/motorist/[UUID]/trips" + CN-Prod: + - metric: "builtin:service.errors.fivexx.successCount" + filter: + - service: "TripService - PROD" + keyRequests: + - "/v2/motorist/[UUID]/trips" + - "/v3/motorist/[UUID]/trips" diff --git a/slos/TP_Vehicle_eRoute.yaml b/slos/TP_Vehicle_eRoute.yaml new file mode 100644 index 0000000..bc4f965 --- /dev/null +++ b/slos/TP_Vehicle_eRoute.yaml @@ -0,0 +1,31 @@ +--- +slo_id: "" +slo_name: 'Wirkkette "eRoute Service" - Reliability of key requests' +displayname: "eRoute" +department: "DE-322" +description: "CoCo QM-Report_Vehicle" +doc_url: "https://atc.bmwgroup.net/confluence/x/YCOqdQ" +slo_definition: + tresholds: + warning: 99 #traffic light orange + failure: 98 #traffic light red + EMEA-Prod: + - metric: "builtin:service.keyRequest.errors.fivexx.rate" + filter: + - service: "Netty on 0.0.0.0:8080 - eroute-ext - prod" + keyRequests: + - "/api/eu/2.5/eroute/protobuf3" + - "/api/eu/2.5/eroute-costs/protobuf3" + - service: "Netty on 0.0.0.0:8080 - eroute-adapter - prod" + keyRequests: + - "/route/eroute" + NA-Prod: + - metric: "builtin:service.keyRequest.errors.fivexx.rate" + filter: + - service: "Netty on 0.0.0.0:8080 - eroute-ext - prod" + keyRequests: + - "/api/eu/2.5/eroute/protobuf3" + - "/api/eu/2.5/eroute-costs/protobuf3" + - service: "Netty on 0.0.0.0:8080 - eroute-adapter - prod" + keyRequests: + - "/route/eroute" diff --git a/Wirkkette_FTS_Service_Reliability_of_key_requests.tf b/slos/Wirkkette_FTS_Service_Reliability_of_key_requests.tf similarity index 67% rename from Wirkkette_FTS_Service_Reliability_of_key_requests.tf rename to slos/Wirkkette_FTS_Service_Reliability_of_key_requests.tf index 2975a54..62bdc13 100644 --- a/Wirkkette_FTS_Service_Reliability_of_key_requests.tf +++ b/slos/Wirkkette_FTS_Service_Reliability_of_key_requests.tf @@ -11,15 +11,16 @@ module Wirkkette_FTS_Service_Reliability_of_key_requests { #metric expression of the calculation as done in data explorer ui metric_expression = </password~", + ~"GET /api/v1/digitalkey//pairing~" + )"))))):splitBy()) + / + (builtin:service.keyRequest.count.server:filter(and(or(in("dt.entity.service_method",entitySelector("type(service_method), + fromRelationship.isServiceMethodOfService( + type(~"SERVICE~"),entityName.in( + ~"TrackApi - smc - PROD~", + ~"digital-key-composite-service - PROD~" + ) + ) + ,entityName.in( + ~"trackKey~", + ~"manageKey~", + ~"GET /api/v1/digitalkey//password~", + ~"GET /api/v1/digitalkey//pairing~" + )"))))):splitBy())) + EOT + + + #if not set (removed) it's defaulted to "-1d" + timeframe = "-1d" + + + #currently the only possible value + evaluation = "AGGREGATE" + + #Target and warning percentage of the SLO as double + target = 98 + #if not set(removed) it's defaulted to 98 + warning = 99 + #if not set(removed) it's defaulted to 99 + +} \ No newline at end of file diff --git a/slos/Wirkkette__Online_Entertainment__-_Reliability_of_key_requests.tf b/slos/Wirkkette__Online_Entertainment__-_Reliability_of_key_requests.tf new file mode 100644 index 0000000..c7fef1b --- /dev/null +++ b/slos/Wirkkette__Online_Entertainment__-_Reliability_of_key_requests.tf @@ -0,0 +1,39 @@ +module Wirkkette__Online_Entertainment__-_Reliability_of_key_requests { + + source = "../../_dynatrace-base-modules/dynatrace-service-level-objective" + + + name = "Wirkkette \"Online Entertainment\" - Reliability of key requests" + + description = "CoCo-QM-Report_Draft" + + + + #entity selector object + filter = "type(\"SERVICE\")" + + + #metric expression of the calculation as done in data explorer ui + metric_expression = <<-EOT + (100)* + ((1)-(((calc:service.vehicleservice_oes_responsecode_998_requestcount:splitBy():sum) + + (calc:service.vehicleservice_oes_responsecode_999_requestcount:splitBy():sum) + + (calc:service.vehicleservice_oes_fivexx_count:splitBy():sum)) / + (calc:service.vehicleservice_oes_request_count_total:splitBy():sum))) + EOT + + + #if not set (removed) it's defaulted to "-1d" + timeframe = "-1d" + + + #currently the only possible value + evaluation = "AGGREGATE" + + #Target and warning percentage of the SLO as double + target = 98 + #if not set(removed) it's defaulted to 98 + warning = 99 + #if not set(removed) it's defaulted to 99 + +} \ No newline at end of file diff --git a/slos/Wirkkette__Vehicle_List__-_Reliability_of_key_requests.tf b/slos/Wirkkette__Vehicle_List__-_Reliability_of_key_requests.tf new file mode 100644 index 0000000..87f5ea6 --- /dev/null +++ b/slos/Wirkkette__Vehicle_List__-_Reliability_of_key_requests.tf @@ -0,0 +1,51 @@ +module Wirkkette__Vehicle_List__-_Reliability_of_key_requests { + + source = "../../_dynatrace-base-modules/dynatrace-service-level-objective" + + + name = "Wirkkette \"Vehicle List\" - Reliability of key requests" + + description = "CoCo-QM-Report_Mobile" + + + + #entity selector object + filter = "" + + + metric_expression = <<-EOT + (100)*((builtin:service.keyRequest.errors.server.successCount:filter(and(or(in("dt.entity.service_method", + entitySelector("type(service_method), + fromRelationship.isServiceMethodOfService( + type(~"SERVICE~"),entityName.in( + ~"btc-vehicle-composite-service - PROD~" + ) + ) + ,entityName.in( + ~"GET /api/v2/vehicles~" + )"))))):splitBy()) + / + (builtin:service.keyRequest.count.server:filter(and(or(in("dt.entity.service_method", + entitySelector("type(service_method), + fromRelationship.isServiceMethodOfService( + type(~"SERVICE~"),entityName.in( + ~"btc-vehicle-composite-service - PROD~" + ) + ) + ,entityName.in( + ~"GET /api/v2/vehicles~" + )"))))):splitBy())) +EOT + + #if not set (removed) it's defaulted to "-1d" + timeframe = "-1d" + + + #currently the only possible value + evaluation = "AGGREGATE" + + #Target and warning percentage of the SLO as double + target = 98 #if not set(removed) it's defaulted to 98 + warning = 99 #if not set(removed) it's defaulted to 99 + +} \ No newline at end of file diff --git a/slos/Wirkkette__Vehicle_List__VURS___-_Reliability_of_key_requests.tf b/slos/Wirkkette__Vehicle_List__VURS___-_Reliability_of_key_requests.tf new file mode 100644 index 0000000..2c9f5f2 --- /dev/null +++ b/slos/Wirkkette__Vehicle_List__VURS___-_Reliability_of_key_requests.tf @@ -0,0 +1,33 @@ +module Wirkkette__Vehicle_List__VURS___-_Reliability_of_key_requests { + + source = "../../_dynatrace-base-modules/dynatrace-service-level-objective" + + + name = "Wirkkette \"Vehicle List (VURS)\" - Reliability of key requests" + + description = "" + + + + #entity selector object + filter = "entityId(\"SERVICE-BACAE8C1A346B7C9\")" + + + #metric expression of the calculation as done in data explorer ui + metric_expression = "builtin:service.successes.server.rate:splitBy()" + + + #if not set (removed) it's defaulted to "-1d" + timeframe = "-1d" + + + #currently the only possible value + evaluation = "AGGREGATE" + + #Target and warning percentage of the SLO as double + target = 98 + #if not set(removed) it's defaulted to 98 + warning = 99 + #if not set(removed) it's defaulted to 99 + +} \ No newline at end of file diff --git a/templates/terraform-template-2.j2 b/templates/terraform-template-2.j2 new file mode 100644 index 0000000..0c50d33 --- /dev/null +++ b/templates/terraform-template-2.j2 @@ -0,0 +1,43 @@ +module {{ module }} { + + source = "../../_dynatrace-base-modules/dynatrace-service-level-objective" + + name = {{ slo_name }} + + description = {{ description }} + + # entity selector object + filter = "" + + # metric expression of the calculation as done in data explorer ui + metric_expression = <