Jenkinsfile edited online with Bitbucket

switching to python 3.9

Jenkinsfile

@@ -1,182 +1,202 @@
- //not required right now as CN is reachable from EMEA as well
- def loopEnvironments(environments){
-     print env.JENKINS_URL 
-     
-     environments.each { key, val -> 
-       
-       //Execute only if you are on the same environment
-       //not required right now as CN is reachable from EMEA as well
-        if (env.JENKINS_URL == environments."${key}"[3].'jenkins') 
-        {
-            envname = environments."${key}"[0].'name'
-            envurl = environments."${key}"[1].'env-url'
-            tokenname = environments."${key}"[2].'env-token-name'
-            
-            sh 'python createReport.py "${envname}"'
-        }
-        
-     }
- }
 def installRetryCount = 0
 def reportRetryCount = 0
- pipeline {
+def checkRetryCount = 0
-		options {
+def checkRetryCountSec = 0
-            ansiColor('xterm')
-        }
-    //label libraryBuild is available in CN JAWS and ROW JAWS, therefore this one was used; no additional intents     
-   // agent {label 'jaws-slaves'}
-   agent {label 'jaws-slaves'}
-    parameters {
-        string(name: 'FROMDATE', defaultValue: '', description: 'Enter from date in format YYYY-MM-DD e.g. 2021-11-01')
 
-        string(name: 'TODATE', defaultValue: '', description: 'Enter to date in format YYYY-MM-DD e.g. 2021-11-30')
+pipeline {
 
-        string(name: 'PRESELECT', defaultValue: 'day', description: 'Enter week | month (without |) - auto selects the last full week or last full month')
+	options {
-
+        ansiColor('xterm')
-        string(name: 'SLICES', defaultValue: 'dh', description: 'Enter h | d | t | y (without spaces or |) - writes the slices hourly, daily, total or year to date into ecxel. given in any order')
     }
 
-    //here comes the trigger according to crontabs - jenkins is in UTC
+    // label libraryBuild is available in CN JAWS and ROW JAWS, therefore this one was used; no additional intents     
+    agent {label 'jaws-slaves'}
 
+    parameters {
+        string(name: 'FROMDATE', defaultValue: '', description: 'Enter from date in format YYYY-MM-DD e.g. 2021-11-01')
+        string(name: 'TODATE', defaultValue: '', description: 'Enter to date in format YYYY-MM-DD e.g. 2021-11-30')
+        string(name: 'PRESELECT', defaultValue: 'week', description: 'Enter day | week | month (without |) - auto selects the last full day, week or month')
+        string(name: 'SLICES', defaultValue: 'dt', description: 'Enter h | d | t | y (without spaces or |) - writes the slices hourly, daily, total or year to date into excel given in any order')
+        string(name: 'KPI_EXTENSION', defaultValue: 'true', description: 'Enter true or false if the extension should be executed')
+        string(name: 'FORMATS', defaultValue: 'xc', description: 'Enter x | c (without spaces or |) - creates xlsx (x) and/or CSV (c) file - the csv file will include a reduced list (depending on target report)')
+    }
+    
+    // here comes the trigger according to crontabs - jenkins is in UTC 
     triggers {
-           //every 1st of every month at 00:00
+       // every 1st of every month at 00:00
-           //cron('0 0 1 * *')
+       // cron('0 0 1 * *')
        
-           //every day at 08:00
+       // every day at 08:00
-           //cron('0 8 * * *')
+       // cron('0 8 * * *')
 
-           //every monday at 06:00
+       cron('0 4 * * 1')
-           cron('0 5 * * 0-7')
-
-           //parameterizedCron('0 10 * * * %PRESELECT=week;SLICES=tdhy')
 
+       // parameterizedCron('0 10 * * * %PRESELECT=week;SLICES=tdhy')
     }
     
     environment {
         //ProxySettings
-	    AUTH = credentials('proxy')
+        AUTH = credentials('jaws_dynatrace_bitbuket_user')
-        proxy_user = "${AUTH_USR}"
-        proxy_pw = "${AUTH_PSW}"
-        //http_proxy="http://${proxy_user}:${proxy_pw}@proxy.muc:8080"
-        //https_proxy="http://${proxy_user}:${proxy_pw}@proxy.muc:8080"
-        //no_proxy="localhost,127.0.0.1,.muc,.bmwgroup.net"
-        //HTTP_PROXY="${http_proxy}"
-        //HTTPS_PROXY="${https_proxy}"
-        //NO_PROXY="${no_proxy}"
-        
         EUPROD_TOKEN_VAR = credentials('EUPROD_TOKEN_VAR')
         EUPREPROD_TOKEN_VAR = credentials('EUPREPROD_TOKEN_VAR')
         NAPROD_TOKEN_VAR = credentials('NAPROD_TOKEN_VAR')
         NAPREPROD_TOKEN_VAR = credentials('NAPREPROD_TOKEN_VAR')
         CNPROD_TOKEN_VAR = credentials('CNPROD_TOKEN_VAR')
         CNPREPROD_TOKEN_VAR = credentials('CNPREPROD_TOKEN_VAR')
+        EUPRODSAAS_TOKEN_VAR = credentials('EUPRODSAAS_TOKEN_VAR')
         FROM_DATE="${params.FROMDATE}"
         TO_DATE="${params.TODATE}"
-        
+        REPORT_TYPE="${params.PRESELECT}"
-        }
+        KPI_EXTENSION="${params.KPI_EXTENSION}"
+    }
         
     stages {
-        stage('install required python packages') {
+        stage('Fetch Repositories') {
             steps {
-                retry(3) {
+                parallel 'Checkout Repositories': {
-                    sleep(300*installRetryCount)
+                    dir("/opt/workspace/shared_configuration/") {
-                    script {
+                        retry(3) {
-                        installRetryCount = installRetryCount+1
+                            sleep(300*checkRetryCount)
+                            script {
+                                checkRetryCount = checkRetryCount + 1
+                            }
+                            git branch: "master", credentialsId: "jaws_dynatrace_bitbuket_user", url: "https://atc.bmwgroup.net/bitbucket/scm/opapm/shared_configuration.git"
+                        }
+                    }
+                    dir("/opt/workspace/qm_report/") {
+                        retry(3) {
+                            sleep(300*checkRetryCountSec)
+                            script {
+                                checkRetryCountSec = checkRetryCountSec + 1
+                            }
+                            git branch: "main", credentialsId: "jaws_dynatrace_bitbuket_user", url: "https://atc.bmwgroup.net/bitbucket/scm/opapm/qm_report.git"
+                        }
                     }
-                    sh '''
-                    pip3 install --user -r requirements.txt 
-                    '''
                 }
+            }
+        }
 
+        stage('Install Required Python Packages') {
+            steps {
+                dir("/opt/workspace/qm_report/") {
+                    retry(3) {
+                        sleep(60*installRetryCount)
+                        script {
+                            installRetryCount = installRetryCount+1
+                        }
+                        sh """
+                        sudo add-apt-repository ppa:deadsnakes/ppa -y && \
+                        sudo apt install python3.9 -y && \
+                        sudo apt install python3.9-distutils -y && \
+                        python3.9 --version
+                        curl https://bootstrap.pypa.io/get-pip.py | python3.9
+                        python3.9 -m pip --version
+                        python3.9 -m pip install -r requirements.txt
+                        python3.9 -m pip install git+https://${AUTH_USR}:${AUTH_PSW}@atc.bmwgroup.net/bitbucket/scm/opapm/keyrequestparser.git
+                        """
+                    }
+                }
             }
         }
 
         stage('Execute Reporting Script') {
             steps {
+                dir("/opt/workspace/qm_report/") {
                     retry(3) {
                         sleep(300*reportRetryCount)
-                        script{
+
+                        script {
                             reportRetryCount = reportRetryCount+1
                             def SCRIPT_PARAMETER = ''
+
                             if(SLICES.isEmpty()) {
                                 currentBuild.result = 'ABORTED'
-                                    error('Aborting due to missing slice parameter')
+                                error('Aborting due to missing slice parameter')
-                                    return
+                                return
+                            } else {
+                                SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -s " + SLICES.toString()
                             }
-                            else {SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -s " + SLICES.toString()}
 
                             if (PRESELECT.isEmpty()) {
                                 if (FROMDATE=='') {
                                     currentBuild.result = 'ABORTED'
                                     error('Aborting due to missing startdate parameter')
                                     return
-                                    
+                                } else { 
+                                    SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -f " + FROMDATE.toString() 
                                 }
-                                else { SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -f " + FROMDATE.toString() }
+
                                 if (TODATE=='') {
                                     currentBuild.result = 'ABORTED'
                                     error('Aborting due to missing enddate parameter')
                                     return
+                                } else {
+                                    SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -t " + TODATE.toString()
                                 }
-                                else {SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -t " + TODATE.toString()}
 
+                            } else { 
+                                SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -p " + PRESELECT.toString()
                             }
-                            else { SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -p " + PRESELECT.toString()}
 
+                            if (FORMATS.isEmpty()) {
+                                currentBuild.result = 'ABORTED'
+                                error('Aborting due to missing report format parameter')
+                                return
+                            } else {
+                                SCRIPT_PARAMETER = SCRIPT_PARAMETER + " -o " + FORMATS.toString()
+                            }
 
+                            echo "${SCRIPT_PARAMETER}"
+                            sh "python3.9 createReport.py ${SCRIPT_PARAMETER}"
+                            sh "python3.9 summary.py"
+                            summary = readFile('summary.txt').trim()
 
+                            if (KPI_EXTENSION=='true') {
+                                echo "Executing KPI Extension"
+                                sh "python3.9 kpi_extension.py"
+                            }
 
-                        echo "${SCRIPT_PARAMETER}"
-                        sh "python3 createReport.py ${SCRIPT_PARAMETER}"
-
-                        sh "python3 summary.py"
-                        summary = readFile('summary.txt').trim()
                         }
                     }
 
-
+                    // Only required once CN is not reachable from EMEA
-                    //Only required once CN is not reachable from EMEA
+                    // loopEnvironments(environments)
-                    //loopEnvironments(environments)
-            }   
-               
-        }
-        stage('Send report') {
-            steps {
-                script {
-                    try {
-                        tmp_body = "Please find the output of the daily QM-Report attached </br></br>${summary}"
-                        emailext subject: env.JOB_NAME,
-                            mimeType: 'text/html',
-                            body: tmp_body,
-                            //to: 'michaela.jaeger@bmw.de, OOC-Support@bmwgroup.com, Andreas.DA.Danzer@partner.bmw.de',
-                            //to: 'rene.forstner@nttdata.com, Andreas.DA.Danzer@partner.bmw.de, linnea.bickeboeller@partner.bmwgroup.com',
-                            //to: 'rene.forstner@nttdata.com, stephan.oertelt@bmw.de, Mohammed.Abadel@bmw.de, michaela.jaeger@bmw.de',
-                            //to: 'rene.forstner@nttdata.com, ermis.wieger@nttdata.com, patryk.gudalewicz.bp@nttdata.com',
-                            //to: 'patryk.gudalewicz.bp@nttdata.com',
-                            //to: 'rene.forstner@nttdata.com, ermis.wieger@nttdata.com, arnel.arnautovic@nttdata.com, patryk.gudalewicz.bp@nttdata.com, stephan.oertelt@bmw.de, Mohammed.Abadel@bmw.de, michaela.jaeger@bmw.de, OOC-Support@bmwgroup.com, Andreas.DB.Danzer@bmwgroup.com',
-                            to: 'Arnel.Arnautovic@nttdata.com',
-                          	replyTo: 'coco-apm@bmw.de',
-                            attachmentsPattern: '*.xlsx'
-                
-                        }
-                    catch ( mailExc ){
-                            echo "Sending Email Failed: ${mailExc}"
-                        }
                 }
             }      
         }
-      }
+
+        stage('Send Report') {
+            steps {
+                script {
+                    dir("/opt/workspace/qm_report/") {
+                        try {
+                            emailext subject: "THE OFFICIAL ${env.JOB_NAME}",
+                            mimeType: 'text/html',
+                            body: "Please find the output of the daily QM-Report attached </br></br>${summary}",
+                            to: 'Ermis.Wieger@nttdata.com, Andreas.DB.Danzer@bmwgroup.com, OOC-Support@bmwgroup.com, Michaela.Jaeger@bmw.de, Thilo.Bindel@bmwgroup.com, Mohammed.Abadel@bmw.de, Jan.SC.Schaefer@bmw.de, omt-xibix@list.bmw.com, johann.wildgruber@bmwgroup.com',
+                            // test:
+                            // to: 'Daniel.Mikula@nttdata.com, Arnel.Arnautovic@nttdata.com',
+                            replyTo: 'coco-apm@bmw.de',
+                            attachmentsPattern: '*.xlsx, *.csv'
+                        } catch ( mailExc ) {
+                            echo "Sending Email Failed: ${mailExc}"
+                        }
+                    }
+                }
+            }
+        }
+    }
 
     post { 
         failure {
-            emailext subject: "${env.JOB_NAME} build ${env.BUILD_ID} failed",
+            emailext subject: "THE OFFICIAL ${env.JOB_NAME} build ${env.BUILD_ID} failed",
-				body: "QM report failed, see logs for details: ${env.BUILD_URL}", 
+            body: "QM report failed, see logs for details: ${env.BUILD_URL}", 
-				to: 'Arnel.Arnautovic@nttdata.com'
+            to: 'Arnel.Arnautovic@nttdata.com, Daniel.Mikula@nttdata.com, ops-xibix@list.bmw.com, omt-xibix@list.bmw.com'
-// to post to the teams channel "0 - APM Service Desk" just uncomment the following command:
+            // to post to the teams channel "0 - APM Service Desk" just uncomment the following command:
-//            office365ConnectorSend webhookUrl: "https://bmwgroup.webhook.office.com/webhookb2/483edc00-c925-4672-8088-8299a0139fca@ce849bab-cc1c-465b-b62e-18f07c9ac198/JenkinsCI/9aca6923685b40f794134853fcbe88f1/ff31bcee-96b3-4481-9bd8-4f74180b263b",
+            // office365ConnectorSend webhookUrl: "https://bmwgroup.webhook.office.com/webhookb2/483edc00-c925-4672-8088-8299a0139fca@ce849bab-cc1c-465b-b62e-18f07c9ac198/JenkinsCI/9aca6923685b40f794134853fcbe88f1/ff31bcee-96b3-4481-9bd8-4f74180b263b",
-//				message: "QM report failed, see logs for details: ${env.BUILD_URL}",
+            // message: "QM report failed, see logs for details: ${env.BUILD_URL}",
-//				status: 'Failure',
+            // status: 'Failure',
-//				color: "d00000"
+            // color: "d00000"
         }
         always {
             cleanWs()