Jenkinsfile

// build-in-container -- CFEngine packages built in containers on the
// CONTAINER_PACKAGES_* docker hosts.
//
// Jenkins finds this file itself: the CFEngine organization folder scans the
// org for a Jenkinsfile in the repository root and makes a job of every branch
// and every pull request, each running its own copy of this file. Nothing in
// infra defines the job.

// Returns the labels to build. build-scripts/labels.txt decides what a branch
// builds. readTrusted reads it from the revision this job is for, without
// cloning the repo.
def selectedLabels() {
  def sel = readTrusted('build-scripts/labels.txt').readLines()
      .collect { it.trim() }
      .findAll { it.startsWith('PACKAGES_') && it ==~ params.LABEL_FILTER }
  if (!sel) {
    error "LABEL_FILTER '${params.LABEL_FILTER}' matches no label in build-scripts/labels.txt."
  }
  return sel
}

// Returns the architecture of the node a label has to build on.
def archOf(String label) {
  return label.contains('_arm_64') ? 'arm64' : 'amd64'
}

// Return the repos we need to check out.
def reposFor(String project) {
  def repos = ['buildscripts', 'core', 'masterfiles']
  if (project == 'nova') { repos.addAll(['enterprise', 'nova', 'mission-portal']) }
  return repos
}

// The revision to use when a repo's parameter is empty. Normally the branch
// this job is for. In a pull request job buildscripts gets the pull request.
// The other repos get the branch it targets.
//
// BASE_BRANCH wins over the job's branch. pr-pipeline passes it, and its matrix
// jobs build it, so this job has to agree with them.
def defaultRev(String repo) {
  if (env.CHANGE_ID) {
    return repo == 'buildscripts' ? "pull/${env.CHANGE_ID}/merge" : env.CHANGE_TARGET
  }
  if (params.BASE_BRANCH?.trim()) { return params.BASE_BRANCH.trim() }
  return env.BRANCH_NAME ?: 'master'
}

// Accepts the same revision forms as the other jobs. A bare number is a pull
// request, which is what cf-bottom sends. tag:NAME is a tag.
def normalizeRev(String rev) {
  if (rev ==~ /\d+/) { return "pull/${rev}/merge" }
  if (rev.startsWith('tag:')) { return rev.substring(4) }
  return rev
}

// Get the revision to build at from <repo>_REV parameter.
def revFor(String repo) {
  def rev = params[repo.toUpperCase().replaceAll('-', '_') + '_REV']
  return (rev && rev.trim()) ? normalizeRev(rev.trim()) : defaultRev(repo)
}

// The refspec the other jobs fetch with. Branch heads are always fetched. Pull
// requests live outside refs/heads, so building one means fetching its ref too,
// or its commit is not in the clone.
def refspec() {
  return '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pull/*'
}

// Runs one build in the workspace of the node the caller allocated.
//
// Cleans up after the previous build. Checks out each repo at its revision from
// revs. Builds, then archives the packages.
//
// opts holds the build-in-container.py flags that vary per build. The flags
// every build shares are added below.
def containerBuild(String opts, List repos, Map revs) {
  // The container hands the directories it writes back to us as it exits, so
  // this only covers a build that never got to exit (e.g. killed).
  sh 'sudo chown -R "$(id -u):$(id -g)" "$WORKSPACE" 2>/dev/null || true'
  cleanWs(deleteDirs: true, notFailBuild: true)

  repos.each { repo ->
    dir("src/${repo}") {
      checkout([$class: 'GitSCM',
                branches: [[name: revs[repo]]],
                userRemoteConfigs: [[url: "git@github.com:cfengine/${repo}.git",
                                     credentialsId: 'jenkins-github',
                                     refspec: refspec()]],
                // Full history on purpose: the build reads SOURCE_DATE_EPOCH and
                // every dependency's revision out of git log, so a shallow clone
                // would change the timestamps it pins.
                extensions: [[$class: 'CloneOption', shallow: false, noTags: false, timeout: 30],
                             [$class: 'SubmoduleOption', recursiveSubmodules: true,
                              parentCredentials: true, timeout: 30]]])
    }
  }

  def cacheDir = '/home/jenkins/cfengine-build-cache'
  def buildType = params.RELEASE_BUILD ? 'RELEASE' : 'DEBUG'
  if (params.EXPLICIT_VERSION?.trim()) { opts += " --version '${params.EXPLICIT_VERSION.trim()}'" }

  withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-build-artifacts-cache',
                                     keyFileVariable: 'JENKINS_SFTP_KEY_PATH')]) {
    sh """set -eu
          ./src/buildscripts/build-in-container.py ${opts} \\
              --build-type '${buildType}' \\
              --build-number '${params.PACKAGE_BUILD_NUMBER}' \\
              --source-dir "\$WORKSPACE/src" \\
              --output-dir "\$WORKSPACE/output" \\
              --cache-dir '${cacheDir}' \\
              --sftp-key "\$JENKINS_SFTP_KEY_PATH"
    """
  }

  archiveArtifacts artifacts: 'output/**', fingerprint: true
}

// All filled in by Resolve refs and read by the build stages, which run on other
// nodes. labels holds the build labels asked for. revs holds the revision each
// repo builds at:
//
//   revs['core'] = pull/1234/merge
def labels = []
def revs = [:]

pipeline {
  agent none

  options {
    timestamps()
    // Without this, every stage would start by checking out the buildscripts
    // branch this job belongs to, which is not necessarily the revision being
    // built. Each stage checks out the repos it builds itself.
    skipDefaultCheckout(true)
    // The build cache under /home/jenkins is shared by every build on a node,
    // and two runs building the same dependency into it would race.
    disableConcurrentBuilds()
    timeout(time: 8, unit: 'HOURS')
    // Packages are large; keep logs longer than the artifacts.
    buildDiscarder(logRotator(numToKeepStr: '50', artifactNumToKeepStr: '10'))
  }

  parameters {
    // Which of the labels in build-scripts/labels.txt to build. The default
    // builds all of them. A regex, matched against the whole label: hubs only
    // is PACKAGES_HUB_.*, one platform is .*redhat_7, one architecture is
    // .*_arm_64_.*.
    string(name: 'LABEL_FILTER', defaultValue: '.*',
           description: 'Regex matching the build labels to build. See build-scripts/labels.txt.')
    booleanParam(name: 'BUILD_TARBALLS', defaultValue: true,
                 description: 'Build the source tarballs.')

    string(name: 'BASE_BRANCH', defaultValue: '',
           description: 'The branch to build the repos whose revision is empty from.')
    string(name: 'BUILDSCRIPTS_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')
    string(name: 'CORE_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')
    string(name: 'MASTERFILES_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')
    string(name: 'ENTERPRISE_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')
    string(name: 'NOVA_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')
    string(name: 'MISSION_PORTAL_REV', defaultValue: '', description: 'Use NUMBER for a pull request, or pull/NUMBER/merge, or tag:SOME_TAG, or a branch or commit. Leave empty for BASE_BRANCH.')

    choice(name: 'PROJECT', choices: ['nova', 'community'], description: 'CFEngine community or enterprise (nova) edition.')
    booleanParam(name: 'RELEASE_BUILD', defaultValue: false,
                 description: 'Whether BUILD_TYPE should be RELEASE or DEBUG.')
    string(name: 'PACKAGE_BUILD_NUMBER', defaultValue: '1',
           description: 'BUILD_NUMBER for the build, which a DEBUG build puts in the package version.')
    string(name: 'EXPLICIT_VERSION', defaultValue: '',
           description: 'Override the version string the build derives from the sources. Leave empty for the usual behaviour.')
  }

  stages {

    stage('Resolve refs') {
      agent { label 'MASTER_x86_64_linux' }
      steps {
        script {
          labels.addAll(selectedLabels())
          def repos = reposFor(params.PROJECT)
          echo "Building ${labels.size()} labels:\n  ${labels.join('\n  ')}"

          repos.each { repo ->
            revs[repo] = revFor(repo)
            echo "${repo}: ${revs[repo]}"
          }

          currentBuild.description = "${params.PROJECT} @ ${revs['core']}: ${labels.size()} labels"
        }
      }
    }

    stage('Tarballs') {
      when { expression { return params.BUILD_TARBALLS } }
      agent { label 'CONTAINER_PACKAGES_amd64' }
      steps {
        script {
          // --tarballs builds core and masterfiles alone, in an image of its own,
          // and forces project and platform itself. Only the build type is ours
          // to pass: it decides the version string.
          containerBuild('--tarballs', ['buildscripts', 'core', 'masterfiles'], revs)
        }
      }
    }

    stage('Build') {
      steps {
        script {
          def repos = reposFor(params.PROJECT)

          parallel labels.collectEntries { label ->
            [(label): {
              node("CONTAINER_PACKAGES_${archOf(label)}") {
                // The label decides the platform, the role and the container
                // architecture, so --arch would only contradict it.
                containerBuild("--label '${label}' --project '${params.PROJECT}'",
                               repos, revs)
              }
            }]
          }
        }
      }
    }
  }
}