Add: [AzurePipeline] introducing a release pipeline

This release pipeline creates all the official release binaries,
and publishes them as artifacts. Currently it can only produce
nightlies and custom builds; stable/testing release binaries are
untested.

This commit also splits up the pipeline in small bits, to both
improve readability, and to share code with the CI pipeline where
possible.

azure-pipelines-ci.yml

@@ -17,37 +17,12 @@ jobs:
         BuildPlatform: 'x64'
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - bash: |
-      set -ex
-      curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
-      unzip windows-dependencies.zip
-      rm -f windows-dependencies.zip
-    displayName: 'Download dependencies'
-    workingDirectory: $(Build.ArtifactStagingDirectory)
-  - script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
-    displayName: 'Install dependencies'
-  - bash: |
-      set -ex
-      cd bin/baseset
-      curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
-      unzip opengfx-0.5.2-all.zip
-      rm -f opengfx-0.5.2-all.zip
-    displayName: 'Install OpenGFX'
-  - task: VSBuild@1
-    displayName: 'Build'
-    inputs:
-      solution: 'projects/openttd_vs141.sln'
-      platform: $(BuildPlatform)
-      configuration: Release
-      maximumCpuCount: true
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  - template: azure-pipelines/templates/windows-dependencies.yml
+  - template: azure-pipelines/templates/ci-opengfx.yml
+  - template: azure-pipelines/templates/windows-build.yml
+    parameters:
+      BuildPlatform: $(BuildPlatform)
   - script: |
       call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86
       cd projects
@@ -67,22 +42,13 @@ jobs:
       linux-i386-gcc-6: {}
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - task: Docker@1
-    displayName: 'Build and test'
-    inputs:
-      command: 'Run an image'
-      imageName: openttd/compile-farm-ci:$(Agent.JobName)
-      volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
-      workingDirectory: '$(Build.SourcesDirectory)'
-      runInBackground: false
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  # The dockers already have the dependencies installed
+  # The dockers already have OpenGFX installed
+  - template: azure-pipelines/templates/linux-build.yml
+    parameters:
+      Image: compile-farm-ci
+      Tag: $(Agent.JobName)
 
 - job: macos
   displayName: 'MacOS'
@@ -90,27 +56,9 @@ jobs:
     vmImage: 'macOS-10.13'
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - script: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
-    displayName: 'Install dependencies'
-  - bash: |
-      set -ex
-      mkdir -p ~/Documents/OpenTTD/baseset
-      cd ~/Documents/OpenTTD/baseset
-      wget https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip
-      unzip opengfx-0.5.2-all.zip
-      rm -f opengfx-0.5.2-all.zip
-    displayName: 'Install OpenGFX'
-  - script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
-    displayName: 'Configure'
-  - script: 'make -j2'
-    displayName: 'Build'
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  - template: azure-pipelines/templates/osx-dependencies.yml
+  - template: azure-pipelines/templates/ci-opengfx.yml
+  - template: azure-pipelines/templates/osx-build.yml
   - script: 'make regression'
     displayName: 'Test'

azure-pipelines-release-stable.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: azure-pipelines/templates/release.yml
+  parameters:
+    IsStableRelease: true

azure-pipelines-release.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: azure-pipelines/templates/release.yml
+  parameters:
+    IsStableRelease: false

azure-pipelines/changelog.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+tag=$(git describe --tags 2>/dev/null)
+
+# If we are a tag, show the part of the changelog that matches the tag.
+# In case of a stable, also show all betas and RCs.
+if [ -n "$tag" ]; then
+    grep="."
+    if [ "$(echo $tag | grep '-')" = "" ]; then
+            grep='^[0-9]\.[0-9]\.[0-9][^-]'
+    fi
+    next=$(cat changelog.txt | grep '^[0-9]' | awk 'BEGIN { show="false" } // { if (show=="true") print $0; if ($1=="'$tag'") show="true"} ' | grep "$grep" | head -n1 | sed 's/ .*//')
+    cat changelog.txt | awk 'BEGIN { show="false" } /^[0-9].[0-9].[0-9]/ { if ($1=="'$next'") show="false"; if ($1=="'$tag'") show="true";} // { if (show=="true") print $0 }'
+    exit 0
+fi
+
+# In all other cases, show the git log of the last 7 days
+revdate=$(git log -1 --pretty=format:"%ci")
+last_week=$(date -u -d "$revdate -7days" +"%Y-%m-%d %H:%M")
+git log --after="${last_week}" --pretty=fuller

azure-pipelines/manifest.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+
+set -ex
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <folder-with-bundles>"
+    exit 1
+fi
+
+FOLDER=$1
+
+if [ ! -e .version ] || [ ! -e .release_date ]; then
+    echo "This script should be executed in the root of an extracted source tarball"
+    exit 1
+fi
+
+# Find the name based on the version
+if [ "${ISSTABLERELEASE}" = "true" ]; then
+    isTesting=$(cat .version | grep "RC\|beta" || true)
+    if [ -z "${isTesting}" ]; then
+        NAME="stable"
+    else
+        NAME="testing"
+    fi
+else
+    NAME=$(cat .version | cut -d- -f2 | cut -d- -f-2)
+fi
+
+# Convert the date to a YAML date
+DATE=$(cat .release_date | tr ' ' T | sed 's/TUTC/:00-00:00/')
+VERSION=$(cat .version)
+BASE="openttd-${VERSION}"
+
+echo "name: ${NAME}" >> manifest.yaml
+echo "date: ${DATE}" >> manifest.yaml
+echo "base: ${BASE}-" >> manifest.yaml
+echo "files:" >> manifest.yaml
+
+error=""
+for i in $(ls ${FOLDER} | grep -v ".txt$\|.md$\|sum$" | sort); do
+    if [ -n "$(echo $i | grep pdb.xz)" ]; then continue; fi
+    if [ -n "$(echo $i | grep dbg.deb)" ]; then continue; fi
+
+    if [ ! -e ${FOLDER}/$i.md5sum ] || [ ! -e ${FOLDER}/$i.sha1sum ] || [ ! -e ${FOLDER}/$i.sha256sum ]; then
+        echo "ERROR: missing checksum file for ${i}" 1>&2
+        error="y"
+        continue
+    fi
+
+    echo "- id: $i" >> manifest.yaml
+    echo "  size: $(stat -c"%s" ${FOLDER}/$i)" >> manifest.yaml
+    echo "  md5sum: $(cat ${FOLDER}/$i.md5sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha1sum: $(cat ${FOLDER}/$i.sha1sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha256sum: $(cat ${FOLDER}/$i.sha256sum | cut -d\  -f1)" >> manifest.yaml
+done
+
+if [ -n "${error}" ]; then
+    echo "ERROR: exiting due to earlier errors" 1>&2
+    exit 1
+fi

azure-pipelines/templates/ci-git-rebase.yml

@@ -0,0 +1,9 @@
+steps:
+# Rebase to origin/master for every PR. This means users don't have to
+# rebase every time master changes. As long as the PR applies cleanly, we
+# will validate it.
+- script: |
+    git config user.email 'info@openttd.org'
+    git config user.name 'OpenTTD CI'
+    git rebase origin/master
+  displayName: 'Rebase to master'

azure-pipelines/templates/ci-opengfx.yml

@@ -0,0 +1,8 @@
+steps:
+- bash: |
+    set -ex
+    cd bin/baseset
+    curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
+    unzip opengfx-0.5.2-all.zip
+    rm -f opengfx-0.5.2-all.zip
+  displayName: 'Install OpenGFX'

azure-pipelines/templates/linux-build.yml

@@ -0,0 +1,18 @@
+parameters:
+  Image: ''
+  Tag: ''
+  ContainerCommand: ''
+
+steps:
+- task: Docker@1
+  ${{ if eq(parameters.Image, 'compile-farm') }}:
+    displayName: 'Build'
+  ${{ if eq(parameters.Image, 'compile-farm-ci') }}:
+    displayName: 'Build and test'
+  inputs:
+    command: 'Run an image'
+    imageName: openttd/${{ parameters.Image }}:${{ parameters.Tag }}
+    volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
+    workingDirectory: '$(Build.SourcesDirectory)'
+    containerCommand: ${{ parameters.ContainerCommand }}
+    runInBackground: false

azure-pipelines/templates/linux-claim-bundles.yml

@@ -0,0 +1,5 @@
+steps:
+# Because we run the compile in a docker (under root), we are not owner
+# of the 'bundles' folder. Fix that by executing a chown on it.
+- bash: sudo chown -R $(id -u):$(id -g) bundles
+  displayName: 'Claim bundles folder back'

azure-pipelines/templates/osx-build.yml

@@ -0,0 +1,5 @@
+steps:
+- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
+  displayName: 'Configure'
+- script: 'make -j2'
+  displayName: 'Build'

azure-pipelines/templates/osx-dependencies.yml

@@ -0,0 +1,11 @@
+steps:
+- script: |
+    set -ex
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
+    # Remove the dynamic libraries of these libraries, to ensure we use
+    # the static versions. That is important, as it is unlikely any
+    # end-user has these brew libraries installed.
+    rm /usr/local/Cellar/lzo/*/lib/*.dylib
+    rm /usr/local/Cellar/xz/*/lib/*.dylib
+    rm /usr/local/Cellar/libpng/*/lib/*.dylib
+  displayName: 'Install dependencies'

azure-pipelines/templates/release-bundles.yml

@@ -0,0 +1,19 @@
+parameters:
+  CalculateChecksums: true
+
+steps:
+- ${{ if eq(parameters.CalculateChecksums, true) }}:
+  - bash: |
+      set -ex
+      cd bundles
+      for i in $(ls); do
+        openssl dgst -r -md5 -hex $i > $i.md5sum
+        openssl dgst -r -sha1 -hex $i > $i.sha1sum
+        openssl dgst -r -sha256 -hex $i > $i.sha256sum
+      done
+    displayName: 'Calculate checksums'
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish bundles'
+  inputs:
+    PathtoPublish: bundles/
+    ArtifactName: bundles

azure-pipelines/templates/release-fetch-source.yml

@@ -0,0 +1,20 @@
+# Fetch the source tarball as prepared by an earlier job. In there is the
+# version predefined. This ensures we are all going to compile the same
+# source with the same version.
+
+steps:
+- checkout: none
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download source'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/openttd-*-source.tar.xz'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- bash: tar --xz -xf ../a/bundles/openttd-*-source.tar.xz --strip-components=1
+  displayName: 'Extracting source'
+- bash: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'

azure-pipelines/templates/release-manifest.yml

@@ -0,0 +1,13 @@
+steps:
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download all bundles'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/*'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- script: |
+    set -ex
+    ./azure-pipelines/manifest.sh ../a/bundles/
+    mkdir -p bundles
+    mv manifest.yaml bundles/
+  displayName: 'Create manifest.yaml'

azure-pipelines/templates/release-prepare-source.yml

@@ -0,0 +1,24 @@
+# Set the revisions, and remove the VCS files.
+# This ensures everything else picks up on the predefined versions, and not
+# that because of some build process the version all of a sudden changes.
+
+steps:
+- script: |
+    set -ex
+    git checkout -B ${BUILD_SOURCEBRANCHNAME}
+    ./findversion.sh > .ottdrev
+    ./azure-pipelines/changelog.sh > .changelog
+    TZ='UTC' date +"%Y-%m-%d %H:%M UTC" > .release_date
+    cat .ottdrev | cut -f 1 -d$'\t' > .version
+    echo "Release Date: $(cat .release_date)"
+    echo "Revision: $(cat .ottdrev)"
+    echo "Version: $(cat .version)"
+  displayName: 'Create version files'
+- script: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'
+- script: find . -iname .hg -or -iname .git -or -iname .svn | xargs rm -rf
+  displayName: 'Remove VCS information'

azure-pipelines/templates/release.yml

@@ -0,0 +1,169 @@
+parameters:
+  # If this is false, not all targets are triggered. For example:
+  # The NSIS installer for Windows and the creation of debs only work for
+  # releases. Not for any other type of binary. So they are skilled if this
+  # is set to false.
+  IsStableRelease: false
+
+jobs:
+- job: source
+  displayName: 'Source'
+  pool:
+    vmImage: 'ubuntu-16.04'
+
+  steps:
+  - template: release-prepare-source.yml
+  - script: |
+      set -ex
+
+      # Rename the folder to openttd-NNN-source
+      mkdir openttd-$(Build.BuildNumber)
+      find . -maxdepth 1 -not -name . -not -name openttd-$(Build.BuildNumber) -exec mv {} openttd-$(Build.BuildNumber)/ \;
+      # Copy back release_date, as it is needed for the template 'release-bundles'
+      cp openttd-$(Build.BuildNumber)/.release_date .release_date
+
+      mkdir bundles
+      tar --xz -cf bundles/openttd-$(Build.BuildNumber)-source.tar.xz openttd-$(Build.BuildNumber)
+      zip -9 -r -q bundles/openttd-$(Build.BuildNumber)-source.zip openttd-$(Build.BuildNumber)
+    displayName: 'Create bundle'
+  - template: release-bundles.yml
+
+- job: meta
+  displayName: 'Metadata'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn: source
+
+  steps:
+  - template: release-fetch-source.yml
+  - script: |
+      set -ex
+
+      mkdir -p bundles
+      cp .changelog bundles/changelog.txt
+      cp .release_date bundles/released.txt
+      cp README.md bundles/README.md
+    displayName: 'Copy meta files'
+  - template: release-bundles.yml
+    parameters:
+      CalculateChecksums: false
+
+- job: docs
+  displayName: 'Docs'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn: source
+
+  steps:
+    - template: release-fetch-source.yml
+    - template: linux-build.yml
+      parameters:
+        Image: compile-farm
+        ContainerCommand: '$(Build.BuildNumber)'
+        Tag: docs
+    - template: linux-claim-bundles.yml
+    - template: release-bundles.yml
+
+- job: windows
+  displayName: 'Windows'
+  pool:
+    vmImage: 'VS2017-Win2016'
+  dependsOn: source
+
+  strategy:
+    matrix:
+      Win32:
+        BuildPlatform: 'Win32'
+        BundlePlatform: 'win32'
+      Win64:
+        BuildPlatform: 'x64'
+        BundlePlatform: 'win64'
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: windows-dependencies.yml
+  - template: windows-dependency-zip.yml
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - template: windows-dependency-nsis.yml
+  - template: windows-build.yml
+    parameters:
+      BuildPlatform: $(BuildPlatform)
+  - bash: |
+      set -ex
+      make -f Makefile.msvc bundle_pdb bundle_zip PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
+    displayName: 'Create bundles'
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - bash: |
+        set -ex
+        # NSIS will be part of the Hosted image in the next update. Till then, we set the PATH ourself
+        export PATH="${PATH}:/c/Program Files (x86)/NSIS"
+        make -f Makefile.msvc bundle_exe PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
+      displayName: 'Create installer bundle'
+  - template: release-bundles.yml
+
+- ${{ if eq(parameters.IsStableRelease, true) }}:
+  - job: linux_stable
+    displayName: 'Linux'
+    pool:
+      vmImage: 'ubuntu-16.04'
+    dependsOn: source
+
+    strategy:
+      matrix:
+        linux-ubuntu-xenial-i386-gcc: {}
+        linux-ubuntu-xenial-amd64-gcc: {}
+        linux-ubuntu-bionic-i386-gcc: {}
+        linux-ubuntu-bionic-amd64-gcc: {}
+        linux-debian-jessie-i386-gcc: {}
+        linux-debian-jessie-amd64-gcc: {}
+        linux-debian-stretch-i386-gcc: {}
+        linux-debian-stretch-amd64-gcc: {}
+
+    steps:
+    - template: release-fetch-source.yml
+    - template: linux-build.yml
+      parameters:
+        Image: compile-farm
+        ContainerCommand: '$(Build.BuildNumber)'
+        Tag: $(Agent.JobName)
+    - template: linux-claim-bundles.yml
+    - template: release-bundles.yml
+
+- job: macos
+  displayName: 'MacOS'
+  pool:
+    vmImage: 'macOS-10.13'
+  dependsOn: source
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: osx-dependencies.yml
+  - template: osx-build.yml
+  - script: 'make bundle_zip bundle_dmg BUNDLE_NAME=openttd-$(Build.BuildNumber)-macosx'
+    displayName: 'Create bundles'
+  - template: release-bundles.yml
+
+- job: manifest
+  displayName: 'Manifest'
+  pool:
+    vmImage: 'ubuntu-16.04'
+  dependsOn:
+  - source
+  - docs
+  - windows
+  - ${{ if eq(parameters.IsStableRelease, true) }}:
+    - linux_stable
+  - macos
+  # "Skipped" is not a status, and is not succeeded. So it seems to be
+  # considered failed. So we trigger if all the earlier jobs are done (which
+  # might be succeeded, failed, or skipped), and run this job. This is not
+  # optimal, but given the rules, it is the only way to get this to work (as
+  # some jobs might be skipped).
+  condition: succeededOrFailed()
+
+  steps:
+  - template: release-fetch-source.yml
+  - template: release-manifest.yml
+  - template: release-bundles.yml
+    parameters:
+      CalculateChecksums: false

azure-pipelines/templates/windows-build.yml

@@ -0,0 +1,11 @@
+parameters:
+  BuildPlatform: ''
+
+steps:
+- task: VSBuild@1
+  displayName: 'Build'
+  inputs:
+    solution: 'projects/openttd_vs141.sln'
+    platform: ${{ parameters.BuildPlatform }}
+    configuration: Release
+    maximumCpuCount: true

azure-pipelines/templates/windows-dependencies.yml

@@ -0,0 +1,10 @@
+steps:
+- bash: |
+    set -ex
+    curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
+    unzip windows-dependencies.zip
+    rm -f windows-dependencies.zip
+  displayName: 'Download dependencies'
+  workingDirectory: $(Build.ArtifactStagingDirectory)
+- script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
+  displayName: 'Install dependencies'

azure-pipelines/templates/windows-dependency-nsis.yml

@@ -0,0 +1,28 @@
+parameters:
+  condition: true
+
+steps:
+- bash: |
+    set -ex
+    # NSIS will be part of the Hosted image in the next update. Till then, we install it ourself
+    choco install nsis -y
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/Nsis7z.zip > Nsis7z.zip
+    unzip Nsis7z.zip
+    cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/"
+    cd ..; rm -rf nsis-plugin
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisGetVersion.zip > NsisGetVersion.zip
+    unzip NsisGetVersion.zip
+    cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
+    cd ..; rm -rf nsis-plugin
+
+    mkdir nsis-plugin; cd nsis-plugin
+    curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisFindProc.zip > NsisFindProc.zip
+    unzip NsisFindProc.zip
+    cp -R *.dll "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
+    cd ..; rm -rf nsis-plugin
+  displayName: 'Install NSIS with the 7z, GetVersion, and FindProc plugins'
+  condition: and(succeeded(), ${{ parameters.condition }})

azure-pipelines/templates/windows-dependency-zip.yml

@@ -0,0 +1,5 @@
+steps:
+- bash: |
+    set -ex
+    choco install zip
+  displayName: 'Install zip'

findversion.sh

@@ -74,8 +74,6 @@ if [ -d "$ROOT_DIR/.git" ]; then
 
 	if [ -n "$TAG" ]; then
 		VERSION="${TAG}"
-	elif [ "${BRANCH}" = "master" ]; then
-		VERSION="${ISODATE}-g${SHORTHASH}"
 	else
 		VERSION="${ISODATE}-${BRANCH}-g${SHORTHASH}"
 	fi