ci: parallelize testing on Windows
authorJohannes Schindelin <johannes.schindelin@gmx.de>
Tue, 29 Jan 2019 14:19:38 +0000 (06:19 -0800)
committerJunio C Hamano <gitster@pobox.com>
Tue, 29 Jan 2019 17:26:47 +0000 (09:26 -0800)
The fact that Git's test suite is implemented in Unix shell script that
is as portable as we can muster, combined with the fact that Unix shell
scripting is foreign to Windows (and therefore has to be emulated),
results in pretty abysmal speed of the test suite on that platform, for
pretty much no other reason than that language choice.

For comparison: while the Linux build & test is typically done within
about 8 minutes, the Windows build & test typically lasts about 80
minutes in Azure Pipelines.

To help with that, let's use the Azure Pipeline feature where you can
parallelize jobs, make jobs depend on each other, and pass artifacts
between them.

The tests are distributed using the following heuristic: listing all
test scripts ordered by size in descending order (as a cheap way to
estimate the overall run time), every Nth script is run (where N is the
total number of parallel jobs), starting at the index corresponding to
the parallel job. This slicing is performed by a new function that is
added to the `test-tool`.

To optimize the overall runtime of the entire Pipeline, we need to move
the Windows jobs to the beginning (otherwise there would be a very
decent chance for the Pipeline to be run only the Windows build, while
all the parallel Windows test jobs wait for this single one).

We use Azure Pipelines Artifacts for both the minimal Git for Windows
SDK as well as the built executables, as deduplication and caching close
to the agents makes that really fast. For comparison: while downloading
and unpacking the minimal Git for Windows SDK via PowerShell takes only
one minute (down from anywhere between 2.5 to 7 when using a shallow
clone), uploading it as Pipeline Artifact takes less than 30s and
downloading and unpacking less than 20s (sometimes even as little as
only twelve seconds).

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Makefile
azure-pipelines.yml
ci/make-test-artifacts.sh [new file with mode: 0755]
ci/run-test-slice.sh [new file with mode: 0755]
t/helper/test-path-utils.c
index 044b4f77bd7be3b4cab1ae78b23afad448c2ce2a..daa318fe1727a4d528c873ff4d75c329df5fd797 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -2927,6 +2927,16 @@ rpm::
        @false
 .PHONY: rpm
 
+artifacts-tar:: $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) $(OTHER_PROGRAMS) \
+               GIT-BUILD-OPTIONS $(TEST_PROGRAMS) $(test_bindir_programs) \
+               $(NO_INSTALL) $(MOFILES)
+       $(QUIET_SUBDIR0)templates $(QUIET_SUBDIR1) \
+               SHELL_PATH='$(SHELL_PATH_SQ)' PERL_PATH='$(PERL_PATH_SQ)'
+       test -n "$(ARTIFACTS_DIRECTORY)"
+       mkdir -p "$(ARTIFACTS_DIRECTORY)"
+       $(TAR) czf "$(ARTIFACTS_DIRECTORY)/artifacts.tar.gz" $^ templates/blt/
+.PHONY: artifacts-tar
+
 htmldocs = git-htmldocs-$(GIT_VERSION)
 manpages = git-manpages-$(GIT_VERSION)
 .PHONY: dist-doc distclean
index 480e841a85dd7067c6fb23ccfd2f3ed52281a0b4..c329b7218bb361c077bb373d6bb04a3196f17256 100644 (file)
@@ -3,8 +3,8 @@ resources:
   fetchDepth: 1
 
 jobs:
-- job: windows
-  displayName: Windows
+- job: windows_build
+  displayName: Windows Build
   condition: succeeded()
   pool: Hosted
   timeoutInMinutes: 240
@@ -30,21 +30,84 @@ jobs:
     displayName: 'Download git-sdk-64-minimal'
   - powershell: |
       & git-sdk-64-minimal\usr\bin\bash.exe -lc @"
-        export DEVELOPER=1
-        export NO_PERL=1
-        export NO_SVN_TESTS=1
-        export GIT_TEST_SKIP_REBASE_P=1
+        ci/make-test-artifacts.sh artifacts
+      "@
+      if (!$?) { exit(1) }
+    displayName: Build
+    env:
+      HOME: $(Build.SourcesDirectory)
+      MSYSTEM: MINGW64
+      DEVELOPER: 1
+      NO_PERL: 1
+  - task: PublishPipelineArtifact@0
+    displayName: 'Publish Pipeline Artifact: test artifacts'
+    inputs:
+      artifactName: 'windows-artifacts'
+      targetPath: '$(Build.SourcesDirectory)\artifacts'
+  - task: PublishPipelineArtifact@0
+    displayName: 'Publish Pipeline Artifact: git-sdk-64-minimal'
+    inputs:
+      artifactName: 'git-sdk-64-minimal'
+      targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal'
+  - powershell: |
+      if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") {
+        cmd /c rmdir "$(Build.SourcesDirectory)\test-cache"
+      }
+    displayName: 'Unmount test-cache'
+    condition: true
+    env:
+      GITFILESHAREPWD: $(gitfileshare.pwd)
+
+- job: windows_test
+  displayName: Windows Test
+  dependsOn: windows_build
+  condition: succeeded()
+  pool: Hosted
+  timeoutInMinutes: 240
+  strategy:
+    parallel: 10
+  steps:
+  - powershell: |
+      if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") {
+        net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no
+        cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\
+      }
+    displayName: 'Mount test-cache'
+    env:
+      GITFILESHAREPWD: $(gitfileshare.pwd)
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact: test artifacts'
+    inputs:
+      artifactName: 'windows-artifacts'
+      targetPath: '$(Build.SourcesDirectory)'
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact: git-sdk-64-minimal'
+    inputs:
+      artifactName: 'git-sdk-64-minimal'
+      targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal'
+  - powershell: |
+      & git-sdk-64-minimal\usr\bin\bash.exe -lc @"
+        test -f artifacts.tar.gz || {
+          echo No test artifacts found\; skipping >&2
+          exit 0
+        }
+        tar xf artifacts.tar.gz || exit 1
+
+        # Let Git ignore the SDK and the test-cache
+        printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude
 
-        ci/run-build-and-tests.sh || {
+        ci/run-test-slice.sh `$SYSTEM_JOBPOSITIONINPHASE `$SYSTEM_TOTALJOBSINPHASE || {
           ci/print-test-failures.sh
           exit 1
         }
       "@
       if (!$?) { exit(1) }
-    displayName: 'Build & Test'
+    displayName: 'Test (parallel)'
     env:
       HOME: $(Build.SourcesDirectory)
       MSYSTEM: MINGW64
+      NO_SVN_TESTS: 1
+      GIT_TEST_SKIP_REBASE_P: 1
   - powershell: |
       if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") {
         cmd /c rmdir "$(Build.SourcesDirectory)\test-cache"
diff --git a/ci/make-test-artifacts.sh b/ci/make-test-artifacts.sh
new file mode 100755 (executable)
index 0000000..6469674
--- /dev/null
@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build Git and store artifacts for testing
+#
+
+mkdir -p "$1" # in case ci/lib.sh decides to quit early
+
+. ${0%/*}/lib.sh
+
+make artifacts-tar ARTIFACTS_DIRECTORY="$1"
+
+check_unignored_build_artifacts
diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh
new file mode 100755 (executable)
index 0000000..f8c2c31
--- /dev/null
@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Test Git in parallel
+#
+
+. ${0%/*}/lib.sh
+
+case "$CI_OS_NAME" in
+windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
+*) ln -s "$cache_dir/.prove" t/.prove;;
+esac
+
+make --quiet -C t T="$(cd t &&
+       ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
+       tr '\n' ' ')"
+
+check_unignored_build_artifacts
index 6efde6f5ba5b7203a73335c51114c2d47156231d..5d543ad21f89c76534e8efe32d6ead512dc7d0c3 100644 (file)
@@ -177,6 +177,14 @@ static int is_dotgitmodules(const char *path)
        return is_hfs_dotgitmodules(path) || is_ntfs_dotgitmodules(path);
 }
 
+static int cmp_by_st_size(const void *a, const void *b)
+{
+       intptr_t x = (intptr_t)((struct string_list_item *)a)->util;
+       intptr_t y = (intptr_t)((struct string_list_item *)b)->util;
+
+       return x > y ? -1 : (x < y ? +1 : 0);
+}
+
 int cmd__path_utils(int argc, const char **argv)
 {
        if (argc == 3 && !strcmp(argv[1], "normalize_path_copy")) {
@@ -324,6 +332,29 @@ int cmd__path_utils(int argc, const char **argv)
                return 0;
        }
 
+       if (argc > 5 && !strcmp(argv[1], "slice-tests")) {
+               int res = 0;
+               long offset, stride, i;
+               struct string_list list = STRING_LIST_INIT_NODUP;
+               struct stat st;
+
+               offset = strtol(argv[2], NULL, 10);
+               stride = strtol(argv[3], NULL, 10);
+               if (stride < 1)
+                       stride = 1;
+               for (i = 4; i < argc; i++)
+                       if (stat(argv[i], &st))
+                               res = error_errno("Cannot stat '%s'", argv[i]);
+                       else
+                               string_list_append(&list, argv[i])->util =
+                                       (void *)(intptr_t)st.st_size;
+               QSORT(list.items, list.nr, cmp_by_st_size);
+               for (i = offset; i < list.nr; i+= stride)
+                       printf("%s\n", list.items[i].string);
+
+               return !!res;
+       }
+
        fprintf(stderr, "%s: unknown function name: %s\n", argv[0],
                argv[1] ? argv[1] : "(there was none)");
        return 1;