pack-objects: Add runtime detection of online CPU's

Packing objects can be done in parallell nowadays, but it's
only done if the config option pack.threads is set to a value
above 1. Because of that, the code-path used is often not the
most optimal one.

This patch adds a routine to detect the number of online CPU's
at runtime (online_cpus()). When pack.threads (or --threads=) is
given a value of 0, the number of threads is set to the number of
online CPU's. This feature is also documented.

As per Nicolas Pitre's recommendations, the default is still to
run pack-objects single-threaded unless explicitly activated,
either by configuration or by command line parameter.

The routine online_cpus() is a rework of "numcpus.c", written by
one Philip Willoughby <pgw99@doc.ic.ac.uk>. numcpus.c is in the
public domain and can presently be downloaded from
http://csgsoft.doc.ic.ac.uk/numcpus/

Signed-off-by: Andreas Ericsson <ae@op5.se>
Signed-off-by: Brandon Casey <casey@nrlssc.navy.mil>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/config.txt b/Documentation/config.txt
index 7b676710bab0709970abe7411d2246c0eb2b34b8..62b697c087e90a182544f05532aac38255c3839a 100644 (file)
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -808,6 +808,8 @@ pack.threads::
        warning. This is meant to reduce packing time on multiprocessor
        machines. The required amount of memory for the delta search window
        is however multiplied by the number of threads.
        warning. This is meant to reduce packing time on multiprocessor
        machines. The required amount of memory for the delta search window
        is however multiplied by the number of threads.

+       Specifying 0 will cause git to auto-detect the number of CPU's
+       and set the number of threads accordingly.

 
 pack.indexVersion::
        Specify the default pack index version.  Valid values are 1 for
 
 pack.indexVersion::
        Specify the default pack index version.  Valid values are 1 for

diff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt
index 8353be186fcc83092acac16b4fc164d6ea669621..5c1bd3b0813a95ee0c1831a2e10f5512a9330793 100644 (file)
--- a/Documentation/git-pack-objects.txt
+++ b/Documentation/git-pack-objects.txt
@@ -177,6 +177,8 @@ base-name::
        This is meant to reduce packing time on multiprocessor machines.
        The required amount of memory for the delta search window is
        however multiplied by the number of threads.
        This is meant to reduce packing time on multiprocessor machines.
        The required amount of memory for the delta search window is
        however multiplied by the number of threads.

+       Specifying 0 will cause git to auto-detect the number of CPU's
+       and set the number of threads accordingly.

 
 --index-version=<version>[,<offset>]::
        This is intended to be used by the test suite only. It allows
 
 --index-version=<version>[,<offset>]::
        This is intended to be used by the test suite only. It allows

diff --git a/Makefile b/Makefile
index d33a556ffed5c2cddb9577ea014d3c017082abb8..2dc82476e577248150a23a6d55dafddfc02ecfca 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -741,6 +741,7 @@ endif
 ifdef THREADED_DELTA_SEARCH
        BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
        EXTLIBS += -lpthread
 ifdef THREADED_DELTA_SEARCH
        BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
        EXTLIBS += -lpthread

+       LIB_OBJS += thread-utils.o

 endif
 
 ifeq ($(TCLTK_PATH),)
 endif
 
 ifeq ($(TCLTK_PATH),)

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index d2bb12e574fb8e18f9b1e3d241f73f732ac740d3..586ae1197570d99fcbb8913003e8aa3c55b31777 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -16,6 +16,7 @@
 #include "progress.h"
 
 #ifdef THREADED_DELTA_SEARCH
 #include "progress.h"
 
 #ifdef THREADED_DELTA_SEARCH

+#include "thread-utils.h"

 #include <pthread.h>
 #endif
 
 #include <pthread.h>
 #endif
 


@@ -1852,11 +1853,11 @@ static int git_pack_config(const char *k, const char *v)
        }
        if (!strcmp(k, "pack.threads")) {
                delta_search_threads = git_config_int(k, v);
        }
        if (!strcmp(k, "pack.threads")) {
                delta_search_threads = git_config_int(k, v);

-               if (delta_search_threads < 1)
+               if (delta_search_threads < 0)

                        die("invalid number of threads specified (%d)",
                            delta_search_threads);
 #ifndef THREADED_DELTA_SEARCH
                        die("invalid number of threads specified (%d)",
                            delta_search_threads);
 #ifndef THREADED_DELTA_SEARCH

-               if (delta_search_threads > 1)
+               if (delta_search_threads != 1)

                        warning("no threads support, ignoring %s", k);
 #endif
                return 0;
                        warning("no threads support, ignoring %s", k);
 #endif
                return 0;


@@ -2122,10 +2123,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                if (!prefixcmp(arg, "--threads=")) {
                        char *end;
                        delta_search_threads = strtoul(arg+10, &end, 0);
                if (!prefixcmp(arg, "--threads=")) {
                        char *end;
                        delta_search_threads = strtoul(arg+10, &end, 0);

-                       if (!arg[10] || *end || delta_search_threads < 1)
+                       if (!arg[10] || *end || delta_search_threads < 0)

                                usage(pack_usage);
 #ifndef THREADED_DELTA_SEARCH
                                usage(pack_usage);
 #ifndef THREADED_DELTA_SEARCH

-                       if (delta_search_threads > 1)
+                       if (delta_search_threads != 1)

                                warning("no threads support, "
                                        "ignoring %s", arg);
 #endif
                                warning("no threads support, "
                                        "ignoring %s", arg);
 #endif


@@ -2235,6 +2236,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        if (!pack_to_stdout && thin)
                die("--thin cannot be used to build an indexable pack.");
 
        if (!pack_to_stdout && thin)
                die("--thin cannot be used to build an indexable pack.");
 

+#ifdef THREADED_DELTA_SEARCH
+       if (!delta_search_threads)      /* --threads=0 means autodetect */
+               delta_search_threads = online_cpus();
+#endif
+

        prepare_packed_git();
 
        if (progress)
        prepare_packed_git();
 
        if (progress)

diff --git a/thread-utils.c b/thread-utils.c
new file mode 100644 (file)
index 0000000..55e7e29
--- /dev/null
+++ b/thread-utils.c
@@ -0,0 +1,48 @@
+#include "cache.h"
+
+#ifdef _WIN32
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h>
+#elif defined(hpux) || defined(__hpux) || defined(_hpux)
+#  include <sys/pstat.h>
+#endif
+
+/*
+ * By doing this in two steps we can at least get
+ * the function to be somewhat coherent, even
+ * with this disgusting nest of #ifdefs.
+ */
+#ifndef _SC_NPROCESSORS_ONLN
+#  ifdef _SC_NPROC_ONLN
+#    define _SC_NPROCESSORS_ONLN _SC_NPROC_ONLN
+#  elif defined _SC_CRAY_NCPU
+#    define _SC_NPROCESSORS_ONLN _SC_CRAY_NCPU
+#  endif
+#endif
+
+int online_cpus(void)
+{
+#ifdef _SC_NPROCESSORS_ONLN
+       long ncpus;
+#endif
+
+#ifdef _WIN32
+       SYSTEM_INFO info;
+       GetSystemInfo(&info);
+
+       if ((int)info.dwNumberOfProcessors > 0)
+               return (int)info.dwNumberOfProcessors;
+#elif defined(hpux) || defined(__hpux) || defined(_hpux)
+       struct pst_dynamic psd;
+
+       if (!pstat_getdynamic(&psd, sizeof(psd), (size_t)1, 0))
+               return (int)psd.psd_proc_cnt;
+#endif
+
+#ifdef _SC_NPROCESSORS_ONLN
+       if ((ncpus = (long)sysconf(_SC_NPROCESSORS_ONLN)) > 0)
+               return (int)ncpus;
+#endif
+
+       return 1;
+}

diff --git a/thread-utils.h b/thread-utils.h
new file mode 100644 (file)
index 0000000..cce4b77
--- /dev/null
+++ b/thread-utils.h
@@ -0,0 +1,6 @@
+#ifndef THREAD_COMPAT_H
+#define THREAD_COMPAT_H
+
+extern int online_cpus(void);
+
+#endif /* THREAD_COMPAT_H */