Merge branch 'bw/protocol-v2' into jt/partial-clone-proto-v2

The beginning of the next-gen transfer protocol.

* bw/protocol-v2: (35 commits)
remote-curl: don't request v2 when pushing
remote-curl: implement stateless-connect command
http: eliminate "# service" line when using protocol v2
http: don't always add Git-Protocol header
http: allow providing extra headers for http requests
remote-curl: store the protocol version the server responded with
remote-curl: create copy of the service name
pkt-line: add packet_buf_write_len function
transport-helper: introduce stateless-connect
transport-helper: refactor process_connect_service
transport-helper: remove name parameter
connect: don't request v2 when pushing
connect: refactor git_connect to only get the protocol version once
fetch-pack: support shallow requests
fetch-pack: perform a fetch using v2
upload-pack: introduce fetch server command
push: pass ref prefixes when pushing
fetch: pass ref prefixes when fetching
ls-remote: pass ref prefixes when requesting a remote's refs
transport: convert transport_get_remote_refs to take a list of ref prefixes
...

diff --cc Documentation/Makefile
Simple merge

diff --cc Documentation/gitremote-helpers.txt
Simple merge

diff --cc Makefile
index 50da82b0169ad7569a900a5b7dbd0b98f147a7f0,fb2175bcf1ed909f8726b14f72e21fd36ae9a658..d23f26563631f85f8ffeb10e994370335af3b5ef
--- 1/Makefile
--- 2/Makefile
+++ b/Makefile
@@@ -660,49 -643,48 +659,50 @@@ X 
  
  PROGRAMS += $(patsubst %.o,git-%$X,$(PROGRAM_OBJS))
  
 -TEST_PROGRAMS_NEED_X += test-chmtime
 -TEST_PROGRAMS_NEED_X += test-ctype
 -TEST_PROGRAMS_NEED_X += test-config
 -TEST_PROGRAMS_NEED_X += test-date
 -TEST_PROGRAMS_NEED_X += test-delta
 -TEST_PROGRAMS_NEED_X += test-drop-caches
 -TEST_PROGRAMS_NEED_X += test-dump-cache-tree
 +TEST_BUILTINS_OBJS += test-chmtime.o
 +TEST_BUILTINS_OBJS += test-config.o
 +TEST_BUILTINS_OBJS += test-ctype.o
 +TEST_BUILTINS_OBJS += test-date.o
 +TEST_BUILTINS_OBJS += test-delta.o
 +TEST_BUILTINS_OBJS += test-drop-caches.o
 +TEST_BUILTINS_OBJS += test-dump-cache-tree.o
 +TEST_BUILTINS_OBJS += test-dump-split-index.o
 +TEST_BUILTINS_OBJS += test-example-decorate.o
 +TEST_BUILTINS_OBJS += test-genrandom.o
 +TEST_BUILTINS_OBJS += test-hashmap.o
 +TEST_BUILTINS_OBJS += test-index-version.o
 +TEST_BUILTINS_OBJS += test-lazy-init-name-hash.o
 +TEST_BUILTINS_OBJS += test-match-trees.o
 +TEST_BUILTINS_OBJS += test-mergesort.o
 +TEST_BUILTINS_OBJS += test-mktemp.o
 +TEST_BUILTINS_OBJS += test-online-cpus.o
 +TEST_BUILTINS_OBJS += test-path-utils.o
 +TEST_BUILTINS_OBJS += test-prio-queue.o
 +TEST_BUILTINS_OBJS += test-read-cache.o
 +TEST_BUILTINS_OBJS += test-ref-store.o
 +TEST_BUILTINS_OBJS += test-regex.o
 +TEST_BUILTINS_OBJS += test-revision-walking.o
 +TEST_BUILTINS_OBJS += test-run-command.o
 +TEST_BUILTINS_OBJS += test-scrap-cache-tree.o
 +TEST_BUILTINS_OBJS += test-sha1-array.o
 +TEST_BUILTINS_OBJS += test-sha1.o
 +TEST_BUILTINS_OBJS += test-sigchain.o
 +TEST_BUILTINS_OBJS += test-strcmp-offset.o
 +TEST_BUILTINS_OBJS += test-string-list.o
 +TEST_BUILTINS_OBJS += test-submodule-config.o
 +TEST_BUILTINS_OBJS += test-subprocess.o
 +TEST_BUILTINS_OBJS += test-urlmatch-normalization.o
 +TEST_BUILTINS_OBJS += test-wildmatch.o
 +TEST_BUILTINS_OBJS += test-write-cache.o
 +
  TEST_PROGRAMS_NEED_X += test-dump-fsmonitor
 -TEST_PROGRAMS_NEED_X += test-dump-split-index
  TEST_PROGRAMS_NEED_X += test-dump-untracked-cache
 -TEST_PROGRAMS_NEED_X += test-example-decorate
  TEST_PROGRAMS_NEED_X += test-fake-ssh
 -TEST_PROGRAMS_NEED_X += test-genrandom
 -TEST_PROGRAMS_NEED_X += test-hashmap
 -TEST_PROGRAMS_NEED_X += test-index-version
 -TEST_PROGRAMS_NEED_X += test-lazy-init-name-hash
  TEST_PROGRAMS_NEED_X += test-line-buffer
 -TEST_PROGRAMS_NEED_X += test-match-trees
 -TEST_PROGRAMS_NEED_X += test-mergesort
 -TEST_PROGRAMS_NEED_X += test-mktemp
 -TEST_PROGRAMS_NEED_X += test-online-cpus
  TEST_PROGRAMS_NEED_X += test-parse-options
 -TEST_PROGRAMS_NEED_X += test-path-utils
+ TEST_PROGRAMS_NEED_X += test-pkt-line
 -TEST_PROGRAMS_NEED_X += test-prio-queue
 -TEST_PROGRAMS_NEED_X += test-read-cache
 -TEST_PROGRAMS_NEED_X += test-write-cache
 -TEST_PROGRAMS_NEED_X += test-ref-store
 -TEST_PROGRAMS_NEED_X += test-regex
 -TEST_PROGRAMS_NEED_X += test-revision-walking
 -TEST_PROGRAMS_NEED_X += test-run-command
 -TEST_PROGRAMS_NEED_X += test-scrap-cache-tree
 -TEST_PROGRAMS_NEED_X += test-sha1
 -TEST_PROGRAMS_NEED_X += test-sha1-array
 -TEST_PROGRAMS_NEED_X += test-sigchain
 -TEST_PROGRAMS_NEED_X += test-strcmp-offset
 -TEST_PROGRAMS_NEED_X += test-string-list
 -TEST_PROGRAMS_NEED_X += test-submodule-config
 -TEST_PROGRAMS_NEED_X += test-subprocess
  TEST_PROGRAMS_NEED_X += test-svn-fe
 -TEST_PROGRAMS_NEED_X += test-urlmatch-normalization
 -TEST_PROGRAMS_NEED_X += test-wildmatch
 +TEST_PROGRAMS_NEED_X += test-tool
  
  TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X))
  

diff --cc builtin/clone.c
index 7df5932b855e874d45b970f6150d65ed25b06f5f,4db3079acc03a2da3bf528ed545bbf465abad767..84f1473d19dc5a521e58c0bc1a7363808888dff1
--- 1/builtin/clone.c
--- 2/builtin/clone.c
+++ b/builtin/clone.c
@@@ -1126,16 -1101,10 +1126,16 @@@ int cmd_clone(int argc, const char **ar
                transport_set_option(transport, TRANS_OPT_UPLOADPACK,
                                     option_upload_pack);
  
 -      if (transport->smart_options && !deepen)
 +      if (filter_options.choice) {
 +              transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER,
 +                                   filter_options.filter_spec);
 +              transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
 +      }
 +
 +      if (transport->smart_options && !deepen && !filter_options.choice)
                transport->smart_options->check_self_contained_and_connected = 1;
  
-       refs = transport_get_remote_refs(transport);
+       refs = transport_get_remote_refs(transport, NULL);
  
        if (refs) {
                mapped_refs = wanted_peer_refs(refs, refspec);

diff --cc builtin/fetch-pack.c
index a7bc1366ab375765c41014640743ef9d77c84c42,f9d7d0b5a519a94317326e0b1baa562aade639a2..1a1bc63566b44bc83c8429463104615d1b2117ff
--- 1/builtin/fetch-pack.c
--- 2/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@@ -52,9 -53,8 +53,10 @@@ int cmd_fetch_pack(int argc, const cha
        struct fetch_pack_args args;
        struct oid_array shallow = OID_ARRAY_INIT;
        struct string_list deepen_not = STRING_LIST_INIT_DUP;
+       struct packet_reader reader;
  
 +      fetch_if_missing = 0;
 +
        packet_trace_identity("fetch-pack");
  
        memset(&args, 0, sizeof(args));

diff --cc builtin/fetch.c
Simple merge

diff --cc builtin/ls-remote.c
Simple merge

diff --cc builtin/receive-pack.c
Simple merge

diff --cc builtin/remote.c
Simple merge

diff --cc builtin/upload-pack.c
index 0000000000000000000000000000000000000000,a757df8da0abe7e979fe6b14aabe88da4cda548a..decde5a3b1b70442e1550027359dab11f4704468
mode 000000,100644..100644
--- /dev/null
--- 2/builtin/upload-pack.c
+++ b/builtin/upload-pack.c
@@@ -1,0 -1,74 +1,74 @@@
 -#include "exec_cmd.h"
+ #include "cache.h"
+ #include "builtin.h"
++#include "exec-cmd.h"
+ #include "pkt-line.h"
+ #include "parse-options.h"
+ #include "protocol.h"
+ #include "upload-pack.h"
+ #include "serve.h"
+ 
+ static const char * const upload_pack_usage[] = {
+       N_("git upload-pack [<options>] <dir>"),
+       NULL
+ };
+ 
+ int cmd_upload_pack(int argc, const char **argv, const char *prefix)
+ {
+       const char *dir;
+       int strict = 0;
+       struct upload_pack_options opts = { 0 };
+       struct serve_options serve_opts = SERVE_OPTIONS_INIT;
+       struct option options[] = {
+               OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc,
+                        N_("quit after a single request/response exchange")),
+               OPT_BOOL(0, "advertise-refs", &opts.advertise_refs,
+                        N_("exit immediately after initial ref advertisement")),
+               OPT_BOOL(0, "strict", &strict,
+                        N_("do not try <directory>/.git/ if <directory> is no Git directory")),
+               OPT_INTEGER(0, "timeout", &opts.timeout,
+                           N_("interrupt transfer after <n> seconds of inactivity")),
+               OPT_END()
+       };
+ 
+       packet_trace_identity("upload-pack");
+       check_replace_refs = 0;
+ 
+       argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0);
+ 
+       if (argc != 1)
+               usage_with_options(upload_pack_usage, options);
+ 
+       if (opts.timeout)
+               opts.daemon_mode = 1;
+ 
+       setup_path();
+ 
+       dir = argv[0];
+ 
+       if (!enter_repo(dir, strict))
+               die("'%s' does not appear to be a git repository", dir);
+ 
+       switch (determine_protocol_version_server()) {
+       case protocol_v2:
+               serve_opts.advertise_capabilities = opts.advertise_refs;
+               serve_opts.stateless_rpc = opts.stateless_rpc;
+               serve(&serve_opts);
+               break;
+       case protocol_v1:
+               /*
+                * v1 is just the original protocol with a version string,
+                * so just fall through after writing the version string.
+                */
+               if (opts.advertise_refs || !opts.stateless_rpc)
+                       packet_write_fmt(1, "version 1\n");
+ 
+               /* fallthrough */
+       case protocol_v0:
+               upload_pack(&opts);
+               break;
+       case protocol_unknown_version:
+               BUG("unknown protocol version");
+       }
+ 
+       return 0;
+ }

diff --cc fetch-pack.c
index 4a8bad8487f611c263bcd7e31febd08eed5bc6d7,837e1fd21d1a9947b209b778d3d2ceccde912454..f93723fec41c772db0c0d6f5a8bb2239c9d370bd
--- 1/fetch-pack.c
--- 2/fetch-pack.c
+++ b/fetch-pack.c
@@@ -1239,9 -1490,13 +1562,13 @@@ struct ref *fetch_pack(struct fetch_pac
                die(_("no matching remote head"));
        }
        prepare_shallow_info(&si, shallow);
-       ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
-                               &si, pack_lockfile);
+       if (version == protocol_v2)
+               ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
+                                          pack_lockfile);
+       else
+               ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
+                                       &si, pack_lockfile);
 -      reprepare_packed_git();
 +      reprepare_packed_git(the_repository);
        update_shallow(args, sought, nr_sought, &si);
        clear_shallow_info(&si);
        return ref_cpy;

diff --cc fetch-pack.h
index 3e224a18226ec6219b09387704baf178821c4c23,7afca730560b7fb62ff7111a044a29d7caac0b8c..6afa08b48bb9f7cc2db1b2fb4644c576fd8d8f3e
--- 1/fetch-pack.h
--- 2/fetch-pack.h
+++ b/fetch-pack.h
@@@ -3,7 -3,7 +3,8 @@@
  
  #include "string-list.h"
  #include "run-command.h"
+ #include "protocol.h"
 +#include "list-objects-filter-options.h"
  
  struct oid_array;
  

diff --cc git.c
index f598fae7b7ab727abedc44decd2f51f107014088,f85d682b62d1ee24e1526576c41982204c5f73c9..158f2cbdb15210a4f05da537690e70300fb62610
--- 1/git.c
--- 2/git.c
+++ b/git.c
@@@ -480,9 -477,10 +481,10 @@@ static struct cmd_struct commands[] = 
        { "update-index", cmd_update_index, RUN_SETUP },
        { "update-ref", cmd_update_ref, RUN_SETUP },
        { "update-server-info", cmd_update_server_info, RUN_SETUP },
 -      { "upload-archive", cmd_upload_archive },
 -      { "upload-archive--writer", cmd_upload_archive_writer },
 +      { "upload-archive", cmd_upload_archive, NO_PARSEOPT },
 +      { "upload-archive--writer", cmd_upload_archive_writer, NO_PARSEOPT },
+       { "upload-pack", cmd_upload_pack },
 -      { "var", cmd_var, RUN_SETUP_GENTLY },
 +      { "var", cmd_var, RUN_SETUP_GENTLY | NO_PARSEOPT },
        { "verify-commit", cmd_verify_commit, RUN_SETUP },
        { "verify-pack", cmd_verify_pack },
        { "verify-tag", cmd_verify_tag, RUN_SETUP },

diff --cc http-backend.c
index cc16cd04ad00a85c0a8a73eec837760b293a3489,5d241e9109f5f860ed9d4ec9f4ea0efedd422b22..adaef16fadfd03f34b8ac5bb496bd51aab292b20
--- 1/http-backend.c
--- 2/http-backend.c
+++ b/http-backend.c
@@@ -11,7 -10,7 +11,8 @@@
  #include "url.h"
  #include "argv-array.h"
  #include "packfile.h"
 +#include "object-store.h"
+ #include "protocol.h"
  
  static const char content_type[] = "Content-Type";
  static const char content_length[] = "Content-Length";

diff --cc http.c
Simple merge

diff --cc refs.c
Simple merge

diff --cc remote-curl.c
index 8d2ffaf8de4a0e2d5c58f3904aee5c1823997ebc,595447b16eb8b175f481b1c8b6d0704d4fa3d650..68e7b1cf5847f2af4147d29920686bfd307b5ba3
--- 1/remote-curl.c
--- 2/remote-curl.c
+++ b/remote-curl.c
@@@ -13,7 -14,7 +14,8 @@@
  #include "credential.h"
  #include "sha1-array.h"
  #include "send-pack.h"
+ #include "protocol.h"
 +#include "quote.h"
  
  static struct remote *remote;
  /* always ends with a trailing slash */

diff --cc remote.h
Simple merge

diff --cc transport-helper.c
Simple merge

diff --cc transport.c
index 94eccf29aa3f9758f5acaa9daddedcd559ad3915,342db492cac5dfaec33de407bfca16aa9951781f..16b2f54f22e910c27ea10c32dfbc398925cc85c1
--- 1/transport.c
--- 2/transport.c
+++ b/transport.c
@@@ -18,7 -18,7 +18,8 @@@
  #include "sha1-array.h"
  #include "sigchain.h"
  #include "transport-internal.h"
+ #include "protocol.h"
 +#include "object-store.h"
  
  static void set_upstreams(struct transport *transport, struct ref *refs,
        int pretend)
@@@ -239,21 -252,29 +262,32 @@@ static int fetch_refs_via_pack(struct t
                data->options.check_self_contained_and_connected;
        args.cloning = transport->cloning;
        args.update_shallow = data->options.update_shallow;
 +      args.from_promisor = data->options.from_promisor;
 +      args.no_dependents = data->options.no_dependents;
 +      args.filter_options = data->options.filter_options;
+       args.stateless_rpc = transport->stateless_rpc;
  
-       if (!data->got_remote_heads) {
-               connect_setup(transport, 0);
-               get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0,
-                                NULL, &data->shallow);
-               data->got_remote_heads = 1;
+       if (!data->got_remote_heads)
+               refs_tmp = get_refs_via_connect(transport, 0, NULL);
+ 
+       switch (data->version) {
+       case protocol_v2:
+               refs = fetch_pack(&args, data->fd, data->conn,
+                                 refs_tmp ? refs_tmp : transport->remote_refs,
+                                 dest, to_fetch, nr_heads, &data->shallow,
+                                 &transport->pack_lockfile, data->version);
+               break;
+       case protocol_v1:
+       case protocol_v0:
+               refs = fetch_pack(&args, data->fd, data->conn,
+                                 refs_tmp ? refs_tmp : transport->remote_refs,
+                                 dest, to_fetch, nr_heads, &data->shallow,
+                                 &transport->pack_lockfile, data->version);
+               break;
+       case protocol_unknown_version:
+               BUG("unknown protocol version");
        }
  
-       refs = fetch_pack(&args, data->fd, data->conn,
-                         refs_tmp ? refs_tmp : transport->remote_refs,
-                         dest, to_fetch, nr_heads, &data->shallow,
-                         &transport->pack_lockfile);
        close(data->fd[0]);
        close(data->fd[1]);
        if (finish_connect(data->conn))

diff --cc transport.h
Simple merge

diff --cc upload-pack.c
index 6261d4fab32999322b2b1b866289b49077c63beb,4c9428c2db64daa2cde9225419ff88d4e2f4fbbb..87b4d32a6e23aed2416a9bb752dfa56b916b141c
--- 1/upload-pack.c
--- 2/upload-pack.c
+++ b/upload-pack.c
@@@ -20,13 -17,8 +19,9 @@@
  #include "argv-array.h"
  #include "prio-queue.h"
  #include "protocol.h"
- 
- static const char * const upload_pack_usage[] = {
-       N_("git upload-pack [<options>] <dir>"),
-       NULL
- };
 +#include "quote.h"
+ #include "upload-pack.h"
+ #include "serve.h"
  
  /* Remember to update object flag allocation in object.h */
  #define THEY_HAVE     (1u << 11)
@@@ -770,55 -860,15 +878,22 @@@ static void receive_needs(void
                if (!line)
                        break;
  
-               if (skip_prefix(line, "shallow ", &arg)) {
-                       struct object_id oid;
-                       struct object *object;
-                       if (get_oid_hex(arg, &oid))
-                               die("invalid shallow line: %s", line);
-                       object = parse_object(&oid);
-                       if (!object)
-                               continue;
-                       if (object->type != OBJ_COMMIT)
-                               die("invalid shallow object %s", oid_to_hex(&oid));
-                       if (!(object->flags & CLIENT_SHALLOW)) {
-                               object->flags |= CLIENT_SHALLOW;
-                               add_object_array(object, NULL, &shallows);
-                       }
+               if (process_shallow(line, &shallows))
                        continue;
-               }
-               if (skip_prefix(line, "deepen ", &arg)) {
-                       char *end = NULL;
-                       depth = strtol(arg, &end, 0);
-                       if (!end || *end || depth <= 0)
-                               die("Invalid deepen: %s", line);
+               if (process_deepen(line, &depth))
                        continue;
-               }
-               if (skip_prefix(line, "deepen-since ", &arg)) {
-                       char *end = NULL;
-                       deepen_since = parse_timestamp(arg, &end, 0);
-                       if (!end || *end || !deepen_since ||
-                           /* revisions.c's max_age -1 is special */
-                           deepen_since == -1)
-                               die("Invalid deepen-since: %s", line);
-                       deepen_rev_list = 1;
+               if (process_deepen_since(line, &deepen_since, &deepen_rev_list))
                        continue;
-               }
-               if (skip_prefix(line, "deepen-not ", &arg)) {
-                       char *ref = NULL;
-                       struct object_id oid;
-                       if (expand_ref(arg, strlen(arg), &oid, &ref) != 1)
-                               die("git upload-pack: ambiguous deepen-not: %s", line);
-                       string_list_append(&deepen_not, ref);
-                       free(ref);
-                       deepen_rev_list = 1;
+               if (process_deepen_not(line, &deepen_not, &deepen_rev_list))
                        continue;
-               }
+ 
 +              if (skip_prefix(line, "filter ", &arg)) {
 +                      if (!filter_capability_requested)
 +                              die("git upload-pack: filtering capability not negotiated");
 +                      parse_list_objects_filter(&filter_options, arg);
 +                      continue;
 +              }
++
                if (!skip_prefix(line, "want ", &arg) ||
                    get_oid_hex(arg, &oid_buf))
                        die("git upload-pack: protocol error, "