From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 13 Feb 2018 21:39:04 +0000 (-0800)
Subject: Merge branch 'jh/partial-clone'
X-Git-Tag: v2.17.0-rc0~132
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/6bed209a20a06f2d6b7142216dabff456de798e1

Merge branch 'jh/partial-clone'

The machinery to clone & fetch, which in turn involves packing and
unpacking objects, have been told how to omit certain objects using
the filtering mechanism introduced by the jh/object-filtering
topic, and also mark the resulting pack as a promisor pack to
tolerate missing objects, taking advantage of the mechanism
introduced by the jh/fsck-promisors topic.

* jh/partial-clone:
t5616: test bulk prefetch after partial fetch
fetch: inherit filter-spec from partial clone
t5616: end-to-end tests for partial clone
fetch-pack: restore save_commit_buffer after use
unpack-trees: batch fetching of missing blobs
clone: partial clone
partial-clone: define partial clone settings in config
fetch: support filters
fetch: refactor calculation of remote list
fetch-pack: test support excluding large blobs
fetch-pack: add --no-filter
fetch-pack, index-pack, transport: partial clone
upload-pack: add object filtering for partial clone
---

6bed209a20a06f2d6b7142216dabff456de798e1
diff --cc t/t5601-clone.sh
index 8c437bf872,13610b70f5..0b62037744
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@@ -611,21 -571,105 +611,122 @@@ test_expect_success 'GIT_TRACE_PACKFIL
  	git -C replay.git index-pack -v --stdin <tmp.pack
  '
  
 +hex2oct () {
 +	perl -ne 'printf "\\%03o", hex for /../g'
 +}
 +
 +test_expect_success 'clone on case-insensitive fs' '
 +	git init icasefs &&
 +	(
 +		cd icasefs
 +		o=$(git hash-object -w --stdin </dev/null | hex2oct) &&
 +		t=$(printf "100644 X\0${o}100644 x\0${o}" |
 +			git hash-object -w -t tree --stdin) &&
 +		c=$(git commit-tree -m bogus $t) &&
 +		git update-ref refs/heads/bogus $c &&
 +		git clone -b bogus . bogus
 +	)
 +'
 +
+ partial_clone () {
+ 	       SERVER="$1" &&
+ 	       URL="$2" &&
+ 
+ 	rm -rf "$SERVER" client &&
+ 	test_create_repo "$SERVER" &&
+ 	test_commit -C "$SERVER" one &&
+ 	HASH1=$(git hash-object "$SERVER/one.t") &&
+ 	git -C "$SERVER" revert HEAD &&
+ 	test_commit -C "$SERVER" two &&
+ 	HASH2=$(git hash-object "$SERVER/two.t") &&
+ 	test_config -C "$SERVER" uploadpack.allowfilter 1 &&
+ 	test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 &&
+ 
+ 	git clone --filter=blob:limit=0 "$URL" client &&
+ 
+ 	git -C client fsck &&
+ 
+ 	# Ensure that unneeded blobs are not inadvertently fetched.
+ 	test_config -C client extensions.partialclone "not a remote" &&
+ 	test_must_fail git -C client cat-file -e "$HASH1" &&
+ 
+ 	# But this blob was fetched, because clone performs an initial checkout
+ 	git -C client cat-file -e "$HASH2"
+ }
+ 
+ test_expect_success 'partial clone' '
+ 	partial_clone server "file://$(pwd)/server"
+ '
+ 
+ test_expect_success 'partial clone: warn if server does not support object filtering' '
+ 	rm -rf server client &&
+ 	test_create_repo server &&
+ 	test_commit -C server one &&
+ 
+ 	git clone --filter=blob:limit=0 "file://$(pwd)/server" client 2> err &&
+ 
+ 	test_i18ngrep "filtering not recognized by server" err
+ '
+ 
+ test_expect_success 'batch missing blob request during checkout' '
+ 	rm -rf server client &&
+ 
+ 	test_create_repo server &&
+ 	echo a >server/a &&
+ 	echo b >server/b &&
+ 	git -C server add a b &&
+ 
+ 	git -C server commit -m x &&
+ 	echo aa >server/a &&
+ 	echo bb >server/b &&
+ 	git -C server add a b &&
+ 	git -C server commit -m x &&
+ 
+ 	test_config -C server uploadpack.allowfilter 1 &&
+ 	test_config -C server uploadpack.allowanysha1inwant 1 &&
+ 
+ 	git clone --filter=blob:limit=0 "file://$(pwd)/server" client &&
+ 
+ 	# Ensure that there is only one negotiation by checking that there is
+ 	# only "done" line sent. ("done" marks the end of negotiation.)
+ 	GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ &&
+ 	grep "git> done" trace >done_lines &&
+ 	test_line_count = 1 done_lines
+ '
+ 
+ test_expect_success 'batch missing blob request does not inadvertently try to fetch gitlinks' '
+ 	rm -rf server client &&
+ 
+ 	test_create_repo repo_for_submodule &&
+ 	test_commit -C repo_for_submodule x &&
+ 
+ 	test_create_repo server &&
+ 	echo a >server/a &&
+ 	echo b >server/b &&
+ 	git -C server add a b &&
+ 	git -C server commit -m x &&
+ 
+ 	echo aa >server/a &&
+ 	echo bb >server/b &&
+ 	# Also add a gitlink pointing to an arbitrary repository
+ 	git -C server submodule add "$(pwd)/repo_for_submodule" c &&
+ 	git -C server add a b c &&
+ 	git -C server commit -m x &&
+ 
+ 	test_config -C server uploadpack.allowfilter 1 &&
+ 	test_config -C server uploadpack.allowanysha1inwant 1 &&
+ 
+ 	# Make sure that it succeeds
+ 	git clone --filter=blob:limit=0 "file://$(pwd)/server" client
+ '
+ 
+ . "$TEST_DIRECTORY"/lib-httpd.sh
+ start_httpd
+ 
+ test_expect_success 'partial clone using HTTP' '
+ 	partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server"
+ '
+ 
+ stop_httpd
+ 
  test_done
diff --cc unpack-trees.c
index 96c3327f19,73a1cdb43e..e6a15bbe44
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@@ -14,7 -14,7 +14,8 @@@
  #include "dir.h"
  #include "submodule.h"
  #include "submodule-config.h"
 +#include "fsmonitor.h"
+ #include "fetch-object.h"
  
  /*
   * Error messages expected by scripts out of plumbing commands such as
diff --cc upload-pack.c
index d5de18127c,15b66051ce..f51b6cfca9
--- a/upload-pack.c
+++ b/upload-pack.c
@@@ -18,7 -20,7 +20,8 @@@
  #include "parse-options.h"
  #include "argv-array.h"
  #include "prio-queue.h"
 +#include "protocol.h"
+ #include "quote.h"
  
  static const char * const upload_pack_usage[] = {
  	N_("git upload-pack [<options>] <dir>"),