Request that various objects from the packfile be omitted
using one of several filtering techniques. These are intended
for use with partial clone and partial fetch operations. See
- `rev-list` for possible "filter-spec" values.
+ `rev-list` for possible "filter-spec" values. When communicating
+ with other processes, senders SHOULD translate scaled integers
+ (e.g. "1k") into a fully-expanded form (e.g. "1024") to aid
+ interoperability with older receivers that may not understand
+ newly-invented scaling suffixes. However, receivers SHOULD
+ accept the following suffixes: 'k', 'm', and 'g' for 1024,
+ 1048576, and 1073741824, respectively.
If the 'ref-in-want' feature is advertised, the following argument can
be included in the client's request as well as the potential addition of
particular ref, where <ref> is the full name of a ref on the
server.
+ If the 'sideband-all' feature is advertised, the following argument can be
+ included in the client's request:
+
+ sideband-all
+ Instruct the server to send the whole response multiplexed, not just
+ the packfile section. All non-flush and non-delim PKT-LINE in the
+ response (not only in the packfile section) will then start with a byte
+ indicating its sideband (1, 2, or 3), and the server may send "0005\2"
+ (a PKT-LINE of sideband 2 with no payload) as a keepalive packet.
+
The response of `fetch` is broken into a number of sections separated by
delimiter packets (0001), with each section beginning with its section
header.
struct oid_array shallow = OID_ARRAY_INIT;
struct string_list deepen_not = STRING_LIST_INIT_DUP;
struct packet_reader reader;
+ enum protocol_version version;
fetch_if_missing = 0;
packet_reader_init(&reader, fd[0], NULL, 0,
PACKET_READ_CHOMP_NEWLINE |
- PACKET_READ_GENTLE_ON_EOF);
+ PACKET_READ_GENTLE_ON_EOF |
+ PACKET_READ_DIE_ON_ERR_PACKET);
- switch (discover_version(&reader)) {
+ version = discover_version(&reader);
+ switch (version) {
case protocol_v2:
- die("support for protocol v2 not implemented yet");
+ get_remote_refs(fd[1], &reader, &ref, 0, NULL, NULL);
+ break;
case protocol_v1:
case protocol_v0:
get_remote_heads(&reader, &ref, 0, NULL, &shallow);
}
ref = fetch_pack(&args, fd, conn, ref, dest, sought, nr_sought,
- &shallow, pack_lockfile_ptr, protocol_v0);
+ &shallow, pack_lockfile_ptr, version);
if (pack_lockfile) {
printf("lock %s\n", pack_lockfile);
fflush(stdout);
ACK_ready
};
- static void consume_shallow_list(struct fetch_pack_args *args, int fd)
+ static void consume_shallow_list(struct fetch_pack_args *args,
+ struct packet_reader *reader)
{
if (args->stateless_rpc && args->deepen) {
/* If we sent a depth we will get back "duplicate"
* shallow and unshallow commands every time there
* is a block of have lines exchanged.
*/
- char *line;
- while ((line = packet_read_line(fd, NULL))) {
- if (starts_with(line, "shallow "))
+ while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
+ if (starts_with(reader->line, "shallow "))
continue;
- if (starts_with(line, "unshallow "))
+ if (starts_with(reader->line, "unshallow "))
continue;
die(_("git fetch-pack: expected shallow list"));
}
+ if (reader->status != PACKET_READ_FLUSH)
+ die(_("git fetch-pack: expected a flush packet after shallow list"));
}
}
- static enum ack_type get_ack(int fd, struct object_id *result_oid)
+ static enum ack_type get_ack(struct packet_reader *reader,
+ struct object_id *result_oid)
{
int len;
- char *line = packet_read_line(fd, &len);
const char *arg;
- if (!line)
+ if (packet_reader_read(reader) != PACKET_READ_NORMAL)
die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
- if (!strcmp(line, "NAK"))
+ len = reader->pktlen;
+
+ if (!strcmp(reader->line, "NAK"))
return NAK;
- if (skip_prefix(line, "ACK ", &arg)) {
+ if (skip_prefix(reader->line, "ACK ", &arg)) {
if (!get_oid_hex(arg, result_oid)) {
arg += 40;
- len -= arg - line;
+ len -= arg - reader->line;
if (len < 1)
return ACK;
if (strstr(arg, "continue"))
return ACK;
}
}
- if (skip_prefix(line, "ERR ", &arg))
- die(_("remote error: %s"), arg);
- die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
+ die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
}
static void send_request(struct fetch_pack_args *args,
int got_ready = 0;
struct strbuf req_buf = STRBUF_INIT;
size_t state_len = 0;
+ struct packet_reader reader;
if (args->stateless_rpc && multi_ack == 1)
die(_("--stateless-rpc requires multi_ack_detailed"));
+ packet_reader_init(&reader, fd[0], NULL, 0,
+ PACKET_READ_CHOMP_NEWLINE |
+ PACKET_READ_DIE_ON_ERR_PACKET);
+
if (!args->no_dependents) {
mark_tips(negotiator, args->negotiation_tips);
for_each_cached_alternate(negotiator, insert_one_alternate_object);
packet_buf_write(&req_buf, "deepen-not %s", s->string);
}
}
- if (server_supports_filtering && args->filter_options.choice)
+ if (server_supports_filtering && args->filter_options.choice) {
+ struct strbuf expanded_filter_spec = STRBUF_INIT;
+ expand_list_objects_filter_spec(&args->filter_options,
+ &expanded_filter_spec);
packet_buf_write(&req_buf, "filter %s",
- args->filter_options.filter_spec);
+ expanded_filter_spec.buf);
+ strbuf_release(&expanded_filter_spec);
+ }
packet_buf_flush(&req_buf);
state_len = req_buf.len;
if (args->deepen) {
- char *line;
const char *arg;
struct object_id oid;
send_request(args, fd[1], &req_buf);
- while ((line = packet_read_line(fd[0], NULL))) {
- if (skip_prefix(line, "shallow ", &arg)) {
+ while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
+ if (skip_prefix(reader.line, "shallow ", &arg)) {
if (get_oid_hex(arg, &oid))
- die(_("invalid shallow line: %s"), line);
+ die(_("invalid shallow line: %s"), reader.line);
register_shallow(the_repository, &oid);
continue;
}
- if (skip_prefix(line, "unshallow ", &arg)) {
+ if (skip_prefix(reader.line, "unshallow ", &arg)) {
if (get_oid_hex(arg, &oid))
- die(_("invalid unshallow line: %s"), line);
+ die(_("invalid unshallow line: %s"), reader.line);
if (!lookup_object(the_repository, oid.hash))
- die(_("object not found: %s"), line);
+ die(_("object not found: %s"), reader.line);
/* make sure that it is parsed as shallow */
if (!parse_object(the_repository, &oid))
- die(_("error in object: %s"), line);
+ die(_("error in object: %s"), reader.line);
if (unregister_shallow(&oid))
- die(_("no shallow found: %s"), line);
+ die(_("no shallow found: %s"), reader.line);
continue;
}
- die(_("expected shallow/unshallow, got %s"), line);
+ die(_("expected shallow/unshallow, got %s"), reader.line);
}
} else if (!args->stateless_rpc)
send_request(args, fd[1], &req_buf);
if (!args->stateless_rpc && count == INITIAL_FLUSH)
continue;
- consume_shallow_list(args, fd[0]);
+ consume_shallow_list(args, &reader);
do {
- ack = get_ack(fd[0], result_oid);
+ ack = get_ack(&reader, result_oid);
if (ack)
print_verbose(args, _("got %s %d %s"), "ack",
ack, oid_to_hex(result_oid));
strbuf_release(&req_buf);
if (!got_ready || !no_done)
- consume_shallow_list(args, fd[0]);
+ consume_shallow_list(args, &reader);
while (flushes || multi_ack) {
- int ack = get_ack(fd[0], result_oid);
+ int ack = get_ack(&reader, result_oid);
if (ack) {
print_verbose(args, _("got %s (%d) %s"), "ack",
ack, oid_to_hex(result_oid));
packet_buf_write(req_buf, "deepen-not %s", s->string);
}
}
+ if (args->deepen_relative)
+ packet_buf_write(req_buf, "deepen-relative\n");
}
static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
const struct fetch_pack_args *args,
const struct ref *wants, struct oidset *common,
- int *haves_to_send, int *in_vain)
+ int *haves_to_send, int *in_vain,
+ int sideband_all)
{
int ret = 0;
struct strbuf req_buf = STRBUF_INIT;
packet_buf_write(&req_buf, "include-tag");
if (prefer_ofs_delta)
packet_buf_write(&req_buf, "ofs-delta");
+ if (sideband_all)
+ packet_buf_write(&req_buf, "sideband-all");
/* Add shallow-info and deepen request */
if (server_supports_feature("fetch", "shallow", 0))
/* Add filter */
if (server_supports_feature("fetch", "filter", 0) &&
args->filter_options.choice) {
+ struct strbuf expanded_filter_spec = STRBUF_INIT;
print_verbose(args, _("Server supports filter"));
+ expand_list_objects_filter_spec(&args->filter_options,
+ &expanded_filter_spec);
packet_buf_write(&req_buf, "filter %s",
- args->filter_options.filter_spec);
+ expanded_filter_spec.buf);
+ strbuf_release(&expanded_filter_spec);
} else if (args->filter_options.choice) {
warning("filtering not recognized by server, ignoring");
}
static void receive_shallow_info(struct fetch_pack_args *args,
struct packet_reader *reader)
{
+ int line_received = 0;
+
process_section_header(reader, "shallow-info", 0);
while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
const char *arg;
if (get_oid_hex(arg, &oid))
die(_("invalid shallow line: %s"), reader->line);
register_shallow(the_repository, &oid);
+ line_received = 1;
continue;
}
if (skip_prefix(reader->line, "unshallow ", &arg)) {
die(_("error in object: %s"), reader->line);
if (unregister_shallow(&oid))
die(_("no shallow found: %s"), reader->line);
+ line_received = 1;
continue;
}
die(_("expected shallow/unshallow, got %s"), reader->line);
reader->status != PACKET_READ_DELIM)
die(_("error processing shallow info: %d"), reader->status);
- setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
- args->deepen = 1;
+ if (line_received) {
+ setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
+ NULL);
+ args->deepen = 1;
+ }
}
static void receive_wanted_refs(struct packet_reader *reader,
struct fetch_negotiator negotiator;
fetch_negotiator_init(&negotiator, negotiation_algorithm);
packet_reader_init(&reader, fd[0], NULL, 0,
- PACKET_READ_CHOMP_NEWLINE);
+ PACKET_READ_CHOMP_NEWLINE |
+ PACKET_READ_DIE_ON_ERR_PACKET);
+ if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
+ server_supports_feature("fetch", "sideband-all", 0)) {
+ reader.use_sideband = 1;
+ reader.me = "fetch-pack";
+ }
while (state != FETCH_DONE) {
switch (state) {
case FETCH_SEND_REQUEST:
if (send_fetch_request(&negotiator, fd[1], args, ref,
&common,
- &haves_to_send, &in_vain))
+ &haves_to_send, &in_vain,
+ reader.use_sideband))
state = FETCH_GET_PACK;
else
state = FETCH_PROCESS_ACKS;
packet_reader_init(&reader, -1, heads->buf, heads->len,
PACKET_READ_CHOMP_NEWLINE |
- PACKET_READ_GENTLE_ON_EOF);
+ PACKET_READ_GENTLE_ON_EOF |
+ PACKET_READ_DIE_ON_ERR_PACKET);
heads->version = discover_version(&reader);
switch (heads->version) {
http_options.extra_headers = &extra_headers;
http_options.initial_request = 1;
http_options.no_cache = 1;
- http_options.keep_error = 1;
http_ret = http_get_strbuf(refs_url.buf, &buffer, &http_options);
switch (http_ret) {
if (maybe_smart &&
(5 <= last->len && last->buf[4] == '#') &&
!strbuf_cmp(&exp, &type)) {
- char *line;
+ struct packet_reader reader;
+ packet_reader_init(&reader, -1, last->buf, last->len,
+ PACKET_READ_CHOMP_NEWLINE |
+ PACKET_READ_DIE_ON_ERR_PACKET);
/*
* smart HTTP response; validate that the service
* pkt-line matches our request.
*/
- line = packet_read_line_buf(&last->buf, &last->len, NULL);
- if (!line)
+ if (packet_reader_read(&reader) != PACKET_READ_NORMAL)
die("invalid server response; expected service, got flush packet");
strbuf_reset(&exp);
strbuf_addf(&exp, "# service=%s", service);
- if (strcmp(line, exp.buf))
- die("invalid server response; got '%s'", line);
+ if (strcmp(reader.line, exp.buf))
+ die("invalid server response; got '%s'", reader.line);
strbuf_release(&exp);
/* The header can include additional metadata lines, up
* until a packet flush marker. Ignore these now, but
* in the future we might start to scan them.
*/
- while (packet_read_line_buf(&last->buf, &last->len, NULL))
- ;
+ for (;;) {
+ packet_reader_read(&reader);
+ if (reader.pktlen <= 0) {
+ break;
+ }
+ }
+
+ last->buf = reader.src_buffer;
+ last->len = reader.src_len;
last->proto_git = 1;
} else if (maybe_smart &&
}
#endif
+struct rpc_in_data {
+ struct rpc_state *rpc;
+ struct active_request_slot *slot;
+};
+
+/*
+ * A callback for CURLOPT_WRITEFUNCTION. The return value is the bytes consumed
+ * from ptr.
+ */
static size_t rpc_in(char *ptr, size_t eltsize,
size_t nmemb, void *buffer_)
{
size_t size = eltsize * nmemb;
- struct rpc_state *rpc = buffer_;
+ struct rpc_in_data *data = buffer_;
+ long response_code;
+
+ if (curl_easy_getinfo(data->slot->curl, CURLINFO_RESPONSE_CODE,
+ &response_code) != CURLE_OK)
+ return size;
+ if (response_code >= 300)
+ return size;
if (size)
- rpc->any_written = 1;
- write_or_die(rpc->in, ptr, size);
+ data->rpc->any_written = 1;
+ write_or_die(data->rpc->in, ptr, size);
return size;
}
return err;
}
-static curl_off_t xcurl_off_t(size_t len) {
+static curl_off_t xcurl_off_t(size_t len)
+{
uintmax_t size = len;
if (size > maximum_signed_value_of_type(curl_off_t))
die("cannot handle pushes this big");
size_t gzip_size = 0;
int err, large_request = 0;
int needs_100_continue = 0;
+ struct rpc_in_data rpc_in_data;
/* Try to load the entire request, if we can fit it into the
* allocated buffer space we can use HTTP/1.0 and avoid the
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in);
- curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc);
+ rpc_in_data.rpc = rpc;
+ rpc_in_data.slot = slot;
+ curl_easy_setopt(slot->curl, CURLOPT_FILE, &rpc_in_data);
+ curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
rpc->any_written = 0;
p->headers = curl_slist_append(p->headers, buf.buf);
packet_reader_init(&p->reader, p->in, NULL, 0,
- PACKET_READ_GENTLE_ON_EOF);
+ PACKET_READ_GENTLE_ON_EOF |
+ PACKET_READ_DIE_ON_ERR_PACKET);
strbuf_release(&buf);
}
#include "cache.h"
#include "color.h"
#include "config.h"
- #include "pkt-line.h"
#include "sideband.h"
#include "help.h"
struct keyword_entry *p = keywords + i;
int len = strlen(p->keyword);
- if (n <= len)
+ if (n < len)
continue;
/*
* Match case insensitively, so we colorize output from existing
* messages. We only highlight the word precisely, so
* "successful" stays uncolored.
*/
- if (!strncasecmp(p->keyword, src, len) && !isalnum(src[len])) {
+ if (!strncasecmp(p->keyword, src, len) &&
+ (len == n || !isalnum(src[len]))) {
strbuf_addstr(dest, p->color);
strbuf_add(dest, src, len);
strbuf_addstr(dest, GIT_COLOR_RESET);
}
- /*
- * Receive multiplexed output stream over git native protocol.
- * in_stream is the input stream from the remote, which carries data
- * in pkt_line format with band designator. Demultiplex it into out
- * and err and return error appropriately. Band #1 carries the
- * primary payload. Things coming over band #2 is not necessarily
- * error; they are usually informative message on the standard error
- * stream, aka "verbose"). A message over band #3 is a signal that
- * the remote died unexpectedly. A flush() concludes the stream.
- */
-
#define DISPLAY_PREFIX "remote: "
#define ANSI_SUFFIX "\033[K"
#define DUMB_SUFFIX " "
- int recv_sideband(const char *me, int in_stream, int out)
+ int demultiplex_sideband(const char *me, char *buf, int len,
+ int die_on_error,
+ struct strbuf *scratch,
+ enum sideband_type *sideband_type)
{
- const char *suffix;
- char buf[LARGE_PACKET_MAX + 1];
- struct strbuf outbuf = STRBUF_INIT;
- int retval = 0;
-
- if (isatty(2) && !is_terminal_dumb())
- suffix = ANSI_SUFFIX;
- else
- suffix = DUMB_SUFFIX;
-
- while (!retval) {
- const char *b, *brk;
- int band, len;
- len = packet_read(in_stream, NULL, NULL, buf, LARGE_PACKET_MAX, 0);
- if (len == 0)
- break;
- if (len < 1) {
- strbuf_addf(&outbuf,
- "%s%s: protocol error: no band designator",
- outbuf.len ? "\n" : "", me);
- retval = SIDEBAND_PROTOCOL_ERROR;
- break;
- }
- band = buf[0] & 0xff;
- buf[len] = '\0';
- len--;
- switch (band) {
- case 3:
- strbuf_addf(&outbuf, "%s%s", outbuf.len ? "\n" : "",
- DISPLAY_PREFIX);
- maybe_colorize_sideband(&outbuf, buf + 1, len);
-
- retval = SIDEBAND_REMOTE_ERROR;
- break;
- case 2:
- b = buf + 1;
-
- /*
- * Append a suffix to each nonempty line to clear the
- * end of the screen line.
- *
- * The output is accumulated in a buffer and
- * each line is printed to stderr using
- * write(2) to ensure inter-process atomicity.
- */
- while ((brk = strpbrk(b, "\n\r"))) {
- int linelen = brk - b;
-
- if (!outbuf.len)
- strbuf_addstr(&outbuf, DISPLAY_PREFIX);
- if (linelen > 0) {
- maybe_colorize_sideband(&outbuf, b, linelen);
- strbuf_addstr(&outbuf, suffix);
- }
-
- strbuf_addch(&outbuf, *brk);
- xwrite(2, outbuf.buf, outbuf.len);
- strbuf_reset(&outbuf);
-
- b = brk + 1;
- }
+ static const char *suffix;
+ const char *b, *brk;
+ int band;
+
+ if (!suffix) {
+ if (isatty(2) && !is_terminal_dumb())
+ suffix = ANSI_SUFFIX;
+ else
+ suffix = DUMB_SUFFIX;
+ }
- if (*b) {
- strbuf_addstr(&outbuf, outbuf.len ?
- "" : DISPLAY_PREFIX);
- maybe_colorize_sideband(&outbuf, b, strlen(b));
+ if (len == 0) {
+ *sideband_type = SIDEBAND_FLUSH;
+ goto cleanup;
+ }
+ if (len < 1) {
+ strbuf_addf(scratch,
+ "%s%s: protocol error: no band designator",
+ scratch->len ? "\n" : "", me);
+ *sideband_type = SIDEBAND_PROTOCOL_ERROR;
+ goto cleanup;
+ }
+ band = buf[0] & 0xff;
+ buf[len] = '\0';
+ len--;
+ switch (band) {
+ case 3:
+ if (die_on_error)
+ die("remote error: %s", buf + 1);
+ strbuf_addf(scratch, "%s%s", scratch->len ? "\n" : "",
+ DISPLAY_PREFIX);
+ maybe_colorize_sideband(scratch, buf + 1, len);
+
+ *sideband_type = SIDEBAND_REMOTE_ERROR;
+ break;
+ case 2:
+ b = buf + 1;
+
+ /*
+ * Append a suffix to each nonempty line to clear the
+ * end of the screen line.
+ *
+ * The output is accumulated in a buffer and
+ * each line is printed to stderr using
+ * write(2) to ensure inter-process atomicity.
+ */
+ while ((brk = strpbrk(b, "\n\r"))) {
+ int linelen = brk - b;
+
+ if (!scratch->len)
+ strbuf_addstr(scratch, DISPLAY_PREFIX);
+ if (linelen > 0) {
+ maybe_colorize_sideband(scratch, b, linelen);
+ strbuf_addstr(scratch, suffix);
}
- break;
- case 1:
- write_or_die(out, buf + 1, len);
- break;
- default:
- strbuf_addf(&outbuf, "%s%s: protocol error: bad band #%d",
- outbuf.len ? "\n" : "", me, band);
- retval = SIDEBAND_PROTOCOL_ERROR;
- break;
+
+ strbuf_addch(scratch, *brk);
+ xwrite(2, scratch->buf, scratch->len);
+ strbuf_reset(scratch);
+
+ b = brk + 1;
+ }
+
+ if (*b) {
+ strbuf_addstr(scratch, scratch->len ?
+ "" : DISPLAY_PREFIX);
+ maybe_colorize_sideband(scratch, b, strlen(b));
}
+ return 0;
+ case 1:
+ *sideband_type = SIDEBAND_PRIMARY;
+ break;
+ default:
+ strbuf_addf(scratch, "%s%s: protocol error: bad band #%d",
+ scratch->len ? "\n" : "", me, band);
+ *sideband_type = SIDEBAND_PROTOCOL_ERROR;
+ break;
}
- if (outbuf.len) {
- strbuf_addch(&outbuf, '\n');
- xwrite(2, outbuf.buf, outbuf.len);
+ cleanup:
+ if (die_on_error && *sideband_type == SIDEBAND_PROTOCOL_ERROR)
+ die("%s", scratch->buf);
+ if (scratch->len) {
+ strbuf_addch(scratch, '\n');
+ xwrite(2, scratch->buf, scratch->len);
}
- strbuf_release(&outbuf);
- return retval;
+ strbuf_release(scratch);
+ return 1;
}
/*
this feature by setting the GIT_TEST_CHAIN_LINT environment
variable to "1" or "0", respectively.
+--stress::
+--stress=<N>::
+ Run the test script repeatedly in multiple parallel jobs until
+ one of them fails. Useful for reproducing rare failures in
+ flaky tests. The number of parallel jobs is, in order of
+ precedence: <N>, or the value of the GIT_TEST_STRESS_LOAD
+ environment variable, or twice the number of available
+ processors (as shown by the 'getconf' utility), or 8.
+ Implies `--verbose -x --immediate` to get the most information
+ about the failure. Note that the verbose output of each test
+ job is saved to 't/test-results/$TEST_NAME.stress-<nr>.out',
+ and only the output of the failed test job is shown on the
+ terminal. The names of the trash directories get a
+ '.stress-<nr>' suffix, and the trash directory of the failed
+ test job is renamed to end with a '.stress-failed' suffix.
+
You can also set the GIT_TEST_INSTALLED environment variable to
the bindir of an existing git installation to test that installation.
You still need to have built this git sandbox, from which various
index to be written after every 'git repack' command, and overrides the
'core.multiPackIndex' setting to true.
+ GIT_TEST_SIDEBAND_ALL=<boolean>, when true, overrides the
+ 'uploadpack.allowSidebandAll' setting to true, and when false, forces
+ fetch-pack to not request sideband-all (even if the server advertises
+ sideband-all).
+
Naming Tests
------------
- Creates an empty test directory with an empty .git/objects database
and chdir(2) into it. This directory is 't/trash
directory.$test_name_without_dotsh', with t/ subject to change by
- the --root option documented above.
+ the --root option documented above, and a '.stress-<N>' suffix
+ appended by the --stress option.
- Defines standard test helper functions for your scripts to
use. These functions are designed to make all scripts behave
PassEnv ASAN_OPTIONS
PassEnv GIT_TRACE
PassEnv GIT_CONFIG_NOSYSTEM
+ PassEnv GIT_TEST_SIDEBAND_ALL
SetEnvIf Git-Protocol ".*" GIT_PROTOCOL=$0
SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
SetEnv GIT_HTTP_EXPORT_ALL
</LocationMatch>
+ScriptAliasMatch /error_git_upload_pack/(.*)/git-upload-pack error.sh/
ScriptAliasMatch /smart_*[^/]*/(.*) ${GIT_EXEC_PATH}/git-http-backend/$1
ScriptAlias /broken_smart/ broken-smart-http.sh/
ScriptAlias /error/ error.sh/
grep "fetch< version 2" trace
'
+test_expect_success 'ensure that multiple fetches in same process from a shallow repo works' '
+ rm -rf server client trace &&
+
+ test_create_repo server &&
+ test_commit -C server one &&
+ test_commit -C server two &&
+ test_commit -C server three &&
+ git clone --shallow-exclude two "file://$(pwd)/server" client &&
+
+ git -C server tag -a -m "an annotated tag" twotag two &&
+
+ # Triggers tag following (thus, 2 fetches in one process)
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ fetch --shallow-exclude one origin &&
+ # Ensure that protocol v2 is used
+ grep "fetch< version 2" trace
+'
+
+test_expect_success 'deepen-relative' '
+ rm -rf server client trace &&
+
+ test_create_repo server &&
+ test_commit -C server one &&
+ test_commit -C server two &&
+ test_commit -C server three &&
+ git clone --depth 1 "file://$(pwd)/server" client &&
+ test_commit -C server four &&
+
+ # Sanity check that only "three" is downloaded
+ git -C client log --pretty=tformat:%s master >actual &&
+ echo three >expected &&
+ test_cmp expected actual &&
+
+ GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \
+ fetch --deepen=1 origin &&
+ # Ensure that protocol v2 is used
+ grep "fetch< version 2" trace &&
+
+ git -C client log --pretty=tformat:%s origin/master >actual &&
+ cat >expected <<-\EOF &&
+ four
+ three
+ two
+ EOF
+ test_cmp expected actual
+'
+
# Test protocol v2 with 'http://' transport
#
. "$TEST_DIRECTORY"/lib-httpd.sh
test_must_fail env GIT_TRACE_PACKET="$(pwd)/log" git -C http_child \
-c protocol.version=2 \
fetch "$HTTPD_URL/one_time_sed/http_parent" 2> err &&
- grep "fetch< acknowledgments" log &&
- ! grep "fetch< ready" log &&
+ grep "fetch< .*acknowledgments" log &&
+ ! grep "fetch< .*ready" log &&
test_i18ngrep "expected no other sections to be sent after no .ready." err
'
static timestamp_t oldest_have;
-static int deepen_relative;
static int multi_ack;
static int no_done;
static int use_thin_pack, use_ofs_delta, use_include_tag;
static int allow_ref_in_want;
static struct list_objects_filter_options filter_options;
+ static int allow_sideband_all;
+
static void reset_timeout(void)
{
alarm(timeout);
if (use_include_tag)
argv_array_push(&pack_objects.args, "--include-tag");
if (filter_options.filter_spec) {
+ struct strbuf expanded_filter_spec = STRBUF_INIT;
+ expand_list_objects_filter_spec(&filter_options,
+ &expanded_filter_spec);
if (pack_objects.use_shell) {
struct strbuf buf = STRBUF_INIT;
- sq_quote_buf(&buf, filter_options.filter_spec);
+ sq_quote_buf(&buf, expanded_filter_spec.buf);
argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf);
strbuf_release(&buf);
} else {
argv_array_pushf(&pack_objects.args, "--filter=%s",
- filter_options.filter_spec);
+ expanded_filter_spec.buf);
}
}
min_generation);
}
- static int get_common_commits(struct object_array *have_obj,
+ static int get_common_commits(struct packet_reader *reader,
+ struct object_array *have_obj,
struct object_array *want_obj)
{
struct object_id oid;
save_commit_buffer = 0;
for (;;) {
- char *line = packet_read_line(0, NULL);
const char *arg;
reset_timeout();
- if (!line) {
+ if (packet_reader_read(reader) != PACKET_READ_NORMAL) {
if (multi_ack == 2 && got_common
&& !got_other && ok_to_give_up(have_obj, want_obj)) {
sent_ready = 1;
got_other = 0;
continue;
}
- if (skip_prefix(line, "have ", &arg)) {
+ if (skip_prefix(reader->line, "have ", &arg)) {
switch (got_oid(arg, &oid, have_obj)) {
case -1: /* they have what we do not */
got_other = 1;
}
continue;
}
- if (!strcmp(line, "done")) {
+ if (!strcmp(reader->line, "done")) {
if (have_obj->nr > 0) {
if (multi_ack)
packet_write_fmt(1, "ACK %s\n", last_hex);
packet_write_fmt(1, "NAK\n");
return -1;
}
- die("git upload-pack: expected SHA1 list, got '%s'", line);
+ die("git upload-pack: expected SHA1 list, got '%s'", reader->line);
}
}
}
}
- static void send_shallow(struct commit_list *result)
+ static void send_shallow(struct packet_writer *writer,
+ struct commit_list *result)
{
while (result) {
struct object *object = &result->item->object;
if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
- packet_write_fmt(1, "shallow %s",
- oid_to_hex(&object->oid));
+ packet_writer_write(writer, "shallow %s",
+ oid_to_hex(&object->oid));
register_shallow(the_repository, &object->oid);
shallow_nr++;
}
}
}
- static void send_unshallow(const struct object_array *shallows,
+ static void send_unshallow(struct packet_writer *writer,
+ const struct object_array *shallows,
struct object_array *want_obj)
{
int i;
struct object *object = shallows->objects[i].item;
if (object->flags & NOT_SHALLOW) {
struct commit_list *parents;
- packet_write_fmt(1, "unshallow %s",
- oid_to_hex(&object->oid));
+ packet_writer_write(writer, "unshallow %s",
+ oid_to_hex(&object->oid));
object->flags &= ~CLIENT_SHALLOW;
/*
* We want to _register_ "object" as shallow, but we
}
}
-
- static void deepen(int depth, int deepen_relative,
+static int check_ref(const char *refname_full, const struct object_id *oid,
+ int flag, void *cb_data);
+ static void deepen(struct packet_writer *writer, int depth, int deepen_relative,
struct object_array *shallows, struct object_array *want_obj)
{
if (depth == INFINITE_DEPTH && !is_repository_shallow(the_repository)) {
struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
struct commit_list *result;
+ /*
+ * Checking for reachable shallows requires that our refs be
+ * marked with OUR_REF.
+ */
+ head_ref_namespaced(check_ref, NULL);
+ for_each_namespaced_ref(check_ref, NULL);
+
get_reachable_list(shallows, &reachable_shallows);
result = get_shallow_commits(&reachable_shallows,
depth + 1,
SHALLOW, NOT_SHALLOW);
- send_shallow(result);
+ send_shallow(writer, result);
free_commit_list(result);
object_array_clear(&reachable_shallows);
} else {
result = get_shallow_commits(want_obj, depth,
SHALLOW, NOT_SHALLOW);
- send_shallow(result);
+ send_shallow(writer, result);
free_commit_list(result);
}
- send_unshallow(shallows, want_obj);
+ send_unshallow(writer, shallows, want_obj);
}
- static void deepen_by_rev_list(int ac, const char **av,
+ static void deepen_by_rev_list(struct packet_writer *writer, int ac,
+ const char **av,
struct object_array *shallows,
struct object_array *want_obj)
{
close_commit_graph(the_repository);
result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
- send_shallow(result);
+ send_shallow(writer, result);
free_commit_list(result);
- send_unshallow(shallows, want_obj);
+ send_unshallow(writer, shallows, want_obj);
}
/* Returns 1 if a shallow list is sent or 0 otherwise */
- static int send_shallow_list(int depth, int deepen_rev_list,
+ static int send_shallow_list(struct packet_writer *writer,
+ int depth, int deepen_rev_list,
timestamp_t deepen_since,
struct string_list *deepen_not,
+ int deepen_relative,
struct object_array *shallows,
struct object_array *want_obj)
{
if (depth > 0 && deepen_rev_list)
die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
if (depth > 0) {
- deepen(depth, deepen_relative, shallows, want_obj);
+ deepen(writer, depth, deepen_relative, shallows, want_obj);
ret = 1;
} else if (deepen_rev_list) {
struct argv_array av = ARGV_ARRAY_INIT;
struct object *o = want_obj->objects[i].item;
argv_array_push(&av, oid_to_hex(&o->oid));
}
- deepen_by_rev_list(av.argc, av.argv, shallows, want_obj);
+ deepen_by_rev_list(writer, av.argc, av.argv, shallows, want_obj);
argv_array_clear(&av);
ret = 1;
} else {
return 0;
}
- static void receive_needs(struct object_array *want_obj)
+ static void receive_needs(struct packet_reader *reader, struct object_array *want_obj)
{
struct object_array shallows = OBJECT_ARRAY_INIT;
struct string_list deepen_not = STRING_LIST_INIT_DUP;
int has_non_tip = 0;
timestamp_t deepen_since = 0;
int deepen_rev_list = 0;
+ int deepen_relative = 0;
+ struct packet_writer writer;
shallow_nr = 0;
+ packet_writer_init(&writer, 1);
for (;;) {
struct object *o;
const char *features;
struct object_id oid_buf;
- char *line = packet_read_line(0, NULL);
const char *arg;
reset_timeout();
- if (!line)
+ if (packet_reader_read(reader) != PACKET_READ_NORMAL)
break;
- if (process_shallow(line, &shallows))
+ if (process_shallow(reader->line, &shallows))
continue;
- if (process_deepen(line, &depth))
+ if (process_deepen(reader->line, &depth))
continue;
- if (process_deepen_since(line, &deepen_since, &deepen_rev_list))
+ if (process_deepen_since(reader->line, &deepen_since, &deepen_rev_list))
continue;
- if (process_deepen_not(line, &deepen_not, &deepen_rev_list))
+ if (process_deepen_not(reader->line, &deepen_not, &deepen_rev_list))
continue;
- if (skip_prefix(line, "filter ", &arg)) {
+ if (skip_prefix(reader->line, "filter ", &arg)) {
if (!filter_capability_requested)
die("git upload-pack: filtering capability not negotiated");
parse_list_objects_filter(&filter_options, arg);
continue;
}
- if (!skip_prefix(line, "want ", &arg) ||
+ if (!skip_prefix(reader->line, "want ", &arg) ||
parse_oid_hex(arg, &oid_buf, &features))
die("git upload-pack: protocol error, "
- "expected to get object ID, not '%s'", line);
+ "expected to get object ID, not '%s'", reader->line);
if (parse_feature_request(features, "deepen-relative"))
deepen_relative = 1;
o = parse_object(the_repository, &oid_buf);
if (!o) {
- packet_write_fmt(1,
- "ERR upload-pack: not our ref %s",
- oid_to_hex(&oid_buf));
+ packet_writer_error(&writer,
+ "upload-pack: not our ref %s",
+ oid_to_hex(&oid_buf));
die("git upload-pack: not our ref %s",
oid_to_hex(&oid_buf));
}
if (depth == 0 && !deepen_rev_list && shallows.nr == 0)
return;
- if (send_shallow_list(depth, deepen_rev_list, deepen_since,
+ if (send_shallow_list(&writer, depth, deepen_rev_list, deepen_since,
- &deepen_not, &shallows, want_obj))
+ &deepen_not, deepen_relative, &shallows,
+ want_obj))
packet_flush(1);
object_array_clear(&shallows);
}
allow_filter = git_config_bool(var, value);
} else if (!strcmp("uploadpack.allowrefinwant", var)) {
allow_ref_in_want = git_config_bool(var, value);
+ } else if (!strcmp("uploadpack.allowsidebandall", var)) {
+ allow_sideband_all = git_config_bool(var, value);
}
if (current_config_scope() != CONFIG_SCOPE_REPO) {
{
struct string_list symref = STRING_LIST_INIT_DUP;
struct object_array want_obj = OBJECT_ARRAY_INIT;
+ struct packet_reader reader;
stateless_rpc = options->stateless_rpc;
timeout = options->timeout;
if (options->advertise_refs)
return;
- receive_needs(&want_obj);
+ packet_reader_init(&reader, 0, NULL, 0,
+ PACKET_READ_CHOMP_NEWLINE |
+ PACKET_READ_DIE_ON_ERR_PACKET);
+
+ receive_needs(&reader, &want_obj);
if (want_obj.nr) {
struct object_array have_obj = OBJECT_ARRAY_INIT;
- get_common_commits(&have_obj, &want_obj);
+ get_common_commits(&reader, &have_obj, &want_obj);
create_pack_file(&have_obj, &want_obj);
}
}
int deepen_rev_list;
int deepen_relative;
+ struct packet_writer writer;
+
unsigned stateless_rpc : 1;
unsigned use_thin_pack : 1;
data->haves = haves;
data->shallows = shallows;
data->deepen_not = deepen_not;
+ packet_writer_init(&data->writer, 1);
}
static void upload_pack_data_clear(struct upload_pack_data *data)
string_list_clear(&data->deepen_not, 0);
}
- static int parse_want(const char *line, struct object_array *want_obj)
+ static int parse_want(struct packet_writer *writer, const char *line,
+ struct object_array *want_obj)
{
const char *arg;
if (skip_prefix(line, "want ", &arg)) {
o = parse_object(the_repository, &oid);
if (!o) {
- packet_write_fmt(1,
- "ERR upload-pack: not our ref %s",
- oid_to_hex(&oid));
+ packet_writer_error(writer,
+ "upload-pack: not our ref %s",
+ oid_to_hex(&oid));
die("git upload-pack: not our ref %s",
oid_to_hex(&oid));
}
return 0;
}
- static int parse_want_ref(const char *line, struct string_list *wanted_refs,
+ static int parse_want_ref(struct packet_writer *writer, const char *line,
+ struct string_list *wanted_refs,
struct object_array *want_obj)
{
const char *arg;
struct object *o;
if (read_ref(arg, &oid)) {
- packet_write_fmt(1, "ERR unknown ref %s", arg);
+ packet_writer_error(writer, "unknown ref %s", arg);
die("unknown ref %s", arg);
}
const char *p;
/* process want */
- if (parse_want(arg, want_obj))
+ if (parse_want(&data->writer, arg, want_obj))
continue;
if (allow_ref_in_want &&
- parse_want_ref(arg, &data->wanted_refs, want_obj))
+ parse_want_ref(&data->writer, arg, &data->wanted_refs,
+ want_obj))
continue;
/* process have line */
if (parse_have(arg, &data->haves))
continue;
}
+ if ((git_env_bool("GIT_TEST_SIDEBAND_ALL", 0) ||
+ allow_sideband_all) &&
+ !strcmp(arg, "sideband-all")) {
+ data->writer.use_sideband = 1;
+ continue;
+ }
+
/* ignore unknown lines maybe? */
die("unexpected line: '%s'", arg);
}
return 0;
}
- static int send_acks(struct oid_array *acks, struct strbuf *response,
+ static int send_acks(struct packet_writer *writer, struct oid_array *acks,
const struct object_array *have_obj,
struct object_array *want_obj)
{
int i;
- packet_buf_write(response, "acknowledgments\n");
+ packet_writer_write(writer, "acknowledgments\n");
/* Send Acks */
if (!acks->nr)
- packet_buf_write(response, "NAK\n");
+ packet_writer_write(writer, "NAK\n");
for (i = 0; i < acks->nr; i++) {
- packet_buf_write(response, "ACK %s\n",
- oid_to_hex(&acks->oid[i]));
+ packet_writer_write(writer, "ACK %s\n",
+ oid_to_hex(&acks->oid[i]));
}
if (ok_to_give_up(have_obj, want_obj)) {
/* Send Ready */
- packet_buf_write(response, "ready\n");
+ packet_writer_write(writer, "ready\n");
return 1;
}
struct object_array *want_obj)
{
struct oid_array common = OID_ARRAY_INIT;
- struct strbuf response = STRBUF_INIT;
int ret = 0;
process_haves(&data->haves, &common, have_obj);
if (data->done) {
ret = 1;
- } else if (send_acks(&common, &response, have_obj, want_obj)) {
- packet_buf_delim(&response);
+ } else if (send_acks(&data->writer, &common, have_obj, want_obj)) {
+ packet_writer_delim(&data->writer);
ret = 1;
} else {
/* Add Flush */
- packet_buf_flush(&response);
+ packet_writer_flush(&data->writer);
ret = 0;
}
- /* Send response */
- write_or_die(1, response.buf, response.len);
- strbuf_release(&response);
-
oid_array_clear(&data->haves);
oid_array_clear(&common);
return ret;
if (!data->wanted_refs.nr)
return;
- packet_write_fmt(1, "wanted-refs\n");
+ packet_writer_write(&data->writer, "wanted-refs\n");
for_each_string_list_item(item, &data->wanted_refs) {
- packet_write_fmt(1, "%s %s\n",
- oid_to_hex(item->util),
- item->string);
+ packet_writer_write(&data->writer, "%s %s\n",
+ oid_to_hex(item->util),
+ item->string);
}
- packet_delim(1);
+ packet_writer_delim(&data->writer);
}
static void send_shallow_info(struct upload_pack_data *data,
!is_repository_shallow(the_repository))
return;
- packet_write_fmt(1, "shallow-info\n");
+ packet_writer_write(&data->writer, "shallow-info\n");
- if (!send_shallow_list(data->depth, data->deepen_rev_list,
+ if (!send_shallow_list(&data->writer, data->depth,
+ data->deepen_rev_list,
data->deepen_since, &data->deepen_not,
+ data->deepen_relative,
&data->shallows, want_obj) &&
is_repository_shallow(the_repository))
- deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows,
- want_obj);
+ deepen(&data->writer, INFINITE_DEPTH, data->deepen_relative,
+ &data->shallows, want_obj);
packet_delim(1);
}
send_wanted_ref_info(&data);
send_shallow_info(&data, &want_obj);
- packet_write_fmt(1, "packfile\n");
+ packet_writer_write(&data.writer, "packfile\n");
create_pack_file(&have_obj, &want_obj);
state = FETCH_DONE;
break;
if (value) {
int allow_filter_value;
int allow_ref_in_want;
+ int allow_sideband_all_value;
strbuf_addstr(value, "shallow");
&allow_ref_in_want) &&
allow_ref_in_want)
strbuf_addstr(value, " ref-in-want");
+
+ if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 0) ||
+ (!repo_config_get_bool(the_repository,
+ "uploadpack.allowsidebandall",
+ &allow_sideband_all_value) &&
+ allow_sideband_all_value))
+ strbuf_addstr(value, " sideband-all");
}
return 1;