index-pack: work around thread-unsafe pread()
[gitweb.git] / builtin / fetch.c
index 5bd00d064a7fbfb828f7a7c5e79f8ed0a5a1e959..025bc3e38d7d8055699ea15f241648602cfaf267 100644 (file)
@@ -36,7 +36,7 @@ static int prune = -1; /* unspecified */
 
 static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
 static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
-static int tags = TAGS_DEFAULT, unshallow;
+static int tags = TAGS_DEFAULT, unshallow, update_shallow;
 static const char *depth;
 static const char *upload_pack;
 static struct strbuf default_rla = STRBUF_INIT;
@@ -44,6 +44,7 @@ static struct transport *gtransport;
 static struct transport *gsecondary;
 static const char *submodule_prefix = "";
 static const char *recurse_submodules_default;
+static int shown_url = 0;
 
 static int option_parse_recurse_submodules(const struct option *opt,
                                   const char *arg, int unset)
@@ -104,6 +105,8 @@ static struct option builtin_fetch_options[] = {
        { OPTION_STRING, 0, "recurse-submodules-default",
                   &recurse_submodules_default, NULL,
                   N_("default mode for recursion"), PARSE_OPT_HIDDEN },
+       OPT_BOOL(0, "update-shallow", &update_shallow,
+                N_("accept refs that update .git/shallow")),
        OPT_END()
 };
 
@@ -160,48 +163,156 @@ static void add_merge_config(struct ref **head,
        }
 }
 
+static int add_existing(const char *refname, const unsigned char *sha1,
+                       int flag, void *cbdata)
+{
+       struct string_list *list = (struct string_list *)cbdata;
+       struct string_list_item *item = string_list_insert(list, refname);
+       item->util = xmalloc(20);
+       hashcpy(item->util, sha1);
+       return 0;
+}
+
+static int will_fetch(struct ref **head, const unsigned char *sha1)
+{
+       struct ref *rm = *head;
+       while (rm) {
+               if (!hashcmp(rm->old_sha1, sha1))
+                       return 1;
+               rm = rm->next;
+       }
+       return 0;
+}
+
 static void find_non_local_tags(struct transport *transport,
                        struct ref **head,
-                       struct ref ***tail);
+                       struct ref ***tail)
+{
+       struct string_list existing_refs = STRING_LIST_INIT_DUP;
+       struct string_list remote_refs = STRING_LIST_INIT_NODUP;
+       const struct ref *ref;
+       struct string_list_item *item = NULL;
+
+       for_each_ref(add_existing, &existing_refs);
+       for (ref = transport_get_remote_refs(transport); ref; ref = ref->next) {
+               if (!starts_with(ref->name, "refs/tags/"))
+                       continue;
+
+               /*
+                * The peeled ref always follows the matching base
+                * ref, so if we see a peeled ref that we don't want
+                * to fetch then we can mark the ref entry in the list
+                * as one to ignore by setting util to NULL.
+                */
+               if (ends_with(ref->name, "^{}")) {
+                       if (item && !has_sha1_file(ref->old_sha1) &&
+                           !will_fetch(head, ref->old_sha1) &&
+                           !has_sha1_file(item->util) &&
+                           !will_fetch(head, item->util))
+                               item->util = NULL;
+                       item = NULL;
+                       continue;
+               }
+
+               /*
+                * If item is non-NULL here, then we previously saw a
+                * ref not followed by a peeled reference, so we need
+                * to check if it is a lightweight tag that we want to
+                * fetch.
+                */
+               if (item && !has_sha1_file(item->util) &&
+                   !will_fetch(head, item->util))
+                       item->util = NULL;
+
+               item = NULL;
+
+               /* skip duplicates and refs that we already have */
+               if (string_list_has_string(&remote_refs, ref->name) ||
+                   string_list_has_string(&existing_refs, ref->name))
+                       continue;
+
+               item = string_list_insert(&remote_refs, ref->name);
+               item->util = (void *)ref->old_sha1;
+       }
+       string_list_clear(&existing_refs, 1);
+
+       /*
+        * We may have a final lightweight tag that needs to be
+        * checked to see if it needs fetching.
+        */
+       if (item && !has_sha1_file(item->util) &&
+           !will_fetch(head, item->util))
+               item->util = NULL;
+
+       /*
+        * For all the tags in the remote_refs string list,
+        * add them to the list of refs to be fetched
+        */
+       for_each_string_list_item(item, &remote_refs) {
+               /* Unless we have already decided to ignore this item... */
+               if (item->util)
+               {
+                       struct ref *rm = alloc_ref(item->string);
+                       rm->peer_ref = alloc_ref(item->string);
+                       hashcpy(rm->old_sha1, item->util);
+                       **tail = rm;
+                       *tail = &rm->next;
+               }
+       }
+
+       string_list_clear(&remote_refs, 0);
+}
 
 static struct ref *get_ref_map(struct transport *transport,
-                              struct refspec *refs, int ref_count, int tags,
-                              int *autotags)
+                              struct refspec *refspecs, int refspec_count,
+                              int tags, int *autotags)
 {
        int i;
        struct ref *rm;
        struct ref *ref_map = NULL;
        struct ref **tail = &ref_map;
 
-       const struct ref *remote_refs = transport_get_remote_refs(transport);
+       /* opportunistically-updated references: */
+       struct ref *orefs = NULL, **oref_tail = &orefs;
 
-       if (ref_count || tags == TAGS_SET) {
-               struct ref **old_tail;
+       const struct ref *remote_refs = transport_get_remote_refs(transport);
 
-               for (i = 0; i < ref_count; i++) {
-                       get_fetch_map(remote_refs, &refs[i], &tail, 0);
-                       if (refs[i].dst && refs[i].dst[0])
+       if (refspec_count) {
+               for (i = 0; i < refspec_count; i++) {
+                       get_fetch_map(remote_refs, &refspecs[i], &tail, 0);
+                       if (refspecs[i].dst && refspecs[i].dst[0])
                                *autotags = 1;
                }
-               /* Merge everything on the command line, but not --tags */
+               /* Merge everything on the command line (but not --tags) */
                for (rm = ref_map; rm; rm = rm->next)
                        rm->fetch_head_status = FETCH_HEAD_MERGE;
-               if (tags == TAGS_SET)
-                       get_fetch_map(remote_refs, tag_refspec, &tail, 0);
 
                /*
-                * For any refs that we happen to be fetching via command-line
-                * arguments, take the opportunity to update their configured
-                * counterparts. However, we do not want to mention these
-                * entries in FETCH_HEAD at all, as they would simply be
-                * duplicates of existing entries.
+                * For any refs that we happen to be fetching via
+                * command-line arguments, the destination ref might
+                * have been missing or have been different than the
+                * remote-tracking ref that would be derived from the
+                * configured refspec.  In these cases, we want to
+                * take the opportunity to update their configured
+                * remote-tracking reference.  However, we do not want
+                * to mention these entries in FETCH_HEAD at all, as
+                * they would simply be duplicates of existing
+                * entries, so we set them FETCH_HEAD_IGNORE below.
+                *
+                * We compute these entries now, based only on the
+                * refspecs specified on the command line.  But we add
+                * them to the list following the refspecs resulting
+                * from the tags option so that one of the latter,
+                * which has FETCH_HEAD_NOT_FOR_MERGE, is not removed
+                * by ref_remove_duplicates() in favor of one of these
+                * opportunistic entries with FETCH_HEAD_IGNORE.
                 */
-               old_tail = tail;
                for (i = 0; i < transport->remote->fetch_refspec_nr; i++)
                        get_fetch_map(ref_map, &transport->remote->fetch[i],
-                                     &tail, 1);
-               for (rm = *old_tail; rm; rm = rm->next)
-                       rm->fetch_head_status = FETCH_HEAD_IGNORE;
+                                     &oref_tail, 1);
+
+               if (tags == TAGS_SET)
+                       get_fetch_map(remote_refs, tag_refspec, &tail, 0);
        } else {
                /* Use the defaults */
                struct remote *remote = transport->remote;
@@ -238,11 +349,21 @@ static struct ref *get_ref_map(struct transport *transport,
                        tail = &ref_map->next;
                }
        }
-       if (tags == TAGS_DEFAULT && *autotags)
+
+       if (tags == TAGS_SET)
+               /* also fetch all tags */
+               get_fetch_map(remote_refs, tag_refspec, &tail, 0);
+       else if (tags == TAGS_DEFAULT && *autotags)
                find_non_local_tags(transport, &ref_map, &tail);
-       ref_remove_duplicates(ref_map);
 
-       return ref_map;
+       /* Now append any refs to be updated opportunistically: */
+       *tail = orefs;
+       for (rm = orefs; rm; rm = rm->next) {
+               rm->fetch_head_status = FETCH_HEAD_IGNORE;
+               tail = &rm->next;
+       }
+
+       return ref_remove_duplicates(ref_map);
 }
 
 #define STORE_REF_ERROR_OTHER 1
@@ -313,7 +434,7 @@ static int update_local_ref(struct ref *ref,
        }
 
        if (!is_null_sha1(ref->old_sha1) &&
-           !prefixcmp(ref->name, "refs/tags/")) {
+           starts_with(ref->name, "refs/tags/")) {
                int r;
                r = s_update_ref("updating tag", ref, 0);
                strbuf_addf(display, "%c %-*s %-*s -> %s%s",
@@ -336,10 +457,10 @@ static int update_local_ref(struct ref *ref,
                 * more likely to follow a standard layout.
                 */
                const char *name = remote_ref ? remote_ref->name : "";
-               if (!prefixcmp(name, "refs/tags/")) {
+               if (starts_with(name, "refs/tags/")) {
                        msg = "storing tag";
                        what = _("[new tag]");
-               } else if (!prefixcmp(name, "refs/heads/")) {
+               } else if (starts_with(name, "refs/heads/")) {
                        msg = "storing head";
                        what = _("[new branch]");
                } else {
@@ -405,6 +526,8 @@ static int iterate_ref_map(void *cb_data, unsigned char sha1[20])
        struct ref **rm = cb_data;
        struct ref *ref = *rm;
 
+       while (ref && ref->status == REF_STATUS_REJECT_SHALLOW)
+               ref = ref->next;
        if (!ref)
                return -1; /* end of the list */
        *rm = ref->next;
@@ -417,7 +540,7 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
 {
        FILE *fp;
        struct commit *commit;
-       int url_len, i, shown_url = 0, rc = 0;
+       int url_len, i, rc = 0;
        struct strbuf note = STRBUF_INIT;
        const char *what, *kind;
        struct ref *rm;
@@ -451,6 +574,13 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
                        struct ref *ref = NULL;
                        const char *merge_status_marker = "";
 
+                       if (rm->status == REF_STATUS_REJECT_SHALLOW) {
+                               if (want_status == FETCH_HEAD_MERGE)
+                                       warning(_("reject %s because shallow roots are not allowed to be updated"),
+                                               rm->peer_ref ? rm->peer_ref->name : rm->name);
+                               continue;
+                       }
+
                        commit = lookup_commit_reference_gently(rm->old_sha1, 1);
                        if (!commit)
                                rm->fetch_head_status = FETCH_HEAD_NOT_FOR_MERGE;
@@ -471,15 +601,15 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
                                kind = "";
                                what = "";
                        }
-                       else if (!prefixcmp(rm->name, "refs/heads/")) {
+                       else if (starts_with(rm->name, "refs/heads/")) {
                                kind = "branch";
                                what = rm->name + 11;
                        }
-                       else if (!prefixcmp(rm->name, "refs/tags/")) {
+                       else if (starts_with(rm->name, "refs/tags/")) {
                                kind = "tag";
                                what = rm->name + 10;
                        }
-                       else if (!prefixcmp(rm->name, "refs/remotes/")) {
+                       else if (starts_with(rm->name, "refs/remotes/")) {
                                kind = "remote-tracking branch";
                                what = rm->name + 13;
                        }
@@ -590,17 +720,36 @@ static int fetch_refs(struct transport *transport, struct ref *ref_map)
        return ret;
 }
 
-static int prune_refs(struct refspec *refs, int ref_count, struct ref *ref_map)
+static int prune_refs(struct refspec *refs, int ref_count, struct ref *ref_map,
+               const char *raw_url)
 {
-       int result = 0;
+       int url_len, i, result = 0;
        struct ref *ref, *stale_refs = get_stale_heads(refs, ref_count, ref_map);
+       char *url;
        const char *dangling_msg = dry_run
                ? _("   (%s will become dangling)")
                : _("   (%s has become dangling)");
 
+       if (raw_url)
+               url = transport_anonymize_url(raw_url);
+       else
+               url = xstrdup("foreign");
+
+       url_len = strlen(url);
+       for (i = url_len - 1; url[i] == '/' && 0 <= i; i--)
+               ;
+
+       url_len = i + 1;
+       if (4 < i && !strncmp(".git", url + i - 3, 4))
+               url_len = i - 3;
+
        for (ref = stale_refs; ref; ref = ref->next) {
                if (!dry_run)
                        result |= delete_ref(ref->name, NULL, 0);
+               if (verbosity >= 0 && !shown_url) {
+                       fprintf(stderr, _("From %.*s\n"), url_len, url);
+                       shown_url = 1;
+               }
                if (verbosity >= 0) {
                        fprintf(stderr, " x %-*s %-*s -> %s\n",
                                TRANSPORT_SUMMARY(_("[deleted]")),
@@ -608,110 +757,11 @@ static int prune_refs(struct refspec *refs, int ref_count, struct ref *ref_map)
                        warn_dangling_symref(stderr, dangling_msg, ref->name);
                }
        }
+       free(url);
        free_refs(stale_refs);
        return result;
 }
 
-static int add_existing(const char *refname, const unsigned char *sha1,
-                       int flag, void *cbdata)
-{
-       struct string_list *list = (struct string_list *)cbdata;
-       struct string_list_item *item = string_list_insert(list, refname);
-       item->util = xmalloc(20);
-       hashcpy(item->util, sha1);
-       return 0;
-}
-
-static int will_fetch(struct ref **head, const unsigned char *sha1)
-{
-       struct ref *rm = *head;
-       while (rm) {
-               if (!hashcmp(rm->old_sha1, sha1))
-                       return 1;
-               rm = rm->next;
-       }
-       return 0;
-}
-
-static void find_non_local_tags(struct transport *transport,
-                       struct ref **head,
-                       struct ref ***tail)
-{
-       struct string_list existing_refs = STRING_LIST_INIT_DUP;
-       struct string_list remote_refs = STRING_LIST_INIT_NODUP;
-       const struct ref *ref;
-       struct string_list_item *item = NULL;
-
-       for_each_ref(add_existing, &existing_refs);
-       for (ref = transport_get_remote_refs(transport); ref; ref = ref->next) {
-               if (prefixcmp(ref->name, "refs/tags/"))
-                       continue;
-
-               /*
-                * The peeled ref always follows the matching base
-                * ref, so if we see a peeled ref that we don't want
-                * to fetch then we can mark the ref entry in the list
-                * as one to ignore by setting util to NULL.
-                */
-               if (!suffixcmp(ref->name, "^{}")) {
-                       if (item && !has_sha1_file(ref->old_sha1) &&
-                           !will_fetch(head, ref->old_sha1) &&
-                           !has_sha1_file(item->util) &&
-                           !will_fetch(head, item->util))
-                               item->util = NULL;
-                       item = NULL;
-                       continue;
-               }
-
-               /*
-                * If item is non-NULL here, then we previously saw a
-                * ref not followed by a peeled reference, so we need
-                * to check if it is a lightweight tag that we want to
-                * fetch.
-                */
-               if (item && !has_sha1_file(item->util) &&
-                   !will_fetch(head, item->util))
-                       item->util = NULL;
-
-               item = NULL;
-
-               /* skip duplicates and refs that we already have */
-               if (string_list_has_string(&remote_refs, ref->name) ||
-                   string_list_has_string(&existing_refs, ref->name))
-                       continue;
-
-               item = string_list_insert(&remote_refs, ref->name);
-               item->util = (void *)ref->old_sha1;
-       }
-       string_list_clear(&existing_refs, 1);
-
-       /*
-        * We may have a final lightweight tag that needs to be
-        * checked to see if it needs fetching.
-        */
-       if (item && !has_sha1_file(item->util) &&
-           !will_fetch(head, item->util))
-               item->util = NULL;
-
-       /*
-        * For all the tags in the remote_refs string list,
-        * add them to the list of refs to be fetched
-        */
-       for_each_string_list_item(item, &remote_refs) {
-               /* Unless we have already decided to ignore this item... */
-               if (item->util)
-               {
-                       struct ref *rm = alloc_ref(item->string);
-                       rm->peer_ref = alloc_ref(item->string);
-                       hashcpy(rm->old_sha1, item->util);
-                       **tail = rm;
-                       *tail = &rm->next;
-               }
-       }
-
-       string_list_clear(&remote_refs, 0);
-}
-
 static void check_not_current_branch(struct ref *ref_map)
 {
        struct branch *current_branch = branch_get(NULL);
@@ -759,6 +809,8 @@ static struct transport *prepare_transport(struct remote *remote)
                set_option(transport, TRANS_OPT_KEEP, "yes");
        if (depth)
                set_option(transport, TRANS_OPT_DEPTH, depth);
+       if (update_shallow)
+               set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
        return transport;
 }
 
@@ -824,39 +876,26 @@ static int do_fetch(struct transport *transport,
 
        if (tags == TAGS_DEFAULT && autotags)
                transport_set_option(transport, TRANS_OPT_FOLLOWTAGS, "1");
-       if (fetch_refs(transport, ref_map)) {
-               free_refs(ref_map);
-               retcode = 1;
-               goto cleanup;
-       }
        if (prune) {
                /*
-                * If --tags was specified, pretend that the user gave us
-                * the canonical tags refspec
+                * We only prune based on refspecs specified
+                * explicitly (via command line or configuration); we
+                * don't care whether --tags was specified.
                 */
-               if (tags == TAGS_SET) {
-                       const char *tags_str = "refs/tags/*:refs/tags/*";
-                       struct refspec *tags_refspec, *refspec;
-
-                       /* Copy the refspec and add the tags to it */
-                       refspec = xcalloc(ref_count + 1, sizeof(struct refspec));
-                       tags_refspec = parse_fetch_refspec(1, &tags_str);
-                       memcpy(refspec, refs, ref_count * sizeof(struct refspec));
-                       memcpy(&refspec[ref_count], tags_refspec, sizeof(struct refspec));
-                       ref_count++;
-
-                       prune_refs(refspec, ref_count, ref_map);
-
-                       ref_count--;
-                       /* The rest of the strings belong to fetch_one */
-                       free_refspec(1, tags_refspec);
-                       free(refspec);
-               } else if (ref_count) {
-                       prune_refs(refs, ref_count, ref_map);
+               if (ref_count) {
+                       prune_refs(refs, ref_count, ref_map, transport->url);
                } else {
-                       prune_refs(transport->remote->fetch, transport->remote->fetch_refspec_nr, ref_map);
+                       prune_refs(transport->remote->fetch,
+                                  transport->remote->fetch_refspec_nr,
+                                  ref_map,
+                                  transport->url);
                }
        }
+       if (fetch_refs(transport, ref_map)) {
+               free_refs(ref_map);
+               retcode = 1;
+               goto cleanup;
+       }
        free_refs(ref_map);
 
        /* if neither --no-tags nor --tags was specified, do automated tag
@@ -892,7 +931,7 @@ static int get_remote_group(const char *key, const char *value, void *priv)
 {
        struct remote_group_data *g = priv;
 
-       if (!prefixcmp(key, "remotes.") &&
+       if (starts_with(key, "remotes.") &&
                        !strcmp(key + 8, g->name)) {
                /* split list by white space */
                int space = strcspn(value, " \t\n");
@@ -930,8 +969,8 @@ static void add_options_to_argv(struct argv_array *argv)
 {
        if (dry_run)
                argv_array_push(argv, "--dry-run");
-       if (prune > 0)
-               argv_array_push(argv, "--prune");
+       if (prune != -1)
+               argv_array_push(argv, prune ? "--prune" : "--no-prune");
        if (update_head_ok)
                argv_array_push(argv, "--update-head-ok");
        if (force)