#include "cache.h" #include "transport.h" #include "run-command.h" #ifndef NO_CURL #include "http.h" #endif #include "pkt-line.h" #include "fetch-pack.h" #include "send-pack.h" #include "walker.h" #include "bundle.h" #include "dir.h" #include "refs.h" /* rsync support */ /* * We copy packed-refs and refs/ into a temporary file, then read the * loose refs recursively (sorting whenever possible), and then inserting * those packed refs that are not yet in the list (not validating, but * assuming that the file is sorted). * * Appears refactoring this from refs.c is too cumbersome. */ static int str_cmp(const void *a, const void *b) { const char *s1 = a; const char *s2 = b; return strcmp(s1, s2); } /* path->buf + name_offset is expected to point to "refs/" */ static int read_loose_refs(struct strbuf *path, int name_offset, struct ref **tail) { DIR *dir = opendir(path->buf); struct dirent *de; struct { char **entries; int nr, alloc; } list; int i, pathlen; if (!dir) return -1; memset (&list, 0, sizeof(list)); while ((de = readdir(dir))) { if (de->d_name[0] == '.' && (de->d_name[1] == '\0' || (de->d_name[1] == '.' && de->d_name[2] == '\0'))) continue; ALLOC_GROW(list.entries, list.nr + 1, list.alloc); list.entries[list.nr++] = xstrdup(de->d_name); } closedir(dir); /* sort the list */ qsort(list.entries, list.nr, sizeof(char *), str_cmp); pathlen = path->len; strbuf_addch(path, '/'); for (i = 0; i < list.nr; i++, strbuf_setlen(path, pathlen + 1)) { strbuf_addstr(path, list.entries[i]); if (read_loose_refs(path, name_offset, tail)) { int fd = open(path->buf, O_RDONLY); char buffer[40]; struct ref *next; if (fd < 0) continue; next = alloc_ref(path->len - name_offset + 1); if (read_in_full(fd, buffer, 40) != 40 || get_sha1_hex(buffer, next->old_sha1)) { close(fd); free(next); continue; } close(fd); strcpy(next->name, path->buf + name_offset); (*tail)->next = next; *tail = next; } } strbuf_setlen(path, pathlen); for (i = 0; i < list.nr; i++) free(list.entries[i]); free(list.entries); return 0; } /* insert the packed refs for which no loose refs were found */ static void insert_packed_refs(const char *packed_refs, struct ref **list) { FILE *f = fopen(packed_refs, "r"); static char buffer[PATH_MAX]; if (!f) return; for (;;) { int cmp = cmp, len; if (!fgets(buffer, sizeof(buffer), f)) { fclose(f); return; } if (hexval(buffer[0]) > 0xf) continue; len = strlen(buffer); if (buffer[len - 1] == '\n') buffer[--len] = '\0'; if (len < 41) continue; while ((*list)->next && (cmp = strcmp(buffer + 41, (*list)->next->name)) > 0) list = &(*list)->next; if (!(*list)->next || cmp < 0) { struct ref *next = alloc_ref(len - 40); buffer[40] = '\0'; if (get_sha1_hex(buffer, next->old_sha1)) { warning ("invalid SHA-1: %s", buffer); free(next); continue; } strcpy(next->name, buffer + 41); next->next = (*list)->next; (*list)->next = next; list = &(*list)->next; } } } static struct ref *get_refs_via_rsync(struct transport *transport) { struct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT; struct ref dummy, *tail = &dummy; struct child_process rsync; const char *args[5]; int temp_dir_len; /* copy the refs to the temporary directory */ strbuf_addstr(&temp_dir, git_path("rsync-refs-XXXXXX")); if (!mkdtemp(temp_dir.buf)) die ("Could not make temporary directory"); temp_dir_len = temp_dir.len; strbuf_addstr(&buf, transport->url); strbuf_addstr(&buf, "/refs"); memset(&rsync, 0, sizeof(rsync)); rsync.argv = args; rsync.stdout_to_stderr = 1; args[0] = "rsync"; args[1] = (transport->verbose > 0) ? "-rv" : "-r"; args[2] = buf.buf; args[3] = temp_dir.buf; args[4] = NULL; if (run_command(&rsync)) die ("Could not run rsync to get refs"); strbuf_reset(&buf); strbuf_addstr(&buf, transport->url); strbuf_addstr(&buf, "/packed-refs"); args[2] = buf.buf; if (run_command(&rsync)) die ("Could not run rsync to get refs"); /* read the copied refs */ strbuf_addstr(&temp_dir, "/refs"); read_loose_refs(&temp_dir, temp_dir_len + 1, &tail); strbuf_setlen(&temp_dir, temp_dir_len); tail = &dummy; strbuf_addstr(&temp_dir, "/packed-refs"); insert_packed_refs(temp_dir.buf, &tail); strbuf_setlen(&temp_dir, temp_dir_len); if (remove_dir_recursively(&temp_dir, 0)) warning ("Error removing temporary directory %s.", temp_dir.buf); strbuf_release(&buf); strbuf_release(&temp_dir); return dummy.next; } static int fetch_objs_via_rsync(struct transport *transport, int nr_objs, struct ref **to_fetch) { struct strbuf buf = STRBUF_INIT; struct child_process rsync; const char *args[8]; int result; strbuf_addstr(&buf, transport->url); strbuf_addstr(&buf, "/objects/"); memset(&rsync, 0, sizeof(rsync)); rsync.argv = args; rsync.stdout_to_stderr = 1; args[0] = "rsync"; args[1] = (transport->verbose > 0) ? "-rv" : "-r"; args[2] = "--ignore-existing"; args[3] = "--exclude"; args[4] = "info"; args[5] = buf.buf; args[6] = get_object_directory(); args[7] = NULL; /* NEEDSWORK: handle one level of alternates */ result = run_command(&rsync); strbuf_release(&buf); return result; } static int write_one_ref(const char *name, const unsigned char *sha1, int flags, void *data) { struct strbuf *buf = data; int len = buf->len; FILE *f; /* when called via for_each_ref(), flags is non-zero */ if (flags && prefixcmp(name, "refs/heads/") && prefixcmp(name, "refs/tags/")) return 0; strbuf_addstr(buf, name); if (safe_create_leading_directories(buf->buf) || !(f = fopen(buf->buf, "w")) || fprintf(f, "%s\n", sha1_to_hex(sha1)) < 0 || fclose(f)) return error("problems writing temporary file %s", buf->buf); strbuf_setlen(buf, len); return 0; } static int write_refs_to_temp_dir(struct strbuf *temp_dir, int refspec_nr, const char **refspec) { int i; for (i = 0; i < refspec_nr; i++) { unsigned char sha1[20]; char *ref; if (dwim_ref(refspec[i], strlen(refspec[i]), sha1, &ref) != 1) return error("Could not get ref %s", refspec[i]); if (write_one_ref(ref, sha1, 0, temp_dir)) { free(ref); return -1; } free(ref); } return 0; } static int rsync_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { struct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT; int result = 0, i; struct child_process rsync; const char *args[10]; if (flags & TRANSPORT_PUSH_MIRROR) return error("rsync transport does not support mirror mode"); /* first push the objects */ strbuf_addstr(&buf, transport->url); strbuf_addch(&buf, '/'); memset(&rsync, 0, sizeof(rsync)); rsync.argv = args; rsync.stdout_to_stderr = 1; i = 0; args[i++] = "rsync"; args[i++] = "-a"; if (flags & TRANSPORT_PUSH_DRY_RUN) args[i++] = "--dry-run"; if (transport->verbose > 0) args[i++] = "-v"; args[i++] = "--ignore-existing"; args[i++] = "--exclude"; args[i++] = "info"; args[i++] = get_object_directory(); args[i++] = buf.buf; args[i++] = NULL; if (run_command(&rsync)) return error("Could not push objects to %s", transport->url); /* copy the refs to the temporary directory; they could be packed. */ strbuf_addstr(&temp_dir, git_path("rsync-refs-XXXXXX")); if (!mkdtemp(temp_dir.buf)) die ("Could not make temporary directory"); strbuf_addch(&temp_dir, '/'); if (flags & TRANSPORT_PUSH_ALL) { if (for_each_ref(write_one_ref, &temp_dir)) return -1; } else if (write_refs_to_temp_dir(&temp_dir, refspec_nr, refspec)) return -1; i = 2; if (flags & TRANSPORT_PUSH_DRY_RUN) args[i++] = "--dry-run"; if (!(flags & TRANSPORT_PUSH_FORCE)) args[i++] = "--ignore-existing"; args[i++] = temp_dir.buf; args[i++] = transport->url; args[i++] = NULL; if (run_command(&rsync)) result = error("Could not push to %s", transport->url); if (remove_dir_recursively(&temp_dir, 0)) warning ("Could not remove temporary directory %s.", temp_dir.buf); strbuf_release(&buf); strbuf_release(&temp_dir); return result; } /* Generic functions for using commit walkers */ static int fetch_objs_via_walker(struct transport *transport, int nr_objs, struct ref **to_fetch) { char *dest = xstrdup(transport->url); struct walker *walker = transport->data; char **objs = xmalloc(nr_objs * sizeof(*objs)); int i; walker->get_all = 1; walker->get_tree = 1; walker->get_history = 1; walker->get_verbosely = transport->verbose >= 0; walker->get_recover = 0; for (i = 0; i < nr_objs; i++) objs[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1)); if (walker_fetch(walker, nr_objs, objs, NULL, NULL)) die("Fetch failed."); for (i = 0; i < nr_objs; i++) free(objs[i]); free(objs); free(dest); return 0; } static int disconnect_walker(struct transport *transport) { struct walker *walker = transport->data; if (walker) walker_free(walker); return 0; } #ifndef NO_CURL static int curl_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { const char **argv; int argc; int err; if (flags & TRANSPORT_PUSH_MIRROR) return error("http transport does not support mirror mode"); argv = xmalloc((refspec_nr + 11) * sizeof(char *)); argv[0] = "http-push"; argc = 1; if (flags & TRANSPORT_PUSH_ALL) argv[argc++] = "--all"; if (flags & TRANSPORT_PUSH_FORCE) argv[argc++] = "--force"; if (flags & TRANSPORT_PUSH_DRY_RUN) argv[argc++] = "--dry-run"; argv[argc++] = transport->url; while (refspec_nr--) argv[argc++] = *refspec++; argv[argc] = NULL; err = run_command_v_opt(argv, RUN_GIT_CMD); switch (err) { case -ERR_RUN_COMMAND_FORK: error("unable to fork for %s", argv[0]); case -ERR_RUN_COMMAND_EXEC: error("unable to exec %s", argv[0]); break; case -ERR_RUN_COMMAND_WAITPID: case -ERR_RUN_COMMAND_WAITPID_WRONG_PID: case -ERR_RUN_COMMAND_WAITPID_SIGNAL: case -ERR_RUN_COMMAND_WAITPID_NOEXIT: error("%s died with strange error", argv[0]); } return !!err; } static int missing__target(int code, int result) { return /* file:// URL -- do we ever use one??? */ (result == CURLE_FILE_COULDNT_READ_FILE) || /* http:// and https:// URL */ (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) || /* ftp:// URL */ (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE) ; } #define missing_target(a) missing__target((a)->http_code, (a)->curl_result) static struct ref *get_refs_via_curl(struct transport *transport) { struct buffer buffer; char *data, *start, *mid; char *ref_name; char *refs_url; int i = 0; struct active_request_slot *slot; struct slot_results results; struct ref *refs = NULL; struct ref *ref = NULL; struct ref *last_ref = NULL; data = xmalloc(4096); buffer.size = 4096; buffer.posn = 0; buffer.buffer = data; refs_url = xmalloc(strlen(transport->url) + 11); sprintf(refs_url, "%s/info/refs", transport->url); http_init(); slot = get_active_slot(); slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_URL, refs_url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { run_active_slot(slot); if (results.curl_result != CURLE_OK) { if (missing_target(&results)) { free(buffer.buffer); return NULL; } else { free(buffer.buffer); error("%s", curl_errorstr); return NULL; } } } else { free(buffer.buffer); error("Unable to start request"); return NULL; } http_cleanup(); data = buffer.buffer; start = NULL; mid = data; while (i < buffer.posn) { if (!start) start = &data[i]; if (data[i] == '\t') mid = &data[i]; if (data[i] == '\n') { data[i] = 0; ref_name = mid + 1; ref = xmalloc(sizeof(struct ref) + strlen(ref_name) + 1); memset(ref, 0, sizeof(struct ref)); strcpy(ref->name, ref_name); get_sha1_hex(start, ref->old_sha1); if (!refs) refs = ref; if (last_ref) last_ref->next = ref; last_ref = ref; start = NULL; } i++; } free(buffer.buffer); return refs; } static int fetch_objs_via_curl(struct transport *transport, int nr_objs, struct ref **to_fetch) { if (!transport->data) transport->data = get_http_walker(transport->url); return fetch_objs_via_walker(transport, nr_objs, to_fetch); } #endif struct bundle_transport_data { int fd; struct bundle_header header; }; static struct ref *get_refs_from_bundle(struct transport *transport) { struct bundle_transport_data *data = transport->data; struct ref *result = NULL; int i; if (data->fd > 0) close(data->fd); data->fd = read_bundle_header(transport->url, &data->header); if (data->fd < 0) die ("Could not read bundle '%s'.", transport->url); for (i = 0; i < data->header.references.nr; i++) { struct ref_list_entry *e = data->header.references.list + i; struct ref *ref = alloc_ref(strlen(e->name) + 1); hashcpy(ref->old_sha1, e->sha1); strcpy(ref->name, e->name); ref->next = result; result = ref; } return result; } static int fetch_refs_from_bundle(struct transport *transport, int nr_heads, struct ref **to_fetch) { struct bundle_transport_data *data = transport->data; return unbundle(&data->header, data->fd); } static int close_bundle(struct transport *transport) { struct bundle_transport_data *data = transport->data; if (data->fd > 0) close(data->fd); free(data); return 0; } struct git_transport_data { unsigned thin : 1; unsigned keep : 1; int depth; const char *uploadpack; const char *receivepack; }; static int set_git_option(struct transport *connection, const char *name, const char *value) { struct git_transport_data *data = connection->data; if (!strcmp(name, TRANS_OPT_UPLOADPACK)) { data->uploadpack = value; return 0; } else if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) { data->receivepack = value; return 0; } else if (!strcmp(name, TRANS_OPT_THIN)) { data->thin = !!value; return 0; } else if (!strcmp(name, TRANS_OPT_KEEP)) { data->keep = !!value; return 0; } else if (!strcmp(name, TRANS_OPT_DEPTH)) { if (!value) data->depth = 0; else data->depth = atoi(value); return 0; } return 1; } static struct ref *get_refs_via_connect(struct transport *transport) { struct git_transport_data *data = transport->data; struct ref *refs; int fd[2]; char *dest = xstrdup(transport->url); struct child_process *conn = git_connect(fd, dest, data->uploadpack, 0); get_remote_heads(fd[0], &refs, 0, NULL, 0); packet_flush(fd[1]); finish_connect(conn); free(dest); return refs; } static int fetch_refs_via_pack(struct transport *transport, int nr_heads, struct ref **to_fetch) { struct git_transport_data *data = transport->data; char **heads = xmalloc(nr_heads * sizeof(*heads)); char **origh = xmalloc(nr_heads * sizeof(*origh)); struct ref *refs; char *dest = xstrdup(transport->url); struct fetch_pack_args args; int i; memset(&args, 0, sizeof(args)); args.uploadpack = data->uploadpack; args.keep_pack = data->keep; args.lock_pack = 1; args.use_thin_pack = data->thin; args.verbose = transport->verbose > 0; args.depth = data->depth; for (i = 0; i < nr_heads; i++) origh[i] = heads[i] = xstrdup(to_fetch[i]->name); refs = fetch_pack(&args, dest, nr_heads, heads, &transport->pack_lockfile); for (i = 0; i < nr_heads; i++) free(origh[i]); free(origh); free(heads); free_refs(refs); free(dest); return 0; } static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { struct git_transport_data *data = transport->data; struct send_pack_args args; args.receivepack = data->receivepack; args.send_all = !!(flags & TRANSPORT_PUSH_ALL); args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR); args.force_update = !!(flags & TRANSPORT_PUSH_FORCE); args.use_thin_pack = data->thin; args.verbose = transport->verbose; args.dry_run = !!(flags & TRANSPORT_PUSH_DRY_RUN); return send_pack(&args, transport->url, transport->remote, refspec_nr, refspec); } static int disconnect_git(struct transport *transport) { free(transport->data); return 0; } static int is_local(const char *url) { const char *colon = strchr(url, ':'); const char *slash = strchr(url, '/'); return !colon || (slash && slash < colon); } static int is_file(const char *url) { struct stat buf; if (stat(url, &buf)) return 0; return S_ISREG(buf.st_mode); } struct transport *transport_get(struct remote *remote, const char *url) { struct transport *ret = xcalloc(1, sizeof(*ret)); ret->remote = remote; ret->url = url; if (!prefixcmp(url, "rsync://")) { ret->get_refs_list = get_refs_via_rsync; ret->fetch = fetch_objs_via_rsync; ret->push = rsync_transport_push; } else if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://") || !prefixcmp(url, "ftp://")) { #ifdef NO_CURL error("git was compiled without libcurl support."); #else ret->get_refs_list = get_refs_via_curl; ret->fetch = fetch_objs_via_curl; ret->push = curl_transport_push; #endif ret->disconnect = disconnect_walker; } else if (is_local(url) && is_file(url)) { struct bundle_transport_data *data = xcalloc(1, sizeof(*data)); ret->data = data; ret->get_refs_list = get_refs_from_bundle; ret->fetch = fetch_refs_from_bundle; ret->disconnect = close_bundle; } else { struct git_transport_data *data = xcalloc(1, sizeof(*data)); ret->data = data; ret->set_option = set_git_option; ret->get_refs_list = get_refs_via_connect; ret->fetch = fetch_refs_via_pack; ret->push = git_transport_push; ret->disconnect = disconnect_git; data->thin = 1; data->uploadpack = "git-upload-pack"; if (remote && remote->uploadpack) data->uploadpack = remote->uploadpack; data->receivepack = "git-receive-pack"; if (remote && remote->receivepack) data->receivepack = remote->receivepack; } return ret; } int transport_set_option(struct transport *transport, const char *name, const char *value) { if (transport->set_option) return transport->set_option(transport, name, value); return 1; } int transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { if (!transport->push) return 1; return transport->push(transport, refspec_nr, refspec, flags); } const struct ref *transport_get_remote_refs(struct transport *transport) { if (!transport->remote_refs) transport->remote_refs = transport->get_refs_list(transport); return transport->remote_refs; } int transport_fetch_refs(struct transport *transport, struct ref *refs) { int rc; int nr_heads = 0, nr_alloc = 0; struct ref **heads = NULL; struct ref *rm; for (rm = refs; rm; rm = rm->next) { if (rm->peer_ref && !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1)) continue; ALLOC_GROW(heads, nr_heads + 1, nr_alloc); heads[nr_heads++] = rm; } rc = transport->fetch(transport, nr_heads, heads); free(heads); return rc; } void transport_unlock_pack(struct transport *transport) { if (transport->pack_lockfile) { unlink(transport->pack_lockfile); free(transport->pack_lockfile); transport->pack_lockfile = NULL; } } int transport_disconnect(struct transport *transport) { int ret = 0; if (transport->disconnect) ret = transport->disconnect(transport); free(transport); return ret; }