Add fetch methods to transport library.

Signed-off-by: Daniel Barkalow <barkalow@iabervon.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/transport.c b/transport.c
index edbdc3c60ed5ec0b49298f9d70e123cf0a5f1e39..1e7374977e041cc4f3423f4c84686723929b36f1 100644 (file)
--- a/transport.c
+++ b/transport.c
@@ -1,6 +1,39 @@
 #include "cache.h"
 #include "transport.h"
 #include "run-command.h"
 #include "cache.h"
 #include "transport.h"
 #include "run-command.h"

+#include "http.h"
+#include "pkt-line.h"
+#include "fetch-pack.h"
+#include "walker.h"
+
+/* Generic functions for using commit walkers */
+
+static int fetch_objs_via_walker(const struct transport *transport,
+                                int nr_objs, char **objs)
+{
+       char *dest = xstrdup(transport->url);
+       struct walker *walker = transport->data;
+
+       walker->get_all = 1;
+       walker->get_tree = 1;
+       walker->get_history = 1;
+       walker->get_verbosely = transport->verbose;
+       walker->get_recover = 0;
+
+       if (walker_fetch(walker, nr_objs, objs, NULL, dest))
+               die("Fetch failed.");
+
+       free(dest);
+       return 0;
+}
+
+static int disconnect_walker(struct transport *transport)
+{
+       struct walker *walker = transport->data;
+       if (walker)
+               walker_free(walker);
+       return 0;
+}

 
 static const struct transport_ops rsync_transport;
 
 
 static const struct transport_ops rsync_transport;
 


@@ -36,9 +69,119 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons
        return !!err;
 }
 
        return !!err;
 }
 

+#ifndef NO_CURL
+static int missing__target(int code, int result)
+{
+       return  /* file:// URL -- do we ever use one??? */
+               (result == CURLE_FILE_COULDNT_READ_FILE) ||
+               /* http:// and https:// URL */
+               (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
+               /* ftp:// URL */
+               (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
+               ;
+}
+
+#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
+
+static struct ref *get_refs_via_curl(const struct transport *transport)
+{
+       struct buffer buffer;
+       char *data, *start, *mid;
+       char *ref_name;
+       char *refs_url;
+       int i = 0;
+
+       struct active_request_slot *slot;
+       struct slot_results results;
+
+       struct ref *refs = NULL;
+       struct ref *ref = NULL;
+       struct ref *last_ref = NULL;
+
+       data = xmalloc(4096);
+       buffer.size = 4096;
+       buffer.posn = 0;
+       buffer.buffer = data;
+
+       refs_url = xmalloc(strlen(transport->url) + 11);
+       sprintf(refs_url, "%s/info/refs", transport->url);
+
+       http_init();
+
+       slot = get_active_slot();
+       slot->results = &results;
+       curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_URL, refs_url);
+       curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
+       if (start_active_slot(slot)) {
+               run_active_slot(slot);
+               if (results.curl_result != CURLE_OK) {
+                       if (missing_target(&results)) {
+                               free(buffer.buffer);
+                               return NULL;
+                       } else {
+                               free(buffer.buffer);
+                               error("%s", curl_errorstr);
+                               return NULL;
+                       }
+               }
+       } else {
+               free(buffer.buffer);
+               error("Unable to start request");
+               return NULL;
+       }
+
+       http_cleanup();
+
+       data = buffer.buffer;
+       start = NULL;
+       mid = data;
+       while (i < buffer.posn) {
+               if (!start)
+                       start = &data[i];
+               if (data[i] == '\t')
+                       mid = &data[i];
+               if (data[i] == '\n') {
+                       data[i] = 0;
+                       ref_name = mid + 1;
+                       ref = xmalloc(sizeof(struct ref) +
+                                     strlen(ref_name) + 1);
+                       memset(ref, 0, sizeof(struct ref));
+                       strcpy(ref->name, ref_name);
+                       get_sha1_hex(start, ref->old_sha1);
+                       if (!refs)
+                               refs = ref;
+                       if (last_ref)
+                               last_ref->next = ref;
+                       last_ref = ref;
+                       start = NULL;
+               }
+               i++;
+       }
+
+       free(buffer.buffer);
+
+       return refs;
+}
+
+#else
+
+static struct ref *get_refs_via_curl(const struct transport *transport)
+{
+       die("Cannot fetch from '%s' without curl ...", transport->url);
+       return NULL;
+}
+
+#endif
+

 static const struct transport_ops curl_transport = {
        /* set_option */        NULL,
 static const struct transport_ops curl_transport = {
        /* set_option */        NULL,

-       /* push */              curl_transport_push
+       /* get_refs_list */     get_refs_via_curl,
+       /* fetch_refs */        NULL,
+       /* fetch_objs */        fetch_objs_via_walker,
+       /* push */              curl_transport_push,
+       /* disconnect */        disconnect_walker

 };
 
 static const struct transport_ops bundle_transport = {
 };
 
 static const struct transport_ops bundle_transport = {


@@ -46,7 +189,13 @@ static const struct transport_ops bundle_transport = {
 
 struct git_transport_data {
        unsigned thin : 1;
 
 struct git_transport_data {
        unsigned thin : 1;

+       unsigned keep : 1;
+
+       int unpacklimit;

 
 

+       int depth;
+
+       const char *uploadpack;

        const char *receivepack;
 };
 
        const char *receivepack;
 };
 


@@ -54,16 +203,86 @@ static int set_git_option(struct transport *connection,
                          const char *name, const char *value)
 {
        struct git_transport_data *data = connection->data;
                          const char *name, const char *value)
 {
        struct git_transport_data *data = connection->data;

-       if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {
+       if (!strcmp(name, TRANS_OPT_UPLOADPACK)) {
+               data->uploadpack = value;
+               return 0;
+       } else if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {

                data->receivepack = value;
                return 0;
        } else if (!strcmp(name, TRANS_OPT_THIN)) {
                data->thin = !!value;
                return 0;
                data->receivepack = value;
                return 0;
        } else if (!strcmp(name, TRANS_OPT_THIN)) {
                data->thin = !!value;
                return 0;

+       } else if (!strcmp(name, TRANS_OPT_KEEP)) {
+               data->keep = !!value;
+               return 0;
+       } else if (!strcmp(name, TRANS_OPT_UNPACKLIMIT)) {
+               data->unpacklimit = atoi(value);
+               return 0;
+       } else if (!strcmp(name, TRANS_OPT_DEPTH)) {
+               if (!value)
+                       data->depth = 0;
+               else
+                       data->depth = atoi(value);
+               return 0;

        }
        return 1;
 }
 
        }
        return 1;
 }
 

+static struct ref *get_refs_via_connect(const struct transport *transport)
+{
+       struct git_transport_data *data = transport->data;
+       struct ref *refs;
+       int fd[2];
+       pid_t pid;
+       char *dest = xstrdup(transport->url);
+
+       pid = git_connect(fd, dest, data->uploadpack, 0);
+
+       if (pid < 0)
+               die("Failed to connect to \"%s\"", transport->url);
+
+       get_remote_heads(fd[0], &refs, 0, NULL, 0);
+       packet_flush(fd[1]);
+
+       finish_connect(pid);
+
+       free(dest);
+
+       return refs;
+}
+
+static int fetch_refs_via_pack(const struct transport *transport,
+                              int nr_heads, char **heads)
+{
+       struct git_transport_data *data = transport->data;
+       struct ref *refs;
+       char *dest = xstrdup(transport->url);
+       struct fetch_pack_args args;
+
+       args.uploadpack = data->uploadpack;
+       args.quiet = 0;
+       args.keep_pack = data->keep;
+       args.unpacklimit = data->unpacklimit;
+       args.use_thin_pack = data->thin;
+       args.fetch_all = 0;
+       args.verbose = transport->verbose;
+       args.depth = data->depth;
+       args.no_progress = 0;
+
+       setup_fetch_pack(&args);
+
+       refs = fetch_pack(dest, nr_heads, heads);
+
+       // ???? check that refs got everything?
+
+       /* free the memory used for the refs list ... */
+
+       free_refs(refs);
+
+       free(dest);
+       return 0;
+}
+

 static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) {
        struct git_transport_data *data = transport->data;
        const char **argv;
 static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) {
        struct git_transport_data *data = transport->data;
        const char **argv;


@@ -110,6 +329,9 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const
 
 static const struct transport_ops git_transport = {
        /* set_option */        set_git_option,
 
 static const struct transport_ops git_transport = {
        /* set_option */        set_git_option,

+       /* get_refs_list */     get_refs_via_connect,
+       /* fetch_refs */        fetch_refs_via_pack,
+       /* fetch_objs */        NULL,

        /* push */              git_transport_push
 };
 
        /* push */              git_transport_push
 };
 


@@ -140,7 +362,10 @@ struct transport *transport_get(struct remote *remote, const char *url,
                   !prefixcmp(url, "ftp://")) {
                ret = xmalloc(sizeof(*ret));
                ret->ops = &curl_transport;
                   !prefixcmp(url, "ftp://")) {
                ret = xmalloc(sizeof(*ret));
                ret->ops = &curl_transport;

-               ret->data = NULL;
+               if (fetch)
+                       ret->data = get_http_walker(url);
+               else
+                       ret->data = NULL;

        } else if (is_local(url) && is_file(url)) {
                ret = xmalloc(sizeof(*ret));
                ret->data = NULL;
        } else if (is_local(url) && is_file(url)) {
                ret = xmalloc(sizeof(*ret));
                ret->data = NULL;


@@ -150,14 +375,19 @@ struct transport *transport_get(struct remote *remote, const char *url,
                ret = xcalloc(1, sizeof(*ret));
                ret->data = data;
                data->thin = 1;
                ret = xcalloc(1, sizeof(*ret));
                ret->data = data;
                data->thin = 1;

+               data->uploadpack = "git-upload-pack";
+               if (remote && remote->uploadpack)
+                       data->uploadpack = remote->uploadpack;

                data->receivepack = "git-receive-pack";
                if (remote && remote->receivepack)
                        data->receivepack = remote->receivepack;
                data->receivepack = "git-receive-pack";
                if (remote && remote->receivepack)
                        data->receivepack = remote->receivepack;

+               data->unpacklimit = -1;

                ret->ops = &git_transport;
        }
        if (ret) {
                ret->remote = remote;
                ret->url = url;
                ret->ops = &git_transport;
        }
        if (ret) {
                ret->remote = remote;
                ret->url = url;

+               ret->remote_refs = NULL;

                ret->fetch = !!fetch;
        }
        return ret;
                ret->fetch = !!fetch;
        }
        return ret;


@@ -186,6 +416,54 @@ int transport_push(struct transport *transport,
        return transport->ops->push(transport, refspec_nr, refspec, flags);
 }
 
        return transport->ops->push(transport, refspec_nr, refspec, flags);
 }
 

+struct ref *transport_get_remote_refs(struct transport *transport)
+{
+       if (!transport->remote_refs)
+               transport->remote_refs =
+                       transport->ops->get_refs_list(transport);
+       return transport->remote_refs;
+}
+
+#define PACK_HEADS_CHUNK_COUNT 256
+
+int transport_fetch_refs(struct transport *transport, struct ref *refs)
+{
+       int i;
+       int nr_heads = 0;
+       char **heads = xmalloc(PACK_HEADS_CHUNK_COUNT * sizeof(char *));
+       struct ref *rm;
+       int use_objs = !transport->ops->fetch_refs;
+
+       for (rm = refs; rm; rm = rm->next) {
+               if (rm->peer_ref &&
+                   !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1))
+                       continue;
+               if (use_objs) {
+                       heads[nr_heads++] = xstrdup(sha1_to_hex(rm->old_sha1));
+               } else {
+                       heads[nr_heads++] = xstrdup(rm->name);
+               }
+               if (nr_heads % PACK_HEADS_CHUNK_COUNT == 0)
+                       heads = xrealloc(heads,
+                                        (nr_heads + PACK_HEADS_CHUNK_COUNT) *
+                                        sizeof(char *));
+       }
+
+       if (use_objs) {
+               if (transport->ops->fetch_objs(transport, nr_heads, heads))
+                       return -1;
+       } else {
+               if (transport->ops->fetch_refs(transport, nr_heads, heads))
+                       return -1;
+       }
+
+       /* free the memory used for the heads list ... */
+       for (i = 0; i < nr_heads; i++)
+               free(heads[i]);
+       free(heads);
+       return 0;
+}
+

 int transport_disconnect(struct transport *transport)
 {
        int ret = 0;
 int transport_disconnect(struct transport *transport)
 {
        int ret = 0;

diff --git a/transport.h b/transport.h
index 5c2eb959b242253a1cece3f9ac2a70b5babb912c..5b0a6b065824c142babcdb28c03bceb48778827c 100644 (file)
--- a/transport.h
+++ b/transport.h
@@ -29,6 +29,11 @@ struct transport_ops {
        int (*set_option)(struct transport *connection, const char *name,
                          const char *value);
 
        int (*set_option)(struct transport *connection, const char *name,
                          const char *value);
 

+       struct ref *(*get_refs_list)(const struct transport *transport);
+       int (*fetch_refs)(const struct transport *transport,
+                         int nr_refs, char **refs);
+       int (*fetch_objs)(const struct transport *transport,
+                         int nr_objs, char **objs);

        int (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags);
 
        int (*disconnect)(struct transport *connection);
        int (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags);
 
        int (*disconnect)(struct transport *connection);


@@ -40,12 +45,24 @@ struct transport *transport_get(struct remote *remote, const char *url,
 
 /* Transport options which apply to git:// and scp-style URLs */
 
 
 /* Transport options which apply to git:// and scp-style URLs */
 

+/* The program to use on the remote side to send a pack */
+#define TRANS_OPT_UPLOADPACK "uploadpack"
+

 /* The program to use on the remote side to receive a pack */
 #define TRANS_OPT_RECEIVEPACK "receivepack"
 
 /* Transfer the data as a thin pack if not null */
 #define TRANS_OPT_THIN "thin"
 
 /* The program to use on the remote side to receive a pack */
 #define TRANS_OPT_RECEIVEPACK "receivepack"
 
 /* Transfer the data as a thin pack if not null */
 #define TRANS_OPT_THIN "thin"
 

+/* Keep the pack that was transferred if not null */
+#define TRANS_OPT_KEEP "keep"
+
+/* Unpack the objects if fewer than this number of objects are fetched */
+#define TRANS_OPT_UNPACKLIMIT "unpacklimit"
+
+/* Limit the depth of the fetch if not null */
+#define TRANS_OPT_DEPTH "depth"
+

 /**
  * Returns 0 if the option was used, non-zero otherwise. Prints a
  * message to stderr if the option is not used.
 /**
  * Returns 0 if the option was used, non-zero otherwise. Prints a
  * message to stderr if the option is not used.


@@ -56,6 +73,10 @@ int transport_set_option(struct transport *transport, const char *name,
 int transport_push(struct transport *connection,
                   int refspec_nr, const char **refspec, int flags);
 
 int transport_push(struct transport *connection,
                   int refspec_nr, const char **refspec, int flags);
 

+struct ref *transport_get_remote_refs(struct transport *transport);
+
+int transport_fetch_refs(struct transport *transport, struct ref *refs);
+

 int transport_disconnect(struct transport *transport);
 
 #endif
 int transport_disconnect(struct transport *transport);
 
 #endif