#include "pack.h"
#include "blob.h"
#include "commit.h"
+#include "run-command.h"
#include "tag.h"
#include "tree.h"
#include "tree-walk.h"
const unsigned char null_sha1[20];
-static int git_open_noatime(const char *name, struct packed_git *p);
-
/*
* This is meant to hold a *small* number of objects that you would
* want read_sha1_file() to be able to return, but yet you do not want
static struct alternate_object_database **alt_odb_tail;
static void read_info_alternates(const char * alternates, int depth);
+static int git_open_noatime(const char *name);
/*
* Prepare alternate object database registry.
int fd;
sprintf(path, "%s/%s", relative_base, alt_file_name);
- fd = git_open_noatime(path, NULL);
+ fd = git_open_noatime(path);
if (fd < 0)
return;
if (fstat(fd, &st) || (st.st_size == 0)) {
struct pack_idx_header *hdr;
size_t idx_size;
uint32_t version, nr, i, *index;
- int fd = git_open_noatime(path, p);
+ int fd = git_open_noatime(path);
struct stat st;
if (fd < 0)
while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))
; /* nothing */
- p->pack_fd = git_open_noatime(p->pack_name, p);
+ p->pack_fd = git_open_noatime(p->pack_name);
if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
return -1;
pack_open_fds++;
return hashcmp(sha1, real_sha1) ? -1 : 0;
}
-static int git_open_noatime(const char *name, struct packed_git *p)
+static int git_open_noatime(const char *name)
{
static int sha1_file_open_flag = O_NOATIME;
char *name = sha1_file_name(sha1);
struct alternate_object_database *alt;
- fd = git_open_noatime(name, NULL);
+ fd = git_open_noatime(name);
if (fd >= 0)
return fd;
for (alt = alt_odb_list; alt; alt = alt->next) {
name = alt->name;
fill_sha1_path(name, sha1);
- fd = git_open_noatime(alt->base, NULL);
+ fd = git_open_noatime(alt->base);
if (fd >= 0)
return fd;
}
/*
* The above condition must be (bytes <= size), not
* (bytes < size). In other words, even though we
- * expect no more output and set avail_out to zer0,
+ * expect no more output and set avail_out to zero,
* the input zlib stream may have bytes that express
* "this concludes the stream", and we *do* want to
* eat that input.
/* forward declaration for a mutually recursive function */
static int packed_object_info(struct packed_git *p, off_t offset,
- unsigned long *sizep);
+ unsigned long *sizep, int *rtype);
static int packed_delta_info(struct packed_git *p,
struct pack_window **w_curs,
base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
if (!base_offset)
return OBJ_BAD;
- type = packed_object_info(p, base_offset, NULL);
+ type = packed_object_info(p, base_offset, NULL, NULL);
if (type <= OBJ_NONE) {
struct revindex_entry *revidx;
const unsigned char *base_sha1;
return type;
}
-static int unpack_object_header(struct packed_git *p,
- struct pack_window **w_curs,
- off_t *curpos,
- unsigned long *sizep)
+int unpack_object_header(struct packed_git *p,
+ struct pack_window **w_curs,
+ off_t *curpos,
+ unsigned long *sizep)
{
unsigned char *base;
unsigned int left;
return type;
}
-const char *packed_object_info_detail(struct packed_git *p,
+int packed_object_info_detail(struct packed_git *p,
off_t obj_offset,
unsigned long *size,
unsigned long *store_size,
case OBJ_BLOB:
case OBJ_TAG:
unuse_pack(&w_curs);
- return typename(type);
+ return type;
case OBJ_OFS_DELTA:
obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
if (!obj_offset)
}
static int packed_object_info(struct packed_git *p, off_t obj_offset,
- unsigned long *sizep)
+ unsigned long *sizep, int *rtype)
{
struct pack_window *w_curs = NULL;
unsigned long size;
enum object_type type;
type = unpack_object_header(p, &w_curs, &curpos, &size);
+ if (rtype)
+ *rtype = type; /* representation type */
switch (type) {
case OBJ_OFS_DELTA:
return hash % MAX_DELTA_CACHE;
}
+static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
+{
+ unsigned long hash = pack_entry_hash(p, base_offset);
+ struct delta_base_cache_entry *ent = delta_base_cache + hash;
+ return (ent->data && ent->p == p && ent->base_offset == base_offset);
+}
+
static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
unsigned long *base_size, enum object_type *type, int keep_cache)
{
return status;
}
-int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
+/* returns enum object_type or negative */
+int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
{
struct cached_object *co;
struct pack_entry e;
- int status;
+ int status, rtype;
co = find_cached_object(sha1);
if (co) {
- if (sizep)
- *sizep = co->size;
+ if (oi->sizep)
+ *(oi->sizep) = co->size;
+ oi->whence = OI_CACHED;
return co->type;
}
if (!find_pack_entry(sha1, &e)) {
/* Most likely it's a loose object. */
- status = sha1_loose_object_info(sha1, sizep);
- if (status >= 0)
+ status = sha1_loose_object_info(sha1, oi->sizep);
+ if (status >= 0) {
+ oi->whence = OI_LOOSE;
return status;
+ }
/* Not a loose object; someone else may have just packed it. */
reprepare_packed_git();
return status;
}
- status = packed_object_info(e.p, e.offset, sizep);
+ status = packed_object_info(e.p, e.offset, oi->sizep, &rtype);
if (status < 0) {
mark_bad_packed_object(e.p, sha1);
- status = sha1_object_info(sha1, sizep);
+ status = sha1_object_info_extended(sha1, oi);
+ } else if (in_delta_base_cache(e.p, e.offset)) {
+ oi->whence = OI_DBCACHED;
+ } else {
+ oi->whence = OI_PACKED;
+ oi->u.packed.offset = e.offset;
+ oi->u.packed.pack = e.p;
+ oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
+ rtype == OBJ_OFS_DELTA);
}
return status;
}
+int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
+{
+ struct object_info oi;
+
+ oi.sizep = sizep;
+ return sha1_object_info_extended(sha1, &oi);
+}
+
static void *read_packed_sha1(const unsigned char *sha1,
enum object_type *type, unsigned long *size)
{
* deal with them should arrange to call read_object() and give error
* messages themselves.
*/
-void *read_sha1_file_repl(const unsigned char *sha1,
- enum object_type *type,
- unsigned long *size,
- const unsigned char **replacement)
+void *read_sha1_file_extended(const unsigned char *sha1,
+ enum object_type *type,
+ unsigned long *size,
+ unsigned flag)
{
- const unsigned char *repl = lookup_replace_object(sha1);
void *data;
char *path;
const struct packed_git *p;
+ const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
+ ? lookup_replace_object(sha1) : sha1;
errno = 0;
data = read_object(repl, type, size);
- if (data) {
- if (replacement)
- *replacement = repl;
+ if (data)
return data;
- }
if (errno && errno != ENOENT)
die_errno("failed to read object %s", sha1_to_hex(sha1));
return ret;
}
+/*
+ * This creates one packfile per large blob, because the caller
+ * immediately wants the result sha1, and fast-import can report the
+ * object name via marks mechanism only by closing the created
+ * packfile.
+ *
+ * This also bypasses the usual "convert-to-git" dance, and that is on
+ * purpose. We could write a streaming version of the converting
+ * functions and insert that before feeding the data to fast-import
+ * (or equivalent in-core API described above), but the primary
+ * motivation for trying to stream from the working tree file and to
+ * avoid mmaping it in core is to deal with large binary blobs, and
+ * by definition they do _not_ want to get any conversion.
+ */
+static int index_stream(unsigned char *sha1, int fd, size_t size,
+ enum object_type type, const char *path,
+ unsigned flags)
+{
+ struct child_process fast_import;
+ char export_marks[512];
+ const char *argv[] = { "fast-import", "--quiet", export_marks, NULL };
+ char tmpfile[512];
+ char fast_import_cmd[512];
+ char buf[512];
+ int len, tmpfd;
+
+ strcpy(tmpfile, git_path("hashstream_XXXXXX"));
+ tmpfd = git_mkstemp_mode(tmpfile, 0600);
+ if (tmpfd < 0)
+ die_errno("cannot create tempfile: %s", tmpfile);
+ if (close(tmpfd))
+ die_errno("cannot close tempfile: %s", tmpfile);
+ sprintf(export_marks, "--export-marks=%s", tmpfile);
+
+ memset(&fast_import, 0, sizeof(fast_import));
+ fast_import.in = -1;
+ fast_import.argv = argv;
+ fast_import.git_cmd = 1;
+ if (start_command(&fast_import))
+ die_errno("index-stream: git fast-import failed");
+
+ len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n",
+ (unsigned long) size);
+ write_or_whine(fast_import.in, fast_import_cmd, len,
+ "index-stream: feeding fast-import");
+ while (size) {
+ char buf[10240];
+ size_t sz = size < sizeof(buf) ? size : sizeof(buf);
+ size_t actual;
+
+ actual = read_in_full(fd, buf, sz);
+ if (actual < 0)
+ die_errno("index-stream: reading input");
+ if (write_in_full(fast_import.in, buf, actual) != actual)
+ die_errno("index-stream: feeding fast-import");
+ size -= actual;
+ }
+ if (close(fast_import.in))
+ die_errno("index-stream: closing fast-import");
+ if (finish_command(&fast_import))
+ die_errno("index-stream: finishing fast-import");
+
+ tmpfd = open(tmpfile, O_RDONLY);
+ if (tmpfd < 0)
+ die_errno("index-stream: cannot open fast-import mark");
+ len = read(tmpfd, buf, sizeof(buf));
+ if (len < 0)
+ die_errno("index-stream: reading fast-import mark");
+ if (close(tmpfd) < 0)
+ die_errno("index-stream: closing fast-import mark");
+ if (unlink(tmpfile))
+ die_errno("index-stream: unlinking fast-import mark");
+ if (len != 44 ||
+ memcmp(":1 ", buf, 3) ||
+ get_sha1_hex(buf + 3, sha1))
+ die_errno("index-stream: unexpected fast-import mark: <%s>", buf);
+ return 0;
+}
+
int index_fd(unsigned char *sha1, int fd, struct stat *st,
enum object_type type, const char *path, unsigned flags)
{
if (!S_ISREG(st->st_mode))
ret = index_pipe(sha1, fd, type, path, flags);
- else
+ else if (size <= big_file_threshold || type != OBJ_BLOB)
ret = index_core(sha1, fd, size, type, path, flags);
+ else
+ ret = index_stream(sha1, fd, size, type, path, flags);
close(fd);
return ret;
}