Make pack creation always fsync() the result

This means that we can depend on packs always being stable on disk,
simplifying a lot of the object serialization worries.  And unlike loose
objects, serializing pack creation IO isn't going to be a performance
killer.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 70d2f5d4161a0bb62f4734be97d0a8dd2a5098dc..4c2e0cd27cd6fb963adb2b5e572312869eae3f5d 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -515,10 +515,12 @@ static void write_pack_file(void)
                 * If so, rewrite it like in fast-import
                 */
                if (pack_to_stdout || nr_written == nr_remaining) {
                 * If so, rewrite it like in fast-import
                 */
                if (pack_to_stdout || nr_written == nr_remaining) {

-                       sha1close(f, sha1, 1);
+                       unsigned flags = pack_to_stdout ? CSUM_CLOSE : CSUM_FSYNC;
+                       sha1close(f, sha1, flags);

                } else {
                        int fd = sha1close(f, NULL, 0);
                        fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);
                } else {
                        int fd = sha1close(f, NULL, 0);
                        fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);

+                       fsync_or_die(fd, pack_tmp_name);

                        close(fd);
                }
 
                        close(fd);
                }
 

diff --git a/cache.h b/cache.h
index eab1a172fe4b2a9d735dc480879fd1034da7942a..092a997b073090e14feaa4825e821af040101eb6 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -761,6 +761,7 @@ extern ssize_t write_in_full(int fd, const void *buf, size_t count);
 extern void write_or_die(int fd, const void *buf, size_t count);
 extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
 extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);
 extern void write_or_die(int fd, const void *buf, size_t count);
 extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
 extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);

+extern void fsync_or_die(int fd, const char *);

 
 /* pager.c */
 extern void setup_pager(void);
 
 /* pager.c */
 extern void setup_pager(void);

diff --git a/csum-file.c b/csum-file.c
index 9728a9954129246b96713d2f3b8dbd52541c416b..ace64f165e4a01fb99892e9b89e7df791a7f4ca1 100644 (file)
--- a/csum-file.c
+++ b/csum-file.c
@@ -32,21 +32,24 @@ static void sha1flush(struct sha1file *f, unsigned int count)
        }
 }
 
        }
 }
 

-int sha1close(struct sha1file *f, unsigned char *result, int final)
+int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags)

 {
        int fd;
        unsigned offset = f->offset;
 {
        int fd;
        unsigned offset = f->offset;

+

        if (offset) {
                SHA1_Update(&f->ctx, f->buffer, offset);
                sha1flush(f, offset);
                f->offset = 0;
        }
        if (offset) {
                SHA1_Update(&f->ctx, f->buffer, offset);
                sha1flush(f, offset);
                f->offset = 0;
        }

-       if (final) {
+       if (flags & (CSUM_CLOSE | CSUM_FSYNC)) {

                /* write checksum and close fd */
                SHA1_Final(f->buffer, &f->ctx);
                if (result)
                        hashcpy(result, f->buffer);
                sha1flush(f, 20);
                /* write checksum and close fd */
                SHA1_Final(f->buffer, &f->ctx);
                if (result)
                        hashcpy(result, f->buffer);
                sha1flush(f, 20);

+               if (flags & CSUM_FSYNC)
+                       fsync_or_die(f->fd, f->name);

                if (close(f->fd))
                        die("%s: sha1 file error on close (%s)",
                            f->name, strerror(errno));
                if (close(f->fd))
                        die("%s: sha1 file error on close (%s)",
                            f->name, strerror(errno));

diff --git a/csum-file.h b/csum-file.h
index 1af76562f31da89e4cd2592079edb9c6a45736e3..72c9487f4fd9fcab5e02fc2dc6afd3cb7f9c036a 100644 (file)
--- a/csum-file.h
+++ b/csum-file.h
@@ -16,9 +16,13 @@ struct sha1file {
        unsigned char buffer[8192];
 };
 
        unsigned char buffer[8192];
 };
 

+/* sha1close flags */
+#define CSUM_CLOSE     1
+#define CSUM_FSYNC     2
+

 extern struct sha1file *sha1fd(int fd, const char *name);
 extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp);
 extern struct sha1file *sha1fd(int fd, const char *name);
 extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp);

-extern int sha1close(struct sha1file *, unsigned char *, int);
+extern int sha1close(struct sha1file *, unsigned char *, unsigned int);

 extern int sha1write(struct sha1file *, void *, unsigned int);
 extern void crc32_begin(struct sha1file *);
 extern uint32_t crc32_end(struct sha1file *);
 extern int sha1write(struct sha1file *, void *, unsigned int);
 extern void crc32_begin(struct sha1file *);
 extern uint32_t crc32_end(struct sha1file *);

diff --git a/fast-import.c b/fast-import.c
index 93119bbd9484504d19f069a5c0038fb1e2f40a26..e72b28679410155cb43968d315f02588e89fd5f1 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -890,7 +890,7 @@ static char *create_index(void)
                SHA1_Update(&ctx, (*c)->sha1, 20);
        }
        sha1write(f, pack_data->sha1, sizeof(pack_data->sha1));
                SHA1_Update(&ctx, (*c)->sha1, 20);
        }
        sha1write(f, pack_data->sha1, sizeof(pack_data->sha1));

-       sha1close(f, NULL, 1);
+       sha1close(f, NULL, CSUM_FSYNC);

        free(idx);
        SHA1_Final(pack_data->sha1, &ctx);
        return tmpfile;
        free(idx);
        SHA1_Final(pack_data->sha1, &ctx);
        return tmpfile;

diff --git a/index-pack.c b/index-pack.c
index aaba9443ccbab1315e67c0e6bedaa82afe5464fc..5ac91baf98615d7e494550155c1c4d7665ed2dcd 100644 (file)
--- a/index-pack.c
+++ b/index-pack.c
@@ -694,6 +694,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
        if (!from_stdin) {
                close(input_fd);
        } else {
        if (!from_stdin) {
                close(input_fd);
        } else {

+               fsync_or_die(output_fd, curr_pack_name);

                err = close(output_fd);
                if (err)
                        die("error while closing pack file: %s", strerror(errno));
                err = close(output_fd);
                if (err)
                        die("error while closing pack file: %s", strerror(errno));

diff --git a/pack-write.c b/pack-write.c
index c66c8af725f6d37b4edaeab75eebac7a387f4922..f52cabe83829289dee7e44673b59a02db38918a5 100644 (file)
--- a/pack-write.c
+++ b/pack-write.c
@@ -139,7 +139,7 @@ char *write_idx_file(char *index_name, struct pack_idx_entry **objects,
        }
 
        sha1write(f, sha1, 20);
        }
 
        sha1write(f, sha1, 20);

-       sha1close(f, NULL, 1);
+       sha1close(f, NULL, CSUM_FSYNC);

        SHA1_Final(sha1, &ctx);
        return index_name;
 }
        SHA1_Final(sha1, &ctx);
        return index_name;
 }

diff --git a/write_or_die.c b/write_or_die.c
index 32f99140205f8969d8a884b3cf9448eec3f1dd16..630be4cb9414b1686a5ad86a4d5166f2828096f1 100644 (file)
--- a/write_or_die.c
+++ b/write_or_die.c
@@ -78,6 +78,13 @@ ssize_t write_in_full(int fd, const void *buf, size_t count)
        return total;
 }
 
        return total;
 }
 

+void fsync_or_die(int fd, const char *msg)
+{
+       if (fsync(fd) < 0) {
+               die("%s: fsync error (%s)", msg, strerror(errno));
+       }
+}
+

 void write_or_die(int fd, const void *buf, size_t count)
 {
        if (write_in_full(fd, buf, count) < 0) {
 void write_or_die(int fd, const void *buf, size_t count)
 {
        if (write_in_full(fd, buf, count) < 0) {