Release pack windows before reporting out of memory.

If we are about to fail because this process has run out of memory we
should first try to automatically control our appetite for address
space by releasing enough least-recently-used pack windows to gain
back enough memory such that we might actually be able to meet the
current allocation request.

This should help users who have fairly large repositories but are
working on systems with relatively small virtual address space.
Many times we see reports on the mailing list of these users running
out of memory during various Git operations.  Dynamically decreasing
the amount of pack memory used when the demand for heap memory is
increasing is an intelligent solution to this problem.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/git-compat-util.h b/git-compat-util.h
index 4764087d8550afa781430461d3fffc3e1bcf1da8..0f856747e578f176dae1ab3ab900a64062fb9292 100644 (file)
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -123,11 +123,17 @@ extern char *gitstrcasestr(const char *haystack, const char *needle);
 extern size_t gitstrlcpy(char *, const char *, size_t);
 #endif
 
 extern size_t gitstrlcpy(char *, const char *, size_t);
 #endif
 

+extern void release_pack_memory(size_t);
+

 static inline char* xstrdup(const char *str)
 {
        char *ret = strdup(str);
 static inline char* xstrdup(const char *str)
 {
        char *ret = strdup(str);

-       if (!ret)
-               die("Out of memory, strdup failed");
+       if (!ret) {
+               release_pack_memory(strlen(str) + 1);
+               ret = strdup(str);
+               if (!ret)
+                       die("Out of memory, strdup failed");
+       }

        return ret;
 }
 
        return ret;
 }
 


@@ -136,8 +142,14 @@ static inline void *xmalloc(size_t size)
        void *ret = malloc(size);
        if (!ret && !size)
                ret = malloc(1);
        void *ret = malloc(size);
        if (!ret && !size)
                ret = malloc(1);

-       if (!ret)
-               die("Out of memory, malloc failed");
+       if (!ret) {
+               release_pack_memory(size);
+               ret = malloc(size);
+               if (!ret && !size)
+                       ret = malloc(1);
+               if (!ret)
+                       die("Out of memory, malloc failed");
+       }

 #ifdef XMALLOC_POISON
        memset(ret, 0xA5, size);
 #endif
 #ifdef XMALLOC_POISON
        memset(ret, 0xA5, size);
 #endif


@@ -149,8 +161,14 @@ static inline void *xrealloc(void *ptr, size_t size)
        void *ret = realloc(ptr, size);
        if (!ret && !size)
                ret = realloc(ptr, 1);
        void *ret = realloc(ptr, size);
        if (!ret && !size)
                ret = realloc(ptr, 1);

-       if (!ret)
-               die("Out of memory, realloc failed");
+       if (!ret) {
+               release_pack_memory(size);
+               ret = realloc(ptr, size);
+               if (!ret && !size)
+                       ret = realloc(ptr, 1);
+               if (!ret)
+                       die("Out of memory, realloc failed");
+       }

        return ret;
 }
 
        return ret;
 }
 


@@ -159,8 +177,14 @@ static inline void *xcalloc(size_t nmemb, size_t size)
        void *ret = calloc(nmemb, size);
        if (!ret && (!nmemb || !size))
                ret = calloc(1, 1);
        void *ret = calloc(nmemb, size);
        if (!ret && (!nmemb || !size))
                ret = calloc(1, 1);

-       if (!ret)
-               die("Out of memory, calloc failed");
+       if (!ret) {
+               release_pack_memory(nmemb * size);
+               ret = calloc(nmemb, size);
+               if (!ret && (!nmemb || !size))
+                       ret = calloc(1, 1);
+               if (!ret)
+                       die("Out of memory, calloc failed");
+       }

        return ret;
 }
 
        return ret;
 }
 

diff --git a/sha1_file.c b/sha1_file.c
index 8de8ce0a728bb5df75557d85c2ad0d55648af90d..fb1032b0fdf9f778441c49664da4f5f6b5da57f5 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -522,6 +522,13 @@ static int unuse_one_window(struct packed_git *current)
        return 0;
 }
 
        return 0;
 }
 

+void release_pack_memory(size_t need)
+{
+       size_t cur = pack_mapped;
+       while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
+               ; /* nothing */
+}
+

 void unuse_pack(struct pack_window **w_cursor)
 {
        struct pack_window *w = *w_cursor;
 void unuse_pack(struct pack_window **w_cursor)
 {
        struct pack_window *w = *w_cursor;