pack-objects: move 'layer' into 'struct packing_data'
[gitweb.git] / builtin / pack-objects.c
index 88d2bb8153e86ccbaa65d93008d5075cebc3ddc7..d5d91eeed5c894ade986e489b9d7f41e6c718a0b 100644 (file)
 #include "streaming.h"
 #include "thread-utils.h"
 #include "pack-bitmap.h"
+#include "delta-islands.h"
 #include "reachable.h"
 #include "sha1-array.h"
 #include "argv-array.h"
 #include "list.h"
 #include "packfile.h"
 #include "object-store.h"
+#include "dir.h"
 
 #define IN_PACK(obj) oe_in_pack(&to_pack, obj)
 #define SIZE(obj) oe_size(&to_pack, obj)
@@ -57,7 +59,8 @@ static const char *pack_usage[] = {
 static struct packing_data to_pack;
 
 static struct pack_idx_entry **written_list;
-static uint32_t nr_result, nr_written;
+static uint32_t nr_result, nr_written, nr_seen;
+static uint32_t write_layer;
 
 static int non_empty;
 static int reuse_delta = 1, reuse_object = 1;
@@ -67,7 +70,8 @@ static int pack_loose_unreachable;
 static int local;
 static int have_non_local_packs;
 static int incremental;
-static int ignore_packed_keep;
+static int ignore_packed_keep_on_disk;
+static int ignore_packed_keep_in_core;
 static int allow_ofs_delta;
 static struct pack_idx_option pack_idx_opts;
 static const char *base_name;
@@ -91,8 +95,10 @@ static uint16_t write_bitmap_options;
 
 static int exclude_promisor_objects;
 
+static int use_delta_islands;
+
 static unsigned long delta_cache_size = 0;
-static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
+static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
 static unsigned long cache_max_small_delta_size = 1000;
 
 static unsigned long window_memory_limit = 0;
@@ -277,6 +283,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
        enum object_type type;
        void *buf;
        struct git_istream *st = NULL;
+       const unsigned hashsz = the_hash_algo->rawsz;
 
        if (!usable_delta) {
                if (oe_type(entry) == OBJ_BLOB &&
@@ -333,7 +340,7 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
                dheader[pos] = ofs & 127;
                while (ofs >>= 7)
                        dheader[--pos] = 128 | (--ofs & 127);
-               if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
+               if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) {
                        if (st)
                                close_istream(st);
                        free(buf);
@@ -345,19 +352,19 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent
        } else if (type == OBJ_REF_DELTA) {
                /*
                 * Deltas with a base reference contain
-                * an additional 20 bytes for the base sha1.
+                * additional bytes for the base object ID.
                 */
-               if (limit && hdrlen + 20 + datalen + 20 >= limit) {
+               if (limit && hdrlen + hashsz + datalen + hashsz >= limit) {
                        if (st)
                                close_istream(st);
                        free(buf);
                        return 0;
                }
                hashwrite(f, header, hdrlen);
-               hashwrite(f, DELTA(entry)->idx.oid.hash, 20);
-               hdrlen += 20;
+               hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
+               hdrlen += hashsz;
        } else {
-               if (limit && hdrlen + datalen + 20 >= limit) {
+               if (limit && hdrlen + datalen + hashsz >= limit) {
                        if (st)
                                close_istream(st);
                        free(buf);
@@ -389,6 +396,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
        unsigned char header[MAX_PACK_OBJECT_HEADER],
                      dheader[MAX_PACK_OBJECT_HEADER];
        unsigned hdrlen;
+       const unsigned hashsz = the_hash_algo->rawsz;
        unsigned long entry_size = SIZE(entry);
 
        if (DELTA(entry))
@@ -425,7 +433,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
                dheader[pos] = ofs & 127;
                while (ofs >>= 7)
                        dheader[--pos] = 128 | (--ofs & 127);
-               if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
+               if (limit && hdrlen + sizeof(dheader) - pos + datalen + hashsz >= limit) {
                        unuse_pack(&w_curs);
                        return 0;
                }
@@ -434,16 +442,16 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry,
                hdrlen += sizeof(dheader) - pos;
                reused_delta++;
        } else if (type == OBJ_REF_DELTA) {
-               if (limit && hdrlen + 20 + datalen + 20 >= limit) {
+               if (limit && hdrlen + hashsz + datalen + hashsz >= limit) {
                        unuse_pack(&w_curs);
                        return 0;
                }
                hashwrite(f, header, hdrlen);
-               hashwrite(f, DELTA(entry)->idx.oid.hash, 20);
-               hdrlen += 20;
+               hashwrite(f, DELTA(entry)->idx.oid.hash, hashsz);
+               hdrlen += hashsz;
                reused_delta++;
        } else {
-               if (limit && hdrlen + datalen + 20 >= limit) {
+               if (limit && hdrlen + datalen + hashsz >= limit) {
                        unuse_pack(&w_curs);
                        return 0;
                }
@@ -603,7 +611,7 @@ static inline void add_to_write_order(struct object_entry **wo,
                               unsigned int *endp,
                               struct object_entry *e)
 {
-       if (e->filled)
+       if (e->filled || oe_layer(&to_pack, e) != write_layer)
                return;
        wo[(*endp)++] = e;
        e->filled = 1;
@@ -663,48 +671,15 @@ static void add_family_to_write_order(struct object_entry **wo,
        add_descendants_to_write_order(wo, endp, root);
 }
 
-static struct object_entry **compute_write_order(void)
+static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end)
 {
-       unsigned int i, wo_end, last_untagged;
-
-       struct object_entry **wo;
+       unsigned int i, last_untagged;
        struct object_entry *objects = to_pack.objects;
 
        for (i = 0; i < to_pack.nr_objects; i++) {
-               objects[i].tagged = 0;
-               objects[i].filled = 0;
-               SET_DELTA_CHILD(&objects[i], NULL);
-               SET_DELTA_SIBLING(&objects[i], NULL);
-       }
-
-       /*
-        * Fully connect delta_child/delta_sibling network.
-        * Make sure delta_sibling is sorted in the original
-        * recency order.
-        */
-       for (i = to_pack.nr_objects; i > 0;) {
-               struct object_entry *e = &objects[--i];
-               if (!DELTA(e))
-                       continue;
-               /* Mark me as the first child */
-               e->delta_sibling_idx = DELTA(e)->delta_child_idx;
-               SET_DELTA_CHILD(DELTA(e), e);
-       }
-
-       /*
-        * Mark objects that are at the tip of tags.
-        */
-       for_each_tag_ref(mark_tagged, NULL);
-
-       /*
-        * Give the objects in the original recency order until
-        * we see a tagged tip.
-        */
-       ALLOC_ARRAY(wo, to_pack.nr_objects);
-       for (i = wo_end = 0; i < to_pack.nr_objects; i++) {
                if (objects[i].tagged)
                        break;
-               add_to_write_order(wo, &wo_end, &objects[i]);
+               add_to_write_order(wo, wo_end, &objects[i]);
        }
        last_untagged = i;
 
@@ -713,7 +688,7 @@ static struct object_entry **compute_write_order(void)
         */
        for (; i < to_pack.nr_objects; i++) {
                if (objects[i].tagged)
-                       add_to_write_order(wo, &wo_end, &objects[i]);
+                       add_to_write_order(wo, wo_end, &objects[i]);
        }
 
        /*
@@ -723,7 +698,7 @@ static struct object_entry **compute_write_order(void)
                if (oe_type(&objects[i]) != OBJ_COMMIT &&
                    oe_type(&objects[i]) != OBJ_TAG)
                        continue;
-               add_to_write_order(wo, &wo_end, &objects[i]);
+               add_to_write_order(wo, wo_end, &objects[i]);
        }
 
        /*
@@ -732,17 +707,61 @@ static struct object_entry **compute_write_order(void)
        for (i = last_untagged; i < to_pack.nr_objects; i++) {
                if (oe_type(&objects[i]) != OBJ_TREE)
                        continue;
-               add_to_write_order(wo, &wo_end, &objects[i]);
+               add_to_write_order(wo, wo_end, &objects[i]);
        }
 
        /*
         * Finally all the rest in really tight order
         */
        for (i = last_untagged; i < to_pack.nr_objects; i++) {
-               if (!objects[i].filled)
-                       add_family_to_write_order(wo, &wo_end, &objects[i]);
+               if (!objects[i].filled && oe_layer(&to_pack, &objects[i]) == write_layer)
+                       add_family_to_write_order(wo, wo_end, &objects[i]);
+       }
+}
+
+static struct object_entry **compute_write_order(void)
+{
+       uint32_t max_layers = 1;
+       unsigned int i, wo_end;
+
+       struct object_entry **wo;
+       struct object_entry *objects = to_pack.objects;
+
+       for (i = 0; i < to_pack.nr_objects; i++) {
+               objects[i].tagged = 0;
+               objects[i].filled = 0;
+               SET_DELTA_CHILD(&objects[i], NULL);
+               SET_DELTA_SIBLING(&objects[i], NULL);
+       }
+
+       /*
+        * Fully connect delta_child/delta_sibling network.
+        * Make sure delta_sibling is sorted in the original
+        * recency order.
+        */
+       for (i = to_pack.nr_objects; i > 0;) {
+               struct object_entry *e = &objects[--i];
+               if (!DELTA(e))
+                       continue;
+               /* Mark me as the first child */
+               e->delta_sibling_idx = DELTA(e)->delta_child_idx;
+               SET_DELTA_CHILD(DELTA(e), e);
        }
 
+       /*
+        * Mark objects that are at the tip of tags.
+        */
+       for_each_tag_ref(mark_tagged, NULL);
+
+       if (use_delta_islands)
+               max_layers = compute_pack_layers(&to_pack);
+
+       ALLOC_ARRAY(wo, to_pack.nr_objects);
+       wo_end = 0;
+
+       for (; write_layer < max_layers; ++write_layer)
+               compute_layer_order(wo, &wo_end);
+
        if (wo_end != to_pack.nr_objects)
                die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects);
 
@@ -767,7 +786,7 @@ static off_t write_reused_pack(struct hashfile *f)
                die_errno("unable to seek in reused packfile");
 
        if (reuse_packfile_offset < 0)
-               reuse_packfile_offset = reuse_packfile->pack_size - 20;
+               reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz;
 
        total = to_write = reuse_packfile_offset - sizeof(struct pack_header);
 
@@ -852,11 +871,11 @@ static void write_pack_file(void)
                 * If so, rewrite it like in fast-import
                 */
                if (pack_to_stdout) {
-                       hashclose(f, oid.hash, CSUM_CLOSE);
+                       finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE);
                } else if (nr_written == nr_remaining) {
-                       hashclose(f, oid.hash, CSUM_FSYNC);
+                       finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
                } else {
-                       int fd = hashclose(f, oid.hash, 0);
+                       int fd = finalize_hashfile(f, oid.hash, 0);
                        fixup_pack_header_footer(fd, oid.hash, pack_tmp_name,
                                                 nr_written, oid.hash, offset);
                        close(fd);
@@ -998,13 +1017,16 @@ static int want_found_object(int exclude, struct packed_git *p)
         * Otherwise, we signal "-1" at the end to tell the caller that we do
         * not know either way, and it needs to check more packs.
         */
-       if (!ignore_packed_keep &&
+       if (!ignore_packed_keep_on_disk &&
+           !ignore_packed_keep_in_core &&
            (!local || !have_non_local_packs))
                return 1;
 
        if (local && !p->pack_local)
                return 0;
-       if (ignore_packed_keep && p->pack_local && p->pack_keep)
+       if (p->pack_local &&
+           ((ignore_packed_keep_on_disk && p->pack_keep) ||
+            (ignore_packed_keep_in_core && p->pack_keep_in_core)))
                return 0;
 
        /* we don't know yet; keep looking for more packs */
@@ -1028,7 +1050,7 @@ static int want_object_in_pack(const struct object_id *oid,
        int want;
        struct list_head *pos;
 
-       if (!exclude && local && has_loose_object_nonlocal(oid->hash))
+       if (!exclude && local && has_loose_object_nonlocal(oid))
                return 0;
 
        /*
@@ -1106,6 +1128,8 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
        off_t found_offset = 0;
        uint32_t index_pos;
 
+       display_progress(progress_state, ++nr_seen);
+
        if (have_duplicate_entry(oid, exclude, &index_pos))
                return 0;
 
@@ -1121,8 +1145,6 @@ static int add_object_entry(const struct object_id *oid, enum object_type type,
        create_object_entry(oid, type, pack_name_hash(name),
                            exclude, name && no_try_delta(name),
                            index_pos, found_pack, found_offset);
-
-       display_progress(progress_state, nr_result);
        return 1;
 }
 
@@ -1133,6 +1155,8 @@ static int add_object_entry_from_bitmap(const struct object_id *oid,
 {
        uint32_t index_pos;
 
+       display_progress(progress_state, ++nr_seen);
+
        if (have_duplicate_entry(oid, 0, &index_pos))
                return 0;
 
@@ -1140,8 +1164,6 @@ static int add_object_entry_from_bitmap(const struct object_id *oid,
                return 0;
 
        create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);
-
-       display_progress(progress_state, nr_result);
        return 1;
 }
 
@@ -1462,7 +1484,7 @@ static void check_object(struct object_entry *entry)
                        if (reuse_delta && !entry->preferred_base)
                                base_ref = use_pack(p, &w_curs,
                                                entry->in_pack_offset + used, NULL);
-                       entry->in_pack_header_size = used + 20;
+                       entry->in_pack_header_size = used + the_hash_algo->rawsz;
                        break;
                case OBJ_OFS_DELTA:
                        buf = use_pack(p, &w_curs,
@@ -1497,7 +1519,8 @@ static void check_object(struct object_entry *entry)
                        break;
                }
 
-               if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
+               if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL)) &&
+                   in_same_island(&entry->idx.oid, &base_entry->idx.oid)) {
                        /*
                         * If base_ref was set above that means we wish to
                         * reuse delta data, and we even found that base
@@ -1544,7 +1567,8 @@ static void check_object(struct object_entry *entry)
                unuse_pack(&w_curs);
        }
 
-       oe_set_type(entry, oid_object_info(&entry->idx.oid, &canonical_size));
+       oe_set_type(entry,
+                   oid_object_info(the_repository, &entry->idx.oid, &canonical_size));
        if (entry->type_valid) {
                SET_SIZE(entry, canonical_size);
        } else {
@@ -1609,14 +1633,15 @@ static void drop_reused_delta(struct object_entry *entry)
 
        oi.sizep = &size;
        oi.typep = &type;
-       if (packed_object_info(IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
+       if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {
                /*
                 * We failed to get the info from this pack for some reason;
                 * fall back to sha1_object_info, which may find another copy.
                 * And if that fails, the error will be recorded in oe_type(entry)
                 * and dealt with in prepare_pack().
                 */
-               oe_set_type(entry, oid_object_info(&entry->idx.oid, &size));
+               oe_set_type(entry,
+                           oid_object_info(the_repository, &entry->idx.oid, &size));
        } else {
                oe_set_type(entry, type);
        }
@@ -1661,7 +1686,7 @@ static void break_delta_chains(struct object_entry *entry)
                 * is a bug.
                 */
                if (cur->dfs_state != DFS_NONE)
-                       die("BUG: confusing delta dfs state in first pass: %d",
+                       BUG("confusing delta dfs state in first pass: %d",
                            cur->dfs_state);
 
                /*
@@ -1718,7 +1743,7 @@ static void break_delta_chains(struct object_entry *entry)
                if (cur->dfs_state == DFS_DONE)
                        break;
                else if (cur->dfs_state != DFS_ACTIVE)
-                       die("BUG: confusing delta dfs state in second pass: %d",
+                       BUG("confusing delta dfs state in second pass: %d",
                            cur->dfs_state);
 
                /*
@@ -1752,6 +1777,10 @@ static void get_object_details(void)
        uint32_t i;
        struct object_entry **sorted_by_offset;
 
+       if (progress)
+               progress_state = start_progress(_("Counting objects"),
+                                               to_pack.nr_objects);
+
        sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *));
        for (i = 0; i < to_pack.nr_objects; i++)
                sorted_by_offset[i] = to_pack.objects + i;
@@ -1763,7 +1792,9 @@ static void get_object_details(void)
                if (entry->type_valid &&
                    oe_size_greater_than(&to_pack, entry, big_file_threshold))
                        entry->no_try_delta = 1;
+               display_progress(progress_state, i + 1);
        }
+       stop_progress(&progress_state);
 
        /*
         * This must happen in a second pass, since we rely on the delta
@@ -1805,6 +1836,11 @@ static int type_size_sort(const void *_a, const void *_b)
                return -1;
        if (a->preferred_base < b->preferred_base)
                return 1;
+       if (use_delta_islands) {
+               int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
+               if (island_cmp)
+                       return island_cmp;
+       }
        if (a_size > b_size)
                return -1;
        if (a_size < b_size)
@@ -1876,7 +1912,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack,
 
        if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {
                read_lock();
-               if (oid_object_info(&e->idx.oid, &size) < 0)
+               if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)
                        die(_("unable to get size of %s"),
                            oid_to_hex(&e->idx.oid));
                read_unlock();
@@ -1936,7 +1972,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
        /* Now some size filtering heuristics. */
        trg_size = SIZE(trg_entry);
        if (!DELTA(trg_entry)) {
-               max_size = trg_size/2 - 20;
+               max_size = trg_size/2 - the_hash_algo->rawsz;
                ref_depth = 1;
        } else {
                max_size = DELTA_SIZE(trg_entry);
@@ -1953,6 +1989,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
        if (trg_size < src_size / 32)
                return 0;
 
+       if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))
+               return 0;
+
        /* Load data if not already done */
        if (!trg->data) {
                read_lock();
@@ -2459,7 +2498,7 @@ static void add_tag_chain(const struct object_id *oid)
        if (packlist_find(&to_pack, oid->hash, NULL))
                return;
 
-       tag = lookup_tag(oid);
+       tag = lookup_tag(the_repository, oid);
        while (1) {
                if (!tag || parse_tag(tag) || !tag->tagged)
                        die("unable to pack objects reachable from tag %s",
@@ -2491,6 +2530,9 @@ static void prepare_pack(int window, int depth)
        uint32_t i, nr_deltas;
        unsigned n;
 
+       if (use_delta_islands)
+               resolve_tree_islands(progress, &to_pack);
+
        get_object_details();
 
        /*
@@ -2654,6 +2696,9 @@ static void show_commit(struct commit *commit, void *data)
 
        if (write_bitmap_index)
                index_commit_for_bitmap(commit);
+
+       if (use_delta_islands)
+               propagate_island_marks(commit);
 }
 
 static void show_object(struct object *obj, const char *name, void *data)
@@ -2661,6 +2706,19 @@ static void show_object(struct object *obj, const char *name, void *data)
        add_preferred_base_object(name);
        add_object_entry(&obj->oid, obj->type, name, 0);
        obj->flags |= OBJECT_ADDED;
+
+       if (use_delta_islands) {
+               const char *p;
+               unsigned depth = 0;
+               struct object_entry *ent;
+
+               for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
+                       depth++;
+
+               ent = packlist_find(&to_pack, obj->oid.hash, NULL);
+               if (ent && depth > oe_tree_depth(&to_pack, ent))
+                       oe_set_tree_depth(&to_pack, ent, depth);
+       }
 }
 
 static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
@@ -2773,7 +2831,7 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
                struct object_id oid;
                struct object *o;
 
-               if (!p->pack_local || p->pack_keep)
+               if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
                        continue;
                if (open_pack_index(p))
                        die("cannot open pack index");
@@ -2804,7 +2862,7 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
 static int add_loose_object(const struct object_id *oid, const char *path,
                            void *data)
 {
-       enum object_type type = oid_object_info(oid, NULL);
+       enum object_type type = oid_object_info(the_repository, oid, NULL);
 
        if (type < 0) {
                warning("loose object at %s could not be examined", path);
@@ -2836,7 +2894,8 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
                                        get_packed_git(the_repository);
 
        while (p) {
-               if ((!p->pack_local || p->pack_keep) &&
+               if ((!p->pack_local || p->pack_keep ||
+                               p->pack_keep_in_core) &&
                        find_pack_entry_one(oid->hash, p)) {
                        last_found = p;
                        return 1;
@@ -2879,7 +2938,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
        struct object_id oid;
 
        for (p = get_packed_git(the_repository); p; p = p->next) {
-               if (!p->pack_local || p->pack_keep)
+               if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
                        continue;
 
                if (open_pack_index(p))
@@ -2905,18 +2964,21 @@ static int pack_options_allow_reuse(void)
 {
        return pack_to_stdout &&
               allow_ofs_delta &&
-              !ignore_packed_keep &&
+              !ignore_packed_keep_on_disk &&
+              !ignore_packed_keep_in_core &&
               (!local || !have_non_local_packs) &&
               !incremental;
 }
 
 static int get_object_list_from_bitmap(struct rev_info *revs)
 {
-       if (prepare_bitmap_walk(revs) < 0)
+       struct bitmap_index *bitmap_git;
+       if (!(bitmap_git = prepare_bitmap_walk(revs)))
                return -1;
 
        if (pack_options_allow_reuse() &&
            !reuse_partial_packfile_from_bitmap(
+                       bitmap_git,
                        &reuse_packfile,
                        &reuse_packfile_objects,
                        &reuse_packfile_offset)) {
@@ -2925,7 +2987,8 @@ static int get_object_list_from_bitmap(struct rev_info *revs)
                display_progress(progress_state, nr_result);
        }
 
-       traverse_bitmap_commit_list(&add_object_entry_from_bitmap);
+       traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);
+       free_bitmap_index(bitmap_git);
        return 0;
 }
 
@@ -2952,7 +3015,7 @@ static void get_object_list(int ac, const char **av)
        setup_revisions(ac, av, &revs, NULL);
 
        /* make sure shallows are read */
-       is_repository_shallow();
+       is_repository_shallow(the_repository);
 
        while (fgets(line, sizeof(line), stdin) != NULL) {
                int len = strlen(line);
@@ -2970,7 +3033,7 @@ static void get_object_list(int ac, const char **av)
                                struct object_id oid;
                                if (get_oid_hex(line + 10, &oid))
                                        die("not an SHA-1 '%s'", line + 10);
-                               register_shallow(&oid);
+                               register_shallow(the_repository, &oid);
                                use_bitmap_index = 0;
                                continue;
                        }
@@ -2983,6 +3046,9 @@ static void get_object_list(int ac, const char **av)
        if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
                return;
 
+       if (use_delta_islands)
+               load_delta_islands();
+
        if (prepare_revision_walk(&revs))
                die("revision walk setup failed");
        mark_edges_uninteresting(&revs, show_edge);
@@ -3014,6 +3080,32 @@ static void get_object_list(int ac, const char **av)
        oid_array_clear(&recent_objects);
 }
 
+static void add_extra_kept_packs(const struct string_list *names)
+{
+       struct packed_git *p;
+
+       if (!names->nr)
+               return;
+
+       for (p = get_packed_git(the_repository); p; p = p->next) {
+               const char *name = basename(p->pack_name);
+               int i;
+
+               if (!p->pack_local)
+                       continue;
+
+               for (i = 0; i < names->nr; i++)
+                       if (!fspathcmp(name, names->items[i].string))
+                               break;
+
+               if (i < names->nr) {
+                       p->pack_keep_in_core = 1;
+                       ignore_packed_keep_in_core = 1;
+                       continue;
+               }
+       }
+}
+
 static int option_parse_index_version(const struct option *opt,
                                      const char *arg, int unset)
 {
@@ -3053,6 +3145,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        struct argv_array rp = ARGV_ARRAY_INIT;
        int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
        int rev_list_index = 0;
+       struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
        struct option pack_objects_options[] = {
                OPT_SET_INT('q', "quiet", &progress,
                            N_("do not show progress meter"), 0),
@@ -3090,18 +3183,18 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                         N_("do not create an empty pack output")),
                OPT_BOOL(0, "revs", &use_internal_rev_list,
                         N_("read revision arguments from standard input")),
-               { OPTION_SET_INT, 0, "unpacked", &rev_list_unpacked, NULL,
-                 N_("limit the objects to those that are not yet packed"),
-                 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },
-               { OPTION_SET_INT, 0, "all", &rev_list_all, NULL,
-                 N_("include objects reachable from any reference"),
-                 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },
-               { OPTION_SET_INT, 0, "reflog", &rev_list_reflog, NULL,
-                 N_("include objects referred by reflog entries"),
-                 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },
-               { OPTION_SET_INT, 0, "indexed-objects", &rev_list_index, NULL,
-                 N_("include objects referred to by the index"),
-                 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },
+               OPT_SET_INT_F(0, "unpacked", &rev_list_unpacked,
+                             N_("limit the objects to those that are not yet packed"),
+                             1, PARSE_OPT_NONEG),
+               OPT_SET_INT_F(0, "all", &rev_list_all,
+                             N_("include objects reachable from any reference"),
+                             1, PARSE_OPT_NONEG),
+               OPT_SET_INT_F(0, "reflog", &rev_list_reflog,
+                             N_("include objects referred by reflog entries"),
+                             1, PARSE_OPT_NONEG),
+               OPT_SET_INT_F(0, "indexed-objects", &rev_list_index,
+                             N_("include objects referred to by the index"),
+                             1, PARSE_OPT_NONEG),
                OPT_BOOL(0, "stdout", &pack_to_stdout,
                         N_("output pack to stdout")),
                OPT_BOOL(0, "include-tag", &include_tag,
@@ -3117,8 +3210,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                         N_("create thin packs")),
                OPT_BOOL(0, "shallow", &shallow,
                         N_("create packs suitable for shallow fetches")),
-               OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep,
+               OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep_on_disk,
                         N_("ignore packs that have companion .keep file")),
+               OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
+                               N_("ignore this pack")),
                OPT_INTEGER(0, "compression", &pack_compression_level,
                            N_("pack compression level")),
                OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents,
@@ -3133,6 +3228,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                  option_parse_missing_action },
                OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
                         N_("do not pack objects in promisor packfiles")),
+               OPT_BOOL(0, "delta-islands", &use_delta_islands,
+                        N_("respect islands during delta compression")),
                OPT_END(),
        };
 
@@ -3197,6 +3294,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                fetch_if_missing = 0;
                argv_array_push(&rp, "--exclude-promisor-objects");
        }
+       if (unpack_unreachable || keep_unreachable || pack_loose_unreachable)
+               use_internal_rev_list = 1;
 
        if (!reuse_object)
                reuse_delta = 0;
@@ -3251,28 +3350,32 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
                use_bitmap_index = use_bitmap_index_default;
 
        /* "hard" reasons not to use bitmaps; these just won't work at all */
-       if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow())
+       if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow(the_repository))
                use_bitmap_index = 0;
 
        if (pack_to_stdout || !rev_list_all)
                write_bitmap_index = 0;
 
+       if (use_delta_islands)
+               argv_array_push(&rp, "--topo-order");
+
        if (progress && all_progress_implied)
                progress = 2;
 
-       if (ignore_packed_keep) {
+       add_extra_kept_packs(&keep_pack_list);
+       if (ignore_packed_keep_on_disk) {
                struct packed_git *p;
                for (p = get_packed_git(the_repository); p; p = p->next)
                        if (p->pack_local && p->pack_keep)
                                break;
                if (!p) /* no keep-able packs found */
-                       ignore_packed_keep = 0;
+                       ignore_packed_keep_on_disk = 0;
        }
        if (local) {
                /*
-                * unlike ignore_packed_keep above, we do not want to
-                * unset "local" based on looking at packs, as it
-                * also covers non-local objects
+                * unlike ignore_packed_keep_on_disk above, we do not
+                * want to unset "local" based on looking at packs, as
+                * it also covers non-local objects
                 */
                struct packed_git *p;
                for (p = get_packed_git(the_repository); p; p = p->next) {
@@ -3286,7 +3389,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
        prepare_packing_data(&to_pack);
 
        if (progress)
-               progress_state = start_progress(_("Counting objects"), 0);
+               progress_state = start_progress(_("Enumerating objects"), 0);
        if (!use_internal_rev_list)
                read_object_list_from_stdin();
        else {