builtin-pack-objects.con commit deprecate the new loose object header format (726f852)
   1#include "builtin.h"
   2#include "cache.h"
   3#include "object.h"
   4#include "blob.h"
   5#include "commit.h"
   6#include "tag.h"
   7#include "tree.h"
   8#include "delta.h"
   9#include "pack.h"
  10#include "csum-file.h"
  11#include "tree-walk.h"
  12#include "diff.h"
  13#include "revision.h"
  14#include "list-objects.h"
  15#include "progress.h"
  16
  17static const char pack_usage[] = "\
  18git-pack-objects [{ -q | --progress | --all-progress }] \n\
  19        [--local] [--incremental] [--window=N] [--depth=N] \n\
  20        [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
  21        [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
  22        [--stdout | base-name] [<ref-list | <object-list]";
  23
  24struct object_entry {
  25        unsigned char sha1[20];
  26        uint32_t crc32;         /* crc of raw pack data for this object */
  27        off_t offset;           /* offset into the final pack file */
  28        unsigned long size;     /* uncompressed size */
  29        unsigned int hash;      /* name hint hash */
  30        unsigned int depth;     /* delta depth */
  31        struct packed_git *in_pack;     /* already in pack */
  32        off_t in_pack_offset;
  33        struct object_entry *delta;     /* delta base object */
  34        struct object_entry *delta_child; /* deltified objects who bases me */
  35        struct object_entry *delta_sibling; /* other deltified objects who
  36                                             * uses the same base as me
  37                                             */
  38        unsigned long delta_size;       /* delta data size (uncompressed) */
  39        enum object_type type;
  40        enum object_type in_pack_type;  /* could be delta */
  41        unsigned char in_pack_header_size;
  42        unsigned char preferred_base; /* we do not pack this, but is available
  43                                       * to be used as the base objectto delta
  44                                       * objects against.
  45                                       */
  46};
  47
  48/*
  49 * Objects we are going to pack are collected in objects array (dynamically
  50 * expanded).  nr_objects & nr_alloc controls this array.  They are stored
  51 * in the order we see -- typically rev-list --objects order that gives us
  52 * nice "minimum seek" order.
  53 */
  54static struct object_entry *objects;
  55static uint32_t nr_objects, nr_alloc, nr_result;
  56
  57static int non_empty;
  58static int no_reuse_delta, no_reuse_object;
  59static int local;
  60static int incremental;
  61static int allow_ofs_delta;
  62static const char *pack_tmp_name, *idx_tmp_name;
  63static char tmpname[PATH_MAX];
  64static unsigned char pack_file_sha1[20];
  65static int progress = 1;
  66static int window = 10;
  67static int depth = 50;
  68static int pack_to_stdout;
  69static int num_preferred_base;
  70static struct progress progress_state;
  71
  72/*
  73 * The object names in objects array are hashed with this hashtable,
  74 * to help looking up the entry by object name.
  75 * This hashtable is built after all the objects are seen.
  76 */
  77static int *object_ix;
  78static int object_ix_hashsz;
  79
  80/*
  81 * Pack index for existing packs give us easy access to the offsets into
  82 * corresponding pack file where each object's data starts, but the entries
  83 * do not store the size of the compressed representation (uncompressed
  84 * size is easily available by examining the pack entry header).  It is
  85 * also rather expensive to find the sha1 for an object given its offset.
  86 *
  87 * We build a hashtable of existing packs (pack_revindex), and keep reverse
  88 * index here -- pack index file is sorted by object name mapping to offset;
  89 * this pack_revindex[].revindex array is a list of offset/index_nr pairs
  90 * ordered by offset, so if you know the offset of an object, next offset
  91 * is where its packed representation ends and the index_nr can be used to
  92 * get the object sha1 from the main index.
  93 */
  94struct revindex_entry {
  95        off_t offset;
  96        unsigned int nr;
  97};
  98struct pack_revindex {
  99        struct packed_git *p;
 100        struct revindex_entry *revindex;
 101};
 102static struct  pack_revindex *pack_revindex;
 103static int pack_revindex_hashsz;
 104
 105/*
 106 * stats
 107 */
 108static uint32_t written, written_delta;
 109static uint32_t reused, reused_delta;
 110
 111static int pack_revindex_ix(struct packed_git *p)
 112{
 113        unsigned long ui = (unsigned long)p;
 114        int i;
 115
 116        ui = ui ^ (ui >> 16); /* defeat structure alignment */
 117        i = (int)(ui % pack_revindex_hashsz);
 118        while (pack_revindex[i].p) {
 119                if (pack_revindex[i].p == p)
 120                        return i;
 121                if (++i == pack_revindex_hashsz)
 122                        i = 0;
 123        }
 124        return -1 - i;
 125}
 126
 127static void prepare_pack_ix(void)
 128{
 129        int num;
 130        struct packed_git *p;
 131        for (num = 0, p = packed_git; p; p = p->next)
 132                num++;
 133        if (!num)
 134                return;
 135        pack_revindex_hashsz = num * 11;
 136        pack_revindex = xcalloc(sizeof(*pack_revindex), pack_revindex_hashsz);
 137        for (p = packed_git; p; p = p->next) {
 138                num = pack_revindex_ix(p);
 139                num = - 1 - num;
 140                pack_revindex[num].p = p;
 141        }
 142        /* revindex elements are lazily initialized */
 143}
 144
 145static int cmp_offset(const void *a_, const void *b_)
 146{
 147        const struct revindex_entry *a = a_;
 148        const struct revindex_entry *b = b_;
 149        return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0;
 150}
 151
 152/*
 153 * Ordered list of offsets of objects in the pack.
 154 */
 155static void prepare_pack_revindex(struct pack_revindex *rix)
 156{
 157        struct packed_git *p = rix->p;
 158        int num_ent = p->num_objects;
 159        int i;
 160        const char *index = p->index_data;
 161
 162        rix->revindex = xmalloc(sizeof(*rix->revindex) * (num_ent + 1));
 163        index += 4 * 256;
 164
 165        if (p->index_version > 1) {
 166                const uint32_t *off_32 =
 167                        (uint32_t *)(index + 8 + p->num_objects * (20 + 4));
 168                const uint32_t *off_64 = off_32 + p->num_objects;
 169                for (i = 0; i < num_ent; i++) {
 170                        uint32_t off = ntohl(*off_32++);
 171                        if (!(off & 0x80000000)) {
 172                                rix->revindex[i].offset = off;
 173                        } else {
 174                                rix->revindex[i].offset =
 175                                        ((uint64_t)ntohl(*off_64++)) << 32;
 176                                rix->revindex[i].offset |=
 177                                        ntohl(*off_64++);
 178                        }
 179                        rix->revindex[i].nr = i;
 180                }
 181        } else {
 182                for (i = 0; i < num_ent; i++) {
 183                        uint32_t hl = *((uint32_t *)(index + 24 * i));
 184                        rix->revindex[i].offset = ntohl(hl);
 185                        rix->revindex[i].nr = i;
 186                }
 187        }
 188
 189        /* This knows the pack format -- the 20-byte trailer
 190         * follows immediately after the last object data.
 191         */
 192        rix->revindex[num_ent].offset = p->pack_size - 20;
 193        rix->revindex[num_ent].nr = -1;
 194        qsort(rix->revindex, num_ent, sizeof(*rix->revindex), cmp_offset);
 195}
 196
 197static struct revindex_entry * find_packed_object(struct packed_git *p,
 198                                                  off_t ofs)
 199{
 200        int num;
 201        int lo, hi;
 202        struct pack_revindex *rix;
 203        struct revindex_entry *revindex;
 204        num = pack_revindex_ix(p);
 205        if (num < 0)
 206                die("internal error: pack revindex uninitialized");
 207        rix = &pack_revindex[num];
 208        if (!rix->revindex)
 209                prepare_pack_revindex(rix);
 210        revindex = rix->revindex;
 211        lo = 0;
 212        hi = p->num_objects + 1;
 213        do {
 214                int mi = (lo + hi) / 2;
 215                if (revindex[mi].offset == ofs) {
 216                        return revindex + mi;
 217                }
 218                else if (ofs < revindex[mi].offset)
 219                        hi = mi;
 220                else
 221                        lo = mi + 1;
 222        } while (lo < hi);
 223        die("internal error: pack revindex corrupt");
 224}
 225
 226static const unsigned char *find_packed_object_name(struct packed_git *p,
 227                                                    off_t ofs)
 228{
 229        struct revindex_entry *entry = find_packed_object(p, ofs);
 230        return nth_packed_object_sha1(p, entry->nr);
 231}
 232
 233static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
 234{
 235        unsigned long othersize, delta_size;
 236        enum object_type type;
 237        void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
 238        void *delta_buf;
 239
 240        if (!otherbuf)
 241                die("unable to read %s", sha1_to_hex(entry->delta->sha1));
 242        delta_buf = diff_delta(otherbuf, othersize,
 243                               buf, size, &delta_size, 0);
 244        if (!delta_buf || delta_size != entry->delta_size)
 245                die("delta size changed");
 246        free(buf);
 247        free(otherbuf);
 248        return delta_buf;
 249}
 250
 251/*
 252 * The per-object header is a pretty dense thing, which is
 253 *  - first byte: low four bits are "size", then three bits of "type",
 254 *    and the high bit is "size continues".
 255 *  - each byte afterwards: low seven bits are size continuation,
 256 *    with the high bit being "size continues"
 257 */
 258static int encode_header(enum object_type type, unsigned long size, unsigned char *hdr)
 259{
 260        int n = 1;
 261        unsigned char c;
 262
 263        if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
 264                die("bad type %d", type);
 265
 266        c = (type << 4) | (size & 15);
 267        size >>= 4;
 268        while (size) {
 269                *hdr++ = c | 0x80;
 270                c = size & 0x7f;
 271                size >>= 7;
 272                n++;
 273        }
 274        *hdr = c;
 275        return n;
 276}
 277
 278/*
 279 * we are going to reuse the existing object data as is.  make
 280 * sure it is not corrupt.
 281 */
 282static int check_pack_inflate(struct packed_git *p,
 283                struct pack_window **w_curs,
 284                off_t offset,
 285                off_t len,
 286                unsigned long expect)
 287{
 288        z_stream stream;
 289        unsigned char fakebuf[4096], *in;
 290        int st;
 291
 292        memset(&stream, 0, sizeof(stream));
 293        inflateInit(&stream);
 294        do {
 295                in = use_pack(p, w_curs, offset, &stream.avail_in);
 296                stream.next_in = in;
 297                stream.next_out = fakebuf;
 298                stream.avail_out = sizeof(fakebuf);
 299                st = inflate(&stream, Z_FINISH);
 300                offset += stream.next_in - in;
 301        } while (st == Z_OK || st == Z_BUF_ERROR);
 302        inflateEnd(&stream);
 303        return (st == Z_STREAM_END &&
 304                stream.total_out == expect &&
 305                stream.total_in == len) ? 0 : -1;
 306}
 307
 308static int check_pack_crc(struct packed_git *p, struct pack_window **w_curs,
 309                          off_t offset, off_t len, unsigned int nr)
 310{
 311        const uint32_t *index_crc;
 312        uint32_t data_crc = crc32(0, Z_NULL, 0);
 313
 314        do {
 315                unsigned int avail;
 316                void *data = use_pack(p, w_curs, offset, &avail);
 317                if (avail > len)
 318                        avail = len;
 319                data_crc = crc32(data_crc, data, avail);
 320                offset += avail;
 321                len -= avail;
 322        } while (len);
 323
 324        index_crc = p->index_data;
 325        index_crc += 2 + 256 + p->num_objects * (20/4) + nr;
 326
 327        return data_crc != ntohl(*index_crc);
 328}
 329
 330static void copy_pack_data(struct sha1file *f,
 331                struct packed_git *p,
 332                struct pack_window **w_curs,
 333                off_t offset,
 334                off_t len)
 335{
 336        unsigned char *in;
 337        unsigned int avail;
 338
 339        while (len) {
 340                in = use_pack(p, w_curs, offset, &avail);
 341                if (avail > len)
 342                        avail = (unsigned int)len;
 343                sha1write(f, in, avail);
 344                offset += avail;
 345                len -= avail;
 346        }
 347}
 348
 349static unsigned long write_object(struct sha1file *f,
 350                                  struct object_entry *entry)
 351{
 352        unsigned long size;
 353        enum object_type type;
 354        void *buf;
 355        unsigned char header[10];
 356        unsigned hdrlen;
 357        off_t datalen;
 358        enum object_type obj_type;
 359        int to_reuse = 0;
 360
 361        if (!pack_to_stdout)
 362                crc32_begin(f);
 363
 364        obj_type = entry->type;
 365        if (no_reuse_object)
 366                to_reuse = 0;   /* explicit */
 367        else if (!entry->in_pack)
 368                to_reuse = 0;   /* can't reuse what we don't have */
 369        else if (obj_type == OBJ_REF_DELTA || obj_type == OBJ_OFS_DELTA)
 370                to_reuse = 1;   /* check_object() decided it for us */
 371        else if (obj_type != entry->in_pack_type)
 372                to_reuse = 0;   /* pack has delta which is unusable */
 373        else if (entry->delta)
 374                to_reuse = 0;   /* we want to pack afresh */
 375        else
 376                to_reuse = 1;   /* we have it in-pack undeltified,
 377                                 * and we do not need to deltify it.
 378                                 */
 379
 380        if (!to_reuse) {
 381                buf = read_sha1_file(entry->sha1, &type, &size);
 382                if (!buf)
 383                        die("unable to read %s", sha1_to_hex(entry->sha1));
 384                if (size != entry->size)
 385                        die("object %s size inconsistency (%lu vs %lu)",
 386                            sha1_to_hex(entry->sha1), size, entry->size);
 387                if (entry->delta) {
 388                        buf = delta_against(buf, size, entry);
 389                        size = entry->delta_size;
 390                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
 391                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
 392                }
 393                /*
 394                 * The object header is a byte of 'type' followed by zero or
 395                 * more bytes of length.
 396                 */
 397                hdrlen = encode_header(obj_type, size, header);
 398                sha1write(f, header, hdrlen);
 399
 400                if (obj_type == OBJ_OFS_DELTA) {
 401                        /*
 402                         * Deltas with relative base contain an additional
 403                         * encoding of the relative offset for the delta
 404                         * base from this object's position in the pack.
 405                         */
 406                        off_t ofs = entry->offset - entry->delta->offset;
 407                        unsigned pos = sizeof(header) - 1;
 408                        header[pos] = ofs & 127;
 409                        while (ofs >>= 7)
 410                                header[--pos] = 128 | (--ofs & 127);
 411                        sha1write(f, header + pos, sizeof(header) - pos);
 412                        hdrlen += sizeof(header) - pos;
 413                } else if (obj_type == OBJ_REF_DELTA) {
 414                        /*
 415                         * Deltas with a base reference contain
 416                         * an additional 20 bytes for the base sha1.
 417                         */
 418                        sha1write(f, entry->delta->sha1, 20);
 419                        hdrlen += 20;
 420                }
 421                datalen = sha1write_compressed(f, buf, size);
 422                free(buf);
 423        }
 424        else {
 425                struct packed_git *p = entry->in_pack;
 426                struct pack_window *w_curs = NULL;
 427                struct revindex_entry *revidx;
 428                off_t offset;
 429
 430                if (entry->delta) {
 431                        obj_type = (allow_ofs_delta && entry->delta->offset) ?
 432                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
 433                        reused_delta++;
 434                }
 435                hdrlen = encode_header(obj_type, entry->size, header);
 436                sha1write(f, header, hdrlen);
 437                if (obj_type == OBJ_OFS_DELTA) {
 438                        off_t ofs = entry->offset - entry->delta->offset;
 439                        unsigned pos = sizeof(header) - 1;
 440                        header[pos] = ofs & 127;
 441                        while (ofs >>= 7)
 442                                header[--pos] = 128 | (--ofs & 127);
 443                        sha1write(f, header + pos, sizeof(header) - pos);
 444                        hdrlen += sizeof(header) - pos;
 445                } else if (obj_type == OBJ_REF_DELTA) {
 446                        sha1write(f, entry->delta->sha1, 20);
 447                        hdrlen += 20;
 448                }
 449
 450                offset = entry->in_pack_offset;
 451                revidx = find_packed_object(p, offset);
 452                datalen = revidx[1].offset - offset;
 453                if (!pack_to_stdout && p->index_version > 1 &&
 454                    check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
 455                        die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
 456                offset += entry->in_pack_header_size;
 457                datalen -= entry->in_pack_header_size;
 458                if (!pack_to_stdout && p->index_version == 1 &&
 459                    check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
 460                        die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
 461                copy_pack_data(f, p, &w_curs, offset, datalen);
 462                unuse_pack(&w_curs);
 463                reused++;
 464        }
 465        if (entry->delta)
 466                written_delta++;
 467        written++;
 468        if (!pack_to_stdout)
 469                entry->crc32 = crc32_end(f);
 470        return hdrlen + datalen;
 471}
 472
 473static off_t write_one(struct sha1file *f,
 474                               struct object_entry *e,
 475                               off_t offset)
 476{
 477        unsigned long size;
 478
 479        /* offset is non zero if object is written already. */
 480        if (e->offset || e->preferred_base)
 481                return offset;
 482
 483        /* if we are deltified, write out base object first. */
 484        if (e->delta)
 485                offset = write_one(f, e->delta, offset);
 486
 487        e->offset = offset;
 488        size = write_object(f, e);
 489
 490        /* make sure off_t is sufficiently large not to wrap */
 491        if (offset > offset + size)
 492                die("pack too large for current definition of off_t");
 493        return offset + size;
 494}
 495
 496static int open_object_dir_tmp(const char *path)
 497{
 498    snprintf(tmpname, sizeof(tmpname), "%s/%s", get_object_directory(), path);
 499    return mkstemp(tmpname);
 500}
 501
 502static off_t write_pack_file(void)
 503{
 504        uint32_t i;
 505        struct sha1file *f;
 506        off_t offset, last_obj_offset = 0;
 507        struct pack_header hdr;
 508        int do_progress = progress;
 509
 510        if (pack_to_stdout) {
 511                f = sha1fd(1, "<stdout>");
 512                do_progress >>= 1;
 513        } else {
 514                int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
 515                if (fd < 0)
 516                        die("unable to create %s: %s\n", tmpname, strerror(errno));
 517                pack_tmp_name = xstrdup(tmpname);
 518                f = sha1fd(fd, pack_tmp_name);
 519        }
 520
 521        if (do_progress)
 522                start_progress(&progress_state, "Writing %u objects...", "", nr_result);
 523
 524        hdr.hdr_signature = htonl(PACK_SIGNATURE);
 525        hdr.hdr_version = htonl(PACK_VERSION);
 526        hdr.hdr_entries = htonl(nr_result);
 527        sha1write(f, &hdr, sizeof(hdr));
 528        offset = sizeof(hdr);
 529        if (!nr_result)
 530                goto done;
 531        for (i = 0; i < nr_objects; i++) {
 532                last_obj_offset = offset;
 533                offset = write_one(f, objects + i, offset);
 534                if (do_progress)
 535                        display_progress(&progress_state, written);
 536        }
 537        if (do_progress)
 538                stop_progress(&progress_state);
 539 done:
 540        if (written != nr_result)
 541                die("wrote %u objects while expecting %u", written, nr_result);
 542        sha1close(f, pack_file_sha1, 1);
 543
 544        return last_obj_offset;
 545}
 546
 547static int sha1_sort(const void *_a, const void *_b)
 548{
 549        const struct object_entry *a = *(struct object_entry **)_a;
 550        const struct object_entry *b = *(struct object_entry **)_b;
 551        return hashcmp(a->sha1, b->sha1);
 552}
 553
 554static uint32_t index_default_version = 1;
 555static uint32_t index_off32_limit = 0x7fffffff;
 556
 557static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
 558{
 559        struct sha1file *f;
 560        struct object_entry **sorted_by_sha, **list, **last;
 561        uint32_t array[256];
 562        uint32_t i, index_version;
 563        SHA_CTX ctx;
 564
 565        int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
 566        if (fd < 0)
 567                die("unable to create %s: %s\n", tmpname, strerror(errno));
 568        idx_tmp_name = xstrdup(tmpname);
 569        f = sha1fd(fd, idx_tmp_name);
 570
 571        if (nr_result) {
 572                uint32_t j = 0;
 573                sorted_by_sha =
 574                        xcalloc(nr_result, sizeof(struct object_entry *));
 575                for (i = 0; i < nr_objects; i++)
 576                        if (!objects[i].preferred_base)
 577                                sorted_by_sha[j++] = objects + i;
 578                if (j != nr_result)
 579                        die("listed %u objects while expecting %u", j, nr_result);
 580                qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
 581                list = sorted_by_sha;
 582                last = sorted_by_sha + nr_result;
 583        } else
 584                sorted_by_sha = list = last = NULL;
 585
 586        /* if last object's offset is >= 2^31 we should use index V2 */
 587        index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
 588
 589        /* index versions 2 and above need a header */
 590        if (index_version >= 2) {
 591                struct pack_idx_header hdr;
 592                hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
 593                hdr.idx_version = htonl(index_version);
 594                sha1write(f, &hdr, sizeof(hdr));
 595        }
 596
 597        /*
 598         * Write the first-level table (the list is sorted,
 599         * but we use a 256-entry lookup to be able to avoid
 600         * having to do eight extra binary search iterations).
 601         */
 602        for (i = 0; i < 256; i++) {
 603                struct object_entry **next = list;
 604                while (next < last) {
 605                        struct object_entry *entry = *next;
 606                        if (entry->sha1[0] != i)
 607                                break;
 608                        next++;
 609                }
 610                array[i] = htonl(next - sorted_by_sha);
 611                list = next;
 612        }
 613        sha1write(f, array, 256 * 4);
 614
 615        /* Compute the SHA1 hash of sorted object names. */
 616        SHA1_Init(&ctx);
 617
 618        /* Write the actual SHA1 entries. */
 619        list = sorted_by_sha;
 620        for (i = 0; i < nr_result; i++) {
 621                struct object_entry *entry = *list++;
 622                if (index_version < 2) {
 623                        uint32_t offset = htonl(entry->offset);
 624                        sha1write(f, &offset, 4);
 625                }
 626                sha1write(f, entry->sha1, 20);
 627                SHA1_Update(&ctx, entry->sha1, 20);
 628        }
 629
 630        if (index_version >= 2) {
 631                unsigned int nr_large_offset = 0;
 632
 633                /* write the crc32 table */
 634                list = sorted_by_sha;
 635                for (i = 0; i < nr_objects; i++) {
 636                        struct object_entry *entry = *list++;
 637                        uint32_t crc32_val = htonl(entry->crc32);
 638                        sha1write(f, &crc32_val, 4);
 639                }
 640
 641                /* write the 32-bit offset table */
 642                list = sorted_by_sha;
 643                for (i = 0; i < nr_objects; i++) {
 644                        struct object_entry *entry = *list++;
 645                        uint32_t offset = (entry->offset <= index_off32_limit) ?
 646                                entry->offset : (0x80000000 | nr_large_offset++);
 647                        offset = htonl(offset);
 648                        sha1write(f, &offset, 4);
 649                }
 650
 651                /* write the large offset table */
 652                list = sorted_by_sha;
 653                while (nr_large_offset) {
 654                        struct object_entry *entry = *list++;
 655                        uint64_t offset = entry->offset;
 656                        if (offset > index_off32_limit) {
 657                                uint32_t split[2];
 658                                split[0]        = htonl(offset >> 32);
 659                                split[1] = htonl(offset & 0xffffffff);
 660                                sha1write(f, split, 8);
 661                                nr_large_offset--;
 662                        }
 663                }
 664        }
 665
 666        sha1write(f, pack_file_sha1, 20);
 667        sha1close(f, NULL, 1);
 668        free(sorted_by_sha);
 669        SHA1_Final(sha1, &ctx);
 670}
 671
 672static int locate_object_entry_hash(const unsigned char *sha1)
 673{
 674        int i;
 675        unsigned int ui;
 676        memcpy(&ui, sha1, sizeof(unsigned int));
 677        i = ui % object_ix_hashsz;
 678        while (0 < object_ix[i]) {
 679                if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
 680                        return i;
 681                if (++i == object_ix_hashsz)
 682                        i = 0;
 683        }
 684        return -1 - i;
 685}
 686
 687static struct object_entry *locate_object_entry(const unsigned char *sha1)
 688{
 689        int i;
 690
 691        if (!object_ix_hashsz)
 692                return NULL;
 693
 694        i = locate_object_entry_hash(sha1);
 695        if (0 <= i)
 696                return &objects[object_ix[i]-1];
 697        return NULL;
 698}
 699
 700static void rehash_objects(void)
 701{
 702        uint32_t i;
 703        struct object_entry *oe;
 704
 705        object_ix_hashsz = nr_objects * 3;
 706        if (object_ix_hashsz < 1024)
 707                object_ix_hashsz = 1024;
 708        object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
 709        memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
 710        for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
 711                int ix = locate_object_entry_hash(oe->sha1);
 712                if (0 <= ix)
 713                        continue;
 714                ix = -1 - ix;
 715                object_ix[ix] = i + 1;
 716        }
 717}
 718
 719static unsigned name_hash(const char *name)
 720{
 721        unsigned char c;
 722        unsigned hash = 0;
 723
 724        /*
 725         * This effectively just creates a sortable number from the
 726         * last sixteen non-whitespace characters. Last characters
 727         * count "most", so things that end in ".c" sort together.
 728         */
 729        while ((c = *name++) != 0) {
 730                if (isspace(c))
 731                        continue;
 732                hash = (hash >> 2) + (c << 24);
 733        }
 734        return hash;
 735}
 736
 737static int add_object_entry(const unsigned char *sha1, enum object_type type,
 738                            unsigned hash, int exclude)
 739{
 740        struct object_entry *entry;
 741        struct packed_git *p, *found_pack = NULL;
 742        off_t found_offset = 0;
 743        int ix;
 744
 745        ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
 746        if (ix >= 0) {
 747                if (exclude) {
 748                        entry = objects + object_ix[ix] - 1;
 749                        if (!entry->preferred_base)
 750                                nr_result--;
 751                        entry->preferred_base = 1;
 752                }
 753                return 0;
 754        }
 755
 756        for (p = packed_git; p; p = p->next) {
 757                off_t offset = find_pack_entry_one(sha1, p);
 758                if (offset) {
 759                        if (!found_pack) {
 760                                found_offset = offset;
 761                                found_pack = p;
 762                        }
 763                        if (exclude)
 764                                break;
 765                        if (incremental)
 766                                return 0;
 767                        if (local && !p->pack_local)
 768                                return 0;
 769                }
 770        }
 771
 772        if (nr_objects >= nr_alloc) {
 773                nr_alloc = (nr_alloc  + 1024) * 3 / 2;
 774                objects = xrealloc(objects, nr_alloc * sizeof(*entry));
 775        }
 776
 777        entry = objects + nr_objects++;
 778        memset(entry, 0, sizeof(*entry));
 779        hashcpy(entry->sha1, sha1);
 780        entry->hash = hash;
 781        if (type)
 782                entry->type = type;
 783        if (exclude)
 784                entry->preferred_base = 1;
 785        else
 786                nr_result++;
 787        if (found_pack) {
 788                entry->in_pack = found_pack;
 789                entry->in_pack_offset = found_offset;
 790        }
 791
 792        if (object_ix_hashsz * 3 <= nr_objects * 4)
 793                rehash_objects();
 794        else
 795                object_ix[-1 - ix] = nr_objects;
 796
 797        if (progress)
 798                display_progress(&progress_state, nr_objects);
 799
 800        return 1;
 801}
 802
 803struct pbase_tree_cache {
 804        unsigned char sha1[20];
 805        int ref;
 806        int temporary;
 807        void *tree_data;
 808        unsigned long tree_size;
 809};
 810
 811static struct pbase_tree_cache *(pbase_tree_cache[256]);
 812static int pbase_tree_cache_ix(const unsigned char *sha1)
 813{
 814        return sha1[0] % ARRAY_SIZE(pbase_tree_cache);
 815}
 816static int pbase_tree_cache_ix_incr(int ix)
 817{
 818        return (ix+1) % ARRAY_SIZE(pbase_tree_cache);
 819}
 820
 821static struct pbase_tree {
 822        struct pbase_tree *next;
 823        /* This is a phony "cache" entry; we are not
 824         * going to evict it nor find it through _get()
 825         * mechanism -- this is for the toplevel node that
 826         * would almost always change with any commit.
 827         */
 828        struct pbase_tree_cache pcache;
 829} *pbase_tree;
 830
 831static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1)
 832{
 833        struct pbase_tree_cache *ent, *nent;
 834        void *data;
 835        unsigned long size;
 836        enum object_type type;
 837        int neigh;
 838        int my_ix = pbase_tree_cache_ix(sha1);
 839        int available_ix = -1;
 840
 841        /* pbase-tree-cache acts as a limited hashtable.
 842         * your object will be found at your index or within a few
 843         * slots after that slot if it is cached.
 844         */
 845        for (neigh = 0; neigh < 8; neigh++) {
 846                ent = pbase_tree_cache[my_ix];
 847                if (ent && !hashcmp(ent->sha1, sha1)) {
 848                        ent->ref++;
 849                        return ent;
 850                }
 851                else if (((available_ix < 0) && (!ent || !ent->ref)) ||
 852                         ((0 <= available_ix) &&
 853                          (!ent && pbase_tree_cache[available_ix])))
 854                        available_ix = my_ix;
 855                if (!ent)
 856                        break;
 857                my_ix = pbase_tree_cache_ix_incr(my_ix);
 858        }
 859
 860        /* Did not find one.  Either we got a bogus request or
 861         * we need to read and perhaps cache.
 862         */
 863        data = read_sha1_file(sha1, &type, &size);
 864        if (!data)
 865                return NULL;
 866        if (type != OBJ_TREE) {
 867                free(data);
 868                return NULL;
 869        }
 870
 871        /* We need to either cache or return a throwaway copy */
 872
 873        if (available_ix < 0)
 874                ent = NULL;
 875        else {
 876                ent = pbase_tree_cache[available_ix];
 877                my_ix = available_ix;
 878        }
 879
 880        if (!ent) {
 881                nent = xmalloc(sizeof(*nent));
 882                nent->temporary = (available_ix < 0);
 883        }
 884        else {
 885                /* evict and reuse */
 886                free(ent->tree_data);
 887                nent = ent;
 888        }
 889        hashcpy(nent->sha1, sha1);
 890        nent->tree_data = data;
 891        nent->tree_size = size;
 892        nent->ref = 1;
 893        if (!nent->temporary)
 894                pbase_tree_cache[my_ix] = nent;
 895        return nent;
 896}
 897
 898static void pbase_tree_put(struct pbase_tree_cache *cache)
 899{
 900        if (!cache->temporary) {
 901                cache->ref--;
 902                return;
 903        }
 904        free(cache->tree_data);
 905        free(cache);
 906}
 907
 908static int name_cmp_len(const char *name)
 909{
 910        int i;
 911        for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)
 912                ;
 913        return i;
 914}
 915
 916static void add_pbase_object(struct tree_desc *tree,
 917                             const char *name,
 918                             int cmplen,
 919                             const char *fullname)
 920{
 921        struct name_entry entry;
 922        int cmp;
 923
 924        while (tree_entry(tree,&entry)) {
 925                cmp = tree_entry_len(entry.path, entry.sha1) != cmplen ? 1 :
 926                      memcmp(name, entry.path, cmplen);
 927                if (cmp > 0)
 928                        continue;
 929                if (cmp < 0)
 930                        return;
 931                if (name[cmplen] != '/') {
 932                        unsigned hash = name_hash(fullname);
 933                        add_object_entry(entry.sha1,
 934                                         S_ISDIR(entry.mode) ? OBJ_TREE : OBJ_BLOB,
 935                                         hash, 1);
 936                        return;
 937                }
 938                if (S_ISDIR(entry.mode)) {
 939                        struct tree_desc sub;
 940                        struct pbase_tree_cache *tree;
 941                        const char *down = name+cmplen+1;
 942                        int downlen = name_cmp_len(down);
 943
 944                        tree = pbase_tree_get(entry.sha1);
 945                        if (!tree)
 946                                return;
 947                        init_tree_desc(&sub, tree->tree_data, tree->tree_size);
 948
 949                        add_pbase_object(&sub, down, downlen, fullname);
 950                        pbase_tree_put(tree);
 951                }
 952        }
 953}
 954
 955static unsigned *done_pbase_paths;
 956static int done_pbase_paths_num;
 957static int done_pbase_paths_alloc;
 958static int done_pbase_path_pos(unsigned hash)
 959{
 960        int lo = 0;
 961        int hi = done_pbase_paths_num;
 962        while (lo < hi) {
 963                int mi = (hi + lo) / 2;
 964                if (done_pbase_paths[mi] == hash)
 965                        return mi;
 966                if (done_pbase_paths[mi] < hash)
 967                        hi = mi;
 968                else
 969                        lo = mi + 1;
 970        }
 971        return -lo-1;
 972}
 973
 974static int check_pbase_path(unsigned hash)
 975{
 976        int pos = (!done_pbase_paths) ? -1 : done_pbase_path_pos(hash);
 977        if (0 <= pos)
 978                return 1;
 979        pos = -pos - 1;
 980        if (done_pbase_paths_alloc <= done_pbase_paths_num) {
 981                done_pbase_paths_alloc = alloc_nr(done_pbase_paths_alloc);
 982                done_pbase_paths = xrealloc(done_pbase_paths,
 983                                            done_pbase_paths_alloc *
 984                                            sizeof(unsigned));
 985        }
 986        done_pbase_paths_num++;
 987        if (pos < done_pbase_paths_num)
 988                memmove(done_pbase_paths + pos + 1,
 989                        done_pbase_paths + pos,
 990                        (done_pbase_paths_num - pos - 1) * sizeof(unsigned));
 991        done_pbase_paths[pos] = hash;
 992        return 0;
 993}
 994
 995static void add_preferred_base_object(const char *name, unsigned hash)
 996{
 997        struct pbase_tree *it;
 998        int cmplen;
 999
1000        if (!num_preferred_base || check_pbase_path(hash))
1001                return;
1002
1003        cmplen = name_cmp_len(name);
1004        for (it = pbase_tree; it; it = it->next) {
1005                if (cmplen == 0) {
1006                        add_object_entry(it->pcache.sha1, OBJ_TREE, 0, 1);
1007                }
1008                else {
1009                        struct tree_desc tree;
1010                        init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);
1011                        add_pbase_object(&tree, name, cmplen, name);
1012                }
1013        }
1014}
1015
1016static void add_preferred_base(unsigned char *sha1)
1017{
1018        struct pbase_tree *it;
1019        void *data;
1020        unsigned long size;
1021        unsigned char tree_sha1[20];
1022
1023        if (window <= num_preferred_base++)
1024                return;
1025
1026        data = read_object_with_reference(sha1, tree_type, &size, tree_sha1);
1027        if (!data)
1028                return;
1029
1030        for (it = pbase_tree; it; it = it->next) {
1031                if (!hashcmp(it->pcache.sha1, tree_sha1)) {
1032                        free(data);
1033                        return;
1034                }
1035        }
1036
1037        it = xcalloc(1, sizeof(*it));
1038        it->next = pbase_tree;
1039        pbase_tree = it;
1040
1041        hashcpy(it->pcache.sha1, tree_sha1);
1042        it->pcache.tree_data = data;
1043        it->pcache.tree_size = size;
1044}
1045
1046static void check_object(struct object_entry *entry)
1047{
1048        if (entry->in_pack) {
1049                struct packed_git *p = entry->in_pack;
1050                struct pack_window *w_curs = NULL;
1051                const unsigned char *base_ref = NULL;
1052                struct object_entry *base_entry;
1053                unsigned long used, used_0;
1054                unsigned int avail;
1055                off_t ofs;
1056                unsigned char *buf, c;
1057
1058                buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
1059
1060                /*
1061                 * We want in_pack_type even if we do not reuse delta
1062                 * since non-delta representations could still be reused.
1063                 */
1064                used = unpack_object_header_gently(buf, avail,
1065                                                   &entry->in_pack_type,
1066                                                   &entry->size);
1067
1068                /*
1069                 * Determine if this is a delta and if so whether we can
1070                 * reuse it or not.  Otherwise let's find out as cheaply as
1071                 * possible what the actual type and size for this object is.
1072                 */
1073                switch (entry->in_pack_type) {
1074                default:
1075                        /* Not a delta hence we've already got all we need. */
1076                        entry->type = entry->in_pack_type;
1077                        entry->in_pack_header_size = used;
1078                        unuse_pack(&w_curs);
1079                        return;
1080                case OBJ_REF_DELTA:
1081                        if (!no_reuse_delta && !entry->preferred_base)
1082                                base_ref = use_pack(p, &w_curs,
1083                                                entry->in_pack_offset + used, NULL);
1084                        entry->in_pack_header_size = used + 20;
1085                        break;
1086                case OBJ_OFS_DELTA:
1087                        buf = use_pack(p, &w_curs,
1088                                       entry->in_pack_offset + used, NULL);
1089                        used_0 = 0;
1090                        c = buf[used_0++];
1091                        ofs = c & 127;
1092                        while (c & 128) {
1093                                ofs += 1;
1094                                if (!ofs || MSB(ofs, 7))
1095                                        die("delta base offset overflow in pack for %s",
1096                                            sha1_to_hex(entry->sha1));
1097                                c = buf[used_0++];
1098                                ofs = (ofs << 7) + (c & 127);
1099                        }
1100                        if (ofs >= entry->in_pack_offset)
1101                                die("delta base offset out of bound for %s",
1102                                    sha1_to_hex(entry->sha1));
1103                        ofs = entry->in_pack_offset - ofs;
1104                        if (!no_reuse_delta && !entry->preferred_base)
1105                                base_ref = find_packed_object_name(p, ofs);
1106                        entry->in_pack_header_size = used + used_0;
1107                        break;
1108                }
1109
1110                if (base_ref && (base_entry = locate_object_entry(base_ref))) {
1111                        /*
1112                         * If base_ref was set above that means we wish to
1113                         * reuse delta data, and we even found that base
1114                         * in the list of objects we want to pack. Goodie!
1115                         *
1116                         * Depth value does not matter - find_deltas() will
1117                         * never consider reused delta as the base object to
1118                         * deltify other objects against, in order to avoid
1119                         * circular deltas.
1120                         */
1121                        entry->type = entry->in_pack_type;
1122                        entry->delta = base_entry;
1123                        entry->delta_sibling = base_entry->delta_child;
1124                        base_entry->delta_child = entry;
1125                        unuse_pack(&w_curs);
1126                        return;
1127                }
1128
1129                if (entry->type) {
1130                        /*
1131                         * This must be a delta and we already know what the
1132                         * final object type is.  Let's extract the actual
1133                         * object size from the delta header.
1134                         */
1135                        entry->size = get_size_from_delta(p, &w_curs,
1136                                        entry->in_pack_offset + entry->in_pack_header_size);
1137                        unuse_pack(&w_curs);
1138                        return;
1139                }
1140
1141                /*
1142                 * No choice but to fall back to the recursive delta walk
1143                 * with sha1_object_info() to find about the object type
1144                 * at this point...
1145                 */
1146                unuse_pack(&w_curs);
1147        }
1148
1149        entry->type = sha1_object_info(entry->sha1, &entry->size);
1150        if (entry->type < 0)
1151                die("unable to get type of object %s",
1152                    sha1_to_hex(entry->sha1));
1153}
1154
1155static int pack_offset_sort(const void *_a, const void *_b)
1156{
1157        const struct object_entry *a = *(struct object_entry **)_a;
1158        const struct object_entry *b = *(struct object_entry **)_b;
1159
1160        /* avoid filesystem trashing with loose objects */
1161        if (!a->in_pack && !b->in_pack)
1162                return hashcmp(a->sha1, b->sha1);
1163
1164        if (a->in_pack < b->in_pack)
1165                return -1;
1166        if (a->in_pack > b->in_pack)
1167                return 1;
1168        return a->in_pack_offset < b->in_pack_offset ? -1 :
1169                        (a->in_pack_offset > b->in_pack_offset);
1170}
1171
1172static void get_object_details(void)
1173{
1174        uint32_t i;
1175        struct object_entry **sorted_by_offset;
1176
1177        sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
1178        for (i = 0; i < nr_objects; i++)
1179                sorted_by_offset[i] = objects + i;
1180        qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
1181
1182        prepare_pack_ix();
1183        for (i = 0; i < nr_objects; i++)
1184                check_object(sorted_by_offset[i]);
1185        free(sorted_by_offset);
1186}
1187
1188static int type_size_sort(const void *_a, const void *_b)
1189{
1190        const struct object_entry *a = *(struct object_entry **)_a;
1191        const struct object_entry *b = *(struct object_entry **)_b;
1192
1193        if (a->type < b->type)
1194                return -1;
1195        if (a->type > b->type)
1196                return 1;
1197        if (a->hash < b->hash)
1198                return -1;
1199        if (a->hash > b->hash)
1200                return 1;
1201        if (a->preferred_base < b->preferred_base)
1202                return -1;
1203        if (a->preferred_base > b->preferred_base)
1204                return 1;
1205        if (a->size < b->size)
1206                return -1;
1207        if (a->size > b->size)
1208                return 1;
1209        return a > b ? -1 : (a < b);  /* newest last */
1210}
1211
1212struct unpacked {
1213        struct object_entry *entry;
1214        void *data;
1215        struct delta_index *index;
1216};
1217
1218/*
1219 * We search for deltas _backwards_ in a list sorted by type and
1220 * by size, so that we see progressively smaller and smaller files.
1221 * That's because we prefer deltas to be from the bigger file
1222 * to the smaller - deletes are potentially cheaper, but perhaps
1223 * more importantly, the bigger file is likely the more recent
1224 * one.
1225 */
1226static int try_delta(struct unpacked *trg, struct unpacked *src,
1227                     unsigned max_depth)
1228{
1229        struct object_entry *trg_entry = trg->entry;
1230        struct object_entry *src_entry = src->entry;
1231        unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
1232        enum object_type type;
1233        void *delta_buf;
1234
1235        /* Don't bother doing diffs between different types */
1236        if (trg_entry->type != src_entry->type)
1237                return -1;
1238
1239        /* We do not compute delta to *create* objects we are not
1240         * going to pack.
1241         */
1242        if (trg_entry->preferred_base)
1243                return -1;
1244
1245        /*
1246         * We do not bother to try a delta that we discarded
1247         * on an earlier try, but only when reusing delta data.
1248         */
1249        if (!no_reuse_delta && trg_entry->in_pack &&
1250            trg_entry->in_pack == src_entry->in_pack &&
1251            trg_entry->in_pack_type != OBJ_REF_DELTA &&
1252            trg_entry->in_pack_type != OBJ_OFS_DELTA)
1253                return 0;
1254
1255        /* Let's not bust the allowed depth. */
1256        if (src_entry->depth >= max_depth)
1257                return 0;
1258
1259        /* Now some size filtering heuristics. */
1260        trg_size = trg_entry->size;
1261        max_size = trg_size/2 - 20;
1262        max_size = max_size * (max_depth - src_entry->depth) / max_depth;
1263        if (max_size == 0)
1264                return 0;
1265        if (trg_entry->delta && trg_entry->delta_size <= max_size)
1266                max_size = trg_entry->delta_size-1;
1267        src_size = src_entry->size;
1268        sizediff = src_size < trg_size ? trg_size - src_size : 0;
1269        if (sizediff >= max_size)
1270                return 0;
1271
1272        /* Load data if not already done */
1273        if (!trg->data) {
1274                trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
1275                if (sz != trg_size)
1276                        die("object %s inconsistent object length (%lu vs %lu)",
1277                            sha1_to_hex(trg_entry->sha1), sz, trg_size);
1278        }
1279        if (!src->data) {
1280                src->data = read_sha1_file(src_entry->sha1, &type, &sz);
1281                if (sz != src_size)
1282                        die("object %s inconsistent object length (%lu vs %lu)",
1283                            sha1_to_hex(src_entry->sha1), sz, src_size);
1284        }
1285        if (!src->index) {
1286                src->index = create_delta_index(src->data, src_size);
1287                if (!src->index)
1288                        die("out of memory");
1289        }
1290
1291        delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
1292        if (!delta_buf)
1293                return 0;
1294
1295        trg_entry->delta = src_entry;
1296        trg_entry->delta_size = delta_size;
1297        trg_entry->depth = src_entry->depth + 1;
1298        free(delta_buf);
1299        return 1;
1300}
1301
1302static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
1303{
1304        struct object_entry *child = me->delta_child;
1305        unsigned int m = n;
1306        while (child) {
1307                unsigned int c = check_delta_limit(child, n + 1);
1308                if (m < c)
1309                        m = c;
1310                child = child->delta_sibling;
1311        }
1312        return m;
1313}
1314
1315static void find_deltas(struct object_entry **list, int window, int depth)
1316{
1317        uint32_t i = nr_objects, idx = 0, processed = 0;
1318        unsigned int array_size = window * sizeof(struct unpacked);
1319        struct unpacked *array;
1320        int max_depth;
1321
1322        if (!nr_objects)
1323                return;
1324        array = xmalloc(array_size);
1325        memset(array, 0, array_size);
1326        if (progress)
1327                start_progress(&progress_state, "Deltifying %u objects...", "", nr_result);
1328
1329        do {
1330                struct object_entry *entry = list[--i];
1331                struct unpacked *n = array + idx;
1332                int j;
1333
1334                if (!entry->preferred_base)
1335                        processed++;
1336
1337                if (progress)
1338                        display_progress(&progress_state, processed);
1339
1340                if (entry->delta)
1341                        /* This happens if we decided to reuse existing
1342                         * delta from a pack.  "!no_reuse_delta &&" is implied.
1343                         */
1344                        continue;
1345
1346                if (entry->size < 50)
1347                        continue;
1348                free_delta_index(n->index);
1349                n->index = NULL;
1350                free(n->data);
1351                n->data = NULL;
1352                n->entry = entry;
1353
1354                /*
1355                 * If the current object is at pack edge, take the depth the
1356                 * objects that depend on the current object into account
1357                 * otherwise they would become too deep.
1358                 */
1359                max_depth = depth;
1360                if (entry->delta_child) {
1361                        max_depth -= check_delta_limit(entry, 0);
1362                        if (max_depth <= 0)
1363                                goto next;
1364                }
1365
1366                j = window;
1367                while (--j > 0) {
1368                        uint32_t other_idx = idx + j;
1369                        struct unpacked *m;
1370                        if (other_idx >= window)
1371                                other_idx -= window;
1372                        m = array + other_idx;
1373                        if (!m->entry)
1374                                break;
1375                        if (try_delta(n, m, max_depth) < 0)
1376                                break;
1377                }
1378
1379                /* if we made n a delta, and if n is already at max
1380                 * depth, leaving it in the window is pointless.  we
1381                 * should evict it first.
1382                 */
1383                if (entry->delta && depth <= entry->depth)
1384                        continue;
1385
1386                next:
1387                idx++;
1388                if (idx >= window)
1389                        idx = 0;
1390        } while (i > 0);
1391
1392        if (progress)
1393                stop_progress(&progress_state);
1394
1395        for (i = 0; i < window; ++i) {
1396                free_delta_index(array[i].index);
1397                free(array[i].data);
1398        }
1399        free(array);
1400}
1401
1402static void prepare_pack(int window, int depth)
1403{
1404        struct object_entry **delta_list;
1405        uint32_t i;
1406
1407        get_object_details();
1408
1409        if (!window || !depth)
1410                return;
1411
1412        delta_list = xmalloc(nr_objects * sizeof(*delta_list));
1413        for (i = 0; i < nr_objects; i++)
1414                delta_list[i] = objects + i;
1415        qsort(delta_list, nr_objects, sizeof(*delta_list), type_size_sort);
1416        find_deltas(delta_list, window+1, depth);
1417        free(delta_list);
1418}
1419
1420static int git_pack_config(const char *k, const char *v)
1421{
1422        if(!strcmp(k, "pack.window")) {
1423                window = git_config_int(k, v);
1424                return 0;
1425        }
1426        if(!strcmp(k, "pack.depth")) {
1427                depth = git_config_int(k, v);
1428                return 0;
1429        }
1430        return git_default_config(k, v);
1431}
1432
1433static void read_object_list_from_stdin(void)
1434{
1435        char line[40 + 1 + PATH_MAX + 2];
1436        unsigned char sha1[20];
1437        unsigned hash;
1438
1439        for (;;) {
1440                if (!fgets(line, sizeof(line), stdin)) {
1441                        if (feof(stdin))
1442                                break;
1443                        if (!ferror(stdin))
1444                                die("fgets returned NULL, not EOF, not error!");
1445                        if (errno != EINTR)
1446                                die("fgets: %s", strerror(errno));
1447                        clearerr(stdin);
1448                        continue;
1449                }
1450                if (line[0] == '-') {
1451                        if (get_sha1_hex(line+1, sha1))
1452                                die("expected edge sha1, got garbage:\n %s",
1453                                    line);
1454                        add_preferred_base(sha1);
1455                        continue;
1456                }
1457                if (get_sha1_hex(line, sha1))
1458                        die("expected sha1, got garbage:\n %s", line);
1459
1460                hash = name_hash(line+41);
1461                add_preferred_base_object(line+41, hash);
1462                add_object_entry(sha1, 0, hash, 0);
1463        }
1464}
1465
1466static void show_commit(struct commit *commit)
1467{
1468        add_object_entry(commit->object.sha1, OBJ_COMMIT, 0, 0);
1469}
1470
1471static void show_object(struct object_array_entry *p)
1472{
1473        unsigned hash = name_hash(p->name);
1474        add_preferred_base_object(p->name, hash);
1475        add_object_entry(p->item->sha1, p->item->type, hash, 0);
1476}
1477
1478static void show_edge(struct commit *commit)
1479{
1480        add_preferred_base(commit->object.sha1);
1481}
1482
1483static void get_object_list(int ac, const char **av)
1484{
1485        struct rev_info revs;
1486        char line[1000];
1487        int flags = 0;
1488
1489        init_revisions(&revs, NULL);
1490        save_commit_buffer = 0;
1491        track_object_refs = 0;
1492        setup_revisions(ac, av, &revs, NULL);
1493
1494        while (fgets(line, sizeof(line), stdin) != NULL) {
1495                int len = strlen(line);
1496                if (line[len - 1] == '\n')
1497                        line[--len] = 0;
1498                if (!len)
1499                        break;
1500                if (*line == '-') {
1501                        if (!strcmp(line, "--not")) {
1502                                flags ^= UNINTERESTING;
1503                                continue;
1504                        }
1505                        die("not a rev '%s'", line);
1506                }
1507                if (handle_revision_arg(line, &revs, flags, 1))
1508                        die("bad revision '%s'", line);
1509        }
1510
1511        prepare_revision_walk(&revs);
1512        mark_edges_uninteresting(revs.commits, &revs, show_edge);
1513        traverse_commit_list(&revs, show_commit, show_object);
1514}
1515
1516static int adjust_perm(const char *path, mode_t mode)
1517{
1518        if (chmod(path, mode))
1519                return -1;
1520        return adjust_shared_perm(path);
1521}
1522
1523int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1524{
1525        int use_internal_rev_list = 0;
1526        int thin = 0;
1527        uint32_t i;
1528        off_t last_obj_offset;
1529        const char *base_name = NULL;
1530        const char **rp_av;
1531        int rp_ac_alloc = 64;
1532        int rp_ac;
1533
1534        rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));
1535
1536        rp_av[0] = "pack-objects";
1537        rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
1538        rp_ac = 2;
1539
1540        git_config(git_pack_config);
1541
1542        progress = isatty(2);
1543        for (i = 1; i < argc; i++) {
1544                const char *arg = argv[i];
1545
1546                if (*arg != '-')
1547                        break;
1548
1549                if (!strcmp("--non-empty", arg)) {
1550                        non_empty = 1;
1551                        continue;
1552                }
1553                if (!strcmp("--local", arg)) {
1554                        local = 1;
1555                        continue;
1556                }
1557                if (!strcmp("--incremental", arg)) {
1558                        incremental = 1;
1559                        continue;
1560                }
1561                if (!prefixcmp(arg, "--window=")) {
1562                        char *end;
1563                        window = strtoul(arg+9, &end, 0);
1564                        if (!arg[9] || *end)
1565                                usage(pack_usage);
1566                        continue;
1567                }
1568                if (!prefixcmp(arg, "--depth=")) {
1569                        char *end;
1570                        depth = strtoul(arg+8, &end, 0);
1571                        if (!arg[8] || *end)
1572                                usage(pack_usage);
1573                        continue;
1574                }
1575                if (!strcmp("--progress", arg)) {
1576                        progress = 1;
1577                        continue;
1578                }
1579                if (!strcmp("--all-progress", arg)) {
1580                        progress = 2;
1581                        continue;
1582                }
1583                if (!strcmp("-q", arg)) {
1584                        progress = 0;
1585                        continue;
1586                }
1587                if (!strcmp("--no-reuse-delta", arg)) {
1588                        no_reuse_delta = 1;
1589                        continue;
1590                }
1591                if (!strcmp("--no-reuse-object", arg)) {
1592                        no_reuse_object = no_reuse_delta = 1;
1593                        continue;
1594                }
1595                if (!strcmp("--delta-base-offset", arg)) {
1596                        allow_ofs_delta = 1;
1597                        continue;
1598                }
1599                if (!strcmp("--stdout", arg)) {
1600                        pack_to_stdout = 1;
1601                        continue;
1602                }
1603                if (!strcmp("--revs", arg)) {
1604                        use_internal_rev_list = 1;
1605                        continue;
1606                }
1607                if (!strcmp("--unpacked", arg) ||
1608                    !prefixcmp(arg, "--unpacked=") ||
1609                    !strcmp("--reflog", arg) ||
1610                    !strcmp("--all", arg)) {
1611                        use_internal_rev_list = 1;
1612                        if (rp_ac >= rp_ac_alloc - 1) {
1613                                rp_ac_alloc = alloc_nr(rp_ac_alloc);
1614                                rp_av = xrealloc(rp_av,
1615                                                 rp_ac_alloc * sizeof(*rp_av));
1616                        }
1617                        rp_av[rp_ac++] = arg;
1618                        continue;
1619                }
1620                if (!strcmp("--thin", arg)) {
1621                        use_internal_rev_list = 1;
1622                        thin = 1;
1623                        rp_av[1] = "--objects-edge";
1624                        continue;
1625                }
1626                if (!prefixcmp(arg, "--index-version=")) {
1627                        char *c;
1628                        index_default_version = strtoul(arg + 16, &c, 10);
1629                        if (index_default_version > 2)
1630                                die("bad %s", arg);
1631                        if (*c == ',')
1632                                index_off32_limit = strtoul(c+1, &c, 0);
1633                        if (*c || index_off32_limit & 0x80000000)
1634                                die("bad %s", arg);
1635                        continue;
1636                }
1637                usage(pack_usage);
1638        }
1639
1640        /* Traditionally "pack-objects [options] base extra" failed;
1641         * we would however want to take refs parameter that would
1642         * have been given to upstream rev-list ourselves, which means
1643         * we somehow want to say what the base name is.  So the
1644         * syntax would be:
1645         *
1646         * pack-objects [options] base <refs...>
1647         *
1648         * in other words, we would treat the first non-option as the
1649         * base_name and send everything else to the internal revision
1650         * walker.
1651         */
1652
1653        if (!pack_to_stdout)
1654                base_name = argv[i++];
1655
1656        if (pack_to_stdout != !base_name)
1657                usage(pack_usage);
1658
1659        if (!pack_to_stdout && thin)
1660                die("--thin cannot be used to build an indexable pack.");
1661
1662        prepare_packed_git();
1663
1664        if (progress)
1665                start_progress(&progress_state, "Generating pack...",
1666                               "Counting objects: ", 0);
1667        if (!use_internal_rev_list)
1668                read_object_list_from_stdin();
1669        else {
1670                rp_av[rp_ac] = NULL;
1671                get_object_list(rp_ac, rp_av);
1672        }
1673        if (progress) {
1674                stop_progress(&progress_state);
1675                fprintf(stderr, "Done counting %u objects.\n", nr_objects);
1676        }
1677
1678        if (non_empty && !nr_result)
1679                return 0;
1680        if (progress && (nr_objects != nr_result))
1681                fprintf(stderr, "Result has %u objects.\n", nr_result);
1682        if (nr_result)
1683                prepare_pack(window, depth);
1684        last_obj_offset = write_pack_file();
1685        if (!pack_to_stdout) {
1686                unsigned char object_list_sha1[20];
1687                mode_t mode = umask(0);
1688
1689                umask(mode);
1690                mode = 0444 & ~mode;
1691
1692                write_index_file(last_obj_offset, object_list_sha1);
1693                snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
1694                         base_name, sha1_to_hex(object_list_sha1));
1695                if (adjust_perm(pack_tmp_name, mode))
1696                        die("unable to make temporary pack file readable: %s",
1697                            strerror(errno));
1698                if (rename(pack_tmp_name, tmpname))
1699                        die("unable to rename temporary pack file: %s",
1700                            strerror(errno));
1701                snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
1702                         base_name, sha1_to_hex(object_list_sha1));
1703                if (adjust_perm(idx_tmp_name, mode))
1704                        die("unable to make temporary index file readable: %s",
1705                            strerror(errno));
1706                if (rename(idx_tmp_name, tmpname))
1707                        die("unable to rename temporary index file: %s",
1708                            strerror(errno));
1709                puts(sha1_to_hex(object_list_sha1));
1710        }
1711        if (progress)
1712                fprintf(stderr, "Total %u (delta %u), reused %u (delta %u)\n",
1713                        written, written_delta, reused, reused_delta);
1714        return 0;
1715}