Andrew's git - gitweb.git/blob

   1#include "builtin.h"
   2#include "cache.h"
   3#include "config.h"
   4#include "dir.h"
   5#include "parse-options.h"
   6#include "run-command.h"
   7#include "sigchain.h"
   8#include "strbuf.h"
   9#include "string-list.h"
  10#include "argv-array.h"
  11#include "midx.h"
  12#include "packfile.h"
  13#include "object-store.h"
  14
  15static int delta_base_offset = 1;
  16static int pack_kept_objects = -1;
  17static int write_bitmaps = -1;
  18static int use_delta_islands;
  19static char *packdir, *packtmp;
  20
  21static const char *const git_repack_usage[] = {
  22        N_("git repack [<options>]"),
  23        NULL
  24};
  25
  26static const char incremental_bitmap_conflict_error[] = N_(
  27"Incremental repacks are incompatible with bitmap indexes.  Use\n"
  28"--no-write-bitmap-index or disable the pack.writebitmaps configuration."
  29);
  30
  31
  32static int repack_config(const char *var, const char *value, void *cb)
  33{
  34        if (!strcmp(var, "repack.usedeltabaseoffset")) {
  35                delta_base_offset = git_config_bool(var, value);
  36                return 0;
  37        }
  38        if (!strcmp(var, "repack.packkeptobjects")) {
  39                pack_kept_objects = git_config_bool(var, value);
  40                return 0;
  41        }
  42        if (!strcmp(var, "repack.writebitmaps") ||
  43            !strcmp(var, "pack.writebitmaps")) {
  44                write_bitmaps = git_config_bool(var, value);
  45                return 0;
  46        }
  47        if (!strcmp(var, "repack.usedeltaislands")) {
  48                use_delta_islands = git_config_bool(var, value);
  49                return 0;
  50        }
  51        return git_default_config(var, value, cb);
  52}
  53
  54/*
  55 * Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files.
  56 */
  57static void remove_temporary_files(void)
  58{
  59        struct strbuf buf = STRBUF_INIT;
  60        size_t dirlen, prefixlen;
  61        DIR *dir;
  62        struct dirent *e;
  63
  64        dir = opendir(packdir);
  65        if (!dir)
  66                return;
  67
  68        /* Point at the slash at the end of ".../objects/pack/" */
  69        dirlen = strlen(packdir) + 1;
  70        strbuf_addstr(&buf, packtmp);
  71        /* Hold the length of  ".tmp-%d-pack-" */
  72        prefixlen = buf.len - dirlen;
  73
  74        while ((e = readdir(dir))) {
  75                if (strncmp(e->d_name, buf.buf + dirlen, prefixlen))
  76                        continue;
  77                strbuf_setlen(&buf, dirlen);
  78                strbuf_addstr(&buf, e->d_name);
  79                unlink(buf.buf);
  80        }
  81        closedir(dir);
  82        strbuf_release(&buf);
  83}
  84
  85static void remove_pack_on_signal(int signo)
  86{
  87        remove_temporary_files();
  88        sigchain_pop(signo);
  89        raise(signo);
  90}
  91
  92static int has_pack_keep_file(void)
  93{
  94        struct packed_git *p;
  95
  96        for (p = get_all_packs(the_repository); p; p = p->next) {
  97                if (p->pack_keep)
  98                        return 1;
  99        }
 100        return 0;
 101}
 102
 103/*
 104 * Adds all packs hex strings to the fname list, which do not
 105 * have a corresponding .keep file. These packs are not to
 106 * be kept if we are going to pack everything into one file.
 107 */
 108static void get_non_kept_pack_filenames(struct string_list *fname_list,
 109                                        const struct string_list *extra_keep)
 110{
 111        DIR *dir;
 112        struct dirent *e;
 113        char *fname;
 114
 115        if (!(dir = opendir(packdir)))
 116                return;
 117
 118        while ((e = readdir(dir)) != NULL) {
 119                size_t len;
 120                int i;
 121
 122                for (i = 0; i < extra_keep->nr; i++)
 123                        if (!fspathcmp(e->d_name, extra_keep->items[i].string))
 124                                break;
 125                if (extra_keep->nr > 0 && i < extra_keep->nr)
 126                        continue;
 127
 128                if (!strip_suffix(e->d_name, ".pack", &len))
 129                        continue;
 130
 131                fname = xmemdupz(e->d_name, len);
 132
 133                if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
 134                        string_list_append_nodup(fname_list, fname);
 135                else
 136                        free(fname);
 137        }
 138        closedir(dir);
 139}
 140
 141static void remove_redundant_pack(const char *dir_name, const char *base_name)
 142{
 143        const char *exts[] = {".pack", ".idx", ".keep", ".bitmap", ".promisor"};
 144        int i;
 145        struct strbuf buf = STRBUF_INIT;
 146        size_t plen;
 147
 148        strbuf_addf(&buf, "%s/%s", dir_name, base_name);
 149        plen = buf.len;
 150
 151        for (i = 0; i < ARRAY_SIZE(exts); i++) {
 152                strbuf_setlen(&buf, plen);
 153                strbuf_addstr(&buf, exts[i]);
 154                unlink(buf.buf);
 155        }
 156        strbuf_release(&buf);
 157}
 158
 159struct pack_objects_args {
 160        const char *window;
 161        const char *window_memory;
 162        const char *depth;
 163        const char *threads;
 164        const char *max_pack_size;
 165        int no_reuse_delta;
 166        int no_reuse_object;
 167        int quiet;
 168        int local;
 169};
 170
 171static void prepare_pack_objects(struct child_process *cmd,
 172                                 const struct pack_objects_args *args)
 173{
 174        argv_array_push(&cmd->args, "pack-objects");
 175        if (args->window)
 176                argv_array_pushf(&cmd->args, "--window=%s", args->window);
 177        if (args->window_memory)
 178                argv_array_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
 179        if (args->depth)
 180                argv_array_pushf(&cmd->args, "--depth=%s", args->depth);
 181        if (args->threads)
 182                argv_array_pushf(&cmd->args, "--threads=%s", args->threads);
 183        if (args->max_pack_size)
 184                argv_array_pushf(&cmd->args, "--max-pack-size=%s", args->max_pack_size);
 185        if (args->no_reuse_delta)
 186                argv_array_pushf(&cmd->args, "--no-reuse-delta");
 187        if (args->no_reuse_object)
 188                argv_array_pushf(&cmd->args, "--no-reuse-object");
 189        if (args->local)
 190                argv_array_push(&cmd->args,  "--local");
 191        if (args->quiet)
 192                argv_array_push(&cmd->args,  "--quiet");
 193        if (delta_base_offset)
 194                argv_array_push(&cmd->args,  "--delta-base-offset");
 195        argv_array_push(&cmd->args, packtmp);
 196        cmd->git_cmd = 1;
 197        cmd->out = -1;
 198}
 199
 200/*
 201 * Write oid to the given struct child_process's stdin, starting it first if
 202 * necessary.
 203 */
 204static int write_oid(const struct object_id *oid, struct packed_git *pack,
 205                     uint32_t pos, void *data)
 206{
 207        struct child_process *cmd = data;
 208
 209        if (cmd->in == -1) {
 210                if (start_command(cmd))
 211                        die(_("could not start pack-objects to repack promisor objects"));
 212        }
 213
 214        xwrite(cmd->in, oid_to_hex(oid), GIT_SHA1_HEXSZ);
 215        xwrite(cmd->in, "\n", 1);
 216        return 0;
 217}
 218
 219static void repack_promisor_objects(const struct pack_objects_args *args,
 220                                    struct string_list *names)
 221{
 222        struct child_process cmd = CHILD_PROCESS_INIT;
 223        FILE *out;
 224        struct strbuf line = STRBUF_INIT;
 225
 226        prepare_pack_objects(&cmd, args);
 227        cmd.in = -1;
 228
 229        /*
 230         * NEEDSWORK: Giving pack-objects only the OIDs without any ordering
 231         * hints may result in suboptimal deltas in the resulting pack. See if
 232         * the OIDs can be sent with fake paths such that pack-objects can use a
 233         * {type -> existing pack order} ordering when computing deltas instead
 234         * of a {type -> size} ordering, which may produce better deltas.
 235         */
 236        for_each_packed_object(write_oid, &cmd,
 237                               FOR_EACH_OBJECT_PROMISOR_ONLY);
 238
 239        if (cmd.in == -1)
 240                /* No packed objects; cmd was never started */
 241                return;
 242
 243        close(cmd.in);
 244
 245        out = xfdopen(cmd.out, "r");
 246        while (strbuf_getline_lf(&line, out) != EOF) {
 247                char *promisor_name;
 248                int fd;
 249                if (line.len != the_hash_algo->hexsz)
 250                        die(_("repack: Expecting full hex object ID lines only from pack-objects."));
 251                string_list_append(names, line.buf);
 252
 253                /*
 254                 * pack-objects creates the .pack and .idx files, but not the
 255                 * .promisor file. Create the .promisor file, which is empty.
 256                 */
 257                promisor_name = mkpathdup("%s-%s.promisor", packtmp,
 258                                          line.buf);
 259                fd = open(promisor_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
 260                if (fd < 0)
 261                        die_errno(_("unable to create '%s'"), promisor_name);
 262                close(fd);
 263                free(promisor_name);
 264        }
 265        fclose(out);
 266        if (finish_command(&cmd))
 267                die(_("could not finish pack-objects to repack promisor objects"));
 268}
 269
 270#define ALL_INTO_ONE 1
 271#define LOOSEN_UNREACHABLE 2
 272
 273int cmd_repack(int argc, const char **argv, const char *prefix)
 274{
 275        struct {
 276                const char *name;
 277                unsigned optional:1;
 278        } exts[] = {
 279                {".pack"},
 280                {".idx"},
 281                {".bitmap", 1},
 282                {".promisor", 1},
 283        };
 284        struct child_process cmd = CHILD_PROCESS_INIT;
 285        struct string_list_item *item;
 286        struct string_list names = STRING_LIST_INIT_DUP;
 287        struct string_list rollback = STRING_LIST_INIT_NODUP;
 288        struct string_list existing_packs = STRING_LIST_INIT_DUP;
 289        struct strbuf line = STRBUF_INIT;
 290        int i, ext, ret, failed;
 291        FILE *out;
 292
 293        /* variables to be filled by option parsing */
 294        int pack_everything = 0;
 295        int delete_redundant = 0;
 296        const char *unpack_unreachable = NULL;
 297        int keep_unreachable = 0;
 298        struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
 299        int no_update_server_info = 0;
 300        int midx_cleared = 0;
 301        struct pack_objects_args po_args = {NULL};
 302
 303        struct option builtin_repack_options[] = {
 304                OPT_BIT('a', NULL, &pack_everything,
 305                                N_("pack everything in a single pack"), ALL_INTO_ONE),
 306                OPT_BIT('A', NULL, &pack_everything,
 307                                N_("same as -a, and turn unreachable objects loose"),
 308                                   LOOSEN_UNREACHABLE | ALL_INTO_ONE),
 309                OPT_BOOL('d', NULL, &delete_redundant,
 310                                N_("remove redundant packs, and run git-prune-packed")),
 311                OPT_BOOL('f', NULL, &po_args.no_reuse_delta,
 312                                N_("pass --no-reuse-delta to git-pack-objects")),
 313                OPT_BOOL('F', NULL, &po_args.no_reuse_object,
 314                                N_("pass --no-reuse-object to git-pack-objects")),
 315                OPT_BOOL('n', NULL, &no_update_server_info,
 316                                N_("do not run git-update-server-info")),
 317                OPT__QUIET(&po_args.quiet, N_("be quiet")),
 318                OPT_BOOL('l', "local", &po_args.local,
 319                                N_("pass --local to git-pack-objects")),
 320                OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
 321                                N_("write bitmap index")),
 322                OPT_BOOL('i', "delta-islands", &use_delta_islands,
 323                                N_("pass --delta-islands to git-pack-objects")),
 324                OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
 325                                N_("with -A, do not loosen objects older than this")),
 326                OPT_BOOL('k', "keep-unreachable", &keep_unreachable,
 327                                N_("with -a, repack unreachable objects")),
 328                OPT_STRING(0, "window", &po_args.window, N_("n"),
 329                                N_("size of the window used for delta compression")),
 330                OPT_STRING(0, "window-memory", &po_args.window_memory, N_("bytes"),
 331                                N_("same as the above, but limit memory size instead of entries count")),
 332                OPT_STRING(0, "depth", &po_args.depth, N_("n"),
 333                                N_("limits the maximum delta depth")),
 334                OPT_STRING(0, "threads", &po_args.threads, N_("n"),
 335                                N_("limits the maximum number of threads")),
 336                OPT_STRING(0, "max-pack-size", &po_args.max_pack_size, N_("bytes"),
 337                                N_("maximum size of each packfile")),
 338                OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
 339                                N_("repack objects in packs marked with .keep")),
 340                OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
 341                                N_("do not repack this pack")),
 342                OPT_END()
 343        };
 344
 345        git_config(repack_config, NULL);
 346
 347        argc = parse_options(argc, argv, prefix, builtin_repack_options,
 348                                git_repack_usage, 0);
 349
 350        if (delete_redundant && repository_format_precious_objects)
 351                die(_("cannot delete packs in a precious-objects repo"));
 352
 353        if (keep_unreachable &&
 354            (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)))
 355                die(_("--keep-unreachable and -A are incompatible"));
 356
 357        if (write_bitmaps < 0) {
 358                write_bitmaps = (pack_everything & ALL_INTO_ONE) &&
 359                                 is_bare_repository() &&
 360                                 keep_pack_list.nr == 0 &&
 361                                 !has_pack_keep_file();
 362        }
 363        if (pack_kept_objects < 0)
 364                pack_kept_objects = write_bitmaps;
 365
 366        if (write_bitmaps && !(pack_everything & ALL_INTO_ONE))
 367                die(_(incremental_bitmap_conflict_error));
 368
 369        packdir = mkpathdup("%s/pack", get_object_directory());
 370        packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
 371
 372        sigchain_push_common(remove_pack_on_signal);
 373
 374        prepare_pack_objects(&cmd, &po_args);
 375
 376        argv_array_push(&cmd.args, "--keep-true-parents");
 377        if (!pack_kept_objects)
 378                argv_array_push(&cmd.args, "--honor-pack-keep");
 379        for (i = 0; i < keep_pack_list.nr; i++)
 380                argv_array_pushf(&cmd.args, "--keep-pack=%s",
 381                                 keep_pack_list.items[i].string);
 382        argv_array_push(&cmd.args, "--non-empty");
 383        argv_array_push(&cmd.args, "--all");
 384        argv_array_push(&cmd.args, "--reflog");
 385        argv_array_push(&cmd.args, "--indexed-objects");
 386        if (repository_format_partial_clone)
 387                argv_array_push(&cmd.args, "--exclude-promisor-objects");
 388        if (write_bitmaps)
 389                argv_array_push(&cmd.args, "--write-bitmap-index");
 390        if (use_delta_islands)
 391                argv_array_push(&cmd.args, "--delta-islands");
 392
 393        if (pack_everything & ALL_INTO_ONE) {
 394                get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
 395
 396                repack_promisor_objects(&po_args, &names);
 397
 398                if (existing_packs.nr && delete_redundant) {
 399                        if (unpack_unreachable) {
 400                                argv_array_pushf(&cmd.args,
 401                                                "--unpack-unreachable=%s",
 402                                                unpack_unreachable);
 403                                argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
 404                        } else if (pack_everything & LOOSEN_UNREACHABLE) {
 405                                argv_array_push(&cmd.args,
 406                                                "--unpack-unreachable");
 407                        } else if (keep_unreachable) {
 408                                argv_array_push(&cmd.args, "--keep-unreachable");
 409                                argv_array_push(&cmd.args, "--pack-loose-unreachable");
 410                        } else {
 411                                argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
 412                        }
 413                }
 414        } else {
 415                argv_array_push(&cmd.args, "--unpacked");
 416                argv_array_push(&cmd.args, "--incremental");
 417        }
 418
 419        cmd.no_stdin = 1;
 420
 421        ret = start_command(&cmd);
 422        if (ret)
 423                return ret;
 424
 425        out = xfdopen(cmd.out, "r");
 426        while (strbuf_getline_lf(&line, out) != EOF) {
 427                if (line.len != the_hash_algo->hexsz)
 428                        die(_("repack: Expecting full hex object ID lines only from pack-objects."));
 429                string_list_append(&names, line.buf);
 430        }
 431        fclose(out);
 432        ret = finish_command(&cmd);
 433        if (ret)
 434                return ret;
 435
 436        if (!names.nr && !po_args.quiet)
 437                printf_ln(_("Nothing new to pack."));
 438
 439        close_all_packs(the_repository->objects);
 440
 441        /*
 442         * Ok we have prepared all new packfiles.
 443         * First see if there are packs of the same name and if so
 444         * if we can move them out of the way (this can happen if we
 445         * repacked immediately after packing fully.
 446         */
 447        failed = 0;
 448        for_each_string_list_item(item, &names) {
 449                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 450                        char *fname, *fname_old;
 451
 452                        if (!midx_cleared) {
 453                                clear_midx_file(the_repository);
 454                                midx_cleared = 1;
 455                        }
 456
 457                        fname = mkpathdup("%s/pack-%s%s", packdir,
 458                                                item->string, exts[ext].name);
 459                        if (!file_exists(fname)) {
 460                                free(fname);
 461                                continue;
 462                        }
 463
 464                        fname_old = mkpathdup("%s/old-%s%s", packdir,
 465                                                item->string, exts[ext].name);
 466                        if (file_exists(fname_old))
 467                                if (unlink(fname_old))
 468                                        failed = 1;
 469
 470                        if (!failed && rename(fname, fname_old)) {
 471                                free(fname);
 472                                free(fname_old);
 473                                failed = 1;
 474                                break;
 475                        } else {
 476                                string_list_append(&rollback, fname);
 477                                free(fname_old);
 478                        }
 479                }
 480                if (failed)
 481                        break;
 482        }
 483        if (failed) {
 484                struct string_list rollback_failure = STRING_LIST_INIT_DUP;
 485                for_each_string_list_item(item, &rollback) {
 486                        char *fname, *fname_old;
 487                        fname = mkpathdup("%s/%s", packdir, item->string);
 488                        fname_old = mkpathdup("%s/old-%s", packdir, item->string);
 489                        if (rename(fname_old, fname))
 490                                string_list_append(&rollback_failure, fname);
 491                        free(fname);
 492                        free(fname_old);
 493                }
 494
 495                if (rollback_failure.nr) {
 496                        int i;
 497                        fprintf(stderr,
 498                                _("WARNING: Some packs in use have been renamed by\n"
 499                                  "WARNING: prefixing old- to their name, in order to\n"
 500                                  "WARNING: replace them with the new version of the\n"
 501                                  "WARNING: file.  But the operation failed, and the\n"
 502                                  "WARNING: attempt to rename them back to their\n"
 503                                  "WARNING: original names also failed.\n"
 504                                  "WARNING: Please rename them in %s manually:\n"), packdir);
 505                        for (i = 0; i < rollback_failure.nr; i++)
 506                                fprintf(stderr, "WARNING:   old-%s -> %s\n",
 507                                        rollback_failure.items[i].string,
 508                                        rollback_failure.items[i].string);
 509                }
 510                exit(1);
 511        }
 512
 513        /* Now the ones with the same name are out of the way... */
 514        for_each_string_list_item(item, &names) {
 515                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 516                        char *fname, *fname_old;
 517                        struct stat statbuffer;
 518                        int exists = 0;
 519                        fname = mkpathdup("%s/pack-%s%s",
 520                                        packdir, item->string, exts[ext].name);
 521                        fname_old = mkpathdup("%s-%s%s",
 522                                        packtmp, item->string, exts[ext].name);
 523                        if (!stat(fname_old, &statbuffer)) {
 524                                statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
 525                                chmod(fname_old, statbuffer.st_mode);
 526                                exists = 1;
 527                        }
 528                        if (exists || !exts[ext].optional) {
 529                                if (rename(fname_old, fname))
 530                                        die_errno(_("renaming '%s' failed"), fname_old);
 531                        }
 532                        free(fname);
 533                        free(fname_old);
 534                }
 535        }
 536
 537        /* Remove the "old-" files */
 538        for_each_string_list_item(item, &names) {
 539                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 540                        char *fname;
 541                        fname = mkpathdup("%s/old-%s%s",
 542                                          packdir,
 543                                          item->string,
 544                                          exts[ext].name);
 545                        if (remove_path(fname))
 546                                warning(_("failed to remove '%s'"), fname);
 547                        free(fname);
 548                }
 549        }
 550
 551        /* End of pack replacement. */
 552
 553        reprepare_packed_git(the_repository);
 554
 555        if (delete_redundant) {
 556                const int hexsz = the_hash_algo->hexsz;
 557                int opts = 0;
 558                string_list_sort(&names);
 559                for_each_string_list_item(item, &existing_packs) {
 560                        char *sha1;
 561                        size_t len = strlen(item->string);
 562                        if (len < hexsz)
 563                                continue;
 564                        sha1 = item->string + len - hexsz;
 565                        if (!string_list_has_string(&names, sha1))
 566                                remove_redundant_pack(packdir, item->string);
 567                }
 568                if (!po_args.quiet && isatty(2))
 569                        opts |= PRUNE_PACKED_VERBOSE;
 570                prune_packed_objects(opts);
 571
 572                if (!keep_unreachable &&
 573                    (!(pack_everything & LOOSEN_UNREACHABLE) ||
 574                     unpack_unreachable) &&
 575                    is_repository_shallow(the_repository))
 576                        prune_shallow(PRUNE_QUICK);
 577        }
 578
 579        if (!no_update_server_info)
 580                update_server_info(0);
 581        remove_temporary_files();
 582
 583        if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
 584                write_midx_file(get_object_directory());
 585
 586        string_list_clear(&names, 0);
 587        string_list_clear(&rollback, 0);
 588        string_list_clear(&existing_packs, 0);
 589        strbuf_release(&line);
 590
 591        return 0;
 592}