Andrew's git / gitweb.git / http-walker.c
?
http-walker.con commit http*: add http_get_info_packs (b8caac2)
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "walker.h"
   5#include "http.h"
   6
   7#define PREV_BUF_SIZE 4096
   8
   9struct alt_base
  10{
  11        char *base;
  12        int got_indices;
  13        struct packed_git *packs;
  14        struct alt_base *next;
  15};
  16
  17enum object_request_state {
  18        WAITING,
  19        ABORTED,
  20        ACTIVE,
  21        COMPLETE,
  22};
  23
  24struct object_request
  25{
  26        struct walker *walker;
  27        unsigned char sha1[20];
  28        struct alt_base *repo;
  29        char *url;
  30        char filename[PATH_MAX];
  31        char tmpfile[PATH_MAX];
  32        int local;
  33        enum object_request_state state;
  34        CURLcode curl_result;
  35        char errorstr[CURL_ERROR_SIZE];
  36        long http_code;
  37        unsigned char real_sha1[20];
  38        git_SHA_CTX c;
  39        z_stream stream;
  40        int zret;
  41        int rename;
  42        struct active_request_slot *slot;
  43        struct object_request *next;
  44};
  45
  46struct alternates_request {
  47        struct walker *walker;
  48        const char *base;
  49        char *url;
  50        struct strbuf *buffer;
  51        struct active_request_slot *slot;
  52        int http_specific;
  53};
  54
  55struct walker_data {
  56        const char *url;
  57        int got_alternates;
  58        struct alt_base *alt;
  59};
  60
  61static struct object_request *object_queue_head;
  62
  63static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  64                               void *data)
  65{
  66        unsigned char expn[4096];
  67        size_t size = eltsize * nmemb;
  68        int posn = 0;
  69        struct object_request *obj_req = (struct object_request *)data;
  70        do {
  71                ssize_t retval = xwrite(obj_req->local,
  72                                        (char *) ptr + posn, size - posn);
  73                if (retval < 0)
  74                        return posn;
  75                posn += retval;
  76        } while (posn < size);
  77
  78        obj_req->stream.avail_in = size;
  79        obj_req->stream.next_in = ptr;
  80        do {
  81                obj_req->stream.next_out = expn;
  82                obj_req->stream.avail_out = sizeof(expn);
  83                obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
  84                git_SHA1_Update(&obj_req->c, expn,
  85                                sizeof(expn) - obj_req->stream.avail_out);
  86        } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  87        data_received++;
  88        return size;
  89}
  90
  91static void fetch_alternates(struct walker *walker, const char *base);
  92
  93static void process_object_response(void *callback_data);
  94
  95static void start_object_request(struct walker *walker,
  96                                 struct object_request *obj_req)
  97{
  98        char *hex = sha1_to_hex(obj_req->sha1);
  99        char prevfile[PATH_MAX];
 100        char *url;
 101        char *posn;
 102        int prevlocal;
 103        unsigned char prev_buf[PREV_BUF_SIZE];
 104        ssize_t prev_read = 0;
 105        long prev_posn = 0;
 106        char range[RANGE_HEADER_SIZE];
 107        struct curl_slist *range_header = NULL;
 108        struct active_request_slot *slot;
 109
 110        snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 111        unlink_or_warn(prevfile);
 112        rename(obj_req->tmpfile, prevfile);
 113        unlink_or_warn(obj_req->tmpfile);
 114
 115        if (obj_req->local != -1)
 116                error("fd leakage in start: %d", obj_req->local);
 117        obj_req->local = open(obj_req->tmpfile,
 118                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 119        /*
 120         * This could have failed due to the "lazy directory creation";
 121         * try to mkdir the last path component.
 122         */
 123        if (obj_req->local < 0 && errno == ENOENT) {
 124                char *dir = strrchr(obj_req->tmpfile, '/');
 125                if (dir) {
 126                        *dir = 0;
 127                        mkdir(obj_req->tmpfile, 0777);
 128                        *dir = '/';
 129                }
 130                obj_req->local = open(obj_req->tmpfile,
 131                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 132        }
 133
 134        if (obj_req->local < 0) {
 135                obj_req->state = ABORTED;
 136                error("Couldn't create temporary file %s for %s: %s",
 137                      obj_req->tmpfile, obj_req->filename, strerror(errno));
 138                return;
 139        }
 140
 141        memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 142
 143        git_inflate_init(&obj_req->stream);
 144
 145        git_SHA1_Init(&obj_req->c);
 146
 147        url = xmalloc(strlen(obj_req->repo->base) + 51);
 148        obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 149        strcpy(url, obj_req->repo->base);
 150        posn = url + strlen(obj_req->repo->base);
 151        strcpy(posn, "/objects/");
 152        posn += 9;
 153        memcpy(posn, hex, 2);
 154        posn += 2;
 155        *(posn++) = '/';
 156        strcpy(posn, hex + 2);
 157        strcpy(obj_req->url, url);
 158
 159        /*
 160         * If a previous temp file is present, process what was already
 161         * fetched.
 162         */
 163        prevlocal = open(prevfile, O_RDONLY);
 164        if (prevlocal != -1) {
 165                do {
 166                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 167                        if (prev_read>0) {
 168                                if (fwrite_sha1_file(prev_buf,
 169                                                     1,
 170                                                     prev_read,
 171                                                     obj_req) == prev_read)
 172                                        prev_posn += prev_read;
 173                                else
 174                                        prev_read = -1;
 175                        }
 176                } while (prev_read > 0);
 177                close(prevlocal);
 178        }
 179        unlink_or_warn(prevfile);
 180
 181        /*
 182         * Reset inflate/SHA1 if there was an error reading the previous temp
 183         * file; also rewind to the beginning of the local file.
 184         */
 185        if (prev_read == -1) {
 186                memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 187                git_inflate_init(&obj_req->stream);
 188                git_SHA1_Init(&obj_req->c);
 189                if (prev_posn>0) {
 190                        prev_posn = 0;
 191                        lseek(obj_req->local, 0, SEEK_SET);
 192                        ftruncate(obj_req->local, 0);
 193                }
 194        }
 195
 196        slot = get_active_slot();
 197        slot->callback_func = process_object_response;
 198        slot->callback_data = obj_req;
 199        obj_req->slot = slot;
 200
 201        curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 202        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 203        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 204        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 205        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 206
 207        /*
 208         * If we have successfully processed data from a previous fetch
 209         * attempt, only fetch the data we don't already have.
 210         */
 211        if (prev_posn>0) {
 212                if (walker->get_verbosely)
 213                        fprintf(stderr,
 214                                "Resuming fetch of object %s at byte %ld\n",
 215                                hex, prev_posn);
 216                sprintf(range, "Range: bytes=%ld-", prev_posn);
 217                range_header = curl_slist_append(range_header, range);
 218                curl_easy_setopt(slot->curl,
 219                                 CURLOPT_HTTPHEADER, range_header);
 220        }
 221
 222        /* Try to get the request started, abort the request on error */
 223        obj_req->state = ACTIVE;
 224        if (!start_active_slot(slot)) {
 225                obj_req->state = ABORTED;
 226                obj_req->slot = NULL;
 227                close(obj_req->local);
 228                obj_req->local = -1;
 229                free(obj_req->url);
 230                return;
 231        }
 232}
 233
 234static void finish_object_request(struct object_request *obj_req)
 235{
 236        struct stat st;
 237
 238        close(obj_req->local);
 239        obj_req->local = -1;
 240
 241        if (obj_req->http_code == 416) {
 242                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 243        } else if (obj_req->curl_result != CURLE_OK) {
 244                if (stat(obj_req->tmpfile, &st) == 0)
 245                        if (st.st_size == 0)
 246                                unlink_or_warn(obj_req->tmpfile);
 247                return;
 248        }
 249
 250        git_inflate_end(&obj_req->stream);
 251        git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
 252        if (obj_req->zret != Z_STREAM_END) {
 253                unlink_or_warn(obj_req->tmpfile);
 254                return;
 255        }
 256        if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 257                unlink_or_warn(obj_req->tmpfile);
 258                return;
 259        }
 260        obj_req->rename =
 261                move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 262
 263        if (obj_req->rename == 0)
 264                walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
 265}
 266
 267static void process_object_response(void *callback_data)
 268{
 269        struct object_request *obj_req =
 270                (struct object_request *)callback_data;
 271        struct walker *walker = obj_req->walker;
 272        struct walker_data *data = walker->data;
 273        struct alt_base *alt = data->alt;
 274
 275        obj_req->curl_result = obj_req->slot->curl_result;
 276        obj_req->http_code = obj_req->slot->http_code;
 277        obj_req->slot = NULL;
 278        obj_req->state = COMPLETE;
 279
 280        /* Use alternates if necessary */
 281        if (missing_target(obj_req)) {
 282                fetch_alternates(walker, alt->base);
 283                if (obj_req->repo->next != NULL) {
 284                        obj_req->repo =
 285                                obj_req->repo->next;
 286                        close(obj_req->local);
 287                        obj_req->local = -1;
 288                        start_object_request(walker, obj_req);
 289                        return;
 290                }
 291        }
 292
 293        finish_object_request(obj_req);
 294}
 295
 296static void release_object_request(struct object_request *obj_req)
 297{
 298        struct object_request *entry = object_queue_head;
 299
 300        if (obj_req->local != -1)
 301                error("fd leakage in release: %d", obj_req->local);
 302        if (obj_req == object_queue_head) {
 303                object_queue_head = obj_req->next;
 304        } else {
 305                while (entry->next != NULL && entry->next != obj_req)
 306                        entry = entry->next;
 307                if (entry->next == obj_req)
 308                        entry->next = entry->next->next;
 309        }
 310
 311        free(obj_req->url);
 312        free(obj_req);
 313}
 314
 315#ifdef USE_CURL_MULTI
 316static int fill_active_slot(struct walker *walker)
 317{
 318        struct object_request *obj_req;
 319
 320        for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 321                if (obj_req->state == WAITING) {
 322                        if (has_sha1_file(obj_req->sha1))
 323                                obj_req->state = COMPLETE;
 324                        else {
 325                                start_object_request(walker, obj_req);
 326                                return 1;
 327                        }
 328                }
 329        }
 330        return 0;
 331}
 332#endif
 333
 334static void prefetch(struct walker *walker, unsigned char *sha1)
 335{
 336        struct object_request *newreq;
 337        struct object_request *tail;
 338        struct walker_data *data = walker->data;
 339        char *filename = sha1_file_name(sha1);
 340
 341        newreq = xmalloc(sizeof(*newreq));
 342        newreq->walker = walker;
 343        hashcpy(newreq->sha1, sha1);
 344        newreq->repo = data->alt;
 345        newreq->url = NULL;
 346        newreq->local = -1;
 347        newreq->state = WAITING;
 348        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 349        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 350                 "%s.temp", filename);
 351        newreq->slot = NULL;
 352        newreq->next = NULL;
 353
 354        http_is_verbose = walker->get_verbosely;
 355
 356        if (object_queue_head == NULL) {
 357                object_queue_head = newreq;
 358        } else {
 359                tail = object_queue_head;
 360                while (tail->next != NULL)
 361                        tail = tail->next;
 362                tail->next = newreq;
 363        }
 364
 365#ifdef USE_CURL_MULTI
 366        fill_active_slots();
 367        step_active_slots();
 368#endif
 369}
 370
 371static void process_alternates_response(void *callback_data)
 372{
 373        struct alternates_request *alt_req =
 374                (struct alternates_request *)callback_data;
 375        struct walker *walker = alt_req->walker;
 376        struct walker_data *cdata = walker->data;
 377        struct active_request_slot *slot = alt_req->slot;
 378        struct alt_base *tail = cdata->alt;
 379        const char *base = alt_req->base;
 380        static const char null_byte = '\0';
 381        char *data;
 382        int i = 0;
 383
 384        if (alt_req->http_specific) {
 385                if (slot->curl_result != CURLE_OK ||
 386                    !alt_req->buffer->len) {
 387
 388                        /* Try reusing the slot to get non-http alternates */
 389                        alt_req->http_specific = 0;
 390                        sprintf(alt_req->url, "%s/objects/info/alternates",
 391                                base);
 392                        curl_easy_setopt(slot->curl, CURLOPT_URL,
 393                                         alt_req->url);
 394                        active_requests++;
 395                        slot->in_use = 1;
 396                        if (slot->finished != NULL)
 397                                (*slot->finished) = 0;
 398                        if (!start_active_slot(slot)) {
 399                                cdata->got_alternates = -1;
 400                                slot->in_use = 0;
 401                                if (slot->finished != NULL)
 402                                        (*slot->finished) = 1;
 403                        }
 404                        return;
 405                }
 406        } else if (slot->curl_result != CURLE_OK) {
 407                if (!missing_target(slot)) {
 408                        cdata->got_alternates = -1;
 409                        return;
 410                }
 411        }
 412
 413        fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 414        alt_req->buffer->len--;
 415        data = alt_req->buffer->buf;
 416
 417        while (i < alt_req->buffer->len) {
 418                int posn = i;
 419                while (posn < alt_req->buffer->len && data[posn] != '\n')
 420                        posn++;
 421                if (data[posn] == '\n') {
 422                        int okay = 0;
 423                        int serverlen = 0;
 424                        struct alt_base *newalt;
 425                        char *target = NULL;
 426                        if (data[i] == '/') {
 427                                /*
 428                                 * This counts
 429                                 * http://git.host/pub/scm/linux.git/
 430                                 * -----------here^
 431                                 * so memcpy(dst, base, serverlen) will
 432                                 * copy up to "...git.host".
 433                                 */
 434                                const char *colon_ss = strstr(base,"://");
 435                                if (colon_ss) {
 436                                        serverlen = (strchr(colon_ss + 3, '/')
 437                                                     - base);
 438                                        okay = 1;
 439                                }
 440                        } else if (!memcmp(data + i, "../", 3)) {
 441                                /*
 442                                 * Relative URL; chop the corresponding
 443                                 * number of subpath from base (and ../
 444                                 * from data), and concatenate the result.
 445                                 *
 446                                 * The code first drops ../ from data, and
 447                                 * then drops one ../ from data and one path
 448                                 * from base.  IOW, one extra ../ is dropped
 449                                 * from data than path is dropped from base.
 450                                 *
 451                                 * This is not wrong.  The alternate in
 452                                 *     http://git.host/pub/scm/linux.git/
 453                                 * to borrow from
 454                                 *     http://git.host/pub/scm/linus.git/
 455                                 * is ../../linus.git/objects/.  You need
 456                                 * two ../../ to borrow from your direct
 457                                 * neighbour.
 458                                 */
 459                                i += 3;
 460                                serverlen = strlen(base);
 461                                while (i + 2 < posn &&
 462                                       !memcmp(data + i, "../", 3)) {
 463                                        do {
 464                                                serverlen--;
 465                                        } while (serverlen &&
 466                                                 base[serverlen - 1] != '/');
 467                                        i += 3;
 468                                }
 469                                /* If the server got removed, give up. */
 470                                okay = strchr(base, ':') - base + 3 <
 471                                       serverlen;
 472                        } else if (alt_req->http_specific) {
 473                                char *colon = strchr(data + i, ':');
 474                                char *slash = strchr(data + i, '/');
 475                                if (colon && slash && colon < data + posn &&
 476                                    slash < data + posn && colon < slash) {
 477                                        okay = 1;
 478                                }
 479                        }
 480                        /* skip "objects\n" at end */
 481                        if (okay) {
 482                                target = xmalloc(serverlen + posn - i - 6);
 483                                memcpy(target, base, serverlen);
 484                                memcpy(target + serverlen, data + i,
 485                                       posn - i - 7);
 486                                target[serverlen + posn - i - 7] = 0;
 487                                if (walker->get_verbosely)
 488                                        fprintf(stderr,
 489                                                "Also look at %s\n", target);
 490                                newalt = xmalloc(sizeof(*newalt));
 491                                newalt->next = NULL;
 492                                newalt->base = target;
 493                                newalt->got_indices = 0;
 494                                newalt->packs = NULL;
 495
 496                                while (tail->next != NULL)
 497                                        tail = tail->next;
 498                                tail->next = newalt;
 499                        }
 500                }
 501                i = posn + 1;
 502        }
 503
 504        cdata->got_alternates = 1;
 505}
 506
 507static void fetch_alternates(struct walker *walker, const char *base)
 508{
 509        struct strbuf buffer = STRBUF_INIT;
 510        char *url;
 511        struct active_request_slot *slot;
 512        struct alternates_request alt_req;
 513        struct walker_data *cdata = walker->data;
 514
 515        /*
 516         * If another request has already started fetching alternates,
 517         * wait for them to arrive and return to processing this request's
 518         * curl message
 519         */
 520#ifdef USE_CURL_MULTI
 521        while (cdata->got_alternates == 0) {
 522                step_active_slots();
 523        }
 524#endif
 525
 526        /* Nothing to do if they've already been fetched */
 527        if (cdata->got_alternates == 1)
 528                return;
 529
 530        /* Start the fetch */
 531        cdata->got_alternates = 0;
 532
 533        if (walker->get_verbosely)
 534                fprintf(stderr, "Getting alternates list for %s\n", base);
 535
 536        url = xmalloc(strlen(base) + 31);
 537        sprintf(url, "%s/objects/info/http-alternates", base);
 538
 539        /*
 540         * Use a callback to process the result, since another request
 541         * may fail and need to have alternates loaded before continuing
 542         */
 543        slot = get_active_slot();
 544        slot->callback_func = process_alternates_response;
 545        alt_req.walker = walker;
 546        slot->callback_data = &alt_req;
 547
 548        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 549        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 550        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 551
 552        alt_req.base = base;
 553        alt_req.url = url;
 554        alt_req.buffer = &buffer;
 555        alt_req.http_specific = 1;
 556        alt_req.slot = slot;
 557
 558        if (start_active_slot(slot))
 559                run_active_slot(slot);
 560        else
 561                cdata->got_alternates = -1;
 562
 563        strbuf_release(&buffer);
 564        free(url);
 565}
 566
 567static int fetch_indices(struct walker *walker, struct alt_base *repo)
 568{
 569        int ret;
 570
 571        if (repo->got_indices)
 572                return 0;
 573
 574        if (walker->get_verbosely)
 575                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 576
 577        switch (http_get_info_packs(repo->base, &repo->packs)) {
 578        case HTTP_OK:
 579        case HTTP_MISSING_TARGET:
 580                repo->got_indices = 1;
 581                ret = 0;
 582                break;
 583        default:
 584                repo->got_indices = 0;
 585                ret = -1;
 586        }
 587
 588        return ret;
 589}
 590
 591static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 592{
 593        char *url;
 594        struct packed_git *target;
 595        struct packed_git **lst;
 596        FILE *packfile;
 597        char *filename;
 598        char tmpfile[PATH_MAX];
 599        int ret;
 600        long prev_posn = 0;
 601        char range[RANGE_HEADER_SIZE];
 602        struct curl_slist *range_header = NULL;
 603
 604        struct active_request_slot *slot;
 605        struct slot_results results;
 606
 607        if (fetch_indices(walker, repo))
 608                return -1;
 609        target = find_sha1_pack(sha1, repo->packs);
 610        if (!target)
 611                return -1;
 612
 613        if (walker->get_verbosely) {
 614                fprintf(stderr, "Getting pack %s\n",
 615                        sha1_to_hex(target->sha1));
 616                fprintf(stderr, " which contains %s\n",
 617                        sha1_to_hex(sha1));
 618        }
 619
 620        url = xmalloc(strlen(repo->base) + 65);
 621        sprintf(url, "%s/objects/pack/pack-%s.pack",
 622                repo->base, sha1_to_hex(target->sha1));
 623
 624        filename = sha1_pack_name(target->sha1);
 625        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 626        packfile = fopen(tmpfile, "a");
 627        if (!packfile)
 628                return error("Unable to open local file %s for pack",
 629                             tmpfile);
 630
 631        slot = get_active_slot();
 632        slot->results = &results;
 633        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 634        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 635        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 636        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 637        slot->local = packfile;
 638
 639        /*
 640         * If there is data present from a previous transfer attempt,
 641         * resume where it left off
 642         */
 643        prev_posn = ftell(packfile);
 644        if (prev_posn>0) {
 645                if (walker->get_verbosely)
 646                        fprintf(stderr,
 647                                "Resuming fetch of pack %s at byte %ld\n",
 648                                sha1_to_hex(target->sha1), prev_posn);
 649                sprintf(range, "Range: bytes=%ld-", prev_posn);
 650                range_header = curl_slist_append(range_header, range);
 651                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 652        }
 653
 654        if (start_active_slot(slot)) {
 655                run_active_slot(slot);
 656                if (results.curl_result != CURLE_OK) {
 657                        fclose(packfile);
 658                        slot->local = NULL;
 659                        return error("Unable to get pack file %s\n%s", url,
 660                                     curl_errorstr);
 661                }
 662        } else {
 663                fclose(packfile);
 664                slot->local = NULL;
 665                return error("Unable to start request");
 666        }
 667
 668        target->pack_size = ftell(packfile);
 669        fclose(packfile);
 670        slot->local = NULL;
 671
 672        ret = move_temp_to_file(tmpfile, filename);
 673        if (ret)
 674                return ret;
 675
 676        lst = &repo->packs;
 677        while (*lst != target)
 678                lst = &((*lst)->next);
 679        *lst = (*lst)->next;
 680
 681        if (verify_pack(target))
 682                return -1;
 683        install_packed_git(target);
 684
 685        return 0;
 686}
 687
 688static void abort_object_request(struct object_request *obj_req)
 689{
 690        if (obj_req->local >= 0) {
 691                close(obj_req->local);
 692                obj_req->local = -1;
 693        }
 694        unlink_or_warn(obj_req->tmpfile);
 695        if (obj_req->slot) {
 696                release_active_slot(obj_req->slot);
 697                obj_req->slot = NULL;
 698        }
 699        release_object_request(obj_req);
 700}
 701
 702static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 703{
 704        char *hex = sha1_to_hex(sha1);
 705        int ret = 0;
 706        struct object_request *obj_req = object_queue_head;
 707
 708        while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 709                obj_req = obj_req->next;
 710        if (obj_req == NULL)
 711                return error("Couldn't find request for %s in the queue", hex);
 712
 713        if (has_sha1_file(obj_req->sha1)) {
 714                abort_object_request(obj_req);
 715                return 0;
 716        }
 717
 718#ifdef USE_CURL_MULTI
 719        while (obj_req->state == WAITING)
 720                step_active_slots();
 721#else
 722        start_object_request(walker, obj_req);
 723#endif
 724
 725        while (obj_req->state == ACTIVE)
 726                run_active_slot(obj_req->slot);
 727
 728        if (obj_req->local != -1) {
 729                close(obj_req->local);
 730                obj_req->local = -1;
 731        }
 732
 733        if (obj_req->state == ABORTED) {
 734                ret = error("Request for %s aborted", hex);
 735        } else if (obj_req->curl_result != CURLE_OK &&
 736                   obj_req->http_code != 416) {
 737                if (missing_target(obj_req))
 738                        ret = -1; /* Be silent, it is probably in a pack. */
 739                else
 740                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 741                                    obj_req->errorstr, obj_req->curl_result,
 742                                    obj_req->http_code, hex);
 743        } else if (obj_req->zret != Z_STREAM_END) {
 744                walker->corrupt_object_found++;
 745                ret = error("File %s (%s) corrupt", hex, obj_req->url);
 746        } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 747                ret = error("File %s has bad hash", hex);
 748        } else if (obj_req->rename < 0) {
 749                ret = error("unable to write sha1 filename %s",
 750                            obj_req->filename);
 751        }
 752
 753        release_object_request(obj_req);
 754        return ret;
 755}
 756
 757static int fetch(struct walker *walker, unsigned char *sha1)
 758{
 759        struct walker_data *data = walker->data;
 760        struct alt_base *altbase = data->alt;
 761
 762        if (!fetch_object(walker, altbase, sha1))
 763                return 0;
 764        while (altbase) {
 765                if (!fetch_pack(walker, altbase, sha1))
 766                        return 0;
 767                fetch_alternates(walker, data->alt->base);
 768                altbase = altbase->next;
 769        }
 770        return error("Unable to find %s under %s", sha1_to_hex(sha1),
 771                     data->alt->base);
 772}
 773
 774static int fetch_ref(struct walker *walker, struct ref *ref)
 775{
 776        struct walker_data *data = walker->data;
 777        return http_fetch_ref(data->alt->base, ref);
 778}
 779
 780static void cleanup(struct walker *walker)
 781{
 782        http_cleanup();
 783}
 784
 785struct walker *get_http_walker(const char *url, struct remote *remote)
 786{
 787        char *s;
 788        struct walker_data *data = xmalloc(sizeof(struct walker_data));
 789        struct walker *walker = xmalloc(sizeof(struct walker));
 790
 791        http_init(remote);
 792
 793        data->alt = xmalloc(sizeof(*data->alt));
 794        data->alt->base = xmalloc(strlen(url) + 1);
 795        strcpy(data->alt->base, url);
 796        for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
 797                *s = 0;
 798
 799        data->alt->got_indices = 0;
 800        data->alt->packs = NULL;
 801        data->alt->next = NULL;
 802        data->got_alternates = -1;
 803
 804        walker->corrupt_object_found = 0;
 805        walker->fetch = fetch;
 806        walker->fetch_ref = fetch_ref;
 807        walker->prefetch = prefetch;
 808        walker->cleanup = cleanup;
 809        walker->data = data;
 810
 811#ifdef USE_CURL_MULTI
 812        add_fill_function(walker, (int (*)(void *)) fill_active_slot);
 813#endif
 814
 815        return walker;
 816}