Andrew's git / gitweb.git / t/t5317-pack-objects-filter-objects.sh
?
t / t5317-pack-objects-filter-objects.shon commit Merge branch 'js/bundle-verify-require-object-store' into maint (abbd504)
   1#!/bin/sh
   2
   3test_description='git pack-objects using object filtering'
   4
   5. ./test-lib.sh
   6
   7# Test blob:none filter.
   8
   9test_expect_success 'setup r1' '
  10        echo "{print \$1}" >print_1.awk &&
  11        echo "{print \$2}" >print_2.awk &&
  12
  13        git init r1 &&
  14        for n in 1 2 3 4 5
  15        do
  16                echo "This is file: $n" > r1/file.$n
  17                git -C r1 add file.$n
  18                git -C r1 commit -m "$n"
  19        done
  20'
  21
  22test_expect_success 'verify blob count in normal packfile' '
  23        git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
  24                >ls_files_result &&
  25        awk -f print_2.awk ls_files_result |
  26        sort >expected &&
  27
  28        git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
  29        HEAD
  30        EOF
  31        git -C r1 index-pack ../all.pack &&
  32
  33        git -C r1 verify-pack -v ../all.pack >verify_result &&
  34        grep blob verify_result |
  35        awk -f print_1.awk |
  36        sort >observed &&
  37
  38        test_cmp expected observed
  39'
  40
  41test_expect_success 'verify blob:none packfile has no blobs' '
  42        git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
  43        HEAD
  44        EOF
  45        git -C r1 index-pack ../filter.pack &&
  46
  47        git -C r1 verify-pack -v ../filter.pack >verify_result &&
  48        grep blob verify_result |
  49        awk -f print_1.awk |
  50        sort >observed &&
  51
  52        nr=$(wc -l <observed) &&
  53        test 0 -eq $nr
  54'
  55
  56test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
  57        git -C r1 verify-pack -v ../all.pack >verify_result &&
  58        grep -E "commit|tree" verify_result |
  59        awk -f print_1.awk |
  60        sort >expected &&
  61
  62        git -C r1 verify-pack -v ../filter.pack >verify_result &&
  63        grep -E "commit|tree" verify_result |
  64        awk -f print_1.awk |
  65        sort >observed &&
  66
  67        test_cmp expected observed
  68'
  69
  70test_expect_success 'get an error for missing tree object' '
  71        git init r5 &&
  72        echo foo >r5/foo &&
  73        git -C r5 add foo &&
  74        git -C r5 commit -m "foo" &&
  75        del=$(git -C r5 rev-parse HEAD^{tree} | sed "s|..|&/|") &&
  76        rm r5/.git/objects/$del &&
  77        test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
  78        HEAD
  79        EOF
  80        grep "bad tree object" bad_tree
  81'
  82
  83test_expect_success 'setup for tests of tree:0' '
  84        mkdir r1/subtree &&
  85        echo "This is a file in a subtree" >r1/subtree/file &&
  86        git -C r1 add subtree/file &&
  87        git -C r1 commit -m subtree
  88'
  89
  90test_expect_success 'verify tree:0 packfile has no blobs or trees' '
  91        git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
  92        HEAD
  93        EOF
  94        git -C r1 index-pack ../commitsonly.pack &&
  95        git -C r1 verify-pack -v ../commitsonly.pack >objs &&
  96        ! grep -E "tree|blob" objs
  97'
  98
  99test_expect_success 'grab tree directly when using tree:0' '
 100        # We should get the tree specified directly but not its blobs or subtrees.
 101        git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
 102        HEAD:
 103        EOF
 104        git -C r1 index-pack ../commitsonly.pack &&
 105        git -C r1 verify-pack -v ../commitsonly.pack >objs &&
 106        awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
 107        git -C r1 rev-parse HEAD: >expected &&
 108        test_cmp expected trees_and_blobs
 109'
 110
 111# Test blob:limit=<n>[kmg] filter.
 112# We boundary test around the size parameter.  The filter is strictly less than
 113# the value, so size 500 and 1000 should have the same results, but 1001 should
 114# filter more.
 115
 116test_expect_success 'setup r2' '
 117        git init r2 &&
 118        for n in 1000 10000
 119        do
 120                printf "%"$n"s" X > r2/large.$n
 121                git -C r2 add large.$n
 122                git -C r2 commit -m "$n"
 123        done
 124'
 125
 126test_expect_success 'verify blob count in normal packfile' '
 127        git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 128        awk -f print_2.awk ls_files_result |
 129        sort >expected &&
 130
 131        git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
 132        HEAD
 133        EOF
 134        git -C r2 index-pack ../all.pack &&
 135
 136        git -C r2 verify-pack -v ../all.pack >verify_result &&
 137        grep blob verify_result |
 138        awk -f print_1.awk |
 139        sort >observed &&
 140
 141        test_cmp expected observed
 142'
 143
 144test_expect_success 'verify blob:limit=500 omits all blobs' '
 145        git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
 146        HEAD
 147        EOF
 148        git -C r2 index-pack ../filter.pack &&
 149
 150        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 151        grep blob verify_result |
 152        awk -f print_1.awk |
 153        sort >observed &&
 154
 155        nr=$(wc -l <observed) &&
 156        test 0 -eq $nr
 157'
 158
 159test_expect_success 'verify blob:limit=1000' '
 160        git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
 161        HEAD
 162        EOF
 163        git -C r2 index-pack ../filter.pack &&
 164
 165        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 166        grep blob verify_result |
 167        awk -f print_1.awk |
 168        sort >observed &&
 169
 170        nr=$(wc -l <observed) &&
 171        test 0 -eq $nr
 172'
 173
 174test_expect_success 'verify blob:limit=1001' '
 175        git -C r2 ls-files -s large.1000 >ls_files_result &&
 176        awk -f print_2.awk ls_files_result |
 177        sort >expected &&
 178
 179        git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
 180        HEAD
 181        EOF
 182        git -C r2 index-pack ../filter.pack &&
 183
 184        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 185        grep blob verify_result |
 186        awk -f print_1.awk |
 187        sort >observed &&
 188
 189        test_cmp expected observed
 190'
 191
 192test_expect_success 'verify blob:limit=10001' '
 193        git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 194        awk -f print_2.awk ls_files_result |
 195        sort >expected &&
 196
 197        git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
 198        HEAD
 199        EOF
 200        git -C r2 index-pack ../filter.pack &&
 201
 202        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 203        grep blob verify_result |
 204        awk -f print_1.awk |
 205        sort >observed &&
 206
 207        test_cmp expected observed
 208'
 209
 210test_expect_success 'verify blob:limit=1k' '
 211        git -C r2 ls-files -s large.1000 >ls_files_result &&
 212        awk -f print_2.awk ls_files_result |
 213        sort >expected &&
 214
 215        git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 216        HEAD
 217        EOF
 218        git -C r2 index-pack ../filter.pack &&
 219
 220        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 221        grep blob verify_result |
 222        awk -f print_1.awk |
 223        sort >observed &&
 224
 225        test_cmp expected observed
 226'
 227
 228test_expect_success 'verify explicitly specifying oversized blob in input' '
 229        git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 230        awk -f print_2.awk ls_files_result |
 231        sort >expected &&
 232
 233        git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 234        HEAD
 235        $(git -C r2 rev-parse HEAD:large.10000)
 236        EOF
 237        git -C r2 index-pack ../filter.pack &&
 238
 239        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 240        grep blob verify_result |
 241        awk -f print_1.awk |
 242        sort >observed &&
 243
 244        test_cmp expected observed
 245'
 246
 247test_expect_success 'verify blob:limit=1m' '
 248        git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 249        awk -f print_2.awk ls_files_result |
 250        sort >expected &&
 251
 252        git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
 253        HEAD
 254        EOF
 255        git -C r2 index-pack ../filter.pack &&
 256
 257        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 258        grep blob verify_result |
 259        awk -f print_1.awk |
 260        sort >observed &&
 261
 262        test_cmp expected observed
 263'
 264
 265test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
 266        git -C r2 verify-pack -v ../all.pack >verify_result &&
 267        grep -E "commit|tree" verify_result |
 268        awk -f print_1.awk |
 269        sort >expected &&
 270
 271        git -C r2 verify-pack -v ../filter.pack >verify_result &&
 272        grep -E "commit|tree" verify_result |
 273        awk -f print_1.awk |
 274        sort >observed &&
 275
 276        test_cmp expected observed
 277'
 278
 279# Test sparse:path=<path> filter.
 280# !!!!
 281# NOTE: sparse:path filter support has been dropped for security reasons,
 282# so the tests have been changed to make sure that using it fails.
 283# !!!!
 284# Use a local file containing a sparse-checkout specification to filter
 285# out blobs not required for the corresponding sparse-checkout.  We do not
 286# require sparse-checkout to actually be enabled.
 287
 288test_expect_success 'setup r3' '
 289        git init r3 &&
 290        mkdir r3/dir1 &&
 291        for n in sparse1 sparse2
 292        do
 293                echo "This is file: $n" > r3/$n
 294                git -C r3 add $n
 295                echo "This is file: dir1/$n" > r3/dir1/$n
 296                git -C r3 add dir1/$n
 297        done &&
 298        git -C r3 commit -m "sparse" &&
 299        echo dir1/ >pattern1 &&
 300        echo sparse1 >pattern2
 301'
 302
 303test_expect_success 'verify blob count in normal packfile' '
 304        git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 305                >ls_files_result &&
 306        awk -f print_2.awk ls_files_result |
 307        sort >expected &&
 308
 309        git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
 310        HEAD
 311        EOF
 312        git -C r3 index-pack ../all.pack &&
 313
 314        git -C r3 verify-pack -v ../all.pack >verify_result &&
 315        grep blob verify_result |
 316        awk -f print_1.awk |
 317        sort >observed &&
 318
 319        test_cmp expected observed
 320'
 321
 322test_expect_success 'verify sparse:path=pattern1 fails' '
 323        test_must_fail git -C r3 pack-objects --revs --stdout \
 324                --filter=sparse:path=../pattern1 <<-EOF
 325        HEAD
 326        EOF
 327'
 328
 329test_expect_success 'verify sparse:path=pattern2 fails' '
 330        test_must_fail git -C r3 pack-objects --revs --stdout \
 331                --filter=sparse:path=../pattern2 <<-EOF
 332        HEAD
 333        EOF
 334'
 335
 336# Test sparse:oid=<oid-ish> filter.
 337# Use a blob containing a sparse-checkout specification to filter
 338# out blobs not required for the corresponding sparse-checkout.  We do not
 339# require sparse-checkout to actually be enabled.
 340
 341test_expect_success 'setup r4' '
 342        git init r4 &&
 343        mkdir r4/dir1 &&
 344        for n in sparse1 sparse2
 345        do
 346                echo "This is file: $n" > r4/$n
 347                git -C r4 add $n
 348                echo "This is file: dir1/$n" > r4/dir1/$n
 349                git -C r4 add dir1/$n
 350        done &&
 351        echo dir1/ >r4/pattern &&
 352        git -C r4 add pattern &&
 353        git -C r4 commit -m "pattern"
 354'
 355
 356test_expect_success 'verify blob count in normal packfile' '
 357        git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 358                >ls_files_result &&
 359        awk -f print_2.awk ls_files_result |
 360        sort >expected &&
 361
 362        git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
 363        HEAD
 364        EOF
 365        git -C r4 index-pack ../all.pack &&
 366
 367        git -C r4 verify-pack -v ../all.pack >verify_result &&
 368        grep blob verify_result |
 369        awk -f print_1.awk |
 370        sort >observed &&
 371
 372        test_cmp expected observed
 373'
 374
 375test_expect_success 'verify sparse:oid=OID' '
 376        git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 377        awk -f print_2.awk ls_files_result |
 378        sort >expected &&
 379
 380        oid=$(git -C r4 ls-files -s pattern | awk -f print_2.awk) &&
 381        git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
 382        HEAD
 383        EOF
 384        git -C r4 index-pack ../filter.pack &&
 385
 386        git -C r4 verify-pack -v ../filter.pack >verify_result &&
 387        grep blob verify_result |
 388        awk -f print_1.awk |
 389        sort >observed &&
 390
 391        test_cmp expected observed
 392'
 393
 394test_expect_success 'verify sparse:oid=oid-ish' '
 395        git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 396        awk -f print_2.awk ls_files_result |
 397        sort >expected &&
 398
 399        git -C r4 pack-objects --revs --stdout --filter=sparse:oid=master:pattern >filter.pack <<-EOF &&
 400        HEAD
 401        EOF
 402        git -C r4 index-pack ../filter.pack &&
 403
 404        git -C r4 verify-pack -v ../filter.pack >verify_result &&
 405        grep blob verify_result |
 406        awk -f print_1.awk |
 407        sort >observed &&
 408
 409        test_cmp expected observed
 410'
 411
 412# Delete some loose objects and use pack-objects, but WITHOUT any filtering.
 413# This models previously omitted objects that we did not receive.
 414
 415test_expect_success 'setup r1 - delete loose blobs' '
 416        git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
 417                >ls_files_result &&
 418        awk -f print_2.awk ls_files_result |
 419        sort >expected &&
 420
 421        for id in `cat expected | sed "s|..|&/|"`
 422        do
 423                rm r1/.git/objects/$id
 424        done
 425'
 426
 427test_expect_success 'verify pack-objects fails w/ missing objects' '
 428        test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
 429        HEAD
 430        EOF
 431'
 432
 433test_expect_success 'verify pack-objects fails w/ --missing=error' '
 434        test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
 435        HEAD
 436        EOF
 437'
 438
 439test_expect_success 'verify pack-objects w/ --missing=allow-any' '
 440        git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
 441        HEAD
 442        EOF
 443'
 444
 445test_done