Merge branch 'ld/p4-changes-block-size'
authorJunio C Hamano <gitster@pobox.com>
Wed, 24 Jun 2015 19:21:56 +0000 (12:21 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 24 Jun 2015 19:21:57 +0000 (12:21 -0700)
More Perforce row number limit workaround for "git p4".

* ld/p4-changes-block-size:
git-p4: fixing --changes-block-size handling
git-p4: add tests for non-numeric revision range
git-p4: test with limited p4 server results
git-p4: additional testing of --changes-block-size

git-p4.py
t/t9800-git-p4-basic.sh
t/t9818-git-p4-block.sh
index 26ad4bcf77e36624e4261ae2dfd93ba637914b9d..073f87bbfdc1dc7ebc3fe610e78709667f597d82 100755 (executable)
--- a/git-p4.py
+++ b/git-p4.py
@@ -43,6 +43,9 @@ def __str__(self):
 # Only labels/tags matching this will be imported/exported
 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
 
+# Grab changes in blocks of this many revisions, unless otherwise requested
+defaultBlockSize = 512
+
 def p4_build_cmd(cmd):
     """Build a suitable p4 command line.
 
@@ -249,6 +252,10 @@ def p4_reopen(type, f):
 def p4_move(src, dest):
     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 
+def p4_last_change():
+    results = p4CmdList(["changes", "-m", "1"])
+    return int(results[0]['change'])
+
 def p4_describe(change):
     """Make sure it returns a valid result by checking for
        the presence of field "time".  Return a dict of the
@@ -742,43 +749,77 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent
 def originP4BranchesExist():
         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 
-def p4ChangesForPaths(depotPaths, changeRange, block_size):
+
+def p4ParseNumericChangeRange(parts):
+    changeStart = int(parts[0][1:])
+    if parts[1] == '#head':
+        changeEnd = p4_last_change()
+    else:
+        changeEnd = int(parts[1])
+
+    return (changeStart, changeEnd)
+
+def chooseBlockSize(blockSize):
+    if blockSize:
+        return blockSize
+    else:
+        return defaultBlockSize
+
+def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
     assert depotPaths
-    assert block_size
 
-    # Parse the change range into start and end
+    # Parse the change range into start and end. Try to find integer
+    # revision ranges as these can be broken up into blocks to avoid
+    # hitting server-side limits (maxrows, maxscanresults). But if
+    # that doesn't work, fall back to using the raw revision specifier
+    # strings, without using block mode.
+
     if changeRange is None or changeRange == '':
-        changeStart = '@1'
-        changeEnd = '#head'
+        changeStart = 1
+        changeEnd = p4_last_change()
+        block_size = chooseBlockSize(requestedBlockSize)
     else:
         parts = changeRange.split(',')
         assert len(parts) == 2
-        changeStart = parts[0]
-        changeEnd = parts[1]
+        try:
+            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
+            block_size = chooseBlockSize(requestedBlockSize)
+        except:
+            changeStart = parts[0][1:]
+            changeEnd = parts[1]
+            if requestedBlockSize:
+                die("cannot use --changes-block-size with non-numeric revisions")
+            block_size = None
 
     # Accumulate change numbers in a dictionary to avoid duplicates
     changes = {}
 
     for p in depotPaths:
         # Retrieve changes a block at a time, to prevent running
-        # into a MaxScanRows error from the server.
-        start = changeStart
-        end = changeEnd
-        get_another_block = True
-        while get_another_block:
-            new_changes = []
+        # into a MaxResults/MaxScanRows error from the server.
+
+        while True:
             cmd = ['changes']
-            cmd += ['-m', str(block_size)]
-            cmd += ["%s...%s,%s" % (p, start, end)]
+
+            if block_size:
+                end = min(changeEnd, changeStart + block_size)
+                revisionRange = "%d,%d" % (changeStart, end)
+            else:
+                revisionRange = "%s,%s" % (changeStart, changeEnd)
+
+            cmd += ["%s...@%s" % (p, revisionRange)]
+
             for line in p4_read_pipe_lines(cmd):
                 changeNum = int(line.split(" ")[1])
-                new_changes.append(changeNum)
                 changes[changeNum] = True
-            if len(new_changes) == block_size:
-                get_another_block = True
-                end = '@' + str(min(new_changes))
-            else:
-                get_another_block = False
+
+            if not block_size:
+                break
+
+            if end >= changeEnd:
+                break
+
+            changeStart = end + 1
 
     changelist = changes.keys()
     changelist.sort()
@@ -1974,7 +2015,7 @@ def __init__(self):
         self.syncWithOrigin = True
         self.importIntoRemotes = True
         self.maxChanges = ""
-        self.changes_block_size = 500
+        self.changes_block_size = None
         self.keepRepoPath = False
         self.depotPaths = None
         self.p4BranchesInGit = []
index 5b562122a16f231b00f7fb8090992f49d4950d82..90d41ed954c829bfb3735f3cea9db336f74adb81 100755 (executable)
@@ -131,6 +131,44 @@ test_expect_success 'clone two dirs, @all, conflicting files' '
        )
 '
 
+revision_ranges="2000/01/01,#head \
+                1,2080/01/01 \
+                2000/01/01,2080/01/01 \
+                2000/01/01,1000 \
+                1,1000"
+
+test_expect_success 'clone using non-numeric revision ranges' '
+       test_when_finished cleanup_git &&
+       for r in $revision_ranges
+       do
+               rm -fr "$git" &&
+               test ! -d "$git" &&
+               git p4 clone --dest="$git" //depot@$r &&
+               (
+                       cd "$git" &&
+                       git ls-files >lines &&
+                       test_line_count = 6 lines
+               )
+       done
+'
+
+test_expect_success 'clone with date range, excluding some changes' '
+       test_when_finished cleanup_git &&
+       before=$(date +%Y/%m/%d:%H:%M:%S) &&
+       sleep 2 &&
+       (
+               cd "$cli" &&
+               :>date_range_test &&
+               p4 add date_range_test &&
+               p4 submit -d "Adding file"
+       ) &&
+       git p4 clone --dest="$git" //depot@1,$before &&
+       (
+               cd "$git" &&
+               test_path_is_missing date_range_test
+       )
+'
+
 test_expect_success 'exit when p4 fails to produce marshaled output' '
        mkdir badp4dir &&
        test_when_finished "rm badp4dir/p4 && rmdir badp4dir" &&
index 153b20afbd420f23e2379e2de1e3f4ea95eecd65..3b3ae1f59a4d354472ce5042aa192313324cc516 100755 (executable)
@@ -8,18 +8,34 @@ test_expect_success 'start p4d' '
        start_p4d
 '
 
-test_expect_success 'Create a repo with ~100 changes' '
+create_restricted_group() {
+       p4 group -i <<-EOF
+       Group: restricted
+       MaxResults: 7
+       MaxScanRows: 40
+       Users: author
+       EOF
+}
+
+test_expect_success 'Create group with limited maxrows' '
+       create_restricted_group
+'
+
+test_expect_success 'Create a repo with many changes' '
        (
-               cd "$cli" &&
+               client_view "//depot/included/... //client/included/..." \
+                           "//depot/excluded/... //client/excluded/..." &&
+               mkdir -p "$cli/included" "$cli/excluded" &&
+               cd "$cli/included" &&
                >file.txt &&
                p4 add file.txt &&
                p4 submit -d "Add file.txt" &&
-               for i in $(test_seq 0 9)
+               for i in $(test_seq 0 5)
                do
                        >outer$i.txt &&
                        p4 add outer$i.txt &&
                        p4 submit -d "Adding outer$i.txt" &&
-                       for j in $(test_seq 0 9)
+                       for j in $(test_seq 0 5)
                        do
                                p4 edit file.txt &&
                                echo $i$j >file.txt &&
@@ -29,34 +45,73 @@ test_expect_success 'Create a repo with ~100 changes' '
        )
 '
 
+test_expect_success 'Default user cannot fetch changes' '
+       ! p4 changes -m 1 //depot/...
+'
+
 test_expect_success 'Clone the repo' '
-       git p4 clone --dest="$git" --changes-block-size=10 --verbose //depot@all
+       git p4 clone --dest="$git" --changes-block-size=7 --verbose //depot/included@all
 '
 
 test_expect_success 'All files are present' '
        echo file.txt >expected &&
        test_write_lines outer0.txt outer1.txt outer2.txt outer3.txt outer4.txt >>expected &&
-       test_write_lines outer5.txt outer6.txt outer7.txt outer8.txt outer9.txt >>expected &&
+       test_write_lines outer5.txt >>expected &&
        ls "$git" >current &&
        test_cmp expected current
 '
 
 test_expect_success 'file.txt is correct' '
-       echo 99 >expected &&
+       echo 55 >expected &&
        test_cmp expected "$git/file.txt"
 '
 
 test_expect_success 'Correct number of commits' '
        (cd "$git" && git log --oneline) >log &&
-       test_line_count = 111 log
+       wc -l log &&
+       test_line_count = 43 log
 '
 
 test_expect_success 'Previous version of file.txt is correct' '
        (cd "$git" && git checkout HEAD^^) &&
-       echo 97 >expected &&
+       echo 53 >expected &&
        test_cmp expected "$git/file.txt"
 '
 
+# Test git-p4 sync, with some files outside the client specification.
+
+p4_add_file() {
+       (cd "$cli" &&
+               >$1 &&
+               p4 add $1 &&
+               p4 submit -d "Added a file" $1
+       )
+}
+
+test_expect_success 'Add some more files' '
+       for i in $(test_seq 0 10)
+       do
+               p4_add_file "included/x$i" &&
+               p4_add_file "excluded/x$i"
+       done &&
+       for i in $(test_seq 0 10)
+       do
+               p4_add_file "excluded/y$i"
+       done
+'
+
+# This should pick up the 10 new files in "included", but not be confused
+# by the additional files in "excluded"
+test_expect_success 'Syncing files' '
+       (
+               cd "$git" &&
+               git p4 sync --changes-block-size=7 &&
+               git checkout p4/master &&
+               ls -l x* > log &&
+               test_line_count = 11 log
+       )
+'
+
 test_expect_success 'kill p4d' '
        kill_p4d
 '