From: Junio C Hamano Date: Mon, 4 Feb 2013 18:25:30 +0000 (-0800) Subject: Merge branch 'pw/git-p4-on-cygwin' X-Git-Tag: v1.8.2-rc0~57 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/9aea11dbc152726bbbe93630cc29b10c29d4f62e?ds=inline;hp=-c Merge branch 'pw/git-p4-on-cygwin' Improve "git p4" on Cygwin. * pw/git-p4-on-cygwin: (21 commits) git p4: introduce gitConfigBool git p4: avoid shell when calling git config git p4: avoid shell when invoking git config --get-all git p4: avoid shell when invoking git rev-list git p4: avoid shell when mapping users git p4: disable read-only attribute before deleting git p4 test: use test_chmod for cygwin git p4: cygwin p4 client does not mark read-only git p4 test: avoid wildcard * in windows git p4 test: use LineEnd unix in windows tests too git p4 test: newline handling git p4: scrub crlf for utf16 files on windows git p4: remove unreachable windows \r\n conversion code git p4 test: translate windows paths for cygwin git p4 test: start p4d inside its db dir git p4 test: use client_view in t9806 git p4 test: avoid loop in client_view git p4 test: use client_view to build the initial client git p4: generate better error message for bad depot path git p4: remove unused imports ... --- 9aea11dbc152726bbbe93630cc29b10c29d4f62e diff --combined git-p4.py index 625a3968c2,955a5dde24..647f110202 --- a/git-p4.py +++ b/git-p4.py @@@ -7,32 -7,22 +7,37 @@@ # 2007 Trolltech ASA # License: MIT # - import sys if sys.hexversion < 0x02040000: # The limiter is the subprocess module sys.stderr.write("git-p4: requires Python 2.4 or later.\n") sys.exit(1) - - import optparse, os, marshal, subprocess, shelve - import tempfile, getopt, os.path, time, platform - import re, shutil + import os + import optparse + import marshal + import subprocess + import tempfile + import time + import platform + import re + import shutil + import stat +try: + from subprocess import CalledProcessError +except ImportError: + # from python2.7:subprocess.py + # Exception classes used by this module. + class CalledProcessError(Exception): + """This exception is raised when a process run by check_call() returns + a non-zero exit status. The exit status will be stored in the + returncode attribute.""" + def __init__(self, returncode, cmd): + self.returncode = returncode + self.cmd = cmd + def __str__(self): + return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + verbose = False # Only labels/tags matching this will be imported/exported @@@ -173,18 -163,30 +178,34 @@@ def system(cmd) expand = isinstance(cmd,basestring) if verbose: sys.stderr.write("executing %s\n" % str(cmd)) - subprocess.check_call(cmd, shell=expand) + retcode = subprocess.call(cmd, shell=expand) + if retcode: + raise CalledProcessError(retcode, cmd) def p4_system(cmd): """Specifically invoke p4 as the system command. """ real_cmd = p4_build_cmd(cmd) expand = isinstance(real_cmd, basestring) - subprocess.check_call(real_cmd, shell=expand) + retcode = subprocess.call(real_cmd, shell=expand) + if retcode: + raise CalledProcessError(retcode, real_cmd) + _p4_version_string = None + def p4_version_string(): + """Read the version string, showing just the last line, which + hopefully is the interesting version bit. + + $ p4 -V + Perforce - The Fast Software Configuration Management System. + Copyright 1995-2011 Perforce Software. All rights reserved. + Rev. P4/NTX86/2011.1/393975 (2011/12/16). + """ + global _p4_version_string + if not _p4_version_string: + a = p4_read_pipe_lines(["-V"]) + _p4_version_string = a[-1].rstrip() + return _p4_version_string + def p4_integrate(src, dest): p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)]) @@@ -558,18 -560,30 +579,30 @@@ def gitBranchExists(branch) return proc.wait() == 0; _gitConfig = {} - def gitConfig(key, args = None): # set args to "--bool", for instance + + def gitConfig(key): + if not _gitConfig.has_key(key): + cmd = [ "git", "config", key ] + s = read_pipe(cmd, ignore_error=True) + _gitConfig[key] = s.strip() + return _gitConfig[key] + + def gitConfigBool(key): + """Return a bool, using git config --bool. It is True only if the + variable is set to true, and False if set to false or not present + in the config.""" + if not _gitConfig.has_key(key): - argsFilter = "" - if args != None: - argsFilter = "%s " % args - cmd = "git config %s%s" % (argsFilter, key) - _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip() + cmd = [ "git", "config", "--bool", key ] + s = read_pipe(cmd, ignore_error=True) + v = s.strip() + _gitConfig[key] = v == "true" return _gitConfig[key] def gitConfigList(key): if not _gitConfig.has_key(key): - _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep) + s = read_pipe(["git", "config", "--get-all", key], ignore_error=True) + _gitConfig[key] = s.strip().split(os.linesep) return _gitConfig[key] def p4BranchesInGit(branchesAreInRemotes=True): @@@ -716,8 -730,7 +749,7 @@@ def p4PathStartsWith(path, prefix) # # we may or may not have a problem. If you have core.ignorecase=true, # we treat DirA and dira as the same directory - ignorecase = gitConfig("core.ignorecase", "--bool") == "true" - if ignorecase: + if gitConfigBool("core.ignorecase"): return path.lower().startswith(prefix.lower()) return path.startswith(prefix) @@@ -787,8 -800,7 +819,8 @@@ def wildcard_encode(path) return path def wildcard_present(path): - return path.translate(None, "*#@%") != path + m = re.search("[*#@%]", path) + return m is not None class Command: def __init__(self): @@@ -954,7 -966,7 +986,7 @@@ class P4Submit(Command, P4UserMap) self.usage += " [name of git branch to submit into perforce depot]" self.origin = "" self.detectRenames = False - self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true" + self.preserveUser = gitConfigBool("git-p4.preserveUser") self.dry_run = False self.prepare_p4_only = False self.conflict_behavior = None @@@ -1049,7 -1061,8 +1081,8 @@@ def p4UserForCommit(self,id): # Return the tuple (perforce user,git email) for a given git commit id self.getUserMapFromPerforceServer() - gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id) + gitEmail = read_pipe(["git", "log", "--max-count=1", + "--format=%ae", id]) gitEmail = gitEmail.strip() if not self.emails.has_key(gitEmail): return (None,gitEmail) @@@ -1062,7 -1075,7 +1095,7 @@@ (user,email) = self.p4UserForCommit(id) if not user: msg = "Cannot find p4 user for email %s in commit %s." % (email, id) - if gitConfig('git-p4.allowMissingP4Users').lower() == "true": + if gitConfigBool("git-p4.allowMissingP4Users"): print "%s" % msg else: die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg) @@@ -1157,7 -1170,7 +1190,7 @@@ message. Return true if okay to continue with the submit.""" # if configured to skip the editing part, just submit - if gitConfig("git-p4.skipSubmitEdit") == "true": + if gitConfigBool("git-p4.skipSubmitEdit"): return True # look at the modification time, to check later if the user saved @@@ -1173,7 -1186,7 +1206,7 @@@ # If the file was not saved, prompt to see if this patch should # be skipped. But skip this verification step if configured so. - if gitConfig("git-p4.skipSubmitEditCheck") == "true": + if gitConfigBool("git-p4.skipSubmitEditCheck"): return True # modification time updated means user saved the file @@@ -1231,6 -1244,9 +1264,9 @@@ p4_edit(dest) pureRenameCopy.discard(dest) filesToChangeExecBit[dest] = diff['dst_mode'] + if self.isWindows: + # turn off read-only attribute + os.chmod(dest, stat.S_IWRITE) os.unlink(dest) editedFiles.add(dest) elif modifier == "R": @@@ -1249,6 -1265,8 +1285,8 @@@ p4_edit(dest) # with move: already open, writable filesToChangeExecBit[dest] = diff['dst_mode'] if not self.p4HasMoveCommand: + if self.isWindows: + os.chmod(dest, stat.S_IWRITE) os.unlink(dest) filesToDelete.add(src) editedFiles.add(dest) @@@ -1268,7 -1286,7 +1306,7 @@@ # Patch failed, maybe it's just RCS keyword woes. Look through # the patch to see if that's possible. - if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true": + if gitConfigBool("git-p4.attemptRCSCleanup"): file = None pattern = None kwfiles = {} @@@ -1289,6 -1307,10 +1327,10 @@@ for file in kwfiles: if verbose: print "zapping %s with %s" % (line,pattern) + # File is being deleted, so not open in p4. Must + # disable the read-only bit on windows. + if self.isWindows and file not in editedFiles: + os.chmod(file, stat.S_IWRITE) self.patchRCSKeywords(file, kwfiles[file]) fixed_rcs_keywords = True @@@ -1559,7 -1581,7 +1601,7 @@@ sys.exit(128) self.useClientSpec = False - if gitConfig("git-p4.useclientspec", "--bool") == "true": + if gitConfigBool("git-p4.useclientspec"): self.useClientSpec = True if self.useClientSpec: self.clientSpecDirs = getClientSpec() @@@ -1595,11 -1617,11 +1637,11 @@@ self.check() commits = [] - for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)): + for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]): commits.append(line.strip()) commits.reverse() - if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"): + if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"): self.checkAuthorship = False else: self.checkAuthorship = True @@@ -1635,7 -1657,7 +1677,7 @@@ else: self.diffOpts += " -C%s" % detectCopies - if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true": + if gitConfigBool("git-p4.detectCopiesHarder"): self.diffOpts += " --find-copies-harder" # @@@ -1719,7 -1741,7 +1761,7 @@@ "--format=format:%h %s", c]) print "You will have to do 'git p4 sync' and rebase." - if gitConfig("git-p4.exportLabels", "--bool") == "true": + if gitConfigBool("git-p4.exportLabels"): self.exportLabels = True if self.exportLabels: @@@ -1989,7 -2011,6 +2031,6 @@@ class P4Sync(Command, P4UserMap) self.syncWithOrigin = True self.importIntoRemotes = True self.maxChanges = "" - self.isWindows = (platform.system() == "Windows") self.keepRepoPath = False self.depotPaths = None self.p4BranchesInGit = [] @@@ -2134,7 -2155,14 +2175,14 @@@ # operations. utf16 is converted to ascii or utf8, perhaps. # But ascii text saved as -t utf16 is completely mangled. # Invoke print -o to get the real contents. + # + # On windows, the newlines will always be mangled by print, so put + # them back too. This is not needed to the cygwin windows version, + # just the native "NT" type. + # text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']]) + if p4_version_string().find("/NT") >= 0: + text = text.replace("\r\n", "\n") contents = [ text ] if type_base == "apple": @@@ -2150,15 -2178,6 +2198,6 @@@ print "\nIgnoring apple filetype file %s" % file['depotFile'] return - # Perhaps windows wants unicode, utf16 newlines translated too; - # but this is not doing it. - if self.isWindows and type_base == "text": - mangled = [] - for data in contents: - data = data.replace("\r\n", "\n") - mangled.append(data) - contents = mangled - # Note that we do not try to de-mangle keywords on utf16 files, # even though in theory somebody may want that. pattern = p4_keywords_regexp_for_type(type_base, type_mods) @@@ -2636,7 -2655,8 +2675,8 @@@ def searchParent(self, parent, branch, target): parentFound = False - for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]): + for blob in read_pipe_lines(["git", "rev-list", "--reverse", + "--no-merges", parent]): blob = blob.strip() if len(read_pipe(["git", "diff-tree", blob, target])) == 0: parentFound = True @@@ -2707,7 -2727,7 +2747,7 @@@ blob = None if len(parent) > 0: - tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change)) + tempBranch = "%s/%d" % (self.tempBranchLocation, change) if self.verbose: print "Creating temporary branch: " + tempBranch self.commit(description, filesForCommit, tempBranch) @@@ -2821,7 -2841,7 +2861,7 @@@ # will use this after clone to set the variable self.useClientSpec_from_options = True else: - if gitConfig("git-p4.useclientspec", "--bool") == "true": + if gitConfigBool("git-p4.useclientspec"): self.useClientSpec = True if self.useClientSpec: self.clientSpecDirs = getClientSpec() @@@ -3061,7 -3081,7 +3101,7 @@@ sys.stdout.write("%s " % b) sys.stdout.write("\n") - if gitConfig("git-p4.importLabels", "--bool") == "true": + if gitConfigBool("git-p4.importLabels"): self.importLabels = True if self.importLabels: @@@ -3179,6 -3199,7 +3219,7 @@@ class P4Clone(P4Sync) self.cloneExclude = ["/"+p for p in self.cloneExclude] for p in depotPaths: if not p.startswith("//"): + sys.stderr.write('Depot paths must start with "//": %s\n' % p) return False if not self.cloneDestination: @@@ -3193,9 -3214,7 +3234,9 @@@ init_cmd = [ "git", "init" ] if self.cloneBare: init_cmd.append("--bare") - subprocess.check_call(init_cmd) + retcode = subprocess.call(init_cmd) + if retcode: + raise CalledProcessError(retcode, init_cmd) if not P4Sync.run(self, depotPaths): return False diff --combined t/t9802-git-p4-filetype.sh index aae1a3f816,c5ab626699..eeefa67904 --- a/t/t9802-git-p4-filetype.sh +++ b/t/t9802-git-p4-filetype.sh @@@ -8,6 -8,123 +8,123 @@@ test_expect_success 'start p4d' start_p4d ' + # + # This series of tests checks newline handling Both p4 and + # git store newlines as \n, and have options to choose how + # newlines appear in checked-out files. + # + test_expect_success 'p4 client newlines, unix' ' + ( + cd "$cli" && + p4 client -o | sed "/LineEnd/s/:.*/:unix/" | p4 client -i && + printf "unix\ncrlf\n" >f-unix && + printf "unix\r\ncrlf\r\n" >f-unix-as-crlf && + p4 add -t text f-unix && + p4 submit -d f-unix && + + # LineEnd: unix; should be no change after sync + cp f-unix f-unix-orig && + p4 sync -f && + test_cmp f-unix-orig f-unix && + + # make sure stored in repo as unix newlines + # use sed to eat python-appened newline + p4 -G print //depot/f-unix | marshal_dump data 2 |\ + sed \$d >f-unix-p4-print && + test_cmp f-unix-orig f-unix-p4-print && + + # switch to win, make sure lf -> crlf + p4 client -o | sed "/LineEnd/s/:.*/:win/" | p4 client -i && + p4 sync -f && + test_cmp f-unix-as-crlf f-unix + ) + ' + + test_expect_success 'p4 client newlines, win' ' + ( + cd "$cli" && + p4 client -o | sed "/LineEnd/s/:.*/:win/" | p4 client -i && + printf "win\r\ncrlf\r\n" >f-win && + printf "win\ncrlf\n" >f-win-as-lf && + p4 add -t text f-win && + p4 submit -d f-win && + + # LineEnd: win; should be no change after sync + cp f-win f-win-orig && + p4 sync -f && + test_cmp f-win-orig f-win && + + # make sure stored in repo as unix newlines + # use sed to eat python-appened newline + p4 -G print //depot/f-win | marshal_dump data 2 |\ + sed \$d >f-win-p4-print && + test_cmp f-win-as-lf f-win-p4-print && + + # switch to unix, make sure lf -> crlf + p4 client -o | sed "/LineEnd/s/:.*/:unix/" | p4 client -i && + p4 sync -f && + test_cmp f-win-as-lf f-win + ) + ' + + test_expect_success 'ensure blobs store only lf newlines' ' + test_when_finished cleanup_git && + ( + cd "$git" && + git init && + git p4 sync //depot@all && + + # verify the files in .git are stored only with newlines + o=$(git ls-tree p4/master -- f-unix | cut -f1 | cut -d\ -f3) && + git cat-file blob $o >f-unix-blob && + test_cmp "$cli"/f-unix-orig f-unix-blob && + + o=$(git ls-tree p4/master -- f-win | cut -f1 | cut -d\ -f3) && + git cat-file blob $o >f-win-blob && + test_cmp "$cli"/f-win-as-lf f-win-blob && + + rm f-unix-blob f-win-blob + ) + ' + + test_expect_success 'gitattributes setting eol=lf produces lf newlines' ' + test_when_finished cleanup_git && + ( + # checkout the files and make sure core.eol works as planned + cd "$git" && + git init && + echo "* eol=lf" >.gitattributes && + git p4 sync //depot@all && + git checkout master && + test_cmp "$cli"/f-unix-orig f-unix && + test_cmp "$cli"/f-win-as-lf f-win + ) + ' + + test_expect_success 'gitattributes setting eol=crlf produces crlf newlines' ' + test_when_finished cleanup_git && + ( + # checkout the files and make sure core.eol works as planned + cd "$git" && + git init && + echo "* eol=crlf" >.gitattributes && + git p4 sync //depot@all && + git checkout master && + test_cmp "$cli"/f-unix-as-crlf f-unix && + test_cmp "$cli"/f-win-orig f-win + ) + ' + + test_expect_success 'crlf cleanup' ' + ( + cd "$cli" && + rm f-unix-orig f-unix-as-crlf && + rm f-win-orig f-win-as-lf && + p4 client -o | sed "/LineEnd/s/:.*/:unix/" | p4 client -i && + p4 sync -f + ) + ' + test_expect_success 'utf-16 file create' ' ( cd "$cli" && @@@ -105,13 -222,12 +222,13 @@@ build_gendouble() cat >gendouble.py <<-\EOF import sys import struct - import array - s = array.array("c", '\0' * 26) - struct.pack_into(">L", s, 0, 0x00051607) # AppleDouble - struct.pack_into(">L", s, 4, 0x00020000) # version 2 - s.tofile(sys.stdout) + s = struct.pack(">LL18s", + 0x00051607, # AppleDouble + 0x00020000, # version 2 + "" # pad to 26 bytes + ) + sys.stdout.write(s) EOF }