import optparse, sys, os, marshal, subprocess, shelve
import tempfile, getopt, os.path, time, platform
-import re
+import re, shutil
verbose = False
location. It means that hooking into the environment, or other configuration
can be done more easily.
"""
- real_cmd = "%s " % "p4"
+ real_cmd = ["p4"]
user = gitConfig("git-p4.user")
if len(user) > 0:
- real_cmd += "-u %s " % user
+ real_cmd += ["-u",user]
password = gitConfig("git-p4.password")
if len(password) > 0:
- real_cmd += "-P %s " % password
+ real_cmd += ["-P", password]
port = gitConfig("git-p4.port")
if len(port) > 0:
- real_cmd += "-p %s " % port
+ real_cmd += ["-p", port]
host = gitConfig("git-p4.host")
if len(host) > 0:
- real_cmd += "-h %s " % host
+ real_cmd += ["-h", host]
client = gitConfig("git-p4.client")
if len(client) > 0:
- real_cmd += "-c %s " % client
+ real_cmd += ["-c", client]
- real_cmd += "%s" % (cmd)
- if verbose:
- print real_cmd
+
+ if isinstance(cmd,basestring):
+ real_cmd = ' '.join(real_cmd) + ' ' + cmd
+ else:
+ real_cmd += cmd
return real_cmd
def chdir(dir):
- if os.name == 'nt':
- os.environ['PWD']=dir
+ # P4 uses the PWD environment variable rather than getcwd(). Since we're
+ # not using the shell, we have to set it ourselves. This path could
+ # be relative, so go there first, then figure out where we ended up.
os.chdir(dir)
+ os.environ['PWD'] = os.getcwd()
def die(msg):
if verbose:
sys.stderr.write(msg + "\n")
sys.exit(1)
-def write_pipe(c, str):
+def write_pipe(c, stdin):
if verbose:
- sys.stderr.write('Writing pipe: %s\n' % c)
+ sys.stderr.write('Writing pipe: %s\n' % str(c))
- pipe = os.popen(c, 'w')
- val = pipe.write(str)
- if pipe.close():
- die('Command failed: %s' % c)
+ expand = isinstance(c,basestring)
+ p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
+ pipe = p.stdin
+ val = pipe.write(stdin)
+ pipe.close()
+ if p.wait():
+ die('Command failed: %s' % str(c))
return val
-def p4_write_pipe(c, str):
+def p4_write_pipe(c, stdin):
real_cmd = p4_build_cmd(c)
- return write_pipe(real_cmd, str)
+ return write_pipe(real_cmd, stdin)
def read_pipe(c, ignore_error=False):
if verbose:
- sys.stderr.write('Reading pipe: %s\n' % c)
+ sys.stderr.write('Reading pipe: %s\n' % str(c))
- pipe = os.popen(c, 'rb')
+ expand = isinstance(c,basestring)
+ p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
+ pipe = p.stdout
val = pipe.read()
- if pipe.close() and not ignore_error:
- die('Command failed: %s' % c)
+ if p.wait() and not ignore_error:
+ die('Command failed: %s' % str(c))
return val
def read_pipe_lines(c):
if verbose:
- sys.stderr.write('Reading pipe: %s\n' % c)
- ## todo: check return status
- pipe = os.popen(c, 'rb')
+ sys.stderr.write('Reading pipe: %s\n' % str(c))
+
+ expand = isinstance(c, basestring)
+ p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
+ pipe = p.stdout
val = pipe.readlines()
- if pipe.close():
- die('Command failed: %s' % c)
+ if pipe.close() or p.wait():
+ die('Command failed: %s' % str(c))
return val
return read_pipe_lines(real_cmd)
def system(cmd):
+ expand = isinstance(cmd,basestring)
if verbose:
- sys.stderr.write("executing %s\n" % cmd)
- if os.system(cmd) != 0:
- die("command failed: %s" % cmd)
+ sys.stderr.write("executing %s\n" % str(cmd))
+ subprocess.check_call(cmd, shell=expand)
def p4_system(cmd):
"""Specifically invoke p4 as the system command. """
real_cmd = p4_build_cmd(cmd)
- return system(real_cmd)
+ expand = isinstance(real_cmd, basestring)
+ subprocess.check_call(real_cmd, shell=expand)
+
+def p4_integrate(src, dest):
+ p4_system(["integrate", "-Dt", src, dest])
-def isP4Exec(kind):
- """Determine if a Perforce 'kind' should have execute permission
+def p4_sync(path):
+ p4_system(["sync", path])
- 'p4 help filetypes' gives a list of the types. If it starts with 'x',
- or x follows one of a few letters. Otherwise, if there is an 'x' after
- a plus sign, it is also executable"""
- return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
+def p4_add(f):
+ p4_system(["add", f])
+
+def p4_delete(f):
+ p4_system(["delete", f])
+
+def p4_edit(f):
+ p4_system(["edit", f])
+
+def p4_revert(f):
+ p4_system(["revert", f])
+
+def p4_reopen(type, file):
+ p4_system(["reopen", "-t", type, file])
+
+#
+# Canonicalize the p4 type and return a tuple of the
+# base type, plus any modifiers. See "p4 help filetypes"
+# for a list and explanation.
+#
+def split_p4_type(p4type):
+
+ p4_filetypes_historical = {
+ "ctempobj": "binary+Sw",
+ "ctext": "text+C",
+ "cxtext": "text+Cx",
+ "ktext": "text+k",
+ "kxtext": "text+kx",
+ "ltext": "text+F",
+ "tempobj": "binary+FSw",
+ "ubinary": "binary+F",
+ "uresource": "resource+F",
+ "uxbinary": "binary+Fx",
+ "xbinary": "binary+x",
+ "xltext": "text+Fx",
+ "xtempobj": "binary+Swx",
+ "xtext": "text+x",
+ "xunicode": "unicode+x",
+ "xutf16": "utf16+x",
+ }
+ if p4type in p4_filetypes_historical:
+ p4type = p4_filetypes_historical[p4type]
+ mods = ""
+ s = p4type.split("+")
+ base = s[0]
+ mods = ""
+ if len(s) > 1:
+ mods = s[1]
+ return (base, mods)
+
+#
+# return the raw p4 type of a file (text, text+ko, etc)
+#
+def p4_type(file):
+ results = p4CmdList(["fstat", "-T", "headType", file])
+ return results[0]['headType']
+
+#
+# Given a type base and modifier, return a regexp matching
+# the keywords that can be expanded in the file
+#
+def p4_keywords_regexp_for_type(base, type_mods):
+ if base in ("text", "unicode", "binary"):
+ kwords = None
+ if "ko" in type_mods:
+ kwords = 'Id|Header'
+ elif "k" in type_mods:
+ kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
+ else:
+ return None
+ pattern = r"""
+ \$ # Starts with a dollar, followed by...
+ (%s) # one of the keywords, followed by...
+ (:[^$]+)? # possibly an old expansion, followed by...
+ \$ # another dollar
+ """ % kwords
+ return pattern
+ else:
+ return None
+
+#
+# Given a file, return a regexp matching the possible
+# RCS keywords that will be expanded, or None for files
+# with kw expansion turned off.
+#
+def p4_keywords_regexp_for_file(file):
+ if not os.path.exists(file):
+ return None
+ else:
+ (type_base, type_mods) = split_p4_type(p4_type(file))
+ return p4_keywords_regexp_for_type(type_base, type_mods)
def setP4ExecBit(file, mode):
# Reopens an already open file and changes the execute bit to match
if p4Type[-1] == "+":
p4Type = p4Type[0:-1]
- p4_system("reopen -t %s %s" % (p4Type, file))
+ p4_reopen(p4Type, file)
def getP4OpenedType(file):
# Returns the perforce file type for the given file.
- result = p4_read_pipe("opened %s" % file)
+ result = p4_read_pipe(["opened", file])
match = re.match(".*\((.+)\)\r?$", result)
if match:
return match.group(1)
return isModeExec(src_mode) != isModeExec(dst_mode)
def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
- cmd = p4_build_cmd("-G %s" % (cmd))
+
+ if isinstance(cmd,basestring):
+ cmd = "-G " + cmd
+ expand = True
+ else:
+ cmd = ["-G"] + cmd
+ expand = False
+
+ cmd = p4_build_cmd(cmd)
if verbose:
- sys.stderr.write("Opening pipe: %s\n" % cmd)
+ sys.stderr.write("Opening pipe: %s\n" % str(cmd))
# Use a temporary file to avoid deadlocks without
# subprocess.communicate(), which would put another copy
stdin_file = None
if stdin is not None:
stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
- stdin_file.write(stdin)
+ if isinstance(stdin,basestring):
+ stdin_file.write(stdin)
+ else:
+ for i in stdin:
+ stdin_file.write(i + '\n')
stdin_file.flush()
stdin_file.seek(0)
- p4 = subprocess.Popen(cmd, shell=True,
+ p4 = subprocess.Popen(cmd,
+ shell=expand,
stdin=stdin_file,
stdout=subprocess.PIPE)
if not depotPath.endswith("/"):
depotPath += "/"
depotPath = depotPath + "..."
- outputList = p4CmdList("where %s" % depotPath)
+ outputList = p4CmdList(["where", depotPath])
output = None
for entry in outputList:
if "depotFile" in entry:
def parseRevision(ref):
return read_pipe("git rev-parse %s" % ref).strip()
+def branchExists(ref):
+ rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
+ ignore_error=True)
+ return len(rev) > 0
+
def extractLogMessageFromGitCommit(commit):
logMessage = ""
def p4ChangesForPaths(depotPaths, changeRange):
assert depotPaths
- output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
- for p in depotPaths]))
+ cmd = ['changes']
+ for p in depotPaths:
+ cmd += ["%s...%s" % (p, changeRange)]
+ output = p4_read_pipe_lines(cmd)
changes = {}
for line in output:
class P4UserMap:
def __init__(self):
self.userMapFromPerforceServer = False
+ self.myP4UserId = None
+
+ def p4UserId(self):
+ if self.myP4UserId:
+ return self.myP4UserId
+
+ results = p4CmdList("user -o")
+ for r in results:
+ if r.has_key('User'):
+ self.myP4UserId = r['User']
+ return r['User']
+ die("Could not find your p4 user id")
+
+ def p4UserIsMe(self, p4User):
+ # return True if the given p4 user is actually me
+ me = self.p4UserId()
+ if not p4User or p4User != me:
+ return False
+ else:
+ return True
def getUserCacheFilename(self):
home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
def run(self, args):
j = 0
- for output in p4CmdList(" ".join(args)):
+ for output in p4CmdList(args):
print 'Element: %d' % j
j += 1
print output
self.verbose = False
self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
self.isWindows = (platform.system() == "Windows")
- self.myP4UserId = None
def check(self):
if len(p4CmdList("opened ...")) > 0:
return result
+ def patchRCSKeywords(self, file, pattern):
+ # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
+ (handle, outFileName) = tempfile.mkstemp(dir='.')
+ try:
+ outFile = os.fdopen(handle, "w+")
+ inFile = open(file, "r")
+ regexp = re.compile(pattern, re.VERBOSE)
+ for line in inFile.readlines():
+ line = regexp.sub(r'$\1$', line)
+ outFile.write(line)
+ inFile.close()
+ outFile.close()
+ # Forcibly overwrite the original file
+ os.unlink(file)
+ shutil.move(outFileName, file)
+ except:
+ # cleanup our temporary file
+ os.unlink(outFileName)
+ print "Failed to strip RCS keywords in %s" % file
+ raise
+
+ print "Patched up RCS keywords in %s" % file
+
def p4UserForCommit(self,id):
# Return the tuple (perforce user,git email) for a given git commit id
self.getUserMapFromPerforceServer()
break
if not client:
die("could not get client spec")
- results = p4CmdList("changes -c %s -m 1" % client)
+ results = p4CmdList(["changes", "-c", client, "-m", "1"])
for r in results:
if r.has_key('change'):
return r['change']
def canChangeChangelists(self):
# check to see if we have p4 admin or super-user permissions, either of
# which are required to modify changelists.
- results = p4CmdList("protects %s" % self.depotPath)
+ results = p4CmdList(["protects", self.depotPath])
for r in results:
if r.has_key('perm'):
if r['perm'] == 'admin':
return 1
return 0
- def p4UserId(self):
- if self.myP4UserId:
- return self.myP4UserId
-
- results = p4CmdList("user -o")
- for r in results:
- if r.has_key('User'):
- self.myP4UserId = r['User']
- return r['User']
- die("Could not find your p4 user id")
-
- def p4UserIsMe(self, p4User):
- # return True if the given p4 user is actually me
- me = self.p4UserId()
- if not p4User or p4User != me:
- return False
- else:
- return True
-
def prepareSubmitTemplate(self):
# remove lines in the Files section that show changes to files outside the depot path we're committing into
template = ""
inFilesSection = False
- for line in p4_read_pipe_lines("change -o"):
+ for line in p4_read_pipe_lines(['change', '-o']):
if line.endswith("\r\n"):
line = line[:-2] + "\n"
if inFilesSection:
return template
+ def edit_template(self, template_file):
+ """Invoke the editor to let the user change the submission
+ message. Return true if okay to continue with the submit."""
+
+ # if configured to skip the editing part, just submit
+ if gitConfig("git-p4.skipSubmitEdit") == "true":
+ return True
+
+ # look at the modification time, to check later if the user saved
+ # the file
+ mtime = os.stat(template_file).st_mtime
+
+ # invoke the editor
+ if os.environ.has_key("P4EDITOR"):
+ editor = os.environ.get("P4EDITOR")
+ else:
+ editor = read_pipe("git var GIT_EDITOR").strip()
+ system(editor + " " + template_file)
+
+ # If the file was not saved, prompt to see if this patch should
+ # be skipped. But skip this verification step if configured so.
+ if gitConfig("git-p4.skipSubmitEditCheck") == "true":
+ return True
+
+ # modification time updated means user saved the file
+ if os.stat(template_file).st_mtime > mtime:
+ return True
+
+ while True:
+ response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
+ if response == 'y':
+ return True
+ if response == 'n':
+ return False
+
def applyCommit(self, id):
print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
filesToDelete = set()
editedFiles = set()
filesToChangeExecBit = {}
+
for line in diff:
diff = parseDiffTreeEntry(line)
modifier = diff['status']
path = diff['src']
if modifier == "M":
- p4_system("edit \"%s\"" % path)
+ p4_edit(path)
if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
filesToChangeExecBit[path] = diff['dst_mode']
editedFiles.add(path)
filesToAdd.remove(path)
elif modifier == "C":
src, dest = diff['src'], diff['dst']
- p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
+ p4_integrate(src, dest)
if diff['src_sha1'] != diff['dst_sha1']:
- p4_system("edit \"%s\"" % (dest))
+ p4_edit(dest)
if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
- p4_system("edit \"%s\"" % (dest))
+ p4_edit(dest)
filesToChangeExecBit[dest] = diff['dst_mode']
os.unlink(dest)
editedFiles.add(dest)
elif modifier == "R":
src, dest = diff['src'], diff['dst']
- p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
+ p4_integrate(src, dest)
if diff['src_sha1'] != diff['dst_sha1']:
- p4_system("edit \"%s\"" % (dest))
+ p4_edit(dest)
if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
- p4_system("edit \"%s\"" % (dest))
+ p4_edit(dest)
filesToChangeExecBit[dest] = diff['dst_mode']
os.unlink(dest)
editedFiles.add(dest)
patchcmd = diffcmd + " | git apply "
tryPatchCmd = patchcmd + "--check -"
applyPatchCmd = patchcmd + "--check --apply -"
+ patch_succeeded = True
if os.system(tryPatchCmd) != 0:
+ fixed_rcs_keywords = False
+ patch_succeeded = False
print "Unfortunately applying the change failed!"
+
+ # Patch failed, maybe it's just RCS keyword woes. Look through
+ # the patch to see if that's possible.
+ if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
+ file = None
+ pattern = None
+ kwfiles = {}
+ for file in editedFiles | filesToDelete:
+ # did this file's delta contain RCS keywords?
+ pattern = p4_keywords_regexp_for_file(file)
+
+ if pattern:
+ # this file is a possibility...look for RCS keywords.
+ regexp = re.compile(pattern, re.VERBOSE)
+ for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
+ if regexp.search(line):
+ if verbose:
+ print "got keyword match on %s in %s in %s" % (pattern, line, file)
+ kwfiles[file] = pattern
+ break
+
+ for file in kwfiles:
+ if verbose:
+ print "zapping %s with %s" % (line,pattern)
+ self.patchRCSKeywords(file, kwfiles[file])
+ fixed_rcs_keywords = True
+
+ if fixed_rcs_keywords:
+ print "Retrying the patch with RCS keywords cleaned up"
+ if os.system(tryPatchCmd) == 0:
+ patch_succeeded = True
+
+ if not patch_succeeded:
print "What do you want to do?"
response = "x"
while response != "s" and response != "a" and response != "w":
if response == "s":
print "Skipping! Good luck with the next patches..."
for f in editedFiles:
- p4_system("revert \"%s\"" % f);
+ p4_revert(f)
for f in filesToAdd:
- system("rm %s" %f)
+ os.remove(f)
return
elif response == "a":
os.system(applyPatchCmd)
system(applyPatchCmd)
for f in filesToAdd:
- p4_system("add \"%s\"" % f)
+ p4_add(f)
for f in filesToDelete:
- p4_system("revert \"%s\"" % f)
- p4_system("delete \"%s\"" % f)
+ p4_revert(f)
+ p4_delete(f)
# Set/clear executable bits
for f in filesToChangeExecBit.keys():
del(os.environ["P4DIFF"])
diff = ""
for editedFile in editedFiles:
- diff += p4_read_pipe("diff -du %r" % editedFile)
+ diff += p4_read_pipe(['diff', '-du', editedFile])
newdiff = ""
for newFile in filesToAdd:
separatorLine = "######## everything below this line is just the diff #######\n"
- [handle, fileName] = tempfile.mkstemp()
+ (handle, fileName) = tempfile.mkstemp()
tmpFile = os.fdopen(handle, "w+")
if self.isWindows:
submitTemplate = submitTemplate.replace("\n", "\r\n")
newdiff = newdiff.replace("\n", "\r\n")
tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
tmpFile.close()
- mtime = os.stat(fileName).st_mtime
- if os.environ.has_key("P4EDITOR"):
- editor = os.environ.get("P4EDITOR")
- else:
- editor = read_pipe("git var GIT_EDITOR").strip()
- system(editor + " " + fileName)
- if gitConfig("git-p4.skipSubmitEditCheck") == "true":
- checkModTime = False
- else:
- checkModTime = True
-
- response = "y"
- if checkModTime and (os.stat(fileName).st_mtime <= mtime):
- response = "x"
- while response != "y" and response != "n":
- response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
-
- if response == "y":
+ if self.edit_template(fileName):
+ # read the edited message and submit
tmpFile = open(fileName, "rb")
message = tmpFile.read()
tmpFile.close()
submitTemplate = message[:message.index(separatorLine)]
if self.isWindows:
submitTemplate = submitTemplate.replace("\r\n", "\n")
- p4_write_pipe("submit -i", submitTemplate)
+ p4_write_pipe(['submit', '-i'], submitTemplate)
if self.preserveUser:
if p4User:
# Get last changelist number. Cannot easily get it from
- # the submit command output as the output is unmarshalled.
+ # the submit command output as the output is
+ # unmarshalled.
changelist = self.lastP4Changelist()
self.modifyChangelistUser(changelist, p4User)
-
else:
+ # skip this patch
+ print "Submission cancelled, undoing p4 changes."
for f in editedFiles:
- p4_system("revert \"%s\"" % f);
+ p4_revert(f)
for f in filesToAdd:
- p4_system("revert \"%s\"" % f);
- system("rm %s" %f)
+ p4_revert(f)
+ os.remove(f)
os.remove(fileName)
else:
die("Detecting current git branch failed!")
elif len(args) == 1:
self.master = args[0]
+ if not branchExists(self.master):
+ die("Branch %s does not exist" % self.master)
else:
return False
print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
self.oldWorkingDirectory = os.getcwd()
+ # ensure the clientPath exists
+ if not os.path.exists(self.clientPath):
+ os.makedirs(self.clientPath)
+
chdir(self.clientPath)
print "Synchronizing p4 checkout..."
- p4_system("sync ...")
-
+ p4_sync("...")
self.check()
commits = []
return True
+class View(object):
+ """Represent a p4 view ("p4 help views"), and map files in a
+ repo according to the view."""
+
+ class Path(object):
+ """A depot or client path, possibly containing wildcards.
+ The only one supported is ... at the end, currently.
+ Initialize with the full path, with //depot or //client."""
+
+ def __init__(self, path, is_depot):
+ self.path = path
+ self.is_depot = is_depot
+ self.find_wildcards()
+ # remember the prefix bit, useful for relative mappings
+ m = re.match("(//[^/]+/)", self.path)
+ if not m:
+ die("Path %s does not start with //prefix/" % self.path)
+ prefix = m.group(1)
+ if not self.is_depot:
+ # strip //client/ on client paths
+ self.path = self.path[len(prefix):]
+
+ def find_wildcards(self):
+ """Make sure wildcards are valid, and set up internal
+ variables."""
+
+ self.ends_triple_dot = False
+ # There are three wildcards allowed in p4 views
+ # (see "p4 help views"). This code knows how to
+ # handle "..." (only at the end), but cannot deal with
+ # "%%n" or "*". Only check the depot_side, as p4 should
+ # validate that the client_side matches too.
+ if re.search(r'%%[1-9]', self.path):
+ die("Can't handle %%n wildcards in view: %s" % self.path)
+ if self.path.find("*") >= 0:
+ die("Can't handle * wildcards in view: %s" % self.path)
+ triple_dot_index = self.path.find("...")
+ if triple_dot_index >= 0:
+ if triple_dot_index != len(self.path) - 3:
+ die("Can handle only single ... wildcard, at end: %s" %
+ self.path)
+ self.ends_triple_dot = True
+
+ def ensure_compatible(self, other_path):
+ """Make sure the wildcards agree."""
+ if self.ends_triple_dot != other_path.ends_triple_dot:
+ die("Both paths must end with ... if either does;\n" +
+ "paths: %s %s" % (self.path, other_path.path))
+
+ def match_wildcards(self, test_path):
+ """See if this test_path matches us, and fill in the value
+ of the wildcards if so. Returns a tuple of
+ (True|False, wildcards[]). For now, only the ... at end
+ is supported, so at most one wildcard."""
+ if self.ends_triple_dot:
+ dotless = self.path[:-3]
+ if test_path.startswith(dotless):
+ wildcard = test_path[len(dotless):]
+ return (True, [ wildcard ])
+ else:
+ if test_path == self.path:
+ return (True, [])
+ return (False, [])
+
+ def match(self, test_path):
+ """Just return if it matches; don't bother with the wildcards."""
+ b, _ = self.match_wildcards(test_path)
+ return b
+
+ def fill_in_wildcards(self, wildcards):
+ """Return the relative path, with the wildcards filled in
+ if there are any."""
+ if self.ends_triple_dot:
+ return self.path[:-3] + wildcards[0]
+ else:
+ return self.path
+
+ class Mapping(object):
+ def __init__(self, depot_side, client_side, overlay, exclude):
+ # depot_side is without the trailing /... if it had one
+ self.depot_side = View.Path(depot_side, is_depot=True)
+ self.client_side = View.Path(client_side, is_depot=False)
+ self.overlay = overlay # started with "+"
+ self.exclude = exclude # started with "-"
+ assert not (self.overlay and self.exclude)
+ self.depot_side.ensure_compatible(self.client_side)
+
+ def __str__(self):
+ c = " "
+ if self.overlay:
+ c = "+"
+ if self.exclude:
+ c = "-"
+ return "View.Mapping: %s%s -> %s" % \
+ (c, self.depot_side.path, self.client_side.path)
+
+ def map_depot_to_client(self, depot_path):
+ """Calculate the client path if using this mapping on the
+ given depot path; does not consider the effect of other
+ mappings in a view. Even excluded mappings are returned."""
+ matches, wildcards = self.depot_side.match_wildcards(depot_path)
+ if not matches:
+ return ""
+ client_path = self.client_side.fill_in_wildcards(wildcards)
+ return client_path
+
+ #
+ # View methods
+ #
+ def __init__(self):
+ self.mappings = []
+
+ def append(self, view_line):
+ """Parse a view line, splitting it into depot and client
+ sides. Append to self.mappings, preserving order."""
+
+ # Split the view line into exactly two words. P4 enforces
+ # structure on these lines that simplifies this quite a bit.
+ #
+ # Either or both words may be double-quoted.
+ # Single quotes do not matter.
+ # Double-quote marks cannot occur inside the words.
+ # A + or - prefix is also inside the quotes.
+ # There are no quotes unless they contain a space.
+ # The line is already white-space stripped.
+ # The two words are separated by a single space.
+ #
+ if view_line[0] == '"':
+ # First word is double quoted. Find its end.
+ close_quote_index = view_line.find('"', 1)
+ if close_quote_index <= 0:
+ die("No first-word closing quote found: %s" % view_line)
+ depot_side = view_line[1:close_quote_index]
+ # skip closing quote and space
+ rhs_index = close_quote_index + 1 + 1
+ else:
+ space_index = view_line.find(" ")
+ if space_index <= 0:
+ die("No word-splitting space found: %s" % view_line)
+ depot_side = view_line[0:space_index]
+ rhs_index = space_index + 1
+
+ if view_line[rhs_index] == '"':
+ # Second word is double quoted. Make sure there is a
+ # double quote at the end too.
+ if not view_line.endswith('"'):
+ die("View line with rhs quote should end with one: %s" %
+ view_line)
+ # skip the quotes
+ client_side = view_line[rhs_index+1:-1]
+ else:
+ client_side = view_line[rhs_index:]
+
+ # prefix + means overlay on previous mapping
+ overlay = False
+ if depot_side.startswith("+"):
+ overlay = True
+ depot_side = depot_side[1:]
+
+ # prefix - means exclude this path
+ exclude = False
+ if depot_side.startswith("-"):
+ exclude = True
+ depot_side = depot_side[1:]
+
+ m = View.Mapping(depot_side, client_side, overlay, exclude)
+ self.mappings.append(m)
+
+ def map_in_client(self, depot_path):
+ """Return the relative location in the client where this
+ depot file should live. Returns "" if the file should
+ not be mapped in the client."""
+
+ paths_filled = []
+ client_path = ""
+
+ # look at later entries first
+ for m in self.mappings[::-1]:
+
+ # see where will this path end up in the client
+ p = m.map_depot_to_client(depot_path)
+
+ if p == "":
+ # Depot path does not belong in client. Must remember
+ # this, as previous items should not cause files to
+ # exist in this path either. Remember that the list is
+ # being walked from the end, which has higher precedence.
+ # Overlap mappings do not exclude previous mappings.
+ if not m.overlay:
+ paths_filled.append(m.client_side)
+
+ else:
+ # This mapping matched; no need to search any further.
+ # But, the mapping could be rejected if the client path
+ # has already been claimed by an earlier mapping (i.e.
+ # one later in the list, which we are walking backwards).
+ already_mapped_in_client = False
+ for f in paths_filled:
+ # this is View.Path.match
+ if f.match(p):
+ already_mapped_in_client = True
+ break
+ if not already_mapped_in_client:
+ # Include this file, unless it is from a line that
+ # explicitly said to exclude it.
+ if not m.exclude:
+ client_path = p
+
+ # a match, even if rejected, always stops the search
+ break
+
+ return client_path
+
class P4Sync(Command, P4UserMap):
delete_actions = ( "delete", "move/delete", "purge" )
self.p4BranchesInGit = []
self.cloneExclude = []
self.useClientSpec = False
- self.clientSpecDirs = []
+ self.clientSpecDirs = None
+ self.tempBranches = []
+ self.tempBranchLocation = "git-p4-tmp"
if gitConfig("git-p4.syncFromOrigin") == "false":
self.syncWithOrigin = False
.replace("%25", "%")
return path
+ # Force a checkpoint in fast-import and wait for it to finish
+ def checkpoint(self):
+ self.gitStream.write("checkpoint\n\n")
+ self.gitStream.write("progress checkpoint\n\n")
+ out = self.gitOutput.readline()
+ if self.verbose:
+ print "checkpoint finished: " + out
+
def extractFilesFromCommit(self, commit):
self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
for path in self.cloneExclude]
def stripRepoPath(self, path, prefixes):
if self.useClientSpec:
-
- # if using the client spec, we use the output directory
- # specified in the client. For example, a view
- # //depot/foo/branch/... //client/branch/foo/...
- # will end up putting all foo/branch files into
- # branch/foo/
- for val in self.clientSpecDirs:
- if path.startswith(val[0]):
- # replace the depot path with the client path
- path = path.replace(val[0], val[1][1])
- # now strip out the client (//client/...)
- path = re.sub("^(//[^/]+/)", '', path)
- # the rest is all path
- return path
+ return self.clientSpecDirs.map_in_client(path)
if self.keepRepoPath:
prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
# - helper for streamP4Files
def streamOneP4File(self, file, contents):
- if file["type"] == "apple":
- print "\nfile %s is a strange apple file that forks. Ignoring" % \
- file['depotFile']
- return
-
relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
relPath = self.wildcard_decode(relPath)
if verbose:
sys.stderr.write("%s\n" % relPath)
- mode = "644"
- if isP4Exec(file["type"]):
- mode = "755"
- elif file["type"] == "symlink":
- mode = "120000"
- # p4 print on a symlink contains "target\n", so strip it off
+ (type_base, type_mods) = split_p4_type(file["type"])
+
+ git_mode = "100644"
+ if "x" in type_mods:
+ git_mode = "100755"
+ if type_base == "symlink":
+ git_mode = "120000"
+ # p4 print on a symlink contains "target\n"; remove the newline
data = ''.join(contents)
contents = [data[:-1]]
- if self.isWindows and file["type"].endswith("text"):
+ if type_base == "utf16":
+ # p4 delivers different text in the python output to -G
+ # than it does when using "print -o", or normal p4 client
+ # operations. utf16 is converted to ascii or utf8, perhaps.
+ # But ascii text saved as -t utf16 is completely mangled.
+ # Invoke print -o to get the real contents.
+ text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
+ contents = [ text ]
+
+ if type_base == "apple":
+ # Apple filetype files will be streamed as a concatenation of
+ # its appledouble header and the contents. This is useless
+ # on both macs and non-macs. If using "print -q -o xx", it
+ # will create "xx" with the data, and "%xx" with the header.
+ # This is also not very useful.
+ #
+ # Ideally, someday, this script can learn how to generate
+ # appledouble files directly and import those to git, but
+ # non-mac machines can never find a use for apple filetype.
+ print "\nIgnoring apple filetype file %s" % file['depotFile']
+ return
+
+ # Perhaps windows wants unicode, utf16 newlines translated too;
+ # but this is not doing it.
+ if self.isWindows and type_base == "text":
mangled = []
for data in contents:
data = data.replace("\r\n", "\n")
mangled.append(data)
contents = mangled
- if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
- contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
- elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
- contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
+ # Note that we do not try to de-mangle keywords on utf16 files,
+ # even though in theory somebody may want that.
+ pattern = p4_keywords_regexp_for_type(type_base, type_mods)
+ if pattern:
+ regexp = re.compile(pattern, re.VERBOSE)
+ text = ''.join(contents)
+ text = regexp.sub(r'$\1$', text)
+ contents = [ text ]
- self.gitStream.write("M %s inline %s\n" % (mode, relPath))
+ self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
# total length...
length = 0
filesToDelete = []
for f in files:
- includeFile = True
- for val in self.clientSpecDirs:
- if f['path'].startswith(val[0]):
- if val[1][0] <= 0:
- includeFile = False
- break
+ # if using a client spec, only add the files that have
+ # a path in the client
+ if self.clientSpecDirs:
+ if self.clientSpecDirs.map_in_client(f['path']) == "":
+ continue
- if includeFile:
- filesForCommit.append(f)
- if f['action'] in self.delete_actions:
- filesToDelete.append(f)
- else:
- filesToRead.append(f)
+ filesForCommit.append(f)
+ if f['action'] in self.delete_actions:
+ filesToDelete.append(f)
+ else:
+ filesToRead.append(f)
# deleted files...
for f in filesToDelete:
def streamP4FilesCbSelf(entry):
self.streamP4FilesCb(entry)
- p4CmdList("-x - print",
- '\n'.join(['%s#%s' % (f['path'], f['rev'])
- for f in filesToRead]),
- cb=streamP4FilesCbSelf)
+ fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
+
+ p4CmdList(["-x", "-", "print"],
+ stdin=fileArgs,
+ cb=streamP4FilesCbSelf)
# do the last chunk
if self.stream_file.has_key('depotFile'):
self.streamOneP4File(self.stream_file, self.stream_contents)
+ def make_email(self, userid):
+ if userid in self.users:
+ return self.users[userid]
+ else:
+ return "%s <a@b>" % userid
+
def commit(self, details, files, branch, branchPrefixes, parent = ""):
epoch = details["time"]
author = details["user"]
committer = ""
if author not in self.users:
self.getUserMapFromPerforceServer()
- if author in self.users:
- committer = "%s %s %s" % (self.users[author], epoch, self.tz)
- else:
- committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
+ committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
self.gitStream.write("committer %s\n" % committer)
if self.verbose:
print "Change %s is labelled %s" % (change, labelDetails)
- files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
- for p in branchPrefixes]))
+ files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
+ for p in branchPrefixes])
if len(files) == len(labelRevisions):
self.gitStream.write("from %s\n" % branch)
owner = labelDetails["Owner"]
- tagger = ""
- if author in self.users:
- tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
+
+ # Try to use the owner of the p4 label, or failing that,
+ # the current p4 user id.
+ if owner:
+ email = self.make_email(owner)
else:
- tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
+ email = self.make_email(self.p4UserId())
+ tagger = "%s %s %s" % (email, epoch, self.tz)
+
self.gitStream.write("tagger %s\n" % tagger)
- self.gitStream.write("data <<EOT\n")
- self.gitStream.write(labelDetails["Description"])
- self.gitStream.write("EOT\n\n")
+
+ description = labelDetails["Description"]
+ self.gitStream.write("data %d\n" % len(description))
+ self.gitStream.write(description)
+ self.gitStream.write("\n")
else:
if not self.silent:
def getLabels(self):
self.labels = {}
- l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
+ l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
if len(l) > 0 and not self.silent:
print "Finding files belonging to labels in %s" % `self.depotPaths`
newestChange = 0
if self.verbose:
print "Querying files for label %s" % label
- for file in p4CmdList("files "
- + ' '.join (["%s...@%s" % (p, label)
- for p in self.depotPaths])):
+ for file in p4CmdList(["files"] +
+ ["%s...@%s" % (p, label)
+ for p in self.depotPaths]):
revisions[file["depotFile"]] = file["rev"]
change = int(file["change"])
if change > newestChange:
command = "branches"
for info in p4CmdList(command):
- details = p4Cmd("branch -o %s" % info["branch"])
+ details = p4Cmd(["branch", "-o", info["branch"]])
viewIdx = 0
while details.has_key("View%s" % viewIdx):
paths = details["View%s" % viewIdx].split(" ")
sourceRef = self.gitRefForBranch(sourceBranch)
#print "source " + sourceBranch
- branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
+ branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
#print "branch parent: %s" % branchParentChange
gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
if len(gitParent) > 0:
self.importChanges(changes)
return True
+ def searchParent(self, parent, branch, target):
+ parentFound = False
+ for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
+ blob = blob.strip()
+ if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
+ parentFound = True
+ if self.verbose:
+ print "Found parent of %s in commit %s" % (branch, blob)
+ break
+ if parentFound:
+ return blob
+ else:
+ return None
+
def importChanges(self, changes):
cnt = 1
for change in changes:
- description = p4Cmd("describe %s" % change)
+ description = p4Cmd(["describe", str(change)])
self.updateOptionDict(description)
if not self.silent:
parent = self.initialParents[branch]
del self.initialParents[branch]
- self.commit(description, filesForCommit, branch, [branchPrefix], parent)
+ blob = None
+ if len(parent) > 0:
+ tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
+ if self.verbose:
+ print "Creating temporary branch: " + tempBranch
+ self.commit(description, filesForCommit, tempBranch, [branchPrefix])
+ self.tempBranches.append(tempBranch)
+ self.checkpoint()
+ blob = self.searchParent(parent, branch, tempBranch)
+ if blob:
+ self.commit(description, filesForCommit, branch, [branchPrefix], blob)
+ else:
+ if self.verbose:
+ print "Parent of %s not found. Committing into head of %s" % (branch, parent)
+ self.commit(description, filesForCommit, branch, [branchPrefix], parent)
else:
files = self.extractFilesFromCommit(description)
self.commit(description, files, self.branch, self.depotPaths,
newestRevision = 0
fileCnt = 0
- for info in p4CmdList("files "
- + ' '.join(["%s...%s"
- % (p, revision)
- for p in self.depotPaths])):
+ fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
+
+ for info in p4CmdList(["files"] + fileArgs):
if 'code' in info and info['code'] == 'error':
sys.stderr.write("p4 returned an error: %s\n"
def getClientSpec(self):
- specList = p4CmdList( "client -o" )
- temp = {}
- for entry in specList:
- for k,v in entry.iteritems():
- if k.startswith("View"):
-
- # p4 has these %%1 to %%9 arguments in specs to
- # reorder paths; which we can't handle (yet :)
- if re.match('%%\d', v) != None:
- print "Sorry, can't handle %%n arguments in client specs"
- sys.exit(1)
-
- if v.startswith('"'):
- start = 1
- else:
- start = 0
- index = v.find("...")
-
- # save the "client view"; i.e the RHS of the view
- # line that tells the client where to put the
- # files for this view.
- cv = v[index+3:].strip() # +3 to remove previous '...'
-
- # if the client view doesn't end with a
- # ... wildcard, then we're going to mess up the
- # output directory, so fail gracefully.
- if not cv.endswith('...'):
- print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
- sys.exit(1)
- cv=cv[:-3]
-
- # now save the view; +index means included, -index
- # means it should be filtered out.
- v = v[start:index]
- if v.startswith("-"):
- v = v[1:]
- include = -len(v)
- else:
- include = len(v)
+ specList = p4CmdList("client -o")
+ if len(specList) != 1:
+ die('Output from "client -o" is %d lines, expecting 1' %
+ len(specList))
- temp[v] = (include, cv)
+ # dictionary of all client parameters
+ entry = specList[0]
- self.clientSpecDirs = temp.items()
- self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
+ # just the keys that start with "View"
+ view_keys = [ k for k in entry.keys() if k.startswith("View") ]
+
+ # hold this new View
+ view = View()
+
+ # append the lines, in order, to the view
+ for view_num in range(len(view_keys)):
+ k = "View%d" % view_num
+ if k not in view_keys:
+ die("Expected view key %s missing" % k)
+ view.append(entry[k])
+
+ self.clientSpecDirs = view
+ if self.verbose:
+ for i, m in enumerate(self.clientSpecDirs.mappings):
+ print "clientSpecDirs %d: %s" % (i, str(m))
def run(self, args):
self.depotPaths = []
if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
- if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
+ if not self.useClientSpec:
+ if gitConfig("git-p4.useclientspec", "--bool") == "true":
+ self.useClientSpec = True
+ if self.useClientSpec:
self.getClientSpec()
# TODO: should always look at previous commits,
revision = ""
self.users = {}
+ # Make sure no revision specifiers are used when --changesfile
+ # is specified.
+ bad_changesfile = False
+ if len(self.changesFile) > 0:
+ for p in self.depotPaths:
+ if p.find("@") >= 0 or p.find("#") >= 0:
+ bad_changesfile = True
+ break
+ if bad_changesfile:
+ die("Option --changesfile is incompatible with revision specifiers")
+
newPaths = []
for p in self.depotPaths:
if p.find("@") != -1:
revision = p[hashIdx:]
p = p[:hashIdx]
elif self.previousDepotPaths == []:
- revision = "#head"
+ # pay attention to changesfile, if given, else import
+ # the entire p4 tree at the head revision
+ if len(self.changesFile) == 0:
+ revision = "#head"
p = re.sub ("\.\.\.$", "", p)
if not p.endswith("/"):
self.gitOutput.close()
self.gitError.close()
+ # Cleanup temporary branches created during import
+ if self.tempBranches != []:
+ for branch in self.tempBranches:
+ read_pipe("git update-ref -d %s" % branch)
+ os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
+
return True
class P4Rebase(Command):
args = sys.argv[2:]
if len(options) > 0:
- options.append(optparse.make_option("--git-dir", dest="gitdir"))
+ if cmd.needsGit:
+ options.append(optparse.make_option("--git-dir", dest="gitdir"))
parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
options,
if not cmd.run(args):
parser.print_help()
+ sys.exit(2)
if __name__ == '__main__':