import stat
import zipfile
import zlib
+import ctypes
+import errno
try:
from subprocess import CalledProcessError
# assume it failed because @... was invalid changelist
return True
-def system(cmd):
+def system(cmd, ignore_error=False):
expand = isinstance(cmd,basestring)
if verbose:
sys.stderr.write("executing %s\n" % str(cmd))
retcode = subprocess.call(cmd, shell=expand)
- if retcode:
+ if retcode and not ignore_error:
raise CalledProcessError(retcode, cmd)
+ return retcode
+
def p4_system(cmd):
"""Specifically invoke p4 as the system command. """
real_cmd = p4_build_cmd(cmd)
def p4_delete(f):
p4_system(["delete", wildcard_encode(f)])
-def p4_edit(f):
- p4_system(["edit", wildcard_encode(f)])
+def p4_edit(f, *options):
+ p4_system(["edit"] + list(options) + [wildcard_encode(f)])
def p4_revert(f):
p4_system(["revert", wildcard_encode(f)])
return clientPath
def currentGitBranch():
- return read_pipe("git name-rev HEAD").split(" ")[1].strip()
+ retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
+ if retcode != 0:
+ # on a detached head
+ return None
+ else:
+ return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
def isValidGitDir(path):
if (os.path.exists(path + "/HEAD")
die("cannot use --changes-block-size with non-numeric revisions")
block_size = None
- # Accumulate change numbers in a dictionary to avoid duplicates
- changes = {}
+ changes = set()
- for p in depotPaths:
- # Retrieve changes a block at a time, to prevent running
- # into a MaxResults/MaxScanRows error from the server.
+ # Retrieve changes a block at a time, to prevent running
+ # into a MaxResults/MaxScanRows error from the server.
- while True:
- cmd = ['changes']
+ while True:
+ cmd = ['changes']
- if block_size:
- end = min(changeEnd, changeStart + block_size)
- revisionRange = "%d,%d" % (changeStart, end)
- else:
- revisionRange = "%s,%s" % (changeStart, changeEnd)
+ if block_size:
+ end = min(changeEnd, changeStart + block_size)
+ revisionRange = "%d,%d" % (changeStart, end)
+ else:
+ revisionRange = "%s,%s" % (changeStart, changeEnd)
+ for p in depotPaths:
cmd += ["%s...@%s" % (p, revisionRange)]
- for line in p4_read_pipe_lines(cmd):
- changeNum = int(line.split(" ")[1])
- changes[changeNum] = True
+ # Insert changes in chronological order
+ for line in reversed(p4_read_pipe_lines(cmd)):
+ changes.add(int(line.split(" ")[1]))
- if not block_size:
- break
+ if not block_size:
+ break
- if end >= changeEnd:
- break
+ if end >= changeEnd:
+ break
- changeStart = end + 1
+ changeStart = end + 1
- changelist = changes.keys()
- changelist.sort()
- return changelist
+ changes = sorted(changes)
+ return changes
def p4PathStartsWith(path, prefix):
# This method tries to remedy a potential mixed-case issue:
steps."""
if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
contentTempFile = self.generateTempFile(contents)
- (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
-
- # Move temp file to final location in large file system
- largeFileDir = os.path.dirname(localLargeFile)
- if not os.path.isdir(largeFileDir):
- os.makedirs(largeFileDir)
- shutil.move(contentTempFile, localLargeFile)
- self.addLargeFile(relPath)
- if gitConfigBool('git-p4.largeFilePush'):
- self.pushFile(localLargeFile)
- if verbose:
- sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
+ (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
+ if pointer_git_mode:
+ git_mode = pointer_git_mode
+ if localLargeFile:
+ # Move temp file to final location in large file system
+ largeFileDir = os.path.dirname(localLargeFile)
+ if not os.path.isdir(largeFileDir):
+ os.makedirs(largeFileDir)
+ shutil.move(contentTempFile, localLargeFile)
+ self.addLargeFile(relPath)
+ if gitConfigBool('git-p4.largeFilePush'):
+ self.pushFile(localLargeFile)
+ if verbose:
+ sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
return (git_mode, contents)
class MockLFS(LargeFileSystem):
the actual content. Return also the new location of the actual
content.
"""
+ if os.path.getsize(contentFile) == 0:
+ return (None, '', None)
+
pointerProcess = subprocess.Popen(
['git', 'lfs', 'pointer', '--file=' + contentFile],
stdout=subprocess.PIPE
if pointerProcess.wait():
os.remove(contentFile)
die('git-lfs pointer command failed. Did you install the extension?')
- pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
- oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
+
+ # Git LFS removed the preamble in the output of the 'pointer' command
+ # starting from version 1.2.0. Check for the preamble here to support
+ # earlier versions.
+ # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
+ if pointerFile.startswith('Git LFS pointer for'):
+ pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
+
+ oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
localLargeFile = os.path.join(
os.getcwd(),
'.git', 'lfs', 'objects', oid[:2], oid[2:4],
)
# LFS Spec states that pointer files should not have the executable bit set.
gitMode = '100644'
- return (gitMode, pointerContents, localLargeFile)
+ return (gitMode, pointerFile, localLargeFile)
def pushFile(self, localLargeFile):
uploadProcess = subprocess.Popen(
self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
self.emails[output["Email"]] = output["User"]
+ mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
+ for mapUserConfig in gitConfigList("git-p4.mapUser"):
+ mapUser = mapUserConfigRegex.findall(mapUserConfig)
+ if mapUser and len(mapUser[0]) == 3:
+ user = mapUser[0][0]
+ fullname = mapUser[0][1]
+ email = mapUser[0][2]
+ self.users[user] = fullname + " <" + email + ">"
+ self.emails[email] = user
s = ''
for (key, val) in self.users.items():
Remove lines in the Files section that show changes to files
outside the depot path we're committing into."""
+ [upstream, settings] = findUpstreamBranchPoint()
+
template = ""
inFilesSection = False
for line in p4_read_pipe_lines(['change', '-o']):
lastTab = path.rfind("\t")
if lastTab != -1:
path = path[:lastTab]
- if not p4PathStartsWith(path, self.depotPath):
- continue
+ if settings.has_key('depot-paths'):
+ if not [p for p in settings['depot-paths']
+ if p4PathStartsWith(path, p)]:
+ continue
+ else:
+ if not p4PathStartsWith(path, self.depotPath):
+ continue
else:
inFilesSection = False
else:
if response == 'n':
return False
- def get_diff_description(self, editedFiles, filesToAdd):
+ def get_diff_description(self, editedFiles, filesToAdd, symlinks):
# diff
if os.environ.has_key("P4DIFF"):
del(os.environ["P4DIFF"])
newdiff += "==== new file ====\n"
newdiff += "--- /dev/null\n"
newdiff += "+++ %s\n" % newFile
- f = open(newFile, "r")
- for line in f.readlines():
- newdiff += "+" + line
- f.close()
+
+ is_link = os.path.islink(newFile)
+ expect_link = newFile in symlinks
+
+ if is_link and expect_link:
+ newdiff += "+%s\n" % os.readlink(newFile)
+ else:
+ f = open(newFile, "r")
+ for line in f.readlines():
+ newdiff += "+" + line
+ f.close()
return (diff + newdiff).replace('\r\n', '\n')
diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
filesToAdd = set()
+ filesToChangeType = set()
filesToDelete = set()
editedFiles = set()
pureRenameCopy = set()
+ symlinks = set()
filesToChangeExecBit = {}
for line in diff:
filesToChangeExecBit[path] = diff['dst_mode']
if path in filesToDelete:
filesToDelete.remove(path)
+
+ dst_mode = int(diff['dst_mode'], 8)
+ if dst_mode == 0120000:
+ symlinks.add(path)
+
elif modifier == "D":
filesToDelete.add(path)
if path in filesToAdd:
os.unlink(dest)
filesToDelete.add(src)
editedFiles.add(dest)
+ elif modifier == "T":
+ filesToChangeType.add(path)
else:
die("unknown modifier %s for %s" % (modifier, path))
#
system(applyPatchCmd)
+ for f in filesToChangeType:
+ p4_edit(f, "-t", "auto")
for f in filesToAdd:
p4_add(f)
for f in filesToDelete:
separatorLine = "######## everything below this line is just the diff #######\n"
if not self.prepare_p4_only:
submitTemplate += separatorLine
- submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
+ submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
(handle, fileName) = tempfile.mkstemp()
tmpFile = os.fdopen(handle, "w+b")
#
# Let the user edit the change description, then submit it.
#
- if self.edit_template(fileName):
- # read the edited message and submit
- ret = True
- tmpFile = open(fileName, "rb")
- message = tmpFile.read()
- tmpFile.close()
- if self.isWindows:
- message = message.replace("\r\n", "\n")
- submitTemplate = message[:message.index(separatorLine)]
- p4_write_pipe(['submit', '-i'], submitTemplate)
-
- if self.preserveUser:
- if p4User:
- # Get last changelist number. Cannot easily get it from
- # the submit command output as the output is
- # unmarshalled.
- changelist = self.lastP4Changelist()
- self.modifyChangelistUser(changelist, p4User)
-
- # The rename/copy happened by applying a patch that created a
- # new file. This leaves it writable, which confuses p4.
- for f in pureRenameCopy:
- p4_sync(f, "-f")
+ submitted = False
- else:
+ try:
+ if self.edit_template(fileName):
+ # read the edited message and submit
+ tmpFile = open(fileName, "rb")
+ message = tmpFile.read()
+ tmpFile.close()
+ if self.isWindows:
+ message = message.replace("\r\n", "\n")
+ submitTemplate = message[:message.index(separatorLine)]
+ p4_write_pipe(['submit', '-i'], submitTemplate)
+
+ if self.preserveUser:
+ if p4User:
+ # Get last changelist number. Cannot easily get it from
+ # the submit command output as the output is
+ # unmarshalled.
+ changelist = self.lastP4Changelist()
+ self.modifyChangelistUser(changelist, p4User)
+
+ # The rename/copy happened by applying a patch that created a
+ # new file. This leaves it writable, which confuses p4.
+ for f in pureRenameCopy:
+ p4_sync(f, "-f")
+ submitted = True
+
+ finally:
# skip this patch
- ret = False
- print "Submission cancelled, undoing p4 changes."
- for f in editedFiles:
- p4_revert(f)
- for f in filesToAdd:
- p4_revert(f)
- os.remove(f)
- for f in filesToDelete:
- p4_revert(f)
+ if not submitted:
+ print "Submission cancelled, undoing p4 changes."
+ for f in editedFiles:
+ p4_revert(f)
+ for f in filesToAdd:
+ p4_revert(f)
+ os.remove(f)
+ for f in filesToDelete:
+ p4_revert(f)
os.remove(fileName)
- return ret
+ return submitted
# Export git tags as p4 labels. Create a p4 label and then tag
# with that.
def run(self, args):
if len(args) == 0:
self.master = currentGitBranch()
- if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
- die("Detecting current git branch failed!")
elif len(args) == 1:
self.master = args[0]
if not branchExists(self.master):
else:
return False
- allowSubmit = gitConfig("git-p4.allowSubmit")
- if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
- die("%s is not in git-p4.allowSubmit" % self.master)
+ if self.master:
+ allowSubmit = gitConfig("git-p4.allowSubmit")
+ if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
+ die("%s is not in git-p4.allowSubmit" % self.master)
[upstream, settings] = findUpstreamBranchPoint()
self.depotPath = settings['depot-paths'][0]
if self.useClientSpec:
self.clientSpecDirs = getClientSpec()
- # Check for the existance of P4 branches
+ # Check for the existence of P4 branches
branchesDetected = (len(p4BranchesInGit().keys()) > 1)
if self.useClientSpec and not branchesDetected:
self.check()
commits = []
- for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
+ if self.master:
+ commitish = self.master
+ else:
+ commitish = 'HEAD'
+
+ for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
commits.append(line.strip())
commits.reverse()
self.useClientSpec_from_options = False
self.clientSpecDirs = None
self.tempBranches = []
- self.tempBranchLocation = "git-p4-tmp"
+ self.tempBranchLocation = "refs/git-p4-tmp"
self.largeFileSystem = None
if gitConfig('git-p4.largeFileSystem'):
fnum = fnum + 1
return files
+ def extractJobsFromCommit(self, commit):
+ jobs = []
+ jnum = 0
+ while commit.has_key("job%s" % jnum):
+ job = commit["job%s" % jnum]
+ jobs.append(job)
+ jnum = jnum + 1
+ return jobs
+
def stripRepoPath(self, path, prefixes):
"""When streaming files, this is called to map a p4 depot path
to where it should go in git. The prefixes are either
filesToDelete = []
for f in files:
- # if using a client spec, only add the files that have
- # a path in the client
- if self.clientSpecDirs:
- if self.clientSpecDirs.map_in_client(f['path']) == "":
- continue
-
filesForCommit.append(f)
if f['action'] in self.delete_actions:
filesToDelete.append(f)
gitStream.write(description)
gitStream.write("\n")
+ def inClientSpec(self, path):
+ if not self.clientSpecDirs:
+ return True
+ inClientSpec = self.clientSpecDirs.map_in_client(path)
+ if not inClientSpec and self.verbose:
+ print('Ignoring file outside of client spec: {0}'.format(path))
+ return inClientSpec
+
+ def hasBranchPrefix(self, path):
+ if not self.branchPrefixes:
+ return True
+ hasPrefix = [p for p in self.branchPrefixes
+ if p4PathStartsWith(path, p)]
+ if not hasPrefix and self.verbose:
+ print('Ignoring file outside of prefix: {0}'.format(path))
+ return hasPrefix
+
def commit(self, details, files, branch, parent = ""):
epoch = details["time"]
author = details["user"]
+ jobs = self.extractJobsFromCommit(details)
if self.verbose:
- print "commit into %s" % branch
-
- # start with reading files; if that fails, we should not
- # create a commit.
- new_files = []
- for f in files:
- if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
- new_files.append (f)
- else:
- sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
+ print('commit into {0}'.format(branch))
if self.clientSpecDirs:
self.clientSpecDirs.update_client_spec_path_cache(files)
+ files = [f for f in files
+ if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
+
+ if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
+ print('Ignoring revision {0} as it would produce an empty commit.'
+ .format(details['change']))
+ return
+
self.gitStream.write("commit %s\n" % branch)
self.gitStream.write("mark :%s\n" % details["change"])
self.committedChanges.add(int(details["change"]))
self.gitStream.write("data <<EOT\n")
self.gitStream.write(details["desc"])
+ if len(jobs) > 0:
+ self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
(','.join(self.branchPrefixes), details["change"]))
if len(details['options']) > 0:
print "parent %s" % parent
self.gitStream.write("from %s\n" % parent)
- self.streamP4Files(new_files)
+ self.streamP4Files(files)
self.gitStream.write("\n")
change = int(details["change"])