import re
import shutil
import stat
+import zipfile
+import zlib
+import ctypes
try:
from subprocess import CalledProcessError
# Only labels/tags matching this will be imported/exported
defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
+# Grab changes in blocks of this many revisions, unless otherwise requested
+defaultBlockSize = 512
+
def p4_build_cmd(cmd):
"""Build a suitable p4 command line.
path = os.getcwd()
os.environ['PWD'] = path
+def calcDiskFree():
+ """Return free space in bytes on the disk of the given dirname."""
+ if platform.system() == 'Windows':
+ free_bytes = ctypes.c_ulonglong(0)
+ ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
+ return free_bytes.value
+ else:
+ st = os.statvfs(os.getcwd())
+ return st.f_bavail * st.f_frsize
+
def die(msg):
if verbose:
raise Exception(msg)
sys.stderr.write('Reading pipe: %s\n' % str(c))
expand = isinstance(c,basestring)
- p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
- pipe = p.stdout
- val = pipe.read()
- if p.wait() and not ignore_error:
- die('Command failed: %s' % str(c))
-
- return val
+ p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
+ (out, err) = p.communicate()
+ if p.returncode != 0 and not ignore_error:
+ die('Command failed: %s\nError: %s' % (str(c), err))
+ return out
def p4_read_pipe(c, ignore_error=False):
real_cmd = p4_build_cmd(c)
# assume it failed because @... was invalid changelist
return True
-def system(cmd):
+def system(cmd, ignore_error=False):
expand = isinstance(cmd,basestring)
if verbose:
sys.stderr.write("executing %s\n" % str(cmd))
retcode = subprocess.call(cmd, shell=expand)
- if retcode:
+ if retcode and not ignore_error:
raise CalledProcessError(retcode, cmd)
+ return retcode
+
def p4_system(cmd):
"""Specifically invoke p4 as the system command. """
real_cmd = p4_build_cmd(cmd)
def p4_delete(f):
p4_system(["delete", wildcard_encode(f)])
-def p4_edit(f):
- p4_system(["edit", wildcard_encode(f)])
+def p4_edit(f, *options):
+ p4_system(["edit"] + list(options) + [wildcard_encode(f)])
def p4_revert(f):
p4_system(["revert", wildcard_encode(f)])
def p4_move(src, dest):
p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
+def p4_last_change():
+ results = p4CmdList(["changes", "-m", "1"])
+ return int(results[0]['change'])
+
def p4_describe(change):
"""Make sure it returns a valid result by checking for
the presence of field "time". Return a dict of the
#
# return the raw p4 type of a file (text, text+ko, etc)
#
-def p4_type(file):
- results = p4CmdList(["fstat", "-T", "headType", file])
+def p4_type(f):
+ results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
return results[0]['headType']
#
# Returns the perforce file type for the given file.
result = p4_read_pipe(["opened", wildcard_encode(file)])
- match = re.match(".*\((.+)\)\r?$", result)
+ match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
if match:
return match.group(1)
else:
def p4Where(depotPath):
if not depotPath.endswith("/"):
depotPath += "/"
- depotPath = depotPath + "..."
- outputList = p4CmdList(["where", depotPath])
+ depotPathLong = depotPath + "..."
+ outputList = p4CmdList(["where", depotPathLong])
output = None
for entry in outputList:
if "depotFile" in entry:
- if entry["depotFile"] == depotPath:
+ # Search for the base client side depot path, as long as it starts with the branch's P4 path.
+ # The base path always ends with "/...".
+ if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
output = entry
break
elif "data" in entry:
return clientPath
def currentGitBranch():
- return read_pipe("git name-rev HEAD").split(" ")[1].strip()
+ retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
+ if retcode != 0:
+ # on a detached head
+ return None
+ else:
+ return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
def isValidGitDir(path):
if (os.path.exists(path + "/HEAD")
_gitConfig = {}
-def gitConfig(key):
+def gitConfig(key, typeSpecifier=None):
if not _gitConfig.has_key(key):
- cmd = [ "git", "config", key ]
+ cmd = [ "git", "config" ]
+ if typeSpecifier:
+ cmd += [ typeSpecifier ]
+ cmd += [ key ]
s = read_pipe(cmd, ignore_error=True)
_gitConfig[key] = s.strip()
return _gitConfig[key]
in the config."""
if not _gitConfig.has_key(key):
- cmd = [ "git", "config", "--bool", key ]
+ _gitConfig[key] = gitConfig(key, '--bool') == "true"
+ return _gitConfig[key]
+
+def gitConfigInt(key):
+ if not _gitConfig.has_key(key):
+ cmd = [ "git", "config", "--int", key ]
s = read_pipe(cmd, ignore_error=True)
v = s.strip()
- _gitConfig[key] = v == "true"
+ try:
+ _gitConfig[key] = int(gitConfig(key, '--int'))
+ except ValueError:
+ _gitConfig[key] = None
return _gitConfig[key]
def gitConfigList(key):
if not _gitConfig.has_key(key):
s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
_gitConfig[key] = s.strip().split(os.linesep)
+ if _gitConfig[key] == ['']:
+ _gitConfig[key] = []
return _gitConfig[key]
def p4BranchesInGit(branchesAreInRemotes=True):
def originP4BranchesExist():
return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
-def p4ChangesForPaths(depotPaths, changeRange):
+
+def p4ParseNumericChangeRange(parts):
+ changeStart = int(parts[0][1:])
+ if parts[1] == '#head':
+ changeEnd = p4_last_change()
+ else:
+ changeEnd = int(parts[1])
+
+ return (changeStart, changeEnd)
+
+def chooseBlockSize(blockSize):
+ if blockSize:
+ return blockSize
+ else:
+ return defaultBlockSize
+
+def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
assert depotPaths
- cmd = ['changes']
- for p in depotPaths:
- cmd += ["%s...%s" % (p, changeRange)]
- output = p4_read_pipe_lines(cmd)
- changes = {}
- for line in output:
- changeNum = int(line.split(" ")[1])
- changes[changeNum] = True
+ # Parse the change range into start and end. Try to find integer
+ # revision ranges as these can be broken up into blocks to avoid
+ # hitting server-side limits (maxrows, maxscanresults). But if
+ # that doesn't work, fall back to using the raw revision specifier
+ # strings, without using block mode.
- changelist = changes.keys()
- changelist.sort()
- return changelist
+ if changeRange is None or changeRange == '':
+ changeStart = 1
+ changeEnd = p4_last_change()
+ block_size = chooseBlockSize(requestedBlockSize)
+ else:
+ parts = changeRange.split(',')
+ assert len(parts) == 2
+ try:
+ (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
+ block_size = chooseBlockSize(requestedBlockSize)
+ except:
+ changeStart = parts[0][1:]
+ changeEnd = parts[1]
+ if requestedBlockSize:
+ die("cannot use --changes-block-size with non-numeric revisions")
+ block_size = None
+
+ changes = []
+
+ # Retrieve changes a block at a time, to prevent running
+ # into a MaxResults/MaxScanRows error from the server.
+
+ while True:
+ cmd = ['changes']
+
+ if block_size:
+ end = min(changeEnd, changeStart + block_size)
+ revisionRange = "%d,%d" % (changeStart, end)
+ else:
+ revisionRange = "%s,%s" % (changeStart, changeEnd)
+
+ for p in depotPaths:
+ cmd += ["%s...@%s" % (p, revisionRange)]
+
+ # Insert changes in chronological order
+ for line in reversed(p4_read_pipe_lines(cmd)):
+ changes.append(int(line.split(" ")[1]))
+
+ if not block_size:
+ break
+
+ if end >= changeEnd:
+ break
+
+ changeStart = end + 1
+
+ changes = sorted(changes)
+ return changes
def p4PathStartsWith(path, prefix):
# This method tries to remedy a potential mixed-case issue:
# dictionary of all client parameters
entry = specList[0]
+ # the //client/ name
+ client_name = entry["Client"]
+
# just the keys that start with "View"
view_keys = [ k for k in entry.keys() if k.startswith("View") ]
# hold this new View
- view = View()
+ view = View(client_name)
# append the lines, in order, to the view
for view_num in range(len(view_keys)):
m = re.search("[*#@%]", path)
return m is not None
+class LargeFileSystem(object):
+ """Base class for large file system support."""
+
+ def __init__(self, writeToGitStream):
+ self.largeFiles = set()
+ self.writeToGitStream = writeToGitStream
+
+ def generatePointer(self, cloneDestination, contentFile):
+ """Return the content of a pointer file that is stored in Git instead of
+ the actual content."""
+ assert False, "Method 'generatePointer' required in " + self.__class__.__name__
+
+ def pushFile(self, localLargeFile):
+ """Push the actual content which is not stored in the Git repository to
+ a server."""
+ assert False, "Method 'pushFile' required in " + self.__class__.__name__
+
+ def hasLargeFileExtension(self, relPath):
+ return reduce(
+ lambda a, b: a or b,
+ [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
+ False
+ )
+
+ def generateTempFile(self, contents):
+ contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
+ for d in contents:
+ contentFile.write(d)
+ contentFile.close()
+ return contentFile.name
+
+ def exceedsLargeFileThreshold(self, relPath, contents):
+ if gitConfigInt('git-p4.largeFileThreshold'):
+ contentsSize = sum(len(d) for d in contents)
+ if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
+ return True
+ if gitConfigInt('git-p4.largeFileCompressedThreshold'):
+ contentsSize = sum(len(d) for d in contents)
+ if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
+ return False
+ contentTempFile = self.generateTempFile(contents)
+ compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
+ zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
+ zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
+ zf.close()
+ compressedContentsSize = zf.infolist()[0].compress_size
+ os.remove(contentTempFile)
+ os.remove(compressedContentFile.name)
+ if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
+ return True
+ return False
+
+ def addLargeFile(self, relPath):
+ self.largeFiles.add(relPath)
+
+ def removeLargeFile(self, relPath):
+ self.largeFiles.remove(relPath)
+
+ def isLargeFile(self, relPath):
+ return relPath in self.largeFiles
+
+ def processContent(self, git_mode, relPath, contents):
+ """Processes the content of git fast import. This method decides if a
+ file is stored in the large file system and handles all necessary
+ steps."""
+ if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
+ contentTempFile = self.generateTempFile(contents)
+ (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
+
+ # Move temp file to final location in large file system
+ largeFileDir = os.path.dirname(localLargeFile)
+ if not os.path.isdir(largeFileDir):
+ os.makedirs(largeFileDir)
+ shutil.move(contentTempFile, localLargeFile)
+ self.addLargeFile(relPath)
+ if gitConfigBool('git-p4.largeFilePush'):
+ self.pushFile(localLargeFile)
+ if verbose:
+ sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
+ return (git_mode, contents)
+
+class MockLFS(LargeFileSystem):
+ """Mock large file system for testing."""
+
+ def generatePointer(self, contentFile):
+ """The pointer content is the original content prefixed with "pointer-".
+ The local filename of the large file storage is derived from the file content.
+ """
+ with open(contentFile, 'r') as f:
+ content = next(f)
+ gitMode = '100644'
+ pointerContents = 'pointer-' + content
+ localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
+ return (gitMode, pointerContents, localLargeFile)
+
+ def pushFile(self, localLargeFile):
+ """The remote filename of the large file storage is the same as the local
+ one but in a different directory.
+ """
+ remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
+ if not os.path.exists(remotePath):
+ os.makedirs(remotePath)
+ shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
+
+class GitLFS(LargeFileSystem):
+ """Git LFS as backend for the git-p4 large file system.
+ See https://git-lfs.github.com/ for details."""
+
+ def __init__(self, *args):
+ LargeFileSystem.__init__(self, *args)
+ self.baseGitAttributes = []
+
+ def generatePointer(self, contentFile):
+ """Generate a Git LFS pointer for the content. Return LFS Pointer file
+ mode and content which is stored in the Git repository instead of
+ the actual content. Return also the new location of the actual
+ content.
+ """
+ pointerProcess = subprocess.Popen(
+ ['git', 'lfs', 'pointer', '--file=' + contentFile],
+ stdout=subprocess.PIPE
+ )
+ pointerFile = pointerProcess.stdout.read()
+ if pointerProcess.wait():
+ os.remove(contentFile)
+ die('git-lfs pointer command failed. Did you install the extension?')
+
+ # Git LFS removed the preamble in the output of the 'pointer' command
+ # starting from version 1.2.0. Check for the preamble here to support
+ # earlier versions.
+ # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
+ if pointerFile.startswith('Git LFS pointer for'):
+ pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
+
+ oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
+ localLargeFile = os.path.join(
+ os.getcwd(),
+ '.git', 'lfs', 'objects', oid[:2], oid[2:4],
+ oid,
+ )
+ # LFS Spec states that pointer files should not have the executable bit set.
+ gitMode = '100644'
+ return (gitMode, pointerFile, localLargeFile)
+
+ def pushFile(self, localLargeFile):
+ uploadProcess = subprocess.Popen(
+ ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
+ )
+ if uploadProcess.wait():
+ die('git-lfs push command failed. Did you define a remote?')
+
+ def generateGitAttributes(self):
+ return (
+ self.baseGitAttributes +
+ [
+ '\n',
+ '#\n',
+ '# Git LFS (see https://git-lfs.github.com/)\n',
+ '#\n',
+ ] +
+ ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
+ for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
+ ] +
+ ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
+ for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
+ ]
+ )
+
+ def addLargeFile(self, relPath):
+ LargeFileSystem.addLargeFile(self, relPath)
+ self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
+
+ def removeLargeFile(self, relPath):
+ LargeFileSystem.removeLargeFile(self, relPath)
+ self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
+
+ def processContent(self, git_mode, relPath, contents):
+ if relPath == '.gitattributes':
+ self.baseGitAttributes = contents
+ return (git_mode, self.generateGitAttributes())
+ else:
+ return LargeFileSystem.processContent(self, git_mode, relPath, contents)
+
class Command:
def __init__(self):
self.usage = "usage: %prog [options]"
self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
self.emails[output["Email"]] = output["User"]
+ mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
+ for mapUserConfig in gitConfigList("git-p4.mapUser"):
+ mapUser = mapUserConfigRegex.findall(mapUserConfig)
+ if mapUser and len(mapUser[0]) == 3:
+ user = mapUser[0][0]
+ fullname = mapUser[0][1]
+ email = mapUser[0][2]
+ self.users[user] = fullname + " <" + email + ">"
+ self.emails[email] = user
s = ''
for (key, val) in self.users.items():
self.p4HasMoveCommand = p4_has_move_command()
self.branch = None
+ if gitConfig('git-p4.largeFileSystem'):
+ die("Large file system not supported for git-p4 submit command. Please remove it from config.")
+
def check(self):
if len(p4CmdList("opened ...")) > 0:
die("You have files opened with perforce! Close them before starting the sync.")
Remove lines in the Files section that show changes to files
outside the depot path we're committing into."""
+ [upstream, settings] = findUpstreamBranchPoint()
+
template = ""
inFilesSection = False
for line in p4_read_pipe_lines(['change', '-o']):
lastTab = path.rfind("\t")
if lastTab != -1:
path = path[:lastTab]
- if not p4PathStartsWith(path, self.depotPath):
- continue
+ if settings.has_key('depot-paths'):
+ if not [p for p in settings['depot-paths']
+ if p4PathStartsWith(path, p)]:
+ continue
+ else:
+ if not p4PathStartsWith(path, self.depotPath):
+ continue
else:
inFilesSection = False
else:
editor = os.environ.get("P4EDITOR")
else:
editor = read_pipe("git var GIT_EDITOR").strip()
- system(editor + " " + template_file)
+ system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
# If the file was not saved, prompt to see if this patch should
# be skipped. But skip this verification step if configured so.
if response == 'n':
return False
+ def get_diff_description(self, editedFiles, filesToAdd):
+ # diff
+ if os.environ.has_key("P4DIFF"):
+ del(os.environ["P4DIFF"])
+ diff = ""
+ for editedFile in editedFiles:
+ diff += p4_read_pipe(['diff', '-du',
+ wildcard_encode(editedFile)])
+
+ # new file diff
+ newdiff = ""
+ for newFile in filesToAdd:
+ newdiff += "==== new file ====\n"
+ newdiff += "--- /dev/null\n"
+ newdiff += "+++ %s\n" % newFile
+ f = open(newFile, "r")
+ for line in f.readlines():
+ newdiff += "+" + line
+ f.close()
+
+ return (diff + newdiff).replace('\r\n', '\n')
+
def applyCommit(self, id):
"""Apply one commit, return True if it succeeded."""
diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
filesToAdd = set()
+ filesToChangeType = set()
filesToDelete = set()
editedFiles = set()
pureRenameCopy = set()
os.unlink(dest)
filesToDelete.add(src)
editedFiles.add(dest)
+ elif modifier == "T":
+ filesToChangeType.add(path)
else:
die("unknown modifier %s for %s" % (modifier, path))
#
system(applyPatchCmd)
+ for f in filesToChangeType:
+ p4_edit(f, "-t", "auto")
for f in filesToAdd:
p4_add(f)
for f in filesToDelete:
submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
separatorLine = "######## everything below this line is just the diff #######\n"
+ if not self.prepare_p4_only:
+ submitTemplate += separatorLine
+ submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
- # diff
- if os.environ.has_key("P4DIFF"):
- del(os.environ["P4DIFF"])
- diff = ""
- for editedFile in editedFiles:
- diff += p4_read_pipe(['diff', '-du',
- wildcard_encode(editedFile)])
-
- # new file diff
- newdiff = ""
- for newFile in filesToAdd:
- newdiff += "==== new file ====\n"
- newdiff += "--- /dev/null\n"
- newdiff += "+++ %s\n" % newFile
- f = open(newFile, "r")
- for line in f.readlines():
- newdiff += "+" + line
- f.close()
-
- # change description file: submitTemplate, separatorLine, diff, newdiff
(handle, fileName) = tempfile.mkstemp()
- tmpFile = os.fdopen(handle, "w+")
+ tmpFile = os.fdopen(handle, "w+b")
if self.isWindows:
submitTemplate = submitTemplate.replace("\n", "\r\n")
- separatorLine = separatorLine.replace("\n", "\r\n")
- newdiff = newdiff.replace("\n", "\r\n")
- tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
+ tmpFile.write(submitTemplate)
tmpFile.close()
if self.prepare_p4_only:
print " " + self.clientPath
print
print "To submit, use \"p4 submit\" to write a new description,"
- print "or \"p4 submit -i %s\" to use the one prepared by" \
+ print "or \"p4 submit -i <%s\" to use the one prepared by" \
" \"git p4\"." % fileName
print "You can delete the file \"%s\" when finished." % fileName
#
# Let the user edit the change description, then submit it.
#
- if self.edit_template(fileName):
- # read the edited message and submit
- ret = True
- tmpFile = open(fileName, "rb")
- message = tmpFile.read()
- tmpFile.close()
- submitTemplate = message[:message.index(separatorLine)]
- if self.isWindows:
- submitTemplate = submitTemplate.replace("\r\n", "\n")
- p4_write_pipe(['submit', '-i'], submitTemplate)
-
- if self.preserveUser:
- if p4User:
- # Get last changelist number. Cannot easily get it from
- # the submit command output as the output is
- # unmarshalled.
- changelist = self.lastP4Changelist()
- self.modifyChangelistUser(changelist, p4User)
-
- # The rename/copy happened by applying a patch that created a
- # new file. This leaves it writable, which confuses p4.
- for f in pureRenameCopy:
- p4_sync(f, "-f")
+ submitted = False
- else:
+ try:
+ if self.edit_template(fileName):
+ # read the edited message and submit
+ tmpFile = open(fileName, "rb")
+ message = tmpFile.read()
+ tmpFile.close()
+ if self.isWindows:
+ message = message.replace("\r\n", "\n")
+ submitTemplate = message[:message.index(separatorLine)]
+ p4_write_pipe(['submit', '-i'], submitTemplate)
+
+ if self.preserveUser:
+ if p4User:
+ # Get last changelist number. Cannot easily get it from
+ # the submit command output as the output is
+ # unmarshalled.
+ changelist = self.lastP4Changelist()
+ self.modifyChangelistUser(changelist, p4User)
+
+ # The rename/copy happened by applying a patch that created a
+ # new file. This leaves it writable, which confuses p4.
+ for f in pureRenameCopy:
+ p4_sync(f, "-f")
+ submitted = True
+
+ finally:
# skip this patch
- ret = False
- print "Submission cancelled, undoing p4 changes."
- for f in editedFiles:
- p4_revert(f)
- for f in filesToAdd:
- p4_revert(f)
- os.remove(f)
- for f in filesToDelete:
- p4_revert(f)
+ if not submitted:
+ print "Submission cancelled, undoing p4 changes."
+ for f in editedFiles:
+ p4_revert(f)
+ for f in filesToAdd:
+ p4_revert(f)
+ os.remove(f)
+ for f in filesToDelete:
+ p4_revert(f)
os.remove(fileName)
- return ret
+ return submitted
# Export git tags as p4 labels. Create a p4 label and then tag
# with that.
for b in body:
labelTemplate += "\t" + b + "\n"
labelTemplate += "View:\n"
- for mapping in clientSpec.mappings:
- labelTemplate += "\t%s\n" % mapping.depot_side.path
+ for depot_side in clientSpec.mappings:
+ labelTemplate += "\t%s\n" % depot_side
if self.dry_run:
print "Would create p4 label %s for tag" % name
# Use the label
p4_system(["tag", "-l", name] +
- ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
+ ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
if verbose:
print "created p4 label for tag %s" % name
def run(self, args):
if len(args) == 0:
self.master = currentGitBranch()
- if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
- die("Detecting current git branch failed!")
elif len(args) == 1:
self.master = args[0]
if not branchExists(self.master):
else:
return False
- allowSubmit = gitConfig("git-p4.allowSubmit")
- if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
- die("%s is not in git-p4.allowSubmit" % self.master)
+ if self.master:
+ allowSubmit = gitConfig("git-p4.allowSubmit")
+ if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
+ die("%s is not in git-p4.allowSubmit" % self.master)
[upstream, settings] = findUpstreamBranchPoint()
self.depotPath = settings['depot-paths'][0]
if self.useClientSpec:
self.clientSpecDirs = getClientSpec()
- if self.useClientSpec:
+ # Check for the existance of P4 branches
+ branchesDetected = (len(p4BranchesInGit().keys()) > 1)
+
+ if self.useClientSpec and not branchesDetected:
# all files are relative to the client spec
self.clientPath = getClientRoot()
else:
self.check()
commits = []
- for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
+ if self.master:
+ commitish = self.master
+ else:
+ commitish = 'HEAD'
+
+ for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
commits.append(line.strip())
commits.reverse()
"""Represent a p4 view ("p4 help views"), and map files in a
repo according to the view."""
- class Path(object):
- """A depot or client path, possibly containing wildcards.
- The only one supported is ... at the end, currently.
- Initialize with the full path, with //depot or //client."""
-
- def __init__(self, path, is_depot):
- self.path = path
- self.is_depot = is_depot
- self.find_wildcards()
- # remember the prefix bit, useful for relative mappings
- m = re.match("(//[^/]+/)", self.path)
- if not m:
- die("Path %s does not start with //prefix/" % self.path)
- prefix = m.group(1)
- if not self.is_depot:
- # strip //client/ on client paths
- self.path = self.path[len(prefix):]
-
- def find_wildcards(self):
- """Make sure wildcards are valid, and set up internal
- variables."""
-
- self.ends_triple_dot = False
- # There are three wildcards allowed in p4 views
- # (see "p4 help views"). This code knows how to
- # handle "..." (only at the end), but cannot deal with
- # "%%n" or "*". Only check the depot_side, as p4 should
- # validate that the client_side matches too.
- if re.search(r'%%[1-9]', self.path):
- die("Can't handle %%n wildcards in view: %s" % self.path)
- if self.path.find("*") >= 0:
- die("Can't handle * wildcards in view: %s" % self.path)
- triple_dot_index = self.path.find("...")
- if triple_dot_index >= 0:
- if triple_dot_index != len(self.path) - 3:
- die("Can handle only single ... wildcard, at end: %s" %
- self.path)
- self.ends_triple_dot = True
-
- def ensure_compatible(self, other_path):
- """Make sure the wildcards agree."""
- if self.ends_triple_dot != other_path.ends_triple_dot:
- die("Both paths must end with ... if either does;\n" +
- "paths: %s %s" % (self.path, other_path.path))
-
- def match_wildcards(self, test_path):
- """See if this test_path matches us, and fill in the value
- of the wildcards if so. Returns a tuple of
- (True|False, wildcards[]). For now, only the ... at end
- is supported, so at most one wildcard."""
- if self.ends_triple_dot:
- dotless = self.path[:-3]
- if test_path.startswith(dotless):
- wildcard = test_path[len(dotless):]
- return (True, [ wildcard ])
- else:
- if test_path == self.path:
- return (True, [])
- return (False, [])
-
- def match(self, test_path):
- """Just return if it matches; don't bother with the wildcards."""
- b, _ = self.match_wildcards(test_path)
- return b
-
- def fill_in_wildcards(self, wildcards):
- """Return the relative path, with the wildcards filled in
- if there are any."""
- if self.ends_triple_dot:
- return self.path[:-3] + wildcards[0]
- else:
- return self.path
-
- class Mapping(object):
- def __init__(self, depot_side, client_side, overlay, exclude):
- # depot_side is without the trailing /... if it had one
- self.depot_side = View.Path(depot_side, is_depot=True)
- self.client_side = View.Path(client_side, is_depot=False)
- self.overlay = overlay # started with "+"
- self.exclude = exclude # started with "-"
- assert not (self.overlay and self.exclude)
- self.depot_side.ensure_compatible(self.client_side)
-
- def __str__(self):
- c = " "
- if self.overlay:
- c = "+"
- if self.exclude:
- c = "-"
- return "View.Mapping: %s%s -> %s" % \
- (c, self.depot_side.path, self.client_side.path)
-
- def map_depot_to_client(self, depot_path):
- """Calculate the client path if using this mapping on the
- given depot path; does not consider the effect of other
- mappings in a view. Even excluded mappings are returned."""
- matches, wildcards = self.depot_side.match_wildcards(depot_path)
- if not matches:
- return ""
- client_path = self.client_side.fill_in_wildcards(wildcards)
- return client_path
-
- #
- # View methods
- #
- def __init__(self):
+ def __init__(self, client_name):
self.mappings = []
+ self.client_prefix = "//%s/" % client_name
+ # cache results of "p4 where" to lookup client file locations
+ self.client_spec_path_cache = {}
def append(self, view_line):
"""Parse a view line, splitting it into depot and client
- sides. Append to self.mappings, preserving order."""
+ sides. Append to self.mappings, preserving order. This
+ is only needed for tag creation."""
# Split the view line into exactly two words. P4 enforces
# structure on these lines that simplifies this quite a bit.
depot_side = view_line[0:space_index]
rhs_index = space_index + 1
- if view_line[rhs_index] == '"':
- # Second word is double quoted. Make sure there is a
- # double quote at the end too.
- if not view_line.endswith('"'):
- die("View line with rhs quote should end with one: %s" %
- view_line)
- # skip the quotes
- client_side = view_line[rhs_index+1:-1]
- else:
- client_side = view_line[rhs_index:]
-
# prefix + means overlay on previous mapping
- overlay = False
if depot_side.startswith("+"):
- overlay = True
depot_side = depot_side[1:]
- # prefix - means exclude this path
+ # prefix - means exclude this path, leave out of mappings
exclude = False
if depot_side.startswith("-"):
exclude = True
depot_side = depot_side[1:]
- m = View.Mapping(depot_side, client_side, overlay, exclude)
- self.mappings.append(m)
+ if not exclude:
+ self.mappings.append(depot_side)
- def map_in_client(self, depot_path):
- """Return the relative location in the client where this
- depot file should live. Returns "" if the file should
- not be mapped in the client."""
+ def convert_client_path(self, clientFile):
+ # chop off //client/ part to make it relative
+ if not clientFile.startswith(self.client_prefix):
+ die("No prefix '%s' on clientFile '%s'" %
+ (self.client_prefix, clientFile))
+ return clientFile[len(self.client_prefix):]
- paths_filled = []
- client_path = ""
+ def update_client_spec_path_cache(self, files):
+ """ Caching file paths by "p4 where" batch query """
- # look at later entries first
- for m in self.mappings[::-1]:
+ # List depot file paths exclude that already cached
+ fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
- # see where will this path end up in the client
- p = m.map_depot_to_client(depot_path)
+ if len(fileArgs) == 0:
+ return # All files in cache
- if p == "":
- # Depot path does not belong in client. Must remember
- # this, as previous items should not cause files to
- # exist in this path either. Remember that the list is
- # being walked from the end, which has higher precedence.
- # Overlap mappings do not exclude previous mappings.
- if not m.overlay:
- paths_filled.append(m.client_side)
+ where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
+ for res in where_result:
+ if "code" in res and res["code"] == "error":
+ # assume error is "... file(s) not in client view"
+ continue
+ if "clientFile" not in res:
+ die("No clientFile in 'p4 where' output")
+ if "unmap" in res:
+ # it will list all of them, but only one not unmap-ped
+ continue
+ if gitConfigBool("core.ignorecase"):
+ res['depotFile'] = res['depotFile'].lower()
+ self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
- else:
- # This mapping matched; no need to search any further.
- # But, the mapping could be rejected if the client path
- # has already been claimed by an earlier mapping (i.e.
- # one later in the list, which we are walking backwards).
- already_mapped_in_client = False
- for f in paths_filled:
- # this is View.Path.match
- if f.match(p):
- already_mapped_in_client = True
- break
- if not already_mapped_in_client:
- # Include this file, unless it is from a line that
- # explicitly said to exclude it.
- if not m.exclude:
- client_path = p
+ # not found files or unmap files set to ""
+ for depotFile in fileArgs:
+ if gitConfigBool("core.ignorecase"):
+ depotFile = depotFile.lower()
+ if depotFile not in self.client_spec_path_cache:
+ self.client_spec_path_cache[depotFile] = ""
- # a match, even if rejected, always stops the search
- break
+ def map_in_client(self, depot_path):
+ """Return the relative location in the client where this
+ depot file should live. Returns "" if the file should
+ not be mapped in the client."""
+
+ if gitConfigBool("core.ignorecase"):
+ depot_path = depot_path.lower()
- return client_path
+ if depot_path in self.client_spec_path_cache:
+ return self.client_spec_path_cache[depot_path]
+
+ die( "Error: %s is not found in client spec path" % depot_path )
+ return ""
class P4Sync(Command, P4UserMap):
delete_actions = ( "delete", "move/delete", "purge" )
optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
help="Import into refs/heads/ , not refs/remotes"),
- optparse.make_option("--max-changes", dest="maxChanges"),
+ optparse.make_option("--max-changes", dest="maxChanges",
+ help="Maximum number of changes to import"),
+ optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
+ help="Internal block size to use when iteratively calling p4 changes"),
optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
- help="Only sync files that are included in the Perforce Client Spec")
+ help="Only sync files that are included in the Perforce Client Spec"),
+ optparse.make_option("-/", dest="cloneExclude",
+ action="append", type="string",
+ help="exclude depot path"),
]
self.description = """Imports from Perforce into a git repository.\n
example:
self.syncWithOrigin = True
self.importIntoRemotes = True
self.maxChanges = ""
+ self.changes_block_size = None
self.keepRepoPath = False
self.depotPaths = None
self.p4BranchesInGit = []
self.clientSpecDirs = None
self.tempBranches = []
self.tempBranchLocation = "git-p4-tmp"
+ self.largeFileSystem = None
+
+ if gitConfig('git-p4.largeFileSystem'):
+ largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
+ self.largeFileSystem = largeFileSystemConstructor(
+ lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
+ )
if gitConfig("git-p4.syncFromOrigin") == "false":
self.syncWithOrigin = False
+ # This is required for the "append" cloneExclude action
+ def ensure_value(self, attr, value):
+ if not hasattr(self, attr) or getattr(self, attr) is None:
+ setattr(self, attr, value)
+ return getattr(self, attr)
+
# Force a checkpoint in fast-import and wait for it to finish
def checkpoint(self):
self.gitStream.write("checkpoint\n\n")
fnum = fnum + 1
return files
+ def extractJobsFromCommit(self, commit):
+ jobs = []
+ jnum = 0
+ while commit.has_key("job%s" % jnum):
+ job = commit["job%s" % jnum]
+ jobs.append(job)
+ jnum = jnum + 1
+ return jobs
+
def stripRepoPath(self, path, prefixes):
"""When streaming files, this is called to map a p4 depot path
to where it should go in git. The prefixes are either
"""Look at each depotFile in the commit to figure out to what
branch it belongs."""
+ if self.clientSpecDirs:
+ files = self.extractFilesFromCommit(commit)
+ self.clientSpecDirs.update_client_spec_path_cache(files)
+
branches = {}
fnum = 0
while commit.has_key("depotFile%s" % fnum):
return branches
+ def writeToGitStream(self, gitMode, relPath, contents):
+ self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
+ self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
+ for d in contents:
+ self.gitStream.write(d)
+ self.gitStream.write('\n')
+
# output one file from the P4 stream
# - helper for streamP4Files
def streamOneP4File(self, file, contents):
relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
if verbose:
- sys.stderr.write("%s\n" % relPath)
+ size = int(self.stream_file['fileSize'])
+ sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
+ sys.stdout.flush()
(type_base, type_mods) = split_p4_type(file["type"])
git_mode = "100755"
if type_base == "symlink":
git_mode = "120000"
- # p4 print on a symlink contains "target\n"; remove the newline
+ # p4 print on a symlink sometimes contains "target\n";
+ # if it does, remove the newline
data = ''.join(contents)
- contents = [data[:-1]]
+ if not data:
+ # Some version of p4 allowed creating a symlink that pointed
+ # to nothing. This causes p4 errors when checking out such
+ # a change, and errors here too. Work around it by ignoring
+ # the bad symlink; hopefully a future change fixes it.
+ print "\nIgnoring empty symlink in %s" % file['depotFile']
+ return
+ elif data[-1] == '\n':
+ contents = [data[:-1]]
+ else:
+ contents = [data]
if type_base == "utf16":
# p4 delivers different text in the python output to -G
# them back too. This is not needed to the cygwin windows version,
# just the native "NT" type.
#
- text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
- if p4_version_string().find("/NT") >= 0:
- text = text.replace("\r\n", "\n")
- contents = [ text ]
+ try:
+ text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
+ except Exception as e:
+ if 'Translation of file content failed' in str(e):
+ type_base = 'binary'
+ else:
+ raise e
+ else:
+ if p4_version_string().find('/NT') >= 0:
+ text = text.replace('\r\n', '\n')
+ contents = [ text ]
if type_base == "apple":
# Apple filetype files will be streamed as a concatenation of
text = regexp.sub(r'$\1$', text)
contents = [ text ]
- self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
+ try:
+ relPath.decode('ascii')
+ except:
+ encoding = 'utf8'
+ if gitConfig('git-p4.pathEncoding'):
+ encoding = gitConfig('git-p4.pathEncoding')
+ relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
+ if self.verbose:
+ print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
- # total length...
- length = 0
- for d in contents:
- length = length + len(d)
+ if self.largeFileSystem:
+ (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
- self.gitStream.write("data %d\n" % length)
- for d in contents:
- self.gitStream.write(d)
- self.gitStream.write("\n")
+ self.writeToGitStream(git_mode, relPath, contents)
def streamOneP4Deletion(self, file):
relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
if verbose:
- sys.stderr.write("delete %s\n" % relPath)
+ sys.stdout.write("delete %s\n" % relPath)
+ sys.stdout.flush()
self.gitStream.write("D %s\n" % relPath)
+ if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
+ self.largeFileSystem.removeLargeFile(relPath)
+
# handle another chunk of streaming data
def streamP4FilesCb(self, marshalled):
if marshalled["code"] == "error":
if "data" in marshalled:
err = marshalled["data"].rstrip()
+
+ if not err and 'fileSize' in self.stream_file:
+ required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
+ if required_bytes > 0:
+ err = 'Not enough space left on %s! Free at least %i MB.' % (
+ os.getcwd(), required_bytes/1024/1024
+ )
+
if err:
f = None
if self.stream_have_file_info:
# 'data' field we need to append to our array
for k in marshalled.keys():
if k == 'data':
+ if 'streamContentSize' not in self.stream_file:
+ self.stream_file['streamContentSize'] = 0
+ self.stream_file['streamContentSize'] += len(marshalled['data'])
self.stream_contents.append(marshalled['data'])
else:
self.stream_file[k] = marshalled[k]
+ if (verbose and
+ 'streamContentSize' in self.stream_file and
+ 'fileSize' in self.stream_file and
+ 'depotFile' in self.stream_file):
+ size = int(self.stream_file["fileSize"])
+ if size > 0:
+ progress = 100*self.stream_file['streamContentSize']/size
+ sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
+ sys.stdout.flush()
+
self.stream_have_file_info = True
# Stream directly from "p4 files" into "git fast-import"
filesToDelete = []
for f in files:
- # if using a client spec, only add the files that have
- # a path in the client
- if self.clientSpecDirs:
- if self.clientSpecDirs.map_in_client(f['path']) == "":
- continue
-
filesForCommit.append(f)
if f['action'] in self.delete_actions:
filesToDelete.append(f)
else:
return "%s <a@b>" % userid
- # Stream a p4 tag
def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
+ """ Stream a p4 tag.
+ commit is either a git commit, or a fast-import mark, ":<p4commit>"
+ """
+
if verbose:
print "writing tag %s for commit %s" % (labelName, commit)
gitStream.write("tag %s\n" % labelName)
gitStream.write(description)
gitStream.write("\n")
+ def inClientSpec(self, path):
+ if not self.clientSpecDirs:
+ return True
+ inClientSpec = self.clientSpecDirs.map_in_client(path)
+ if not inClientSpec and self.verbose:
+ print('Ignoring file outside of client spec: {0}'.format(path))
+ return inClientSpec
+
+ def hasBranchPrefix(self, path):
+ if not self.branchPrefixes:
+ return True
+ hasPrefix = [p for p in self.branchPrefixes
+ if p4PathStartsWith(path, p)]
+ if not hasPrefix and self.verbose:
+ print('Ignoring file outside of prefix: {0}'.format(path))
+ return hasPrefix
+
def commit(self, details, files, branch, parent = ""):
epoch = details["time"]
author = details["user"]
+ jobs = self.extractJobsFromCommit(details)
if self.verbose:
- print "commit into %s" % branch
+ print('commit into {0}'.format(branch))
- # start with reading files; if that fails, we should not
- # create a commit.
- new_files = []
- for f in files:
- if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
- new_files.append (f)
- else:
- sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
+ if self.clientSpecDirs:
+ self.clientSpecDirs.update_client_spec_path_cache(files)
+
+ files = [f for f in files
+ if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
+
+ if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
+ print('Ignoring revision {0} as it would produce an empty commit.'
+ .format(details['change']))
+ return
self.gitStream.write("commit %s\n" % branch)
-# gitStream.write("mark :%s\n" % details["change"])
+ self.gitStream.write("mark :%s\n" % details["change"])
self.committedChanges.add(int(details["change"]))
committer = ""
if author not in self.users:
self.gitStream.write("data <<EOT\n")
self.gitStream.write(details["desc"])
+ if len(jobs) > 0:
+ self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
(','.join(self.branchPrefixes), details["change"]))
if len(details['options']) > 0:
print "parent %s" % parent
self.gitStream.write("from %s\n" % parent)
- self.streamP4Files(new_files)
+ self.streamP4Files(files)
self.gitStream.write("\n")
change = int(details["change"])
if change.has_key('change'):
# find the corresponding git commit; take the oldest commit
changelist = int(change['change'])
- gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
- "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
- if len(gitCommit) == 0:
- print "could not find git commit for changelist %d" % changelist
- else:
- gitCommit = gitCommit.strip()
+ if changelist in self.committedChanges:
+ gitCommit = ":%d" % changelist # use a fast-import mark
commitFound = True
+ else:
+ gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
+ "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
+ if len(gitCommit) == 0:
+ print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
+ else:
+ commitFound = True
+ gitCommit = gitCommit.strip()
+
+ if commitFound:
# Convert from p4 time format
try:
tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
branchPrefix = self.depotPaths[0] + branch + "/"
range = "@1,%s" % maxChange
#print "prefix" + branchPrefix
- changes = p4ChangesForPaths([branchPrefix], range)
+ changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
if len(changes) <= 0:
return False
firstChange = changes[0]
if self.verbose:
print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
self.changeRange)
- changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
+ changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
if len(self.maxChanges) > 0:
changes = changes[:min(int(self.maxChanges), len(changes))]
print "Rebasing the current branch onto %s" % upstream
oldHead = read_pipe("git rev-parse HEAD").strip()
system("git rebase %s" % upstream)
- system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
+ system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
return True
class P4Clone(P4Sync):
optparse.make_option("--destination", dest="cloneDestination",
action='store', default=None,
help="where to leave result of the clone"),
- optparse.make_option("-/", dest="cloneExclude",
- action="append", type="string",
- help="exclude depot path"),
optparse.make_option("--bare", dest="cloneBare",
action="store_true", default=False),
]
self.needsGit = False
self.cloneBare = False
- # This is required for the "append" cloneExclude action
- def ensure_value(self, attr, value):
- if not hasattr(self, attr) or getattr(self, attr) is None:
- setattr(self, attr, value)
- return getattr(self, attr)
-
def defaultDestination(self, args):
## TODO: use common prefix of args?
depotPath = args[0]