1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25import zipfile 26import zlib 27import ctypes 28 29try: 30from subprocess import CalledProcessError 31exceptImportError: 32# from python2.7:subprocess.py 33# Exception classes used by this module. 34classCalledProcessError(Exception): 35"""This exception is raised when a process run by check_call() returns 36 a non-zero exit status. The exit status will be stored in the 37 returncode attribute.""" 38def__init__(self, returncode, cmd): 39 self.returncode = returncode 40 self.cmd = cmd 41def__str__(self): 42return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 43 44verbose =False 45 46# Only labels/tags matching this will be imported/exported 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 48 49# Grab changes in blocks of this many revisions, unless otherwise requested 50defaultBlockSize =512 51 52defp4_build_cmd(cmd): 53"""Build a suitable p4 command line. 54 55 This consolidates building and returning a p4 command line into one 56 location. It means that hooking into the environment, or other configuration 57 can be done more easily. 58 """ 59 real_cmd = ["p4"] 60 61 user =gitConfig("git-p4.user") 62iflen(user) >0: 63 real_cmd += ["-u",user] 64 65 password =gitConfig("git-p4.password") 66iflen(password) >0: 67 real_cmd += ["-P", password] 68 69 port =gitConfig("git-p4.port") 70iflen(port) >0: 71 real_cmd += ["-p", port] 72 73 host =gitConfig("git-p4.host") 74iflen(host) >0: 75 real_cmd += ["-H", host] 76 77 client =gitConfig("git-p4.client") 78iflen(client) >0: 79 real_cmd += ["-c", client] 80 81 82ifisinstance(cmd,basestring): 83 real_cmd =' '.join(real_cmd) +' '+ cmd 84else: 85 real_cmd += cmd 86return real_cmd 87 88defchdir(path, is_client_path=False): 89"""Do chdir to the given path, and set the PWD environment 90 variable for use by P4. It does not look at getcwd() output. 91 Since we're not using the shell, it is necessary to set the 92 PWD environment variable explicitly. 93 94 Normally, expand the path to force it to be absolute. This 95 addresses the use of relative path names inside P4 settings, 96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 97 as given; it looks for .p4config using PWD. 98 99 If is_client_path, the path was handed to us directly by p4, 100 and may be a symbolic link. Do not call os.getcwd() in this 101 case, because it will cause p4 to think that PWD is not inside 102 the client path. 103 """ 104 105 os.chdir(path) 106if not is_client_path: 107 path = os.getcwd() 108 os.environ['PWD'] = path 109 110defcalcDiskFree(): 111"""Return free space in bytes on the disk of the given dirname.""" 112if platform.system() =='Windows': 113 free_bytes = ctypes.c_ulonglong(0) 114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()),None,None, ctypes.pointer(free_bytes)) 115return free_bytes.value 116else: 117 st = os.statvfs(os.getcwd()) 118return st.f_bavail * st.f_frsize 119 120defdie(msg): 121if verbose: 122raiseException(msg) 123else: 124 sys.stderr.write(msg +"\n") 125 sys.exit(1) 126 127defwrite_pipe(c, stdin): 128if verbose: 129 sys.stderr.write('Writing pipe:%s\n'%str(c)) 130 131 expand =isinstance(c,basestring) 132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 133 pipe = p.stdin 134 val = pipe.write(stdin) 135 pipe.close() 136if p.wait(): 137die('Command failed:%s'%str(c)) 138 139return val 140 141defp4_write_pipe(c, stdin): 142 real_cmd =p4_build_cmd(c) 143returnwrite_pipe(real_cmd, stdin) 144 145defread_pipe(c, ignore_error=False): 146if verbose: 147 sys.stderr.write('Reading pipe:%s\n'%str(c)) 148 149 expand =isinstance(c,basestring) 150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) 151(out, err) = p.communicate() 152if p.returncode !=0and not ignore_error: 153die('Command failed:%s\nError:%s'% (str(c), err)) 154return out 155 156defp4_read_pipe(c, ignore_error=False): 157 real_cmd =p4_build_cmd(c) 158returnread_pipe(real_cmd, ignore_error) 159 160defread_pipe_lines(c): 161if verbose: 162 sys.stderr.write('Reading pipe:%s\n'%str(c)) 163 164 expand =isinstance(c, basestring) 165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 166 pipe = p.stdout 167 val = pipe.readlines() 168if pipe.close()or p.wait(): 169die('Command failed:%s'%str(c)) 170 171return val 172 173defp4_read_pipe_lines(c): 174"""Specifically invoke p4 on the command supplied. """ 175 real_cmd =p4_build_cmd(c) 176returnread_pipe_lines(real_cmd) 177 178defp4_has_command(cmd): 179"""Ask p4 for help on this command. If it returns an error, the 180 command does not exist in this version of p4.""" 181 real_cmd =p4_build_cmd(["help", cmd]) 182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 183 stderr=subprocess.PIPE) 184 p.communicate() 185return p.returncode ==0 186 187defp4_has_move_command(): 188"""See if the move command exists, that it supports -k, and that 189 it has not been administratively disabled. The arguments 190 must be correct, but the filenames do not have to exist. Use 191 ones with wildcards so even if they exist, it will fail.""" 192 193if notp4_has_command("move"): 194return False 195 cmd =p4_build_cmd(["move","-k","@from","@to"]) 196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 197(out, err) = p.communicate() 198# return code will be 1 in either case 199if err.find("Invalid option") >=0: 200return False 201if err.find("disabled") >=0: 202return False 203# assume it failed because @... was invalid changelist 204return True 205 206defsystem(cmd, ignore_error=False): 207 expand =isinstance(cmd,basestring) 208if verbose: 209 sys.stderr.write("executing%s\n"%str(cmd)) 210 retcode = subprocess.call(cmd, shell=expand) 211if retcode and not ignore_error: 212raiseCalledProcessError(retcode, cmd) 213 214return retcode 215 216defp4_system(cmd): 217"""Specifically invoke p4 as the system command. """ 218 real_cmd =p4_build_cmd(cmd) 219 expand =isinstance(real_cmd, basestring) 220 retcode = subprocess.call(real_cmd, shell=expand) 221if retcode: 222raiseCalledProcessError(retcode, real_cmd) 223 224_p4_version_string =None 225defp4_version_string(): 226"""Read the version string, showing just the last line, which 227 hopefully is the interesting version bit. 228 229 $ p4 -V 230 Perforce - The Fast Software Configuration Management System. 231 Copyright 1995-2011 Perforce Software. All rights reserved. 232 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 233 """ 234global _p4_version_string 235if not _p4_version_string: 236 a =p4_read_pipe_lines(["-V"]) 237 _p4_version_string = a[-1].rstrip() 238return _p4_version_string 239 240defp4_integrate(src, dest): 241p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 242 243defp4_sync(f, *options): 244p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 245 246defp4_add(f): 247# forcibly add file names with wildcards 248ifwildcard_present(f): 249p4_system(["add","-f", f]) 250else: 251p4_system(["add", f]) 252 253defp4_delete(f): 254p4_system(["delete",wildcard_encode(f)]) 255 256defp4_edit(f, *options): 257p4_system(["edit"] +list(options) + [wildcard_encode(f)]) 258 259defp4_revert(f): 260p4_system(["revert",wildcard_encode(f)]) 261 262defp4_reopen(type, f): 263p4_system(["reopen","-t",type,wildcard_encode(f)]) 264 265defp4_move(src, dest): 266p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 267 268defp4_last_change(): 269 results =p4CmdList(["changes","-m","1"]) 270returnint(results[0]['change']) 271 272defp4_describe(change): 273"""Make sure it returns a valid result by checking for 274 the presence of field "time". Return a dict of the 275 results.""" 276 277 ds =p4CmdList(["describe","-s",str(change)]) 278iflen(ds) !=1: 279die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 280 281 d = ds[0] 282 283if"p4ExitCode"in d: 284die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 285str(d))) 286if"code"in d: 287if d["code"] =="error": 288die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 289 290if"time"not in d: 291die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 292 293return d 294 295# 296# Canonicalize the p4 type and return a tuple of the 297# base type, plus any modifiers. See "p4 help filetypes" 298# for a list and explanation. 299# 300defsplit_p4_type(p4type): 301 302 p4_filetypes_historical = { 303"ctempobj":"binary+Sw", 304"ctext":"text+C", 305"cxtext":"text+Cx", 306"ktext":"text+k", 307"kxtext":"text+kx", 308"ltext":"text+F", 309"tempobj":"binary+FSw", 310"ubinary":"binary+F", 311"uresource":"resource+F", 312"uxbinary":"binary+Fx", 313"xbinary":"binary+x", 314"xltext":"text+Fx", 315"xtempobj":"binary+Swx", 316"xtext":"text+x", 317"xunicode":"unicode+x", 318"xutf16":"utf16+x", 319} 320if p4type in p4_filetypes_historical: 321 p4type = p4_filetypes_historical[p4type] 322 mods ="" 323 s = p4type.split("+") 324 base = s[0] 325 mods ="" 326iflen(s) >1: 327 mods = s[1] 328return(base, mods) 329 330# 331# return the raw p4 type of a file (text, text+ko, etc) 332# 333defp4_type(f): 334 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 335return results[0]['headType'] 336 337# 338# Given a type base and modifier, return a regexp matching 339# the keywords that can be expanded in the file 340# 341defp4_keywords_regexp_for_type(base, type_mods): 342if base in("text","unicode","binary"): 343 kwords =None 344if"ko"in type_mods: 345 kwords ='Id|Header' 346elif"k"in type_mods: 347 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 348else: 349return None 350 pattern = r""" 351 \$ # Starts with a dollar, followed by... 352 (%s) # one of the keywords, followed by... 353 (:[^$\n]+)? # possibly an old expansion, followed by... 354 \$ # another dollar 355 """% kwords 356return pattern 357else: 358return None 359 360# 361# Given a file, return a regexp matching the possible 362# RCS keywords that will be expanded, or None for files 363# with kw expansion turned off. 364# 365defp4_keywords_regexp_for_file(file): 366if not os.path.exists(file): 367return None 368else: 369(type_base, type_mods) =split_p4_type(p4_type(file)) 370returnp4_keywords_regexp_for_type(type_base, type_mods) 371 372defsetP4ExecBit(file, mode): 373# Reopens an already open file and changes the execute bit to match 374# the execute bit setting in the passed in mode. 375 376 p4Type ="+x" 377 378if notisModeExec(mode): 379 p4Type =getP4OpenedType(file) 380 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 381 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 382if p4Type[-1] =="+": 383 p4Type = p4Type[0:-1] 384 385p4_reopen(p4Type,file) 386 387defgetP4OpenedType(file): 388# Returns the perforce file type for the given file. 389 390 result =p4_read_pipe(["opened",wildcard_encode(file)]) 391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 392if match: 393return match.group(1) 394else: 395die("Could not determine file type for%s(result: '%s')"% (file, result)) 396 397# Return the set of all p4 labels 398defgetP4Labels(depotPaths): 399 labels =set() 400ifisinstance(depotPaths,basestring): 401 depotPaths = [depotPaths] 402 403for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 404 label = l['label'] 405 labels.add(label) 406 407return labels 408 409# Return the set of all git tags 410defgetGitTags(): 411 gitTags =set() 412for line inread_pipe_lines(["git","tag"]): 413 tag = line.strip() 414 gitTags.add(tag) 415return gitTags 416 417defdiffTreePattern(): 418# This is a simple generator for the diff tree regex pattern. This could be 419# a class variable if this and parseDiffTreeEntry were a part of a class. 420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 421while True: 422yield pattern 423 424defparseDiffTreeEntry(entry): 425"""Parses a single diff tree entry into its component elements. 426 427 See git-diff-tree(1) manpage for details about the format of the diff 428 output. This method returns a dictionary with the following elements: 429 430 src_mode - The mode of the source file 431 dst_mode - The mode of the destination file 432 src_sha1 - The sha1 for the source file 433 dst_sha1 - The sha1 fr the destination file 434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 435 status_score - The score for the status (applicable for 'C' and 'R' 436 statuses). This is None if there is no score. 437 src - The path for the source file. 438 dst - The path for the destination file. This is only present for 439 copy or renames. If it is not present, this is None. 440 441 If the pattern is not matched, None is returned.""" 442 443 match =diffTreePattern().next().match(entry) 444if match: 445return{ 446'src_mode': match.group(1), 447'dst_mode': match.group(2), 448'src_sha1': match.group(3), 449'dst_sha1': match.group(4), 450'status': match.group(5), 451'status_score': match.group(6), 452'src': match.group(7), 453'dst': match.group(10) 454} 455return None 456 457defisModeExec(mode): 458# Returns True if the given git mode represents an executable file, 459# otherwise False. 460return mode[-3:] =="755" 461 462defisModeExecChanged(src_mode, dst_mode): 463returnisModeExec(src_mode) !=isModeExec(dst_mode) 464 465defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 466 467ifisinstance(cmd,basestring): 468 cmd ="-G "+ cmd 469 expand =True 470else: 471 cmd = ["-G"] + cmd 472 expand =False 473 474 cmd =p4_build_cmd(cmd) 475if verbose: 476 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 477 478# Use a temporary file to avoid deadlocks without 479# subprocess.communicate(), which would put another copy 480# of stdout into memory. 481 stdin_file =None 482if stdin is not None: 483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 484ifisinstance(stdin,basestring): 485 stdin_file.write(stdin) 486else: 487for i in stdin: 488 stdin_file.write(i +'\n') 489 stdin_file.flush() 490 stdin_file.seek(0) 491 492 p4 = subprocess.Popen(cmd, 493 shell=expand, 494 stdin=stdin_file, 495 stdout=subprocess.PIPE) 496 497 result = [] 498try: 499while True: 500 entry = marshal.load(p4.stdout) 501if cb is not None: 502cb(entry) 503else: 504 result.append(entry) 505exceptEOFError: 506pass 507 exitCode = p4.wait() 508if exitCode !=0: 509 entry = {} 510 entry["p4ExitCode"] = exitCode 511 result.append(entry) 512 513return result 514 515defp4Cmd(cmd): 516list=p4CmdList(cmd) 517 result = {} 518for entry inlist: 519 result.update(entry) 520return result; 521 522defp4Where(depotPath): 523if not depotPath.endswith("/"): 524 depotPath +="/" 525 depotPathLong = depotPath +"..." 526 outputList =p4CmdList(["where", depotPathLong]) 527 output =None 528for entry in outputList: 529if"depotFile"in entry: 530# Search for the base client side depot path, as long as it starts with the branch's P4 path. 531# The base path always ends with "/...". 532if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 533 output = entry 534break 535elif"data"in entry: 536 data = entry.get("data") 537 space = data.find(" ") 538if data[:space] == depotPath: 539 output = entry 540break 541if output ==None: 542return"" 543if output["code"] =="error": 544return"" 545 clientPath ="" 546if"path"in output: 547 clientPath = output.get("path") 548elif"data"in output: 549 data = output.get("data") 550 lastSpace = data.rfind(" ") 551 clientPath = data[lastSpace +1:] 552 553if clientPath.endswith("..."): 554 clientPath = clientPath[:-3] 555return clientPath 556 557defcurrentGitBranch(): 558 retcode =system(["git","symbolic-ref","-q","HEAD"], ignore_error=True) 559if retcode !=0: 560# on a detached head 561return None 562else: 563returnread_pipe(["git","name-rev","HEAD"]).split(" ")[1].strip() 564 565defisValidGitDir(path): 566if(os.path.exists(path +"/HEAD") 567and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 568return True; 569return False 570 571defparseRevision(ref): 572returnread_pipe("git rev-parse%s"% ref).strip() 573 574defbranchExists(ref): 575 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 576 ignore_error=True) 577returnlen(rev) >0 578 579defextractLogMessageFromGitCommit(commit): 580 logMessage ="" 581 582## fixme: title is first line of commit, not 1st paragraph. 583 foundTitle =False 584for log inread_pipe_lines("git cat-file commit%s"% commit): 585if not foundTitle: 586iflen(log) ==1: 587 foundTitle =True 588continue 589 590 logMessage += log 591return logMessage 592 593defextractSettingsGitLog(log): 594 values = {} 595for line in log.split("\n"): 596 line = line.strip() 597 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 598if not m: 599continue 600 601 assignments = m.group(1).split(':') 602for a in assignments: 603 vals = a.split('=') 604 key = vals[0].strip() 605 val = ('='.join(vals[1:])).strip() 606if val.endswith('\"')and val.startswith('"'): 607 val = val[1:-1] 608 609 values[key] = val 610 611 paths = values.get("depot-paths") 612if not paths: 613 paths = values.get("depot-path") 614if paths: 615 values['depot-paths'] = paths.split(',') 616return values 617 618defgitBranchExists(branch): 619 proc = subprocess.Popen(["git","rev-parse", branch], 620 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 621return proc.wait() ==0; 622 623_gitConfig = {} 624 625defgitConfig(key, typeSpecifier=None): 626if not _gitConfig.has_key(key): 627 cmd = ["git","config"] 628if typeSpecifier: 629 cmd += [ typeSpecifier ] 630 cmd += [ key ] 631 s =read_pipe(cmd, ignore_error=True) 632 _gitConfig[key] = s.strip() 633return _gitConfig[key] 634 635defgitConfigBool(key): 636"""Return a bool, using git config --bool. It is True only if the 637 variable is set to true, and False if set to false or not present 638 in the config.""" 639 640if not _gitConfig.has_key(key): 641 _gitConfig[key] =gitConfig(key,'--bool') =="true" 642return _gitConfig[key] 643 644defgitConfigInt(key): 645if not _gitConfig.has_key(key): 646 cmd = ["git","config","--int", key ] 647 s =read_pipe(cmd, ignore_error=True) 648 v = s.strip() 649try: 650 _gitConfig[key] =int(gitConfig(key,'--int')) 651exceptValueError: 652 _gitConfig[key] =None 653return _gitConfig[key] 654 655defgitConfigList(key): 656if not _gitConfig.has_key(key): 657 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 658 _gitConfig[key] = s.strip().split(os.linesep) 659if _gitConfig[key] == ['']: 660 _gitConfig[key] = [] 661return _gitConfig[key] 662 663defp4BranchesInGit(branchesAreInRemotes=True): 664"""Find all the branches whose names start with "p4/", looking 665 in remotes or heads as specified by the argument. Return 666 a dictionary of{ branch: revision }for each one found. 667 The branch names are the short names, without any 668 "p4/" prefix.""" 669 670 branches = {} 671 672 cmdline ="git rev-parse --symbolic " 673if branchesAreInRemotes: 674 cmdline +="--remotes" 675else: 676 cmdline +="--branches" 677 678for line inread_pipe_lines(cmdline): 679 line = line.strip() 680 681# only import to p4/ 682if not line.startswith('p4/'): 683continue 684# special symbolic ref to p4/master 685if line =="p4/HEAD": 686continue 687 688# strip off p4/ prefix 689 branch = line[len("p4/"):] 690 691 branches[branch] =parseRevision(line) 692 693return branches 694 695defbranch_exists(branch): 696"""Make sure that the given ref name really exists.""" 697 698 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 700 out, _ = p.communicate() 701if p.returncode: 702return False 703# expect exactly one line of output: the branch name 704return out.rstrip() == branch 705 706deffindUpstreamBranchPoint(head ="HEAD"): 707 branches =p4BranchesInGit() 708# map from depot-path to branch name 709 branchByDepotPath = {} 710for branch in branches.keys(): 711 tip = branches[branch] 712 log =extractLogMessageFromGitCommit(tip) 713 settings =extractSettingsGitLog(log) 714if settings.has_key("depot-paths"): 715 paths =",".join(settings["depot-paths"]) 716 branchByDepotPath[paths] ="remotes/p4/"+ branch 717 718 settings =None 719 parent =0 720while parent <65535: 721 commit = head +"~%s"% parent 722 log =extractLogMessageFromGitCommit(commit) 723 settings =extractSettingsGitLog(log) 724if settings.has_key("depot-paths"): 725 paths =",".join(settings["depot-paths"]) 726if branchByDepotPath.has_key(paths): 727return[branchByDepotPath[paths], settings] 728 729 parent = parent +1 730 731return["", settings] 732 733defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 734if not silent: 735print("Creating/updating branch(es) in%sbased on origin branch(es)" 736% localRefPrefix) 737 738 originPrefix ="origin/p4/" 739 740for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 741 line = line.strip() 742if(not line.startswith(originPrefix))or line.endswith("HEAD"): 743continue 744 745 headName = line[len(originPrefix):] 746 remoteHead = localRefPrefix + headName 747 originHead = line 748 749 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 750if(not original.has_key('depot-paths') 751or not original.has_key('change')): 752continue 753 754 update =False 755if notgitBranchExists(remoteHead): 756if verbose: 757print"creating%s"% remoteHead 758 update =True 759else: 760 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 761if settings.has_key('change') >0: 762if settings['depot-paths'] == original['depot-paths']: 763 originP4Change =int(original['change']) 764 p4Change =int(settings['change']) 765if originP4Change > p4Change: 766print("%s(%s) is newer than%s(%s). " 767"Updating p4 branch from origin." 768% (originHead, originP4Change, 769 remoteHead, p4Change)) 770 update =True 771else: 772print("Ignoring:%swas imported from%swhile " 773"%swas imported from%s" 774% (originHead,','.join(original['depot-paths']), 775 remoteHead,','.join(settings['depot-paths']))) 776 777if update: 778system("git update-ref%s %s"% (remoteHead, originHead)) 779 780deforiginP4BranchesExist(): 781returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 782 783 784defp4ParseNumericChangeRange(parts): 785 changeStart =int(parts[0][1:]) 786if parts[1] =='#head': 787 changeEnd =p4_last_change() 788else: 789 changeEnd =int(parts[1]) 790 791return(changeStart, changeEnd) 792 793defchooseBlockSize(blockSize): 794if blockSize: 795return blockSize 796else: 797return defaultBlockSize 798 799defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 800assert depotPaths 801 802# Parse the change range into start and end. Try to find integer 803# revision ranges as these can be broken up into blocks to avoid 804# hitting server-side limits (maxrows, maxscanresults). But if 805# that doesn't work, fall back to using the raw revision specifier 806# strings, without using block mode. 807 808if changeRange is None or changeRange =='': 809 changeStart =1 810 changeEnd =p4_last_change() 811 block_size =chooseBlockSize(requestedBlockSize) 812else: 813 parts = changeRange.split(',') 814assertlen(parts) ==2 815try: 816(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 817 block_size =chooseBlockSize(requestedBlockSize) 818except: 819 changeStart = parts[0][1:] 820 changeEnd = parts[1] 821if requestedBlockSize: 822die("cannot use --changes-block-size with non-numeric revisions") 823 block_size =None 824 825 changes = [] 826 827# Retrieve changes a block at a time, to prevent running 828# into a MaxResults/MaxScanRows error from the server. 829 830while True: 831 cmd = ['changes'] 832 833if block_size: 834 end =min(changeEnd, changeStart + block_size) 835 revisionRange ="%d,%d"% (changeStart, end) 836else: 837 revisionRange ="%s,%s"% (changeStart, changeEnd) 838 839for p in depotPaths: 840 cmd += ["%s...@%s"% (p, revisionRange)] 841 842# Insert changes in chronological order 843for line inreversed(p4_read_pipe_lines(cmd)): 844 changes.append(int(line.split(" ")[1])) 845 846if not block_size: 847break 848 849if end >= changeEnd: 850break 851 852 changeStart = end +1 853 854 changes =sorted(changes) 855return changes 856 857defp4PathStartsWith(path, prefix): 858# This method tries to remedy a potential mixed-case issue: 859# 860# If UserA adds //depot/DirA/file1 861# and UserB adds //depot/dira/file2 862# 863# we may or may not have a problem. If you have core.ignorecase=true, 864# we treat DirA and dira as the same directory 865ifgitConfigBool("core.ignorecase"): 866return path.lower().startswith(prefix.lower()) 867return path.startswith(prefix) 868 869defgetClientSpec(): 870"""Look at the p4 client spec, create a View() object that contains 871 all the mappings, and return it.""" 872 873 specList =p4CmdList("client -o") 874iflen(specList) !=1: 875die('Output from "client -o" is%dlines, expecting 1'% 876len(specList)) 877 878# dictionary of all client parameters 879 entry = specList[0] 880 881# the //client/ name 882 client_name = entry["Client"] 883 884# just the keys that start with "View" 885 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 886 887# hold this new View 888 view =View(client_name) 889 890# append the lines, in order, to the view 891for view_num inrange(len(view_keys)): 892 k ="View%d"% view_num 893if k not in view_keys: 894die("Expected view key%smissing"% k) 895 view.append(entry[k]) 896 897return view 898 899defgetClientRoot(): 900"""Grab the client directory.""" 901 902 output =p4CmdList("client -o") 903iflen(output) !=1: 904die('Output from "client -o" is%dlines, expecting 1'%len(output)) 905 906 entry = output[0] 907if"Root"not in entry: 908die('Client has no "Root"') 909 910return entry["Root"] 911 912# 913# P4 wildcards are not allowed in filenames. P4 complains 914# if you simply add them, but you can force it with "-f", in 915# which case it translates them into %xx encoding internally. 916# 917defwildcard_decode(path): 918# Search for and fix just these four characters. Do % last so 919# that fixing it does not inadvertently create new %-escapes. 920# Cannot have * in a filename in windows; untested as to 921# what p4 would do in such a case. 922if not platform.system() =="Windows": 923 path = path.replace("%2A","*") 924 path = path.replace("%23","#") \ 925.replace("%40","@") \ 926.replace("%25","%") 927return path 928 929defwildcard_encode(path): 930# do % first to avoid double-encoding the %s introduced here 931 path = path.replace("%","%25") \ 932.replace("*","%2A") \ 933.replace("#","%23") \ 934.replace("@","%40") 935return path 936 937defwildcard_present(path): 938 m = re.search("[*#@%]", path) 939return m is not None 940 941classLargeFileSystem(object): 942"""Base class for large file system support.""" 943 944def__init__(self, writeToGitStream): 945 self.largeFiles =set() 946 self.writeToGitStream = writeToGitStream 947 948defgeneratePointer(self, cloneDestination, contentFile): 949"""Return the content of a pointer file that is stored in Git instead of 950 the actual content.""" 951assert False,"Method 'generatePointer' required in "+ self.__class__.__name__ 952 953defpushFile(self, localLargeFile): 954"""Push the actual content which is not stored in the Git repository to 955 a server.""" 956assert False,"Method 'pushFile' required in "+ self.__class__.__name__ 957 958defhasLargeFileExtension(self, relPath): 959returnreduce( 960lambda a, b: a or b, 961[relPath.endswith('.'+ e)for e ingitConfigList('git-p4.largeFileExtensions')], 962False 963) 964 965defgenerateTempFile(self, contents): 966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 967for d in contents: 968 contentFile.write(d) 969 contentFile.close() 970return contentFile.name 971 972defexceedsLargeFileThreshold(self, relPath, contents): 973ifgitConfigInt('git-p4.largeFileThreshold'): 974 contentsSize =sum(len(d)for d in contents) 975if contentsSize >gitConfigInt('git-p4.largeFileThreshold'): 976return True 977ifgitConfigInt('git-p4.largeFileCompressedThreshold'): 978 contentsSize =sum(len(d)for d in contents) 979if contentsSize <=gitConfigInt('git-p4.largeFileCompressedThreshold'): 980return False 981 contentTempFile = self.generateTempFile(contents) 982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w') 984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) 985 zf.close() 986 compressedContentsSize = zf.infolist()[0].compress_size 987 os.remove(contentTempFile) 988 os.remove(compressedContentFile.name) 989if compressedContentsSize >gitConfigInt('git-p4.largeFileCompressedThreshold'): 990return True 991return False 992 993defaddLargeFile(self, relPath): 994 self.largeFiles.add(relPath) 995 996defremoveLargeFile(self, relPath): 997 self.largeFiles.remove(relPath) 998 999defisLargeFile(self, relPath):1000return relPath in self.largeFiles10011002defprocessContent(self, git_mode, relPath, contents):1003"""Processes the content of git fast import. This method decides if a1004 file is stored in the large file system and handles all necessary1005 steps."""1006if self.exceedsLargeFileThreshold(relPath, contents)or self.hasLargeFileExtension(relPath):1007 contentTempFile = self.generateTempFile(contents)1008(git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)10091010# Move temp file to final location in large file system1011 largeFileDir = os.path.dirname(localLargeFile)1012if not os.path.isdir(largeFileDir):1013 os.makedirs(largeFileDir)1014 shutil.move(contentTempFile, localLargeFile)1015 self.addLargeFile(relPath)1016ifgitConfigBool('git-p4.largeFilePush'):1017 self.pushFile(localLargeFile)1018if verbose:1019 sys.stderr.write("%smoved to large file system (%s)\n"% (relPath, localLargeFile))1020return(git_mode, contents)10211022classMockLFS(LargeFileSystem):1023"""Mock large file system for testing."""10241025defgeneratePointer(self, contentFile):1026"""The pointer content is the original content prefixed with "pointer-".1027 The local filename of the large file storage is derived from the file content.1028 """1029withopen(contentFile,'r')as f:1030 content =next(f)1031 gitMode ='100644'1032 pointerContents ='pointer-'+ content1033 localLargeFile = os.path.join(os.getcwd(),'.git','mock-storage','local', content[:-1])1034return(gitMode, pointerContents, localLargeFile)10351036defpushFile(self, localLargeFile):1037"""The remote filename of the large file storage is the same as the local1038 one but in a different directory.1039 """1040 remotePath = os.path.join(os.path.dirname(localLargeFile),'..','remote')1041if not os.path.exists(remotePath):1042 os.makedirs(remotePath)1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))10441045classGitLFS(LargeFileSystem):1046"""Git LFS as backend for the git-p4 large file system.1047 See https://git-lfs.github.com/ for details."""10481049def__init__(self, *args):1050 LargeFileSystem.__init__(self, *args)1051 self.baseGitAttributes = []10521053defgeneratePointer(self, contentFile):1054"""Generate a Git LFS pointer for the content. Return LFS Pointer file1055 mode and content which is stored in the Git repository instead of1056 the actual content. Return also the new location of the actual1057 content.1058 """1059 pointerProcess = subprocess.Popen(1060['git','lfs','pointer','--file='+ contentFile],1061 stdout=subprocess.PIPE1062)1063 pointerFile = pointerProcess.stdout.read()1064if pointerProcess.wait():1065 os.remove(contentFile)1066die('git-lfs pointer command failed. Did you install the extension?')10671068# Git LFS removed the preamble in the output of the 'pointer' command1069# starting from version 1.2.0. Check for the preamble here to support1070# earlier versions.1071# c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b431072if pointerFile.startswith('Git LFS pointer for'):1073 pointerFile = re.sub(r'Git LFS pointer for.*\n\n','', pointerFile)10741075 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)1076 localLargeFile = os.path.join(1077 os.getcwd(),1078'.git','lfs','objects', oid[:2], oid[2:4],1079 oid,1080)1081# LFS Spec states that pointer files should not have the executable bit set.1082 gitMode ='100644'1083return(gitMode, pointerFile, localLargeFile)10841085defpushFile(self, localLargeFile):1086 uploadProcess = subprocess.Popen(1087['git','lfs','push','--object-id','origin', os.path.basename(localLargeFile)]1088)1089if uploadProcess.wait():1090die('git-lfs push command failed. Did you define a remote?')10911092defgenerateGitAttributes(self):1093return(1094 self.baseGitAttributes +1095[1096'\n',1097'#\n',1098'# Git LFS (see https://git-lfs.github.com/)\n',1099'#\n',1100] +1101['*.'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1102for f insorted(gitConfigList('git-p4.largeFileExtensions'))1103] +1104['/'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1105for f insorted(self.largeFiles)if not self.hasLargeFileExtension(f)1106]1107)11081109defaddLargeFile(self, relPath):1110 LargeFileSystem.addLargeFile(self, relPath)1111 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11121113defremoveLargeFile(self, relPath):1114 LargeFileSystem.removeLargeFile(self, relPath)1115 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11161117defprocessContent(self, git_mode, relPath, contents):1118if relPath =='.gitattributes':1119 self.baseGitAttributes = contents1120return(git_mode, self.generateGitAttributes())1121else:1122return LargeFileSystem.processContent(self, git_mode, relPath, contents)11231124class Command:1125def__init__(self):1126 self.usage ="usage: %prog [options]"1127 self.needsGit =True1128 self.verbose =False11291130class P4UserMap:1131def__init__(self):1132 self.userMapFromPerforceServer =False1133 self.myP4UserId =None11341135defp4UserId(self):1136if self.myP4UserId:1137return self.myP4UserId11381139 results =p4CmdList("user -o")1140for r in results:1141if r.has_key('User'):1142 self.myP4UserId = r['User']1143return r['User']1144die("Could not find your p4 user id")11451146defp4UserIsMe(self, p4User):1147# return True if the given p4 user is actually me1148 me = self.p4UserId()1149if not p4User or p4User != me:1150return False1151else:1152return True11531154defgetUserCacheFilename(self):1155 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))1156return home +"/.gitp4-usercache.txt"11571158defgetUserMapFromPerforceServer(self):1159if self.userMapFromPerforceServer:1160return1161 self.users = {}1162 self.emails = {}11631164for output inp4CmdList("users"):1165if not output.has_key("User"):1166continue1167 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">"1168 self.emails[output["Email"]] = output["User"]11691170 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)1171for mapUserConfig ingitConfigList("git-p4.mapUser"):1172 mapUser = mapUserConfigRegex.findall(mapUserConfig)1173if mapUser andlen(mapUser[0]) ==3:1174 user = mapUser[0][0]1175 fullname = mapUser[0][1]1176 email = mapUser[0][2]1177 self.users[user] = fullname +" <"+ email +">"1178 self.emails[email] = user11791180 s =''1181for(key, val)in self.users.items():1182 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1))11831184open(self.getUserCacheFilename(),"wb").write(s)1185 self.userMapFromPerforceServer =True11861187defloadUserMapFromCache(self):1188 self.users = {}1189 self.userMapFromPerforceServer =False1190try:1191 cache =open(self.getUserCacheFilename(),"rb")1192 lines = cache.readlines()1193 cache.close()1194for line in lines:1195 entry = line.strip().split("\t")1196 self.users[entry[0]] = entry[1]1197exceptIOError:1198 self.getUserMapFromPerforceServer()11991200classP4Debug(Command):1201def__init__(self):1202 Command.__init__(self)1203 self.options = []1204 self.description ="A tool to debug the output of p4 -G."1205 self.needsGit =False12061207defrun(self, args):1208 j =01209for output inp4CmdList(args):1210print'Element:%d'% j1211 j +=11212print output1213return True12141215classP4RollBack(Command):1216def__init__(self):1217 Command.__init__(self)1218 self.options = [1219 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1220]1221 self.description ="A tool to debug the multi-branch import. Don't use :)"1222 self.rollbackLocalBranches =False12231224defrun(self, args):1225iflen(args) !=1:1226return False1227 maxChange =int(args[0])12281229if"p4ExitCode"inp4Cmd("changes -m 1"):1230die("Problems executing p4");12311232if self.rollbackLocalBranches:1233 refPrefix ="refs/heads/"1234 lines =read_pipe_lines("git rev-parse --symbolic --branches")1235else:1236 refPrefix ="refs/remotes/"1237 lines =read_pipe_lines("git rev-parse --symbolic --remotes")12381239for line in lines:1240if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1241 line = line.strip()1242 ref = refPrefix + line1243 log =extractLogMessageFromGitCommit(ref)1244 settings =extractSettingsGitLog(log)12451246 depotPaths = settings['depot-paths']1247 change = settings['change']12481249 changed =False12501251iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1252for p in depotPaths]))) ==0:1253print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange)1254system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1255continue12561257while change andint(change) > maxChange:1258 changed =True1259if self.verbose:1260print"%sis at%s; rewinding towards%s"% (ref, change, maxChange)1261system("git update-ref%s\"%s^\""% (ref, ref))1262 log =extractLogMessageFromGitCommit(ref)1263 settings =extractSettingsGitLog(log)126412651266 depotPaths = settings['depot-paths']1267 change = settings['change']12681269if changed:1270print"%srewound to%s"% (ref, change)12711272return True12731274classP4Submit(Command, P4UserMap):12751276 conflict_behavior_choices = ("ask","skip","quit")12771278def__init__(self):1279 Command.__init__(self)1280 P4UserMap.__init__(self)1281 self.options = [1282 optparse.make_option("--origin", dest="origin"),1283 optparse.make_option("-M", dest="detectRenames", action="store_true"),1284# preserve the user, requires relevant p4 permissions1285 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1286 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1287 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1288 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1289 optparse.make_option("--conflict", dest="conflict_behavior",1290 choices=self.conflict_behavior_choices),1291 optparse.make_option("--branch", dest="branch"),1292 optparse.make_option("--shelve", dest="shelve", action="store_true",1293help="Shelve instead of submit. Shelved files are reverted, "1294"restoring the workspace to the state before the shelve"),1295]1296 self.description ="Submit changes from git to the perforce depot."1297 self.usage +=" [name of git branch to submit into perforce depot]"1298 self.origin =""1299 self.detectRenames =False1300 self.preserveUser =gitConfigBool("git-p4.preserveUser")1301 self.dry_run =False1302 self.shelve =False1303 self.prepare_p4_only =False1304 self.conflict_behavior =None1305 self.isWindows = (platform.system() =="Windows")1306 self.exportLabels =False1307 self.p4HasMoveCommand =p4_has_move_command()1308 self.branch =None13091310ifgitConfig('git-p4.largeFileSystem'):1311die("Large file system not supported for git-p4 submit command. Please remove it from config.")13121313defcheck(self):1314iflen(p4CmdList("opened ...")) >0:1315die("You have files opened with perforce! Close them before starting the sync.")13161317defseparate_jobs_from_description(self, message):1318"""Extract and return a possible Jobs field in the commit1319 message. It goes into a separate section in the p4 change1320 specification.13211322 A jobs line starts with "Jobs:" and looks like a new field1323 in a form. Values are white-space separated on the same1324 line or on following lines that start with a tab.13251326 This does not parse and extract the full git commit message1327 like a p4 form. It just sees the Jobs: line as a marker1328 to pass everything from then on directly into the p4 form,1329 but outside the description section.13301331 Return a tuple (stripped log message, jobs string)."""13321333 m = re.search(r'^Jobs:', message, re.MULTILINE)1334if m is None:1335return(message,None)13361337 jobtext = message[m.start():]1338 stripped_message = message[:m.start()].rstrip()1339return(stripped_message, jobtext)13401341defprepareLogMessage(self, template, message, jobs):1342"""Edits the template returned from "p4 change -o" to insert1343 the message in the Description field, and the jobs text in1344 the Jobs field."""1345 result =""13461347 inDescriptionSection =False13481349for line in template.split("\n"):1350if line.startswith("#"):1351 result += line +"\n"1352continue13531354if inDescriptionSection:1355if line.startswith("Files:")or line.startswith("Jobs:"):1356 inDescriptionSection =False1357# insert Jobs section1358if jobs:1359 result += jobs +"\n"1360else:1361continue1362else:1363if line.startswith("Description:"):1364 inDescriptionSection =True1365 line +="\n"1366for messageLine in message.split("\n"):1367 line +="\t"+ messageLine +"\n"13681369 result += line +"\n"13701371return result13721373defpatchRCSKeywords(self,file, pattern):1374# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1375(handle, outFileName) = tempfile.mkstemp(dir='.')1376try:1377 outFile = os.fdopen(handle,"w+")1378 inFile =open(file,"r")1379 regexp = re.compile(pattern, re.VERBOSE)1380for line in inFile.readlines():1381 line = regexp.sub(r'$\1$', line)1382 outFile.write(line)1383 inFile.close()1384 outFile.close()1385# Forcibly overwrite the original file1386 os.unlink(file)1387 shutil.move(outFileName,file)1388except:1389# cleanup our temporary file1390 os.unlink(outFileName)1391print"Failed to strip RCS keywords in%s"%file1392raise13931394print"Patched up RCS keywords in%s"%file13951396defp4UserForCommit(self,id):1397# Return the tuple (perforce user,git email) for a given git commit id1398 self.getUserMapFromPerforceServer()1399 gitEmail =read_pipe(["git","log","--max-count=1",1400"--format=%ae",id])1401 gitEmail = gitEmail.strip()1402if not self.emails.has_key(gitEmail):1403return(None,gitEmail)1404else:1405return(self.emails[gitEmail],gitEmail)14061407defcheckValidP4Users(self,commits):1408# check if any git authors cannot be mapped to p4 users1409foridin commits:1410(user,email) = self.p4UserForCommit(id)1411if not user:1412 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1413ifgitConfigBool("git-p4.allowMissingP4Users"):1414print"%s"% msg1415else:1416die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)14171418deflastP4Changelist(self):1419# Get back the last changelist number submitted in this client spec. This1420# then gets used to patch up the username in the change. If the same1421# client spec is being used by multiple processes then this might go1422# wrong.1423 results =p4CmdList("client -o")# find the current client1424 client =None1425for r in results:1426if r.has_key('Client'):1427 client = r['Client']1428break1429if not client:1430die("could not get client spec")1431 results =p4CmdList(["changes","-c", client,"-m","1"])1432for r in results:1433if r.has_key('change'):1434return r['change']1435die("Could not get changelist number for last submit - cannot patch up user details")14361437defmodifyChangelistUser(self, changelist, newUser):1438# fixup the user field of a changelist after it has been submitted.1439 changes =p4CmdList("change -o%s"% changelist)1440iflen(changes) !=1:1441die("Bad output from p4 change modifying%sto user%s"%1442(changelist, newUser))14431444 c = changes[0]1445if c['User'] == newUser:return# nothing to do1446 c['User'] = newUser1447input= marshal.dumps(c)14481449 result =p4CmdList("change -f -i", stdin=input)1450for r in result:1451if r.has_key('code'):1452if r['code'] =='error':1453die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1454if r.has_key('data'):1455print("Updated user field for changelist%sto%s"% (changelist, newUser))1456return1457die("Could not modify user field of changelist%sto%s"% (changelist, newUser))14581459defcanChangeChangelists(self):1460# check to see if we have p4 admin or super-user permissions, either of1461# which are required to modify changelists.1462 results =p4CmdList(["protects", self.depotPath])1463for r in results:1464if r.has_key('perm'):1465if r['perm'] =='admin':1466return11467if r['perm'] =='super':1468return11469return014701471defprepareSubmitTemplate(self):1472"""Run "p4 change -o" to grab a change specification template.1473 This does not use "p4 -G", as it is nice to keep the submission1474 template in original order, since a human might edit it.14751476 Remove lines in the Files section that show changes to files1477 outside the depot path we're committing into."""14781479[upstream, settings] =findUpstreamBranchPoint()14801481 template =""1482 inFilesSection =False1483for line inp4_read_pipe_lines(['change','-o']):1484if line.endswith("\r\n"):1485 line = line[:-2] +"\n"1486if inFilesSection:1487if line.startswith("\t"):1488# path starts and ends with a tab1489 path = line[1:]1490 lastTab = path.rfind("\t")1491if lastTab != -1:1492 path = path[:lastTab]1493if settings.has_key('depot-paths'):1494if not[p for p in settings['depot-paths']1495ifp4PathStartsWith(path, p)]:1496continue1497else:1498if notp4PathStartsWith(path, self.depotPath):1499continue1500else:1501 inFilesSection =False1502else:1503if line.startswith("Files:"):1504 inFilesSection =True15051506 template += line15071508return template15091510defedit_template(self, template_file):1511"""Invoke the editor to let the user change the submission1512 message. Return true if okay to continue with the submit."""15131514# if configured to skip the editing part, just submit1515ifgitConfigBool("git-p4.skipSubmitEdit"):1516return True15171518# look at the modification time, to check later if the user saved1519# the file1520 mtime = os.stat(template_file).st_mtime15211522# invoke the editor1523if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1524 editor = os.environ.get("P4EDITOR")1525else:1526 editor =read_pipe("git var GIT_EDITOR").strip()1527system(["sh","-c", ('%s"$@"'% editor), editor, template_file])15281529# If the file was not saved, prompt to see if this patch should1530# be skipped. But skip this verification step if configured so.1531ifgitConfigBool("git-p4.skipSubmitEditCheck"):1532return True15331534# modification time updated means user saved the file1535if os.stat(template_file).st_mtime > mtime:1536return True15371538while True:1539 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1540if response =='y':1541return True1542if response =='n':1543return False15441545defget_diff_description(self, editedFiles, filesToAdd):1546# diff1547if os.environ.has_key("P4DIFF"):1548del(os.environ["P4DIFF"])1549 diff =""1550for editedFile in editedFiles:1551 diff +=p4_read_pipe(['diff','-du',1552wildcard_encode(editedFile)])15531554# new file diff1555 newdiff =""1556for newFile in filesToAdd:1557 newdiff +="==== new file ====\n"1558 newdiff +="--- /dev/null\n"1559 newdiff +="+++%s\n"% newFile1560 f =open(newFile,"r")1561for line in f.readlines():1562 newdiff +="+"+ line1563 f.close()15641565return(diff + newdiff).replace('\r\n','\n')15661567defapplyCommit(self,id):1568"""Apply one commit, return True if it succeeded."""15691570print"Applying",read_pipe(["git","show","-s",1571"--format=format:%h%s",id])15721573(p4User, gitEmail) = self.p4UserForCommit(id)15741575 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1576 filesToAdd =set()1577 filesToChangeType =set()1578 filesToDelete =set()1579 editedFiles =set()1580 pureRenameCopy =set()1581 filesToChangeExecBit = {}15821583for line in diff:1584 diff =parseDiffTreeEntry(line)1585 modifier = diff['status']1586 path = diff['src']1587if modifier =="M":1588p4_edit(path)1589ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1590 filesToChangeExecBit[path] = diff['dst_mode']1591 editedFiles.add(path)1592elif modifier =="A":1593 filesToAdd.add(path)1594 filesToChangeExecBit[path] = diff['dst_mode']1595if path in filesToDelete:1596 filesToDelete.remove(path)1597elif modifier =="D":1598 filesToDelete.add(path)1599if path in filesToAdd:1600 filesToAdd.remove(path)1601elif modifier =="C":1602 src, dest = diff['src'], diff['dst']1603p4_integrate(src, dest)1604 pureRenameCopy.add(dest)1605if diff['src_sha1'] != diff['dst_sha1']:1606p4_edit(dest)1607 pureRenameCopy.discard(dest)1608ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1609p4_edit(dest)1610 pureRenameCopy.discard(dest)1611 filesToChangeExecBit[dest] = diff['dst_mode']1612if self.isWindows:1613# turn off read-only attribute1614 os.chmod(dest, stat.S_IWRITE)1615 os.unlink(dest)1616 editedFiles.add(dest)1617elif modifier =="R":1618 src, dest = diff['src'], diff['dst']1619if self.p4HasMoveCommand:1620p4_edit(src)# src must be open before move1621p4_move(src, dest)# opens for (move/delete, move/add)1622else:1623p4_integrate(src, dest)1624if diff['src_sha1'] != diff['dst_sha1']:1625p4_edit(dest)1626else:1627 pureRenameCopy.add(dest)1628ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1629if not self.p4HasMoveCommand:1630p4_edit(dest)# with move: already open, writable1631 filesToChangeExecBit[dest] = diff['dst_mode']1632if not self.p4HasMoveCommand:1633if self.isWindows:1634 os.chmod(dest, stat.S_IWRITE)1635 os.unlink(dest)1636 filesToDelete.add(src)1637 editedFiles.add(dest)1638elif modifier =="T":1639 filesToChangeType.add(path)1640else:1641die("unknown modifier%sfor%s"% (modifier, path))16421643 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1644 patchcmd = diffcmd +" | git apply "1645 tryPatchCmd = patchcmd +"--check -"1646 applyPatchCmd = patchcmd +"--check --apply -"1647 patch_succeeded =True16481649if os.system(tryPatchCmd) !=0:1650 fixed_rcs_keywords =False1651 patch_succeeded =False1652print"Unfortunately applying the change failed!"16531654# Patch failed, maybe it's just RCS keyword woes. Look through1655# the patch to see if that's possible.1656ifgitConfigBool("git-p4.attemptRCSCleanup"):1657file=None1658 pattern =None1659 kwfiles = {}1660forfilein editedFiles | filesToDelete:1661# did this file's delta contain RCS keywords?1662 pattern =p4_keywords_regexp_for_file(file)16631664if pattern:1665# this file is a possibility...look for RCS keywords.1666 regexp = re.compile(pattern, re.VERBOSE)1667for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1668if regexp.search(line):1669if verbose:1670print"got keyword match on%sin%sin%s"% (pattern, line,file)1671 kwfiles[file] = pattern1672break16731674forfilein kwfiles:1675if verbose:1676print"zapping%swith%s"% (line,pattern)1677# File is being deleted, so not open in p4. Must1678# disable the read-only bit on windows.1679if self.isWindows andfilenot in editedFiles:1680 os.chmod(file, stat.S_IWRITE)1681 self.patchRCSKeywords(file, kwfiles[file])1682 fixed_rcs_keywords =True16831684if fixed_rcs_keywords:1685print"Retrying the patch with RCS keywords cleaned up"1686if os.system(tryPatchCmd) ==0:1687 patch_succeeded =True16881689if not patch_succeeded:1690for f in editedFiles:1691p4_revert(f)1692return False16931694#1695# Apply the patch for real, and do add/delete/+x handling.1696#1697system(applyPatchCmd)16981699for f in filesToChangeType:1700p4_edit(f,"-t","auto")1701for f in filesToAdd:1702p4_add(f)1703for f in filesToDelete:1704p4_revert(f)1705p4_delete(f)17061707# Set/clear executable bits1708for f in filesToChangeExecBit.keys():1709 mode = filesToChangeExecBit[f]1710setP4ExecBit(f, mode)17111712#1713# Build p4 change description, starting with the contents1714# of the git commit message.1715#1716 logMessage =extractLogMessageFromGitCommit(id)1717 logMessage = logMessage.strip()1718(logMessage, jobs) = self.separate_jobs_from_description(logMessage)17191720 template = self.prepareSubmitTemplate()1721 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)17221723if self.preserveUser:1724 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User17251726if self.checkAuthorship and not self.p4UserIsMe(p4User):1727 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1728 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1729 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"17301731 separatorLine ="######## everything below this line is just the diff #######\n"1732if not self.prepare_p4_only:1733 submitTemplate += separatorLine1734 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)17351736(handle, fileName) = tempfile.mkstemp()1737 tmpFile = os.fdopen(handle,"w+b")1738if self.isWindows:1739 submitTemplate = submitTemplate.replace("\n","\r\n")1740 tmpFile.write(submitTemplate)1741 tmpFile.close()17421743if self.prepare_p4_only:1744#1745# Leave the p4 tree prepared, and the submit template around1746# and let the user decide what to do next1747#1748print1749print"P4 workspace prepared for submission."1750print"To submit or revert, go to client workspace"1751print" "+ self.clientPath1752print1753print"To submit, use\"p4 submit\"to write a new description,"1754print"or\"p4 submit -i <%s\"to use the one prepared by" \1755"\"git p4\"."% fileName1756print"You can delete the file\"%s\"when finished."% fileName17571758if self.preserveUser and p4User and not self.p4UserIsMe(p4User):1759print"To preserve change ownership by user%s, you must\n" \1760"do\"p4 change -f <change>\"after submitting and\n" \1761"edit the User field."1762if pureRenameCopy:1763print"After submitting, renamed files must be re-synced."1764print"Invoke\"p4 sync -f\"on each of these files:"1765for f in pureRenameCopy:1766print" "+ f17671768print1769print"To revert the changes, use\"p4 revert ...\", and delete"1770print"the submit template file\"%s\""% fileName1771if filesToAdd:1772print"Since the commit adds new files, they must be deleted:"1773for f in filesToAdd:1774print" "+ f1775print1776return True17771778#1779# Let the user edit the change description, then submit it.1780#1781 submitted =False17821783try:1784if self.edit_template(fileName):1785# read the edited message and submit1786 tmpFile =open(fileName,"rb")1787 message = tmpFile.read()1788 tmpFile.close()1789if self.isWindows:1790 message = message.replace("\r\n","\n")1791 submitTemplate = message[:message.index(separatorLine)]1792if self.shelve:1793p4_write_pipe(['shelve','-i'], submitTemplate)1794else:1795p4_write_pipe(['submit','-i'], submitTemplate)1796# The rename/copy happened by applying a patch that created a1797# new file. This leaves it writable, which confuses p4.1798for f in pureRenameCopy:1799p4_sync(f,"-f")18001801if self.preserveUser:1802if p4User:1803# Get last changelist number. Cannot easily get it from1804# the submit command output as the output is1805# unmarshalled.1806 changelist = self.lastP4Changelist()1807 self.modifyChangelistUser(changelist, p4User)18081809 submitted =True18101811finally:1812# skip this patch1813if not submitted or self.shelve:1814if self.shelve:1815print("Reverting shelved files.")1816else:1817print("Submission cancelled, undoing p4 changes.")1818for f in editedFiles | filesToDelete:1819p4_revert(f)1820for f in filesToAdd:1821p4_revert(f)1822 os.remove(f)18231824 os.remove(fileName)1825return submitted18261827# Export git tags as p4 labels. Create a p4 label and then tag1828# with that.1829defexportGitTags(self, gitTags):1830 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1831iflen(validLabelRegexp) ==0:1832 validLabelRegexp = defaultLabelRegexp1833 m = re.compile(validLabelRegexp)18341835for name in gitTags:18361837if not m.match(name):1838if verbose:1839print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1840continue18411842# Get the p4 commit this corresponds to1843 logMessage =extractLogMessageFromGitCommit(name)1844 values =extractSettingsGitLog(logMessage)18451846if not values.has_key('change'):1847# a tag pointing to something not sent to p4; ignore1848if verbose:1849print"git tag%sdoes not give a p4 commit"% name1850continue1851else:1852 changelist = values['change']18531854# Get the tag details.1855 inHeader =True1856 isAnnotated =False1857 body = []1858for l inread_pipe_lines(["git","cat-file","-p", name]):1859 l = l.strip()1860if inHeader:1861if re.match(r'tag\s+', l):1862 isAnnotated =True1863elif re.match(r'\s*$', l):1864 inHeader =False1865continue1866else:1867 body.append(l)18681869if not isAnnotated:1870 body = ["lightweight tag imported by git p4\n"]18711872# Create the label - use the same view as the client spec we are using1873 clientSpec =getClientSpec()18741875 labelTemplate ="Label:%s\n"% name1876 labelTemplate +="Description:\n"1877for b in body:1878 labelTemplate +="\t"+ b +"\n"1879 labelTemplate +="View:\n"1880for depot_side in clientSpec.mappings:1881 labelTemplate +="\t%s\n"% depot_side18821883if self.dry_run:1884print"Would create p4 label%sfor tag"% name1885elif self.prepare_p4_only:1886print"Not creating p4 label%sfor tag due to option" \1887" --prepare-p4-only"% name1888else:1889p4_write_pipe(["label","-i"], labelTemplate)18901891# Use the label1892p4_system(["tag","-l", name] +1893["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])18941895if verbose:1896print"created p4 label for tag%s"% name18971898defrun(self, args):1899iflen(args) ==0:1900 self.master =currentGitBranch()1901eliflen(args) ==1:1902 self.master = args[0]1903if notbranchExists(self.master):1904die("Branch%sdoes not exist"% self.master)1905else:1906return False19071908if self.master:1909 allowSubmit =gitConfig("git-p4.allowSubmit")1910iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1911die("%sis not in git-p4.allowSubmit"% self.master)19121913[upstream, settings] =findUpstreamBranchPoint()1914 self.depotPath = settings['depot-paths'][0]1915iflen(self.origin) ==0:1916 self.origin = upstream19171918if self.preserveUser:1919if not self.canChangeChangelists():1920die("Cannot preserve user names without p4 super-user or admin permissions")19211922# if not set from the command line, try the config file1923if self.conflict_behavior is None:1924 val =gitConfig("git-p4.conflict")1925if val:1926if val not in self.conflict_behavior_choices:1927die("Invalid value '%s' for config git-p4.conflict"% val)1928else:1929 val ="ask"1930 self.conflict_behavior = val19311932if self.verbose:1933print"Origin branch is "+ self.origin19341935iflen(self.depotPath) ==0:1936print"Internal error: cannot locate perforce depot path from existing branches"1937 sys.exit(128)19381939 self.useClientSpec =False1940ifgitConfigBool("git-p4.useclientspec"):1941 self.useClientSpec =True1942if self.useClientSpec:1943 self.clientSpecDirs =getClientSpec()19441945# Check for the existence of P4 branches1946 branchesDetected = (len(p4BranchesInGit().keys()) >1)19471948if self.useClientSpec and not branchesDetected:1949# all files are relative to the client spec1950 self.clientPath =getClientRoot()1951else:1952 self.clientPath =p4Where(self.depotPath)19531954if self.clientPath =="":1955die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)19561957print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1958 self.oldWorkingDirectory = os.getcwd()19591960# ensure the clientPath exists1961 new_client_dir =False1962if not os.path.exists(self.clientPath):1963 new_client_dir =True1964 os.makedirs(self.clientPath)19651966chdir(self.clientPath, is_client_path=True)1967if self.dry_run:1968print"Would synchronize p4 checkout in%s"% self.clientPath1969else:1970print"Synchronizing p4 checkout..."1971if new_client_dir:1972# old one was destroyed, and maybe nobody told p41973p4_sync("...","-f")1974else:1975p4_sync("...")1976 self.check()19771978 commits = []1979if self.master:1980 commitish = self.master1981else:1982 commitish ='HEAD'19831984for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, commitish)]):1985 commits.append(line.strip())1986 commits.reverse()19871988if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):1989 self.checkAuthorship =False1990else:1991 self.checkAuthorship =True19921993if self.preserveUser:1994 self.checkValidP4Users(commits)19951996#1997# Build up a set of options to be passed to diff when1998# submitting each commit to p4.1999#2000if self.detectRenames:2001# command-line -M arg2002 self.diffOpts ="-M"2003else:2004# If not explicitly set check the config variable2005 detectRenames =gitConfig("git-p4.detectRenames")20062007if detectRenames.lower() =="false"or detectRenames =="":2008 self.diffOpts =""2009elif detectRenames.lower() =="true":2010 self.diffOpts ="-M"2011else:2012 self.diffOpts ="-M%s"% detectRenames20132014# no command-line arg for -C or --find-copies-harder, just2015# config variables2016 detectCopies =gitConfig("git-p4.detectCopies")2017if detectCopies.lower() =="false"or detectCopies =="":2018pass2019elif detectCopies.lower() =="true":2020 self.diffOpts +=" -C"2021else:2022 self.diffOpts +=" -C%s"% detectCopies20232024ifgitConfigBool("git-p4.detectCopiesHarder"):2025 self.diffOpts +=" --find-copies-harder"20262027#2028# Apply the commits, one at a time. On failure, ask if should2029# continue to try the rest of the patches, or quit.2030#2031if self.dry_run:2032print"Would apply"2033 applied = []2034 last =len(commits) -12035for i, commit inenumerate(commits):2036if self.dry_run:2037print" ",read_pipe(["git","show","-s",2038"--format=format:%h%s", commit])2039 ok =True2040else:2041 ok = self.applyCommit(commit)2042if ok:2043 applied.append(commit)2044else:2045if self.prepare_p4_only and i < last:2046print"Processing only the first commit due to option" \2047" --prepare-p4-only"2048break2049if i < last:2050 quit =False2051while True:2052# prompt for what to do, or use the option/variable2053if self.conflict_behavior =="ask":2054print"What do you want to do?"2055 response =raw_input("[s]kip this commit but apply"2056" the rest, or [q]uit? ")2057if not response:2058continue2059elif self.conflict_behavior =="skip":2060 response ="s"2061elif self.conflict_behavior =="quit":2062 response ="q"2063else:2064die("Unknown conflict_behavior '%s'"%2065 self.conflict_behavior)20662067if response[0] =="s":2068print"Skipping this commit, but applying the rest"2069break2070if response[0] =="q":2071print"Quitting"2072 quit =True2073break2074if quit:2075break20762077chdir(self.oldWorkingDirectory)2078 shelved_applied ="shelved"if self.shelve else"applied"2079if self.dry_run:2080pass2081elif self.prepare_p4_only:2082pass2083eliflen(commits) ==len(applied):2084print("All commits{0}!".format(shelved_applied))20852086 sync =P4Sync()2087if self.branch:2088 sync.branch = self.branch2089 sync.run([])20902091 rebase =P4Rebase()2092 rebase.rebase()20932094else:2095iflen(applied) ==0:2096print("No commits{0}.".format(shelved_applied))2097else:2098print("{0}only the commits marked with '*':".format(shelved_applied.capitalize()))2099for c in commits:2100if c in applied:2101 star ="*"2102else:2103 star =" "2104print star,read_pipe(["git","show","-s",2105"--format=format:%h%s", c])2106print"You will have to do 'git p4 sync' and rebase."21072108ifgitConfigBool("git-p4.exportLabels"):2109 self.exportLabels =True21102111if self.exportLabels:2112 p4Labels =getP4Labels(self.depotPath)2113 gitTags =getGitTags()21142115 missingGitTags = gitTags - p4Labels2116 self.exportGitTags(missingGitTags)21172118# exit with error unless everything applied perfectly2119iflen(commits) !=len(applied):2120 sys.exit(1)21212122return True21232124classView(object):2125"""Represent a p4 view ("p4 help views"), and map files in a2126 repo according to the view."""21272128def__init__(self, client_name):2129 self.mappings = []2130 self.client_prefix ="//%s/"% client_name2131# cache results of "p4 where" to lookup client file locations2132 self.client_spec_path_cache = {}21332134defappend(self, view_line):2135"""Parse a view line, splitting it into depot and client2136 sides. Append to self.mappings, preserving order. This2137 is only needed for tag creation."""21382139# Split the view line into exactly two words. P4 enforces2140# structure on these lines that simplifies this quite a bit.2141#2142# Either or both words may be double-quoted.2143# Single quotes do not matter.2144# Double-quote marks cannot occur inside the words.2145# A + or - prefix is also inside the quotes.2146# There are no quotes unless they contain a space.2147# The line is already white-space stripped.2148# The two words are separated by a single space.2149#2150if view_line[0] =='"':2151# First word is double quoted. Find its end.2152 close_quote_index = view_line.find('"',1)2153if close_quote_index <=0:2154die("No first-word closing quote found:%s"% view_line)2155 depot_side = view_line[1:close_quote_index]2156# skip closing quote and space2157 rhs_index = close_quote_index +1+12158else:2159 space_index = view_line.find(" ")2160if space_index <=0:2161die("No word-splitting space found:%s"% view_line)2162 depot_side = view_line[0:space_index]2163 rhs_index = space_index +121642165# prefix + means overlay on previous mapping2166if depot_side.startswith("+"):2167 depot_side = depot_side[1:]21682169# prefix - means exclude this path, leave out of mappings2170 exclude =False2171if depot_side.startswith("-"):2172 exclude =True2173 depot_side = depot_side[1:]21742175if not exclude:2176 self.mappings.append(depot_side)21772178defconvert_client_path(self, clientFile):2179# chop off //client/ part to make it relative2180if not clientFile.startswith(self.client_prefix):2181die("No prefix '%s' on clientFile '%s'"%2182(self.client_prefix, clientFile))2183return clientFile[len(self.client_prefix):]21842185defupdate_client_spec_path_cache(self, files):2186""" Caching file paths by "p4 where" batch query """21872188# List depot file paths exclude that already cached2189 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]21902191iflen(fileArgs) ==0:2192return# All files in cache21932194 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)2195for res in where_result:2196if"code"in res and res["code"] =="error":2197# assume error is "... file(s) not in client view"2198continue2199if"clientFile"not in res:2200die("No clientFile in 'p4 where' output")2201if"unmap"in res:2202# it will list all of them, but only one not unmap-ped2203continue2204ifgitConfigBool("core.ignorecase"):2205 res['depotFile'] = res['depotFile'].lower()2206 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])22072208# not found files or unmap files set to ""2209for depotFile in fileArgs:2210ifgitConfigBool("core.ignorecase"):2211 depotFile = depotFile.lower()2212if depotFile not in self.client_spec_path_cache:2213 self.client_spec_path_cache[depotFile] =""22142215defmap_in_client(self, depot_path):2216"""Return the relative location in the client where this2217 depot file should live. Returns "" if the file should2218 not be mapped in the client."""22192220ifgitConfigBool("core.ignorecase"):2221 depot_path = depot_path.lower()22222223if depot_path in self.client_spec_path_cache:2224return self.client_spec_path_cache[depot_path]22252226die("Error:%sis not found in client spec path"% depot_path )2227return""22282229classP4Sync(Command, P4UserMap):2230 delete_actions = ("delete","move/delete","purge")22312232def__init__(self):2233 Command.__init__(self)2234 P4UserMap.__init__(self)2235 self.options = [2236 optparse.make_option("--branch", dest="branch"),2237 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),2238 optparse.make_option("--changesfile", dest="changesFile"),2239 optparse.make_option("--silent", dest="silent", action="store_true"),2240 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),2241 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2242 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",2243help="Import into refs/heads/ , not refs/remotes"),2244 optparse.make_option("--max-changes", dest="maxChanges",2245help="Maximum number of changes to import"),2246 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",2247help="Internal block size to use when iteratively calling p4 changes"),2248 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2249help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2250 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2251help="Only sync files that are included in the Perforce Client Spec"),2252 optparse.make_option("-/", dest="cloneExclude",2253 action="append",type="string",2254help="exclude depot path"),2255]2256 self.description ="""Imports from Perforce into a git repository.\n2257 example:2258 //depot/my/project/ -- to import the current head2259 //depot/my/project/@all -- to import everything2260 //depot/my/project/@1,6 -- to import only from revision 1 to 622612262 (a ... is not needed in the path p4 specification, it's added implicitly)"""22632264 self.usage +=" //depot/path[@revRange]"2265 self.silent =False2266 self.createdBranches =set()2267 self.committedChanges =set()2268 self.branch =""2269 self.detectBranches =False2270 self.detectLabels =False2271 self.importLabels =False2272 self.changesFile =""2273 self.syncWithOrigin =True2274 self.importIntoRemotes =True2275 self.maxChanges =""2276 self.changes_block_size =None2277 self.keepRepoPath =False2278 self.depotPaths =None2279 self.p4BranchesInGit = []2280 self.cloneExclude = []2281 self.useClientSpec =False2282 self.useClientSpec_from_options =False2283 self.clientSpecDirs =None2284 self.tempBranches = []2285 self.tempBranchLocation ="refs/git-p4-tmp"2286 self.largeFileSystem =None22872288ifgitConfig('git-p4.largeFileSystem'):2289 largeFileSystemConstructor =globals()[gitConfig('git-p4.largeFileSystem')]2290 self.largeFileSystem =largeFileSystemConstructor(2291lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)2292)22932294ifgitConfig("git-p4.syncFromOrigin") =="false":2295 self.syncWithOrigin =False22962297# This is required for the "append" cloneExclude action2298defensure_value(self, attr, value):2299if nothasattr(self, attr)orgetattr(self, attr)is None:2300setattr(self, attr, value)2301returngetattr(self, attr)23022303# Force a checkpoint in fast-import and wait for it to finish2304defcheckpoint(self):2305 self.gitStream.write("checkpoint\n\n")2306 self.gitStream.write("progress checkpoint\n\n")2307 out = self.gitOutput.readline()2308if self.verbose:2309print"checkpoint finished: "+ out23102311defextractFilesFromCommit(self, commit):2312 self.cloneExclude = [re.sub(r"\.\.\.$","", path)2313for path in self.cloneExclude]2314 files = []2315 fnum =02316while commit.has_key("depotFile%s"% fnum):2317 path = commit["depotFile%s"% fnum]23182319if[p for p in self.cloneExclude2320ifp4PathStartsWith(path, p)]:2321 found =False2322else:2323 found = [p for p in self.depotPaths2324ifp4PathStartsWith(path, p)]2325if not found:2326 fnum = fnum +12327continue23282329file= {}2330file["path"] = path2331file["rev"] = commit["rev%s"% fnum]2332file["action"] = commit["action%s"% fnum]2333file["type"] = commit["type%s"% fnum]2334 files.append(file)2335 fnum = fnum +12336return files23372338defextractJobsFromCommit(self, commit):2339 jobs = []2340 jnum =02341while commit.has_key("job%s"% jnum):2342 job = commit["job%s"% jnum]2343 jobs.append(job)2344 jnum = jnum +12345return jobs23462347defstripRepoPath(self, path, prefixes):2348"""When streaming files, this is called to map a p4 depot path2349 to where it should go in git. The prefixes are either2350 self.depotPaths, or self.branchPrefixes in the case of2351 branch detection."""23522353if self.useClientSpec:2354# branch detection moves files up a level (the branch name)2355# from what client spec interpretation gives2356 path = self.clientSpecDirs.map_in_client(path)2357if self.detectBranches:2358for b in self.knownBranches:2359if path.startswith(b +"/"):2360 path = path[len(b)+1:]23612362elif self.keepRepoPath:2363# Preserve everything in relative path name except leading2364# //depot/; just look at first prefix as they all should2365# be in the same depot.2366 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2367ifp4PathStartsWith(path, depot):2368 path = path[len(depot):]23692370else:2371for p in prefixes:2372ifp4PathStartsWith(path, p):2373 path = path[len(p):]2374break23752376 path =wildcard_decode(path)2377return path23782379defsplitFilesIntoBranches(self, commit):2380"""Look at each depotFile in the commit to figure out to what2381 branch it belongs."""23822383if self.clientSpecDirs:2384 files = self.extractFilesFromCommit(commit)2385 self.clientSpecDirs.update_client_spec_path_cache(files)23862387 branches = {}2388 fnum =02389while commit.has_key("depotFile%s"% fnum):2390 path = commit["depotFile%s"% fnum]2391 found = [p for p in self.depotPaths2392ifp4PathStartsWith(path, p)]2393if not found:2394 fnum = fnum +12395continue23962397file= {}2398file["path"] = path2399file["rev"] = commit["rev%s"% fnum]2400file["action"] = commit["action%s"% fnum]2401file["type"] = commit["type%s"% fnum]2402 fnum = fnum +124032404# start with the full relative path where this file would2405# go in a p4 client2406if self.useClientSpec:2407 relPath = self.clientSpecDirs.map_in_client(path)2408else:2409 relPath = self.stripRepoPath(path, self.depotPaths)24102411for branch in self.knownBranches.keys():2412# add a trailing slash so that a commit into qt/4.2foo2413# doesn't end up in qt/4.2, e.g.2414if relPath.startswith(branch +"/"):2415if branch not in branches:2416 branches[branch] = []2417 branches[branch].append(file)2418break24192420return branches24212422defwriteToGitStream(self, gitMode, relPath, contents):2423 self.gitStream.write('M%sinline%s\n'% (gitMode, relPath))2424 self.gitStream.write('data%d\n'%sum(len(d)for d in contents))2425for d in contents:2426 self.gitStream.write(d)2427 self.gitStream.write('\n')24282429# output one file from the P4 stream2430# - helper for streamP4Files24312432defstreamOneP4File(self,file, contents):2433 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2434if verbose:2435 size =int(self.stream_file['fileSize'])2436 sys.stdout.write('\r%s-->%s(%iMB)\n'% (file['depotFile'], relPath, size/1024/1024))2437 sys.stdout.flush()24382439(type_base, type_mods) =split_p4_type(file["type"])24402441 git_mode ="100644"2442if"x"in type_mods:2443 git_mode ="100755"2444if type_base =="symlink":2445 git_mode ="120000"2446# p4 print on a symlink sometimes contains "target\n";2447# if it does, remove the newline2448 data =''.join(contents)2449if not data:2450# Some version of p4 allowed creating a symlink that pointed2451# to nothing. This causes p4 errors when checking out such2452# a change, and errors here too. Work around it by ignoring2453# the bad symlink; hopefully a future change fixes it.2454print"\nIgnoring empty symlink in%s"%file['depotFile']2455return2456elif data[-1] =='\n':2457 contents = [data[:-1]]2458else:2459 contents = [data]24602461if type_base =="utf16":2462# p4 delivers different text in the python output to -G2463# than it does when using "print -o", or normal p4 client2464# operations. utf16 is converted to ascii or utf8, perhaps.2465# But ascii text saved as -t utf16 is completely mangled.2466# Invoke print -o to get the real contents.2467#2468# On windows, the newlines will always be mangled by print, so put2469# them back too. This is not needed to the cygwin windows version,2470# just the native "NT" type.2471#2472try:2473 text =p4_read_pipe(['print','-q','-o','-','%s@%s'% (file['depotFile'],file['change'])])2474exceptExceptionas e:2475if'Translation of file content failed'instr(e):2476 type_base ='binary'2477else:2478raise e2479else:2480ifp4_version_string().find('/NT') >=0:2481 text = text.replace('\r\n','\n')2482 contents = [ text ]24832484if type_base =="apple":2485# Apple filetype files will be streamed as a concatenation of2486# its appledouble header and the contents. This is useless2487# on both macs and non-macs. If using "print -q -o xx", it2488# will create "xx" with the data, and "%xx" with the header.2489# This is also not very useful.2490#2491# Ideally, someday, this script can learn how to generate2492# appledouble files directly and import those to git, but2493# non-mac machines can never find a use for apple filetype.2494print"\nIgnoring apple filetype file%s"%file['depotFile']2495return24962497# Note that we do not try to de-mangle keywords on utf16 files,2498# even though in theory somebody may want that.2499 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2500if pattern:2501 regexp = re.compile(pattern, re.VERBOSE)2502 text =''.join(contents)2503 text = regexp.sub(r'$\1$', text)2504 contents = [ text ]25052506try:2507 relPath.decode('ascii')2508except:2509 encoding ='utf8'2510ifgitConfig('git-p4.pathEncoding'):2511 encoding =gitConfig('git-p4.pathEncoding')2512 relPath = relPath.decode(encoding,'replace').encode('utf8','replace')2513if self.verbose:2514print'Path with non-ASCII characters detected. Used%sto encode:%s'% (encoding, relPath)25152516if self.largeFileSystem:2517(git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)25182519 self.writeToGitStream(git_mode, relPath, contents)25202521defstreamOneP4Deletion(self,file):2522 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2523if verbose:2524 sys.stdout.write("delete%s\n"% relPath)2525 sys.stdout.flush()2526 self.gitStream.write("D%s\n"% relPath)25272528if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):2529 self.largeFileSystem.removeLargeFile(relPath)25302531# handle another chunk of streaming data2532defstreamP4FilesCb(self, marshalled):25332534# catch p4 errors and complain2535 err =None2536if"code"in marshalled:2537if marshalled["code"] =="error":2538if"data"in marshalled:2539 err = marshalled["data"].rstrip()25402541if not err and'fileSize'in self.stream_file:2542 required_bytes =int((4*int(self.stream_file["fileSize"])) -calcDiskFree())2543if required_bytes >0:2544 err ='Not enough space left on%s! Free at least%iMB.'% (2545 os.getcwd(), required_bytes/1024/10242546)25472548if err:2549 f =None2550if self.stream_have_file_info:2551if"depotFile"in self.stream_file:2552 f = self.stream_file["depotFile"]2553# force a failure in fast-import, else an empty2554# commit will be made2555 self.gitStream.write("\n")2556 self.gitStream.write("die-now\n")2557 self.gitStream.close()2558# ignore errors, but make sure it exits first2559 self.importProcess.wait()2560if f:2561die("Error from p4 print for%s:%s"% (f, err))2562else:2563die("Error from p4 print:%s"% err)25642565if marshalled.has_key('depotFile')and self.stream_have_file_info:2566# start of a new file - output the old one first2567 self.streamOneP4File(self.stream_file, self.stream_contents)2568 self.stream_file = {}2569 self.stream_contents = []2570 self.stream_have_file_info =False25712572# pick up the new file information... for the2573# 'data' field we need to append to our array2574for k in marshalled.keys():2575if k =='data':2576if'streamContentSize'not in self.stream_file:2577 self.stream_file['streamContentSize'] =02578 self.stream_file['streamContentSize'] +=len(marshalled['data'])2579 self.stream_contents.append(marshalled['data'])2580else:2581 self.stream_file[k] = marshalled[k]25822583if(verbose and2584'streamContentSize'in self.stream_file and2585'fileSize'in self.stream_file and2586'depotFile'in self.stream_file):2587 size =int(self.stream_file["fileSize"])2588if size >0:2589 progress =100*self.stream_file['streamContentSize']/size2590 sys.stdout.write('\r%s %d%%(%iMB)'% (self.stream_file['depotFile'], progress,int(size/1024/1024)))2591 sys.stdout.flush()25922593 self.stream_have_file_info =True25942595# Stream directly from "p4 files" into "git fast-import"2596defstreamP4Files(self, files):2597 filesForCommit = []2598 filesToRead = []2599 filesToDelete = []26002601for f in files:2602 filesForCommit.append(f)2603if f['action']in self.delete_actions:2604 filesToDelete.append(f)2605else:2606 filesToRead.append(f)26072608# deleted files...2609for f in filesToDelete:2610 self.streamOneP4Deletion(f)26112612iflen(filesToRead) >0:2613 self.stream_file = {}2614 self.stream_contents = []2615 self.stream_have_file_info =False26162617# curry self argument2618defstreamP4FilesCbSelf(entry):2619 self.streamP4FilesCb(entry)26202621 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]26222623p4CmdList(["-x","-","print"],2624 stdin=fileArgs,2625 cb=streamP4FilesCbSelf)26262627# do the last chunk2628if self.stream_file.has_key('depotFile'):2629 self.streamOneP4File(self.stream_file, self.stream_contents)26302631defmake_email(self, userid):2632if userid in self.users:2633return self.users[userid]2634else:2635return"%s<a@b>"% userid26362637defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2638""" Stream a p4 tag.2639 commit is either a git commit, or a fast-import mark, ":<p4commit>"2640 """26412642if verbose:2643print"writing tag%sfor commit%s"% (labelName, commit)2644 gitStream.write("tag%s\n"% labelName)2645 gitStream.write("from%s\n"% commit)26462647if labelDetails.has_key('Owner'):2648 owner = labelDetails["Owner"]2649else:2650 owner =None26512652# Try to use the owner of the p4 label, or failing that,2653# the current p4 user id.2654if owner:2655 email = self.make_email(owner)2656else:2657 email = self.make_email(self.p4UserId())2658 tagger ="%s %s %s"% (email, epoch, self.tz)26592660 gitStream.write("tagger%s\n"% tagger)26612662print"labelDetails=",labelDetails2663if labelDetails.has_key('Description'):2664 description = labelDetails['Description']2665else:2666 description ='Label from git p4'26672668 gitStream.write("data%d\n"%len(description))2669 gitStream.write(description)2670 gitStream.write("\n")26712672definClientSpec(self, path):2673if not self.clientSpecDirs:2674return True2675 inClientSpec = self.clientSpecDirs.map_in_client(path)2676if not inClientSpec and self.verbose:2677print('Ignoring file outside of client spec:{0}'.format(path))2678return inClientSpec26792680defhasBranchPrefix(self, path):2681if not self.branchPrefixes:2682return True2683 hasPrefix = [p for p in self.branchPrefixes2684ifp4PathStartsWith(path, p)]2685if not hasPrefix and self.verbose:2686print('Ignoring file outside of prefix:{0}'.format(path))2687return hasPrefix26882689defcommit(self, details, files, branch, parent =""):2690 epoch = details["time"]2691 author = details["user"]2692 jobs = self.extractJobsFromCommit(details)26932694if self.verbose:2695print('commit into{0}'.format(branch))26962697if self.clientSpecDirs:2698 self.clientSpecDirs.update_client_spec_path_cache(files)26992700 files = [f for f in files2701if self.inClientSpec(f['path'])and self.hasBranchPrefix(f['path'])]27022703if not files and notgitConfigBool('git-p4.keepEmptyCommits'):2704print('Ignoring revision{0}as it would produce an empty commit.'2705.format(details['change']))2706return27072708 self.gitStream.write("commit%s\n"% branch)2709 self.gitStream.write("mark :%s\n"% details["change"])2710 self.committedChanges.add(int(details["change"]))2711 committer =""2712if author not in self.users:2713 self.getUserMapFromPerforceServer()2714 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)27152716 self.gitStream.write("committer%s\n"% committer)27172718 self.gitStream.write("data <<EOT\n")2719 self.gitStream.write(details["desc"])2720iflen(jobs) >0:2721 self.gitStream.write("\nJobs:%s"% (' '.join(jobs)))2722 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2723(','.join(self.branchPrefixes), details["change"]))2724iflen(details['options']) >0:2725 self.gitStream.write(": options =%s"% details['options'])2726 self.gitStream.write("]\nEOT\n\n")27272728iflen(parent) >0:2729if self.verbose:2730print"parent%s"% parent2731 self.gitStream.write("from%s\n"% parent)27322733 self.streamP4Files(files)2734 self.gitStream.write("\n")27352736 change =int(details["change"])27372738if self.labels.has_key(change):2739 label = self.labels[change]2740 labelDetails = label[0]2741 labelRevisions = label[1]2742if self.verbose:2743print"Change%sis labelled%s"% (change, labelDetails)27442745 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2746for p in self.branchPrefixes])27472748iflen(files) ==len(labelRevisions):27492750 cleanedFiles = {}2751for info in files:2752if info["action"]in self.delete_actions:2753continue2754 cleanedFiles[info["depotFile"]] = info["rev"]27552756if cleanedFiles == labelRevisions:2757 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)27582759else:2760if not self.silent:2761print("Tag%sdoes not match with change%s: files do not match."2762% (labelDetails["label"], change))27632764else:2765if not self.silent:2766print("Tag%sdoes not match with change%s: file count is different."2767% (labelDetails["label"], change))27682769# Build a dictionary of changelists and labels, for "detect-labels" option.2770defgetLabels(self):2771 self.labels = {}27722773 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2774iflen(l) >0and not self.silent:2775print"Finding files belonging to labels in%s"% `self.depotPaths`27762777for output in l:2778 label = output["label"]2779 revisions = {}2780 newestChange =02781if self.verbose:2782print"Querying files for label%s"% label2783forfileinp4CmdList(["files"] +2784["%s...@%s"% (p, label)2785for p in self.depotPaths]):2786 revisions[file["depotFile"]] =file["rev"]2787 change =int(file["change"])2788if change > newestChange:2789 newestChange = change27902791 self.labels[newestChange] = [output, revisions]27922793if self.verbose:2794print"Label changes:%s"% self.labels.keys()27952796# Import p4 labels as git tags. A direct mapping does not2797# exist, so assume that if all the files are at the same revision2798# then we can use that, or it's something more complicated we should2799# just ignore.2800defimportP4Labels(self, stream, p4Labels):2801if verbose:2802print"import p4 labels: "+' '.join(p4Labels)28032804 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2805 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2806iflen(validLabelRegexp) ==0:2807 validLabelRegexp = defaultLabelRegexp2808 m = re.compile(validLabelRegexp)28092810for name in p4Labels:2811 commitFound =False28122813if not m.match(name):2814if verbose:2815print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2816continue28172818if name in ignoredP4Labels:2819continue28202821 labelDetails =p4CmdList(['label',"-o", name])[0]28222823# get the most recent changelist for each file in this label2824 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2825for p in self.depotPaths])28262827if change.has_key('change'):2828# find the corresponding git commit; take the oldest commit2829 changelist =int(change['change'])2830if changelist in self.committedChanges:2831 gitCommit =":%d"% changelist # use a fast-import mark2832 commitFound =True2833else:2834 gitCommit =read_pipe(["git","rev-list","--max-count=1",2835"--reverse",":/\[git-p4:.*change =%d\]"% changelist], ignore_error=True)2836iflen(gitCommit) ==0:2837print"importing label%s: could not find git commit for changelist%d"% (name, changelist)2838else:2839 commitFound =True2840 gitCommit = gitCommit.strip()28412842if commitFound:2843# Convert from p4 time format2844try:2845 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2846exceptValueError:2847print"Could not convert label time%s"% labelDetails['Update']2848 tmwhen =128492850 when =int(time.mktime(tmwhen))2851 self.streamTag(stream, name, labelDetails, gitCommit, when)2852if verbose:2853print"p4 label%smapped to git commit%s"% (name, gitCommit)2854else:2855if verbose:2856print"Label%shas no changelists - possibly deleted?"% name28572858if not commitFound:2859# We can't import this label; don't try again as it will get very2860# expensive repeatedly fetching all the files for labels that will2861# never be imported. If the label is moved in the future, the2862# ignore will need to be removed manually.2863system(["git","config","--add","git-p4.ignoredP4Labels", name])28642865defguessProjectName(self):2866for p in self.depotPaths:2867if p.endswith("/"):2868 p = p[:-1]2869 p = p[p.strip().rfind("/") +1:]2870if not p.endswith("/"):2871 p +="/"2872return p28732874defgetBranchMapping(self):2875 lostAndFoundBranches =set()28762877 user =gitConfig("git-p4.branchUser")2878iflen(user) >0:2879 command ="branches -u%s"% user2880else:2881 command ="branches"28822883for info inp4CmdList(command):2884 details =p4Cmd(["branch","-o", info["branch"]])2885 viewIdx =02886while details.has_key("View%s"% viewIdx):2887 paths = details["View%s"% viewIdx].split(" ")2888 viewIdx = viewIdx +12889# require standard //depot/foo/... //depot/bar/... mapping2890iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2891continue2892 source = paths[0]2893 destination = paths[1]2894## HACK2895ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2896 source = source[len(self.depotPaths[0]):-4]2897 destination = destination[len(self.depotPaths[0]):-4]28982899if destination in self.knownBranches:2900if not self.silent:2901print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2902print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2903continue29042905 self.knownBranches[destination] = source29062907 lostAndFoundBranches.discard(destination)29082909if source not in self.knownBranches:2910 lostAndFoundBranches.add(source)29112912# Perforce does not strictly require branches to be defined, so we also2913# check git config for a branch list.2914#2915# Example of branch definition in git config file:2916# [git-p4]2917# branchList=main:branchA2918# branchList=main:branchB2919# branchList=branchA:branchC2920 configBranches =gitConfigList("git-p4.branchList")2921for branch in configBranches:2922if branch:2923(source, destination) = branch.split(":")2924 self.knownBranches[destination] = source29252926 lostAndFoundBranches.discard(destination)29272928if source not in self.knownBranches:2929 lostAndFoundBranches.add(source)293029312932for branch in lostAndFoundBranches:2933 self.knownBranches[branch] = branch29342935defgetBranchMappingFromGitBranches(self):2936 branches =p4BranchesInGit(self.importIntoRemotes)2937for branch in branches.keys():2938if branch =="master":2939 branch ="main"2940else:2941 branch = branch[len(self.projectName):]2942 self.knownBranches[branch] = branch29432944defupdateOptionDict(self, d):2945 option_keys = {}2946if self.keepRepoPath:2947 option_keys['keepRepoPath'] =129482949 d["options"] =' '.join(sorted(option_keys.keys()))29502951defreadOptions(self, d):2952 self.keepRepoPath = (d.has_key('options')2953and('keepRepoPath'in d['options']))29542955defgitRefForBranch(self, branch):2956if branch =="main":2957return self.refPrefix +"master"29582959iflen(branch) <=0:2960return branch29612962return self.refPrefix + self.projectName + branch29632964defgitCommitByP4Change(self, ref, change):2965if self.verbose:2966print"looking in ref "+ ref +" for change%susing bisect..."% change29672968 earliestCommit =""2969 latestCommit =parseRevision(ref)29702971while True:2972if self.verbose:2973print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2974 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2975iflen(next) ==0:2976if self.verbose:2977print"argh"2978return""2979 log =extractLogMessageFromGitCommit(next)2980 settings =extractSettingsGitLog(log)2981 currentChange =int(settings['change'])2982if self.verbose:2983print"current change%s"% currentChange29842985if currentChange == change:2986if self.verbose:2987print"found%s"% next2988return next29892990if currentChange < change:2991 earliestCommit ="^%s"% next2992else:2993 latestCommit ="%s"% next29942995return""29962997defimportNewBranch(self, branch, maxChange):2998# make fast-import flush all changes to disk and update the refs using the checkpoint2999# command so that we can try to find the branch parent in the git history3000 self.gitStream.write("checkpoint\n\n");3001 self.gitStream.flush();3002 branchPrefix = self.depotPaths[0] + branch +"/"3003range="@1,%s"% maxChange3004#print "prefix" + branchPrefix3005 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)3006iflen(changes) <=0:3007return False3008 firstChange = changes[0]3009#print "first change in branch: %s" % firstChange3010 sourceBranch = self.knownBranches[branch]3011 sourceDepotPath = self.depotPaths[0] + sourceBranch3012 sourceRef = self.gitRefForBranch(sourceBranch)3013#print "source " + sourceBranch30143015 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])3016#print "branch parent: %s" % branchParentChange3017 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)3018iflen(gitParent) >0:3019 self.initialParents[self.gitRefForBranch(branch)] = gitParent3020#print "parent git commit: %s" % gitParent30213022 self.importChanges(changes)3023return True30243025defsearchParent(self, parent, branch, target):3026 parentFound =False3027for blob inread_pipe_lines(["git","rev-list","--reverse",3028"--no-merges", parent]):3029 blob = blob.strip()3030iflen(read_pipe(["git","diff-tree", blob, target])) ==0:3031 parentFound =True3032if self.verbose:3033print"Found parent of%sin commit%s"% (branch, blob)3034break3035if parentFound:3036return blob3037else:3038return None30393040defimportChanges(self, changes):3041 cnt =13042for change in changes:3043 description =p4_describe(change)3044 self.updateOptionDict(description)30453046if not self.silent:3047 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))3048 sys.stdout.flush()3049 cnt = cnt +130503051try:3052if self.detectBranches:3053 branches = self.splitFilesIntoBranches(description)3054for branch in branches.keys():3055## HACK --hwn3056 branchPrefix = self.depotPaths[0] + branch +"/"3057 self.branchPrefixes = [ branchPrefix ]30583059 parent =""30603061 filesForCommit = branches[branch]30623063if self.verbose:3064print"branch is%s"% branch30653066 self.updatedBranches.add(branch)30673068if branch not in self.createdBranches:3069 self.createdBranches.add(branch)3070 parent = self.knownBranches[branch]3071if parent == branch:3072 parent =""3073else:3074 fullBranch = self.projectName + branch3075if fullBranch not in self.p4BranchesInGit:3076if not self.silent:3077print("\nImporting new branch%s"% fullBranch);3078if self.importNewBranch(branch, change -1):3079 parent =""3080 self.p4BranchesInGit.append(fullBranch)3081if not self.silent:3082print("\nResuming with change%s"% change);30833084if self.verbose:3085print"parent determined through known branches:%s"% parent30863087 branch = self.gitRefForBranch(branch)3088 parent = self.gitRefForBranch(parent)30893090if self.verbose:3091print"looking for initial parent for%s; current parent is%s"% (branch, parent)30923093iflen(parent) ==0and branch in self.initialParents:3094 parent = self.initialParents[branch]3095del self.initialParents[branch]30963097 blob =None3098iflen(parent) >0:3099 tempBranch ="%s/%d"% (self.tempBranchLocation, change)3100if self.verbose:3101print"Creating temporary branch: "+ tempBranch3102 self.commit(description, filesForCommit, tempBranch)3103 self.tempBranches.append(tempBranch)3104 self.checkpoint()3105 blob = self.searchParent(parent, branch, tempBranch)3106if blob:3107 self.commit(description, filesForCommit, branch, blob)3108else:3109if self.verbose:3110print"Parent of%snot found. Committing into head of%s"% (branch, parent)3111 self.commit(description, filesForCommit, branch, parent)3112else:3113 files = self.extractFilesFromCommit(description)3114 self.commit(description, files, self.branch,3115 self.initialParent)3116# only needed once, to connect to the previous commit3117 self.initialParent =""3118exceptIOError:3119print self.gitError.read()3120 sys.exit(1)31213122defimportHeadRevision(self, revision):3123print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)31243125 details = {}3126 details["user"] ="git perforce import user"3127 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"3128% (' '.join(self.depotPaths), revision))3129 details["change"] = revision3130 newestRevision =031313132 fileCnt =03133 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]31343135for info inp4CmdList(["files"] + fileArgs):31363137if'code'in info and info['code'] =='error':3138 sys.stderr.write("p4 returned an error:%s\n"3139% info['data'])3140if info['data'].find("must refer to client") >=0:3141 sys.stderr.write("This particular p4 error is misleading.\n")3142 sys.stderr.write("Perhaps the depot path was misspelled.\n");3143 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))3144 sys.exit(1)3145if'p4ExitCode'in info:3146 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])3147 sys.exit(1)314831493150 change =int(info["change"])3151if change > newestRevision:3152 newestRevision = change31533154if info["action"]in self.delete_actions:3155# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!3156#fileCnt = fileCnt + 13157continue31583159for prop in["depotFile","rev","action","type"]:3160 details["%s%s"% (prop, fileCnt)] = info[prop]31613162 fileCnt = fileCnt +131633164 details["change"] = newestRevision31653166# Use time from top-most change so that all git p4 clones of3167# the same p4 repo have the same commit SHA1s.3168 res =p4_describe(newestRevision)3169 details["time"] = res["time"]31703171 self.updateOptionDict(details)3172try:3173 self.commit(details, self.extractFilesFromCommit(details), self.branch)3174exceptIOError:3175print"IO error with git fast-import. Is your git version recent enough?"3176print self.gitError.read()317731783179defrun(self, args):3180 self.depotPaths = []3181 self.changeRange =""3182 self.previousDepotPaths = []3183 self.hasOrigin =False31843185# map from branch depot path to parent branch3186 self.knownBranches = {}3187 self.initialParents = {}31883189if self.importIntoRemotes:3190 self.refPrefix ="refs/remotes/p4/"3191else:3192 self.refPrefix ="refs/heads/p4/"31933194if self.syncWithOrigin:3195 self.hasOrigin =originP4BranchesExist()3196if self.hasOrigin:3197if not self.silent:3198print'Syncing with origin first, using "git fetch origin"'3199system("git fetch origin")32003201 branch_arg_given =bool(self.branch)3202iflen(self.branch) ==0:3203 self.branch = self.refPrefix +"master"3204ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:3205system("git update-ref%srefs/heads/p4"% self.branch)3206system("git branch -D p4")32073208# accept either the command-line option, or the configuration variable3209if self.useClientSpec:3210# will use this after clone to set the variable3211 self.useClientSpec_from_options =True3212else:3213ifgitConfigBool("git-p4.useclientspec"):3214 self.useClientSpec =True3215if self.useClientSpec:3216 self.clientSpecDirs =getClientSpec()32173218# TODO: should always look at previous commits,3219# merge with previous imports, if possible.3220if args == []:3221if self.hasOrigin:3222createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)32233224# branches holds mapping from branch name to sha13225 branches =p4BranchesInGit(self.importIntoRemotes)32263227# restrict to just this one, disabling detect-branches3228if branch_arg_given:3229 short = self.branch.split("/")[-1]3230if short in branches:3231 self.p4BranchesInGit = [ short ]3232else:3233 self.p4BranchesInGit = branches.keys()32343235iflen(self.p4BranchesInGit) >1:3236if not self.silent:3237print"Importing from/into multiple branches"3238 self.detectBranches =True3239for branch in branches.keys():3240 self.initialParents[self.refPrefix + branch] = \3241 branches[branch]32423243if self.verbose:3244print"branches:%s"% self.p4BranchesInGit32453246 p4Change =03247for branch in self.p4BranchesInGit:3248 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)32493250 settings =extractSettingsGitLog(logMsg)32513252 self.readOptions(settings)3253if(settings.has_key('depot-paths')3254and settings.has_key('change')):3255 change =int(settings['change']) +13256 p4Change =max(p4Change, change)32573258 depotPaths =sorted(settings['depot-paths'])3259if self.previousDepotPaths == []:3260 self.previousDepotPaths = depotPaths3261else:3262 paths = []3263for(prev, cur)inzip(self.previousDepotPaths, depotPaths):3264 prev_list = prev.split("/")3265 cur_list = cur.split("/")3266for i inrange(0,min(len(cur_list),len(prev_list))):3267if cur_list[i] <> prev_list[i]:3268 i = i -13269break32703271 paths.append("/".join(cur_list[:i +1]))32723273 self.previousDepotPaths = paths32743275if p4Change >0:3276 self.depotPaths =sorted(self.previousDepotPaths)3277 self.changeRange ="@%s,#head"% p4Change3278if not self.silent and not self.detectBranches:3279print"Performing incremental import into%sgit branch"% self.branch32803281# accept multiple ref name abbreviations:3282# refs/foo/bar/branch -> use it exactly3283# p4/branch -> prepend refs/remotes/ or refs/heads/3284# branch -> prepend refs/remotes/p4/ or refs/heads/p4/3285if not self.branch.startswith("refs/"):3286if self.importIntoRemotes:3287 prepend ="refs/remotes/"3288else:3289 prepend ="refs/heads/"3290if not self.branch.startswith("p4/"):3291 prepend +="p4/"3292 self.branch = prepend + self.branch32933294iflen(args) ==0and self.depotPaths:3295if not self.silent:3296print"Depot paths:%s"%' '.join(self.depotPaths)3297else:3298if self.depotPaths and self.depotPaths != args:3299print("previous import used depot path%sand now%swas specified. "3300"This doesn't work!"% (' '.join(self.depotPaths),3301' '.join(args)))3302 sys.exit(1)33033304 self.depotPaths =sorted(args)33053306 revision =""3307 self.users = {}33083309# Make sure no revision specifiers are used when --changesfile3310# is specified.3311 bad_changesfile =False3312iflen(self.changesFile) >0:3313for p in self.depotPaths:3314if p.find("@") >=0or p.find("#") >=0:3315 bad_changesfile =True3316break3317if bad_changesfile:3318die("Option --changesfile is incompatible with revision specifiers")33193320 newPaths = []3321for p in self.depotPaths:3322if p.find("@") != -1:3323 atIdx = p.index("@")3324 self.changeRange = p[atIdx:]3325if self.changeRange =="@all":3326 self.changeRange =""3327elif','not in self.changeRange:3328 revision = self.changeRange3329 self.changeRange =""3330 p = p[:atIdx]3331elif p.find("#") != -1:3332 hashIdx = p.index("#")3333 revision = p[hashIdx:]3334 p = p[:hashIdx]3335elif self.previousDepotPaths == []:3336# pay attention to changesfile, if given, else import3337# the entire p4 tree at the head revision3338iflen(self.changesFile) ==0:3339 revision ="#head"33403341 p = re.sub("\.\.\.$","", p)3342if not p.endswith("/"):3343 p +="/"33443345 newPaths.append(p)33463347 self.depotPaths = newPaths33483349# --detect-branches may change this for each branch3350 self.branchPrefixes = self.depotPaths33513352 self.loadUserMapFromCache()3353 self.labels = {}3354if self.detectLabels:3355 self.getLabels();33563357if self.detectBranches:3358## FIXME - what's a P4 projectName ?3359 self.projectName = self.guessProjectName()33603361if self.hasOrigin:3362 self.getBranchMappingFromGitBranches()3363else:3364 self.getBranchMapping()3365if self.verbose:3366print"p4-git branches:%s"% self.p4BranchesInGit3367print"initial parents:%s"% self.initialParents3368for b in self.p4BranchesInGit:3369if b !="master":33703371## FIXME3372 b = b[len(self.projectName):]3373 self.createdBranches.add(b)33743375 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))33763377 self.importProcess = subprocess.Popen(["git","fast-import"],3378 stdin=subprocess.PIPE,3379 stdout=subprocess.PIPE,3380 stderr=subprocess.PIPE);3381 self.gitOutput = self.importProcess.stdout3382 self.gitStream = self.importProcess.stdin3383 self.gitError = self.importProcess.stderr33843385if revision:3386 self.importHeadRevision(revision)3387else:3388 changes = []33893390iflen(self.changesFile) >0:3391 output =open(self.changesFile).readlines()3392 changeSet =set()3393for line in output:3394 changeSet.add(int(line))33953396for change in changeSet:3397 changes.append(change)33983399 changes.sort()3400else:3401# catch "git p4 sync" with no new branches, in a repo that3402# does not have any existing p4 branches3403iflen(args) ==0:3404if not self.p4BranchesInGit:3405die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")34063407# The default branch is master, unless --branch is used to3408# specify something else. Make sure it exists, or complain3409# nicely about how to use --branch.3410if not self.detectBranches:3411if notbranch_exists(self.branch):3412if branch_arg_given:3413die("Error: branch%sdoes not exist."% self.branch)3414else:3415die("Error: no branch%s; perhaps specify one with --branch."%3416 self.branch)34173418if self.verbose:3419print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3420 self.changeRange)3421 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)34223423iflen(self.maxChanges) >0:3424 changes = changes[:min(int(self.maxChanges),len(changes))]34253426iflen(changes) ==0:3427if not self.silent:3428print"No changes to import!"3429else:3430if not self.silent and not self.detectBranches:3431print"Import destination:%s"% self.branch34323433 self.updatedBranches =set()34343435if not self.detectBranches:3436if args:3437# start a new branch3438 self.initialParent =""3439else:3440# build on a previous revision3441 self.initialParent =parseRevision(self.branch)34423443 self.importChanges(changes)34443445if not self.silent:3446print""3447iflen(self.updatedBranches) >0:3448 sys.stdout.write("Updated branches: ")3449for b in self.updatedBranches:3450 sys.stdout.write("%s"% b)3451 sys.stdout.write("\n")34523453ifgitConfigBool("git-p4.importLabels"):3454 self.importLabels =True34553456if self.importLabels:3457 p4Labels =getP4Labels(self.depotPaths)3458 gitTags =getGitTags()34593460 missingP4Labels = p4Labels - gitTags3461 self.importP4Labels(self.gitStream, missingP4Labels)34623463 self.gitStream.close()3464if self.importProcess.wait() !=0:3465die("fast-import failed:%s"% self.gitError.read())3466 self.gitOutput.close()3467 self.gitError.close()34683469# Cleanup temporary branches created during import3470if self.tempBranches != []:3471for branch in self.tempBranches:3472read_pipe("git update-ref -d%s"% branch)3473 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))34743475# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3476# a convenient shortcut refname "p4".3477if self.importIntoRemotes:3478 head_ref = self.refPrefix +"HEAD"3479if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3480system(["git","symbolic-ref", head_ref, self.branch])34813482return True34833484classP4Rebase(Command):3485def__init__(self):3486 Command.__init__(self)3487 self.options = [3488 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3489]3490 self.importLabels =False3491 self.description = ("Fetches the latest revision from perforce and "3492+"rebases the current work (branch) against it")34933494defrun(self, args):3495 sync =P4Sync()3496 sync.importLabels = self.importLabels3497 sync.run([])34983499return self.rebase()35003501defrebase(self):3502if os.system("git update-index --refresh") !=0:3503die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");3504iflen(read_pipe("git diff-index HEAD --")) >0:3505die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");35063507[upstream, settings] =findUpstreamBranchPoint()3508iflen(upstream) ==0:3509die("Cannot find upstream branchpoint for rebase")35103511# the branchpoint may be p4/foo~3, so strip off the parent3512 upstream = re.sub("~[0-9]+$","", upstream)35133514print"Rebasing the current branch onto%s"% upstream3515 oldHead =read_pipe("git rev-parse HEAD").strip()3516system("git rebase%s"% upstream)3517system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3518return True35193520classP4Clone(P4Sync):3521def__init__(self):3522 P4Sync.__init__(self)3523 self.description ="Creates a new git repository and imports from Perforce into it"3524 self.usage ="usage: %prog [options] //depot/path[@revRange]"3525 self.options += [3526 optparse.make_option("--destination", dest="cloneDestination",3527 action='store', default=None,3528help="where to leave result of the clone"),3529 optparse.make_option("--bare", dest="cloneBare",3530 action="store_true", default=False),3531]3532 self.cloneDestination =None3533 self.needsGit =False3534 self.cloneBare =False35353536defdefaultDestination(self, args):3537## TODO: use common prefix of args?3538 depotPath = args[0]3539 depotDir = re.sub("(@[^@]*)$","", depotPath)3540 depotDir = re.sub("(#[^#]*)$","", depotDir)3541 depotDir = re.sub(r"\.\.\.$","", depotDir)3542 depotDir = re.sub(r"/$","", depotDir)3543return os.path.split(depotDir)[1]35443545defrun(self, args):3546iflen(args) <1:3547return False35483549if self.keepRepoPath and not self.cloneDestination:3550 sys.stderr.write("Must specify destination for --keep-path\n")3551 sys.exit(1)35523553 depotPaths = args35543555if not self.cloneDestination andlen(depotPaths) >1:3556 self.cloneDestination = depotPaths[-1]3557 depotPaths = depotPaths[:-1]35583559 self.cloneExclude = ["/"+p for p in self.cloneExclude]3560for p in depotPaths:3561if not p.startswith("//"):3562 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3563return False35643565if not self.cloneDestination:3566 self.cloneDestination = self.defaultDestination(args)35673568print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)35693570if not os.path.exists(self.cloneDestination):3571 os.makedirs(self.cloneDestination)3572chdir(self.cloneDestination)35733574 init_cmd = ["git","init"]3575if self.cloneBare:3576 init_cmd.append("--bare")3577 retcode = subprocess.call(init_cmd)3578if retcode:3579raiseCalledProcessError(retcode, init_cmd)35803581if not P4Sync.run(self, depotPaths):3582return False35833584# create a master branch and check out a work tree3585ifgitBranchExists(self.branch):3586system(["git","branch","master", self.branch ])3587if not self.cloneBare:3588system(["git","checkout","-f"])3589else:3590print'Not checking out any branch, use ' \3591'"git checkout -q -b master <branch>"'35923593# auto-set this variable if invoked with --use-client-spec3594if self.useClientSpec_from_options:3595system("git config --bool git-p4.useclientspec true")35963597return True35983599classP4Branches(Command):3600def__init__(self):3601 Command.__init__(self)3602 self.options = [ ]3603 self.description = ("Shows the git branches that hold imports and their "3604+"corresponding perforce depot paths")3605 self.verbose =False36063607defrun(self, args):3608iforiginP4BranchesExist():3609createOrUpdateBranchesFromOrigin()36103611 cmdline ="git rev-parse --symbolic "3612 cmdline +=" --remotes"36133614for line inread_pipe_lines(cmdline):3615 line = line.strip()36163617if not line.startswith('p4/')or line =="p4/HEAD":3618continue3619 branch = line36203621 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)3622 settings =extractSettingsGitLog(log)36233624print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])3625return True36263627classHelpFormatter(optparse.IndentedHelpFormatter):3628def__init__(self):3629 optparse.IndentedHelpFormatter.__init__(self)36303631defformat_description(self, description):3632if description:3633return description +"\n"3634else:3635return""36363637defprintUsage(commands):3638print"usage:%s<command> [options]"% sys.argv[0]3639print""3640print"valid commands:%s"%", ".join(commands)3641print""3642print"Try%s<command> --help for command specific help."% sys.argv[0]3643print""36443645commands = {3646"debug": P4Debug,3647"submit": P4Submit,3648"commit": P4Submit,3649"sync": P4Sync,3650"rebase": P4Rebase,3651"clone": P4Clone,3652"rollback": P4RollBack,3653"branches": P4Branches3654}365536563657defmain():3658iflen(sys.argv[1:]) ==0:3659printUsage(commands.keys())3660 sys.exit(2)36613662 cmdName = sys.argv[1]3663try:3664 klass = commands[cmdName]3665 cmd =klass()3666exceptKeyError:3667print"unknown command%s"% cmdName3668print""3669printUsage(commands.keys())3670 sys.exit(2)36713672 options = cmd.options3673 cmd.gitdir = os.environ.get("GIT_DIR",None)36743675 args = sys.argv[2:]36763677 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))3678if cmd.needsGit:3679 options.append(optparse.make_option("--git-dir", dest="gitdir"))36803681 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3682 options,3683 description = cmd.description,3684 formatter =HelpFormatter())36853686(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3687global verbose3688 verbose = cmd.verbose3689if cmd.needsGit:3690if cmd.gitdir ==None:3691 cmd.gitdir = os.path.abspath(".git")3692if notisValidGitDir(cmd.gitdir):3693 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3694if os.path.exists(cmd.gitdir):3695 cdup =read_pipe("git rev-parse --show-cdup").strip()3696iflen(cdup) >0:3697chdir(cdup);36983699if notisValidGitDir(cmd.gitdir):3700ifisValidGitDir(cmd.gitdir +"/.git"):3701 cmd.gitdir +="/.git"3702else:3703die("fatal: cannot locate git repository at%s"% cmd.gitdir)37043705 os.environ["GIT_DIR"] = cmd.gitdir37063707if not cmd.run(args):3708 parser.print_help()3709 sys.exit(2)371037113712if __name__ =='__main__':3713main()