1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10 11import optparse, sys, os, marshal, subprocess, shelve 12import tempfile, getopt, os.path, time, platform 13import re, shutil 14 15verbose =False 16 17 18defp4_build_cmd(cmd): 19"""Build a suitable p4 command line. 20 21 This consolidates building and returning a p4 command line into one 22 location. It means that hooking into the environment, or other configuration 23 can be done more easily. 24 """ 25 real_cmd = ["p4"] 26 27 user =gitConfig("git-p4.user") 28iflen(user) >0: 29 real_cmd += ["-u",user] 30 31 password =gitConfig("git-p4.password") 32iflen(password) >0: 33 real_cmd += ["-P", password] 34 35 port =gitConfig("git-p4.port") 36iflen(port) >0: 37 real_cmd += ["-p", port] 38 39 host =gitConfig("git-p4.host") 40iflen(host) >0: 41 real_cmd += ["-H", host] 42 43 client =gitConfig("git-p4.client") 44iflen(client) >0: 45 real_cmd += ["-c", client] 46 47 48ifisinstance(cmd,basestring): 49 real_cmd =' '.join(real_cmd) +' '+ cmd 50else: 51 real_cmd += cmd 52return real_cmd 53 54defchdir(dir): 55# P4 uses the PWD environment variable rather than getcwd(). Since we're 56# not using the shell, we have to set it ourselves. This path could 57# be relative, so go there first, then figure out where we ended up. 58 os.chdir(dir) 59 os.environ['PWD'] = os.getcwd() 60 61defdie(msg): 62if verbose: 63raiseException(msg) 64else: 65 sys.stderr.write(msg +"\n") 66 sys.exit(1) 67 68defwrite_pipe(c, stdin): 69if verbose: 70 sys.stderr.write('Writing pipe:%s\n'%str(c)) 71 72 expand =isinstance(c,basestring) 73 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 74 pipe = p.stdin 75 val = pipe.write(stdin) 76 pipe.close() 77if p.wait(): 78die('Command failed:%s'%str(c)) 79 80return val 81 82defp4_write_pipe(c, stdin): 83 real_cmd =p4_build_cmd(c) 84returnwrite_pipe(real_cmd, stdin) 85 86defread_pipe(c, ignore_error=False): 87if verbose: 88 sys.stderr.write('Reading pipe:%s\n'%str(c)) 89 90 expand =isinstance(c,basestring) 91 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 92 pipe = p.stdout 93 val = pipe.read() 94if p.wait()and not ignore_error: 95die('Command failed:%s'%str(c)) 96 97return val 98 99defp4_read_pipe(c, ignore_error=False): 100 real_cmd =p4_build_cmd(c) 101returnread_pipe(real_cmd, ignore_error) 102 103defread_pipe_lines(c): 104if verbose: 105 sys.stderr.write('Reading pipe:%s\n'%str(c)) 106 107 expand =isinstance(c, basestring) 108 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 109 pipe = p.stdout 110 val = pipe.readlines() 111if pipe.close()or p.wait(): 112die('Command failed:%s'%str(c)) 113 114return val 115 116defp4_read_pipe_lines(c): 117"""Specifically invoke p4 on the command supplied. """ 118 real_cmd =p4_build_cmd(c) 119returnread_pipe_lines(real_cmd) 120 121defsystem(cmd): 122 expand =isinstance(cmd,basestring) 123if verbose: 124 sys.stderr.write("executing%s\n"%str(cmd)) 125 subprocess.check_call(cmd, shell=expand) 126 127defp4_system(cmd): 128"""Specifically invoke p4 as the system command. """ 129 real_cmd =p4_build_cmd(cmd) 130 expand =isinstance(real_cmd, basestring) 131 subprocess.check_call(real_cmd, shell=expand) 132 133defp4_integrate(src, dest): 134p4_system(["integrate","-Dt", src, dest]) 135 136defp4_sync(path): 137p4_system(["sync", path]) 138 139defp4_add(f): 140p4_system(["add", f]) 141 142defp4_delete(f): 143p4_system(["delete", f]) 144 145defp4_edit(f): 146p4_system(["edit", f]) 147 148defp4_revert(f): 149p4_system(["revert", f]) 150 151defp4_reopen(type,file): 152p4_system(["reopen","-t",type,file]) 153 154# 155# Canonicalize the p4 type and return a tuple of the 156# base type, plus any modifiers. See "p4 help filetypes" 157# for a list and explanation. 158# 159defsplit_p4_type(p4type): 160 161 p4_filetypes_historical = { 162"ctempobj":"binary+Sw", 163"ctext":"text+C", 164"cxtext":"text+Cx", 165"ktext":"text+k", 166"kxtext":"text+kx", 167"ltext":"text+F", 168"tempobj":"binary+FSw", 169"ubinary":"binary+F", 170"uresource":"resource+F", 171"uxbinary":"binary+Fx", 172"xbinary":"binary+x", 173"xltext":"text+Fx", 174"xtempobj":"binary+Swx", 175"xtext":"text+x", 176"xunicode":"unicode+x", 177"xutf16":"utf16+x", 178} 179if p4type in p4_filetypes_historical: 180 p4type = p4_filetypes_historical[p4type] 181 mods ="" 182 s = p4type.split("+") 183 base = s[0] 184 mods ="" 185iflen(s) >1: 186 mods = s[1] 187return(base, mods) 188 189# 190# return the raw p4 type of a file (text, text+ko, etc) 191# 192defp4_type(file): 193 results =p4CmdList(["fstat","-T","headType",file]) 194return results[0]['headType'] 195 196# 197# Given a type base and modifier, return a regexp matching 198# the keywords that can be expanded in the file 199# 200defp4_keywords_regexp_for_type(base, type_mods): 201if base in("text","unicode","binary"): 202 kwords =None 203if"ko"in type_mods: 204 kwords ='Id|Header' 205elif"k"in type_mods: 206 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 207else: 208return None 209 pattern = r""" 210 \$ # Starts with a dollar, followed by... 211 (%s) # one of the keywords, followed by... 212 (:[^$]+)? # possibly an old expansion, followed by... 213 \$ # another dollar 214 """% kwords 215return pattern 216else: 217return None 218 219# 220# Given a file, return a regexp matching the possible 221# RCS keywords that will be expanded, or None for files 222# with kw expansion turned off. 223# 224defp4_keywords_regexp_for_file(file): 225if not os.path.exists(file): 226return None 227else: 228(type_base, type_mods) =split_p4_type(p4_type(file)) 229returnp4_keywords_regexp_for_type(type_base, type_mods) 230 231defsetP4ExecBit(file, mode): 232# Reopens an already open file and changes the execute bit to match 233# the execute bit setting in the passed in mode. 234 235 p4Type ="+x" 236 237if notisModeExec(mode): 238 p4Type =getP4OpenedType(file) 239 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 240 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 241if p4Type[-1] =="+": 242 p4Type = p4Type[0:-1] 243 244p4_reopen(p4Type,file) 245 246defgetP4OpenedType(file): 247# Returns the perforce file type for the given file. 248 249 result =p4_read_pipe(["opened",file]) 250 match = re.match(".*\((.+)\)\r?$", result) 251if match: 252return match.group(1) 253else: 254die("Could not determine file type for%s(result: '%s')"% (file, result)) 255 256defdiffTreePattern(): 257# This is a simple generator for the diff tree regex pattern. This could be 258# a class variable if this and parseDiffTreeEntry were a part of a class. 259 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 260while True: 261yield pattern 262 263defparseDiffTreeEntry(entry): 264"""Parses a single diff tree entry into its component elements. 265 266 See git-diff-tree(1) manpage for details about the format of the diff 267 output. This method returns a dictionary with the following elements: 268 269 src_mode - The mode of the source file 270 dst_mode - The mode of the destination file 271 src_sha1 - The sha1 for the source file 272 dst_sha1 - The sha1 fr the destination file 273 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 274 status_score - The score for the status (applicable for 'C' and 'R' 275 statuses). This is None if there is no score. 276 src - The path for the source file. 277 dst - The path for the destination file. This is only present for 278 copy or renames. If it is not present, this is None. 279 280 If the pattern is not matched, None is returned.""" 281 282 match =diffTreePattern().next().match(entry) 283if match: 284return{ 285'src_mode': match.group(1), 286'dst_mode': match.group(2), 287'src_sha1': match.group(3), 288'dst_sha1': match.group(4), 289'status': match.group(5), 290'status_score': match.group(6), 291'src': match.group(7), 292'dst': match.group(10) 293} 294return None 295 296defisModeExec(mode): 297# Returns True if the given git mode represents an executable file, 298# otherwise False. 299return mode[-3:] =="755" 300 301defisModeExecChanged(src_mode, dst_mode): 302returnisModeExec(src_mode) !=isModeExec(dst_mode) 303 304defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 305 306ifisinstance(cmd,basestring): 307 cmd ="-G "+ cmd 308 expand =True 309else: 310 cmd = ["-G"] + cmd 311 expand =False 312 313 cmd =p4_build_cmd(cmd) 314if verbose: 315 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 316 317# Use a temporary file to avoid deadlocks without 318# subprocess.communicate(), which would put another copy 319# of stdout into memory. 320 stdin_file =None 321if stdin is not None: 322 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 323ifisinstance(stdin,basestring): 324 stdin_file.write(stdin) 325else: 326for i in stdin: 327 stdin_file.write(i +'\n') 328 stdin_file.flush() 329 stdin_file.seek(0) 330 331 p4 = subprocess.Popen(cmd, 332 shell=expand, 333 stdin=stdin_file, 334 stdout=subprocess.PIPE) 335 336 result = [] 337try: 338while True: 339 entry = marshal.load(p4.stdout) 340if cb is not None: 341cb(entry) 342else: 343 result.append(entry) 344exceptEOFError: 345pass 346 exitCode = p4.wait() 347if exitCode !=0: 348 entry = {} 349 entry["p4ExitCode"] = exitCode 350 result.append(entry) 351 352return result 353 354defp4Cmd(cmd): 355list=p4CmdList(cmd) 356 result = {} 357for entry inlist: 358 result.update(entry) 359return result; 360 361defp4Where(depotPath): 362if not depotPath.endswith("/"): 363 depotPath +="/" 364 depotPath = depotPath +"..." 365 outputList =p4CmdList(["where", depotPath]) 366 output =None 367for entry in outputList: 368if"depotFile"in entry: 369if entry["depotFile"] == depotPath: 370 output = entry 371break 372elif"data"in entry: 373 data = entry.get("data") 374 space = data.find(" ") 375if data[:space] == depotPath: 376 output = entry 377break 378if output ==None: 379return"" 380if output["code"] =="error": 381return"" 382 clientPath ="" 383if"path"in output: 384 clientPath = output.get("path") 385elif"data"in output: 386 data = output.get("data") 387 lastSpace = data.rfind(" ") 388 clientPath = data[lastSpace +1:] 389 390if clientPath.endswith("..."): 391 clientPath = clientPath[:-3] 392return clientPath 393 394defcurrentGitBranch(): 395returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 396 397defisValidGitDir(path): 398if(os.path.exists(path +"/HEAD") 399and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 400return True; 401return False 402 403defparseRevision(ref): 404returnread_pipe("git rev-parse%s"% ref).strip() 405 406defbranchExists(ref): 407 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 408 ignore_error=True) 409returnlen(rev) >0 410 411defextractLogMessageFromGitCommit(commit): 412 logMessage ="" 413 414## fixme: title is first line of commit, not 1st paragraph. 415 foundTitle =False 416for log inread_pipe_lines("git cat-file commit%s"% commit): 417if not foundTitle: 418iflen(log) ==1: 419 foundTitle =True 420continue 421 422 logMessage += log 423return logMessage 424 425defextractSettingsGitLog(log): 426 values = {} 427for line in log.split("\n"): 428 line = line.strip() 429 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 430if not m: 431continue 432 433 assignments = m.group(1).split(':') 434for a in assignments: 435 vals = a.split('=') 436 key = vals[0].strip() 437 val = ('='.join(vals[1:])).strip() 438if val.endswith('\"')and val.startswith('"'): 439 val = val[1:-1] 440 441 values[key] = val 442 443 paths = values.get("depot-paths") 444if not paths: 445 paths = values.get("depot-path") 446if paths: 447 values['depot-paths'] = paths.split(',') 448return values 449 450defgitBranchExists(branch): 451 proc = subprocess.Popen(["git","rev-parse", branch], 452 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 453return proc.wait() ==0; 454 455_gitConfig = {} 456defgitConfig(key, args =None):# set args to "--bool", for instance 457if not _gitConfig.has_key(key): 458 argsFilter ="" 459if args !=None: 460 argsFilter ="%s"% args 461 cmd ="git config%s%s"% (argsFilter, key) 462 _gitConfig[key] =read_pipe(cmd, ignore_error=True).strip() 463return _gitConfig[key] 464 465defgitConfigList(key): 466if not _gitConfig.has_key(key): 467 _gitConfig[key] =read_pipe("git config --get-all%s"% key, ignore_error=True).strip().split(os.linesep) 468return _gitConfig[key] 469 470defp4BranchesInGit(branchesAreInRemotes =True): 471 branches = {} 472 473 cmdline ="git rev-parse --symbolic " 474if branchesAreInRemotes: 475 cmdline +=" --remotes" 476else: 477 cmdline +=" --branches" 478 479for line inread_pipe_lines(cmdline): 480 line = line.strip() 481 482## only import to p4/ 483if not line.startswith('p4/')or line =="p4/HEAD": 484continue 485 branch = line 486 487# strip off p4 488 branch = re.sub("^p4/","", line) 489 490 branches[branch] =parseRevision(line) 491return branches 492 493deffindUpstreamBranchPoint(head ="HEAD"): 494 branches =p4BranchesInGit() 495# map from depot-path to branch name 496 branchByDepotPath = {} 497for branch in branches.keys(): 498 tip = branches[branch] 499 log =extractLogMessageFromGitCommit(tip) 500 settings =extractSettingsGitLog(log) 501if settings.has_key("depot-paths"): 502 paths =",".join(settings["depot-paths"]) 503 branchByDepotPath[paths] ="remotes/p4/"+ branch 504 505 settings =None 506 parent =0 507while parent <65535: 508 commit = head +"~%s"% parent 509 log =extractLogMessageFromGitCommit(commit) 510 settings =extractSettingsGitLog(log) 511if settings.has_key("depot-paths"): 512 paths =",".join(settings["depot-paths"]) 513if branchByDepotPath.has_key(paths): 514return[branchByDepotPath[paths], settings] 515 516 parent = parent +1 517 518return["", settings] 519 520defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 521if not silent: 522print("Creating/updating branch(es) in%sbased on origin branch(es)" 523% localRefPrefix) 524 525 originPrefix ="origin/p4/" 526 527for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 528 line = line.strip() 529if(not line.startswith(originPrefix))or line.endswith("HEAD"): 530continue 531 532 headName = line[len(originPrefix):] 533 remoteHead = localRefPrefix + headName 534 originHead = line 535 536 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 537if(not original.has_key('depot-paths') 538or not original.has_key('change')): 539continue 540 541 update =False 542if notgitBranchExists(remoteHead): 543if verbose: 544print"creating%s"% remoteHead 545 update =True 546else: 547 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 548if settings.has_key('change') >0: 549if settings['depot-paths'] == original['depot-paths']: 550 originP4Change =int(original['change']) 551 p4Change =int(settings['change']) 552if originP4Change > p4Change: 553print("%s(%s) is newer than%s(%s). " 554"Updating p4 branch from origin." 555% (originHead, originP4Change, 556 remoteHead, p4Change)) 557 update =True 558else: 559print("Ignoring:%swas imported from%swhile " 560"%swas imported from%s" 561% (originHead,','.join(original['depot-paths']), 562 remoteHead,','.join(settings['depot-paths']))) 563 564if update: 565system("git update-ref%s %s"% (remoteHead, originHead)) 566 567deforiginP4BranchesExist(): 568returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 569 570defp4ChangesForPaths(depotPaths, changeRange): 571assert depotPaths 572 cmd = ['changes'] 573for p in depotPaths: 574 cmd += ["%s...%s"% (p, changeRange)] 575 output =p4_read_pipe_lines(cmd) 576 577 changes = {} 578for line in output: 579 changeNum =int(line.split(" ")[1]) 580 changes[changeNum] =True 581 582 changelist = changes.keys() 583 changelist.sort() 584return changelist 585 586defp4PathStartsWith(path, prefix): 587# This method tries to remedy a potential mixed-case issue: 588# 589# If UserA adds //depot/DirA/file1 590# and UserB adds //depot/dira/file2 591# 592# we may or may not have a problem. If you have core.ignorecase=true, 593# we treat DirA and dira as the same directory 594 ignorecase =gitConfig("core.ignorecase","--bool") =="true" 595if ignorecase: 596return path.lower().startswith(prefix.lower()) 597return path.startswith(prefix) 598 599defgetClientSpec(): 600"""Look at the p4 client spec, create a View() object that contains 601 all the mappings, and return it.""" 602 603 specList =p4CmdList("client -o") 604iflen(specList) !=1: 605die('Output from "client -o" is%dlines, expecting 1'% 606len(specList)) 607 608# dictionary of all client parameters 609 entry = specList[0] 610 611# just the keys that start with "View" 612 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 613 614# hold this new View 615 view =View() 616 617# append the lines, in order, to the view 618for view_num inrange(len(view_keys)): 619 k ="View%d"% view_num 620if k not in view_keys: 621die("Expected view key%smissing"% k) 622 view.append(entry[k]) 623 624return view 625 626defgetClientRoot(): 627"""Grab the client directory.""" 628 629 output =p4CmdList("client -o") 630iflen(output) !=1: 631die('Output from "client -o" is%dlines, expecting 1'%len(output)) 632 633 entry = output[0] 634if"Root"not in entry: 635die('Client has no "Root"') 636 637return entry["Root"] 638 639class Command: 640def__init__(self): 641 self.usage ="usage: %prog [options]" 642 self.needsGit =True 643 644class P4UserMap: 645def__init__(self): 646 self.userMapFromPerforceServer =False 647 self.myP4UserId =None 648 649defp4UserId(self): 650if self.myP4UserId: 651return self.myP4UserId 652 653 results =p4CmdList("user -o") 654for r in results: 655if r.has_key('User'): 656 self.myP4UserId = r['User'] 657return r['User'] 658die("Could not find your p4 user id") 659 660defp4UserIsMe(self, p4User): 661# return True if the given p4 user is actually me 662 me = self.p4UserId() 663if not p4User or p4User != me: 664return False 665else: 666return True 667 668defgetUserCacheFilename(self): 669 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 670return home +"/.gitp4-usercache.txt" 671 672defgetUserMapFromPerforceServer(self): 673if self.userMapFromPerforceServer: 674return 675 self.users = {} 676 self.emails = {} 677 678for output inp4CmdList("users"): 679if not output.has_key("User"): 680continue 681 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 682 self.emails[output["Email"]] = output["User"] 683 684 685 s ='' 686for(key, val)in self.users.items(): 687 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 688 689open(self.getUserCacheFilename(),"wb").write(s) 690 self.userMapFromPerforceServer =True 691 692defloadUserMapFromCache(self): 693 self.users = {} 694 self.userMapFromPerforceServer =False 695try: 696 cache =open(self.getUserCacheFilename(),"rb") 697 lines = cache.readlines() 698 cache.close() 699for line in lines: 700 entry = line.strip().split("\t") 701 self.users[entry[0]] = entry[1] 702exceptIOError: 703 self.getUserMapFromPerforceServer() 704 705classP4Debug(Command): 706def__init__(self): 707 Command.__init__(self) 708 self.options = [ 709 optparse.make_option("--verbose", dest="verbose", action="store_true", 710 default=False), 711] 712 self.description ="A tool to debug the output of p4 -G." 713 self.needsGit =False 714 self.verbose =False 715 716defrun(self, args): 717 j =0 718for output inp4CmdList(args): 719print'Element:%d'% j 720 j +=1 721print output 722return True 723 724classP4RollBack(Command): 725def__init__(self): 726 Command.__init__(self) 727 self.options = [ 728 optparse.make_option("--verbose", dest="verbose", action="store_true"), 729 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") 730] 731 self.description ="A tool to debug the multi-branch import. Don't use :)" 732 self.verbose =False 733 self.rollbackLocalBranches =False 734 735defrun(self, args): 736iflen(args) !=1: 737return False 738 maxChange =int(args[0]) 739 740if"p4ExitCode"inp4Cmd("changes -m 1"): 741die("Problems executing p4"); 742 743if self.rollbackLocalBranches: 744 refPrefix ="refs/heads/" 745 lines =read_pipe_lines("git rev-parse --symbolic --branches") 746else: 747 refPrefix ="refs/remotes/" 748 lines =read_pipe_lines("git rev-parse --symbolic --remotes") 749 750for line in lines: 751if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"): 752 line = line.strip() 753 ref = refPrefix + line 754 log =extractLogMessageFromGitCommit(ref) 755 settings =extractSettingsGitLog(log) 756 757 depotPaths = settings['depot-paths'] 758 change = settings['change'] 759 760 changed =False 761 762iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange) 763for p in depotPaths]))) ==0: 764print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange) 765system("git update-ref -d%s`git rev-parse%s`"% (ref, ref)) 766continue 767 768while change andint(change) > maxChange: 769 changed =True 770if self.verbose: 771print"%sis at%s; rewinding towards%s"% (ref, change, maxChange) 772system("git update-ref%s\"%s^\""% (ref, ref)) 773 log =extractLogMessageFromGitCommit(ref) 774 settings =extractSettingsGitLog(log) 775 776 777 depotPaths = settings['depot-paths'] 778 change = settings['change'] 779 780if changed: 781print"%srewound to%s"% (ref, change) 782 783return True 784 785classP4Submit(Command, P4UserMap): 786def__init__(self): 787 Command.__init__(self) 788 P4UserMap.__init__(self) 789 self.options = [ 790 optparse.make_option("--verbose", dest="verbose", action="store_true"), 791 optparse.make_option("--origin", dest="origin"), 792 optparse.make_option("-M", dest="detectRenames", action="store_true"), 793# preserve the user, requires relevant p4 permissions 794 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), 795] 796 self.description ="Submit changes from git to the perforce depot." 797 self.usage +=" [name of git branch to submit into perforce depot]" 798 self.interactive =True 799 self.origin ="" 800 self.detectRenames =False 801 self.verbose =False 802 self.preserveUser =gitConfig("git-p4.preserveUser").lower() =="true" 803 self.isWindows = (platform.system() =="Windows") 804 805defcheck(self): 806iflen(p4CmdList("opened ...")) >0: 807die("You have files opened with perforce! Close them before starting the sync.") 808 809# replaces everything between 'Description:' and the next P4 submit template field with the 810# commit message 811defprepareLogMessage(self, template, message): 812 result ="" 813 814 inDescriptionSection =False 815 816for line in template.split("\n"): 817if line.startswith("#"): 818 result += line +"\n" 819continue 820 821if inDescriptionSection: 822if line.startswith("Files:")or line.startswith("Jobs:"): 823 inDescriptionSection =False 824else: 825continue 826else: 827if line.startswith("Description:"): 828 inDescriptionSection =True 829 line +="\n" 830for messageLine in message.split("\n"): 831 line +="\t"+ messageLine +"\n" 832 833 result += line +"\n" 834 835return result 836 837defpatchRCSKeywords(self,file, pattern): 838# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern 839(handle, outFileName) = tempfile.mkstemp(dir='.') 840try: 841 outFile = os.fdopen(handle,"w+") 842 inFile =open(file,"r") 843 regexp = re.compile(pattern, re.VERBOSE) 844for line in inFile.readlines(): 845 line = regexp.sub(r'$\1$', line) 846 outFile.write(line) 847 inFile.close() 848 outFile.close() 849# Forcibly overwrite the original file 850 os.unlink(file) 851 shutil.move(outFileName,file) 852except: 853# cleanup our temporary file 854 os.unlink(outFileName) 855print"Failed to strip RCS keywords in%s"%file 856raise 857 858print"Patched up RCS keywords in%s"%file 859 860defp4UserForCommit(self,id): 861# Return the tuple (perforce user,git email) for a given git commit id 862 self.getUserMapFromPerforceServer() 863 gitEmail =read_pipe("git log --max-count=1 --format='%%ae'%s"%id) 864 gitEmail = gitEmail.strip() 865if not self.emails.has_key(gitEmail): 866return(None,gitEmail) 867else: 868return(self.emails[gitEmail],gitEmail) 869 870defcheckValidP4Users(self,commits): 871# check if any git authors cannot be mapped to p4 users 872foridin commits: 873(user,email) = self.p4UserForCommit(id) 874if not user: 875 msg ="Cannot find p4 user for email%sin commit%s."% (email,id) 876ifgitConfig('git-p4.allowMissingP4Users').lower() =="true": 877print"%s"% msg 878else: 879die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg) 880 881deflastP4Changelist(self): 882# Get back the last changelist number submitted in this client spec. This 883# then gets used to patch up the username in the change. If the same 884# client spec is being used by multiple processes then this might go 885# wrong. 886 results =p4CmdList("client -o")# find the current client 887 client =None 888for r in results: 889if r.has_key('Client'): 890 client = r['Client'] 891break 892if not client: 893die("could not get client spec") 894 results =p4CmdList(["changes","-c", client,"-m","1"]) 895for r in results: 896if r.has_key('change'): 897return r['change'] 898die("Could not get changelist number for last submit - cannot patch up user details") 899 900defmodifyChangelistUser(self, changelist, newUser): 901# fixup the user field of a changelist after it has been submitted. 902 changes =p4CmdList("change -o%s"% changelist) 903iflen(changes) !=1: 904die("Bad output from p4 change modifying%sto user%s"% 905(changelist, newUser)) 906 907 c = changes[0] 908if c['User'] == newUser:return# nothing to do 909 c['User'] = newUser 910input= marshal.dumps(c) 911 912 result =p4CmdList("change -f -i", stdin=input) 913for r in result: 914if r.has_key('code'): 915if r['code'] =='error': 916die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data'])) 917if r.has_key('data'): 918print("Updated user field for changelist%sto%s"% (changelist, newUser)) 919return 920die("Could not modify user field of changelist%sto%s"% (changelist, newUser)) 921 922defcanChangeChangelists(self): 923# check to see if we have p4 admin or super-user permissions, either of 924# which are required to modify changelists. 925 results =p4CmdList(["protects", self.depotPath]) 926for r in results: 927if r.has_key('perm'): 928if r['perm'] =='admin': 929return1 930if r['perm'] =='super': 931return1 932return0 933 934defprepareSubmitTemplate(self): 935# remove lines in the Files section that show changes to files outside the depot path we're committing into 936 template ="" 937 inFilesSection =False 938for line inp4_read_pipe_lines(['change','-o']): 939if line.endswith("\r\n"): 940 line = line[:-2] +"\n" 941if inFilesSection: 942if line.startswith("\t"): 943# path starts and ends with a tab 944 path = line[1:] 945 lastTab = path.rfind("\t") 946if lastTab != -1: 947 path = path[:lastTab] 948if notp4PathStartsWith(path, self.depotPath): 949continue 950else: 951 inFilesSection =False 952else: 953if line.startswith("Files:"): 954 inFilesSection =True 955 956 template += line 957 958return template 959 960defedit_template(self, template_file): 961"""Invoke the editor to let the user change the submission 962 message. Return true if okay to continue with the submit.""" 963 964# if configured to skip the editing part, just submit 965ifgitConfig("git-p4.skipSubmitEdit") =="true": 966return True 967 968# look at the modification time, to check later if the user saved 969# the file 970 mtime = os.stat(template_file).st_mtime 971 972# invoke the editor 973if os.environ.has_key("P4EDITOR"): 974 editor = os.environ.get("P4EDITOR") 975else: 976 editor =read_pipe("git var GIT_EDITOR").strip() 977system(editor +" "+ template_file) 978 979# If the file was not saved, prompt to see if this patch should 980# be skipped. But skip this verification step if configured so. 981ifgitConfig("git-p4.skipSubmitEditCheck") =="true": 982return True 983 984# modification time updated means user saved the file 985if os.stat(template_file).st_mtime > mtime: 986return True 987 988while True: 989 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ") 990if response =='y': 991return True 992if response =='n': 993return False 994 995defapplyCommit(self,id): 996print"Applying%s"% (read_pipe("git log --max-count=1 --pretty=oneline%s"%id)) 997 998(p4User, gitEmail) = self.p4UserForCommit(id) 9991000if not self.detectRenames:1001# If not explicitly set check the config variable1002 self.detectRenames =gitConfig("git-p4.detectRenames")10031004if self.detectRenames.lower() =="false"or self.detectRenames =="":1005 diffOpts =""1006elif self.detectRenames.lower() =="true":1007 diffOpts ="-M"1008else:1009 diffOpts ="-M%s"% self.detectRenames10101011 detectCopies =gitConfig("git-p4.detectCopies")1012if detectCopies.lower() =="true":1013 diffOpts +=" -C"1014elif detectCopies !=""and detectCopies.lower() !="false":1015 diffOpts +=" -C%s"% detectCopies10161017ifgitConfig("git-p4.detectCopiesHarder","--bool") =="true":1018 diffOpts +=" --find-copies-harder"10191020 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (diffOpts,id,id))1021 filesToAdd =set()1022 filesToDelete =set()1023 editedFiles =set()1024 filesToChangeExecBit = {}10251026for line in diff:1027 diff =parseDiffTreeEntry(line)1028 modifier = diff['status']1029 path = diff['src']1030if modifier =="M":1031p4_edit(path)1032ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1033 filesToChangeExecBit[path] = diff['dst_mode']1034 editedFiles.add(path)1035elif modifier =="A":1036 filesToAdd.add(path)1037 filesToChangeExecBit[path] = diff['dst_mode']1038if path in filesToDelete:1039 filesToDelete.remove(path)1040elif modifier =="D":1041 filesToDelete.add(path)1042if path in filesToAdd:1043 filesToAdd.remove(path)1044elif modifier =="C":1045 src, dest = diff['src'], diff['dst']1046p4_integrate(src, dest)1047if diff['src_sha1'] != diff['dst_sha1']:1048p4_edit(dest)1049ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1050p4_edit(dest)1051 filesToChangeExecBit[dest] = diff['dst_mode']1052 os.unlink(dest)1053 editedFiles.add(dest)1054elif modifier =="R":1055 src, dest = diff['src'], diff['dst']1056p4_integrate(src, dest)1057if diff['src_sha1'] != diff['dst_sha1']:1058p4_edit(dest)1059ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1060p4_edit(dest)1061 filesToChangeExecBit[dest] = diff['dst_mode']1062 os.unlink(dest)1063 editedFiles.add(dest)1064 filesToDelete.add(src)1065else:1066die("unknown modifier%sfor%s"% (modifier, path))10671068 diffcmd ="git format-patch -k --stdout\"%s^\"..\"%s\""% (id,id)1069 patchcmd = diffcmd +" | git apply "1070 tryPatchCmd = patchcmd +"--check -"1071 applyPatchCmd = patchcmd +"--check --apply -"1072 patch_succeeded =True10731074if os.system(tryPatchCmd) !=0:1075 fixed_rcs_keywords =False1076 patch_succeeded =False1077print"Unfortunately applying the change failed!"10781079# Patch failed, maybe it's just RCS keyword woes. Look through1080# the patch to see if that's possible.1081ifgitConfig("git-p4.attemptRCSCleanup","--bool") =="true":1082file=None1083 pattern =None1084 kwfiles = {}1085forfilein editedFiles | filesToDelete:1086# did this file's delta contain RCS keywords?1087 pattern =p4_keywords_regexp_for_file(file)10881089if pattern:1090# this file is a possibility...look for RCS keywords.1091 regexp = re.compile(pattern, re.VERBOSE)1092for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1093if regexp.search(line):1094if verbose:1095print"got keyword match on%sin%sin%s"% (pattern, line,file)1096 kwfiles[file] = pattern1097break10981099forfilein kwfiles:1100if verbose:1101print"zapping%swith%s"% (line,pattern)1102 self.patchRCSKeywords(file, kwfiles[file])1103 fixed_rcs_keywords =True11041105if fixed_rcs_keywords:1106print"Retrying the patch with RCS keywords cleaned up"1107if os.system(tryPatchCmd) ==0:1108 patch_succeeded =True11091110if not patch_succeeded:1111print"What do you want to do?"1112 response ="x"1113while response !="s"and response !="a"and response !="w":1114 response =raw_input("[s]kip this patch / [a]pply the patch forcibly "1115"and with .rej files / [w]rite the patch to a file (patch.txt) ")1116if response =="s":1117print"Skipping! Good luck with the next patches..."1118for f in editedFiles:1119p4_revert(f)1120for f in filesToAdd:1121 os.remove(f)1122return1123elif response =="a":1124 os.system(applyPatchCmd)1125iflen(filesToAdd) >0:1126print"You may also want to call p4 add on the following files:"1127print" ".join(filesToAdd)1128iflen(filesToDelete):1129print"The following files should be scheduled for deletion with p4 delete:"1130print" ".join(filesToDelete)1131die("Please resolve and submit the conflict manually and "1132+"continue afterwards with git-p4 submit --continue")1133elif response =="w":1134system(diffcmd +" > patch.txt")1135print"Patch saved to patch.txt in%s!"% self.clientPath1136die("Please resolve and submit the conflict manually and "1137"continue afterwards with git-p4 submit --continue")11381139system(applyPatchCmd)11401141for f in filesToAdd:1142p4_add(f)1143for f in filesToDelete:1144p4_revert(f)1145p4_delete(f)11461147# Set/clear executable bits1148for f in filesToChangeExecBit.keys():1149 mode = filesToChangeExecBit[f]1150setP4ExecBit(f, mode)11511152 logMessage =extractLogMessageFromGitCommit(id)1153 logMessage = logMessage.strip()11541155 template = self.prepareSubmitTemplate()11561157if self.interactive:1158 submitTemplate = self.prepareLogMessage(template, logMessage)11591160if self.preserveUser:1161 submitTemplate = submitTemplate + ("\n######## Actual user%s, modified after commit\n"% p4User)11621163if os.environ.has_key("P4DIFF"):1164del(os.environ["P4DIFF"])1165 diff =""1166for editedFile in editedFiles:1167 diff +=p4_read_pipe(['diff','-du', editedFile])11681169 newdiff =""1170for newFile in filesToAdd:1171 newdiff +="==== new file ====\n"1172 newdiff +="--- /dev/null\n"1173 newdiff +="+++%s\n"% newFile1174 f =open(newFile,"r")1175for line in f.readlines():1176 newdiff +="+"+ line1177 f.close()11781179if self.checkAuthorship and not self.p4UserIsMe(p4User):1180 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1181 submitTemplate +="######## Use git-p4 option --preserve-user to modify authorship\n"1182 submitTemplate +="######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"11831184 separatorLine ="######## everything below this line is just the diff #######\n"11851186(handle, fileName) = tempfile.mkstemp()1187 tmpFile = os.fdopen(handle,"w+")1188if self.isWindows:1189 submitTemplate = submitTemplate.replace("\n","\r\n")1190 separatorLine = separatorLine.replace("\n","\r\n")1191 newdiff = newdiff.replace("\n","\r\n")1192 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)1193 tmpFile.close()11941195if self.edit_template(fileName):1196# read the edited message and submit1197 tmpFile =open(fileName,"rb")1198 message = tmpFile.read()1199 tmpFile.close()1200 submitTemplate = message[:message.index(separatorLine)]1201if self.isWindows:1202 submitTemplate = submitTemplate.replace("\r\n","\n")1203p4_write_pipe(['submit','-i'], submitTemplate)12041205if self.preserveUser:1206if p4User:1207# Get last changelist number. Cannot easily get it from1208# the submit command output as the output is1209# unmarshalled.1210 changelist = self.lastP4Changelist()1211 self.modifyChangelistUser(changelist, p4User)1212else:1213# skip this patch1214print"Submission cancelled, undoing p4 changes."1215for f in editedFiles:1216p4_revert(f)1217for f in filesToAdd:1218p4_revert(f)1219 os.remove(f)12201221 os.remove(fileName)1222else:1223 fileName ="submit.txt"1224file=open(fileName,"w+")1225file.write(self.prepareLogMessage(template, logMessage))1226file.close()1227print("Perforce submit template written as%s. "1228+"Please review/edit and then use p4 submit -i <%sto submit directly!"1229% (fileName, fileName))12301231defrun(self, args):1232iflen(args) ==0:1233 self.master =currentGitBranch()1234iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1235die("Detecting current git branch failed!")1236eliflen(args) ==1:1237 self.master = args[0]1238if notbranchExists(self.master):1239die("Branch%sdoes not exist"% self.master)1240else:1241return False12421243 allowSubmit =gitConfig("git-p4.allowSubmit")1244iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1245die("%sis not in git-p4.allowSubmit"% self.master)12461247[upstream, settings] =findUpstreamBranchPoint()1248 self.depotPath = settings['depot-paths'][0]1249iflen(self.origin) ==0:1250 self.origin = upstream12511252if self.preserveUser:1253if not self.canChangeChangelists():1254die("Cannot preserve user names without p4 super-user or admin permissions")12551256if self.verbose:1257print"Origin branch is "+ self.origin12581259iflen(self.depotPath) ==0:1260print"Internal error: cannot locate perforce depot path from existing branches"1261 sys.exit(128)12621263 self.useClientSpec =False1264ifgitConfig("git-p4.useclientspec","--bool") =="true":1265 self.useClientSpec =True1266if self.useClientSpec:1267 self.clientSpecDirs =getClientSpec()12681269if self.useClientSpec:1270# all files are relative to the client spec1271 self.clientPath =getClientRoot()1272else:1273 self.clientPath =p4Where(self.depotPath)12741275if self.clientPath =="":1276die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)12771278print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1279 self.oldWorkingDirectory = os.getcwd()12801281# ensure the clientPath exists1282if not os.path.exists(self.clientPath):1283 os.makedirs(self.clientPath)12841285chdir(self.clientPath)1286print"Synchronizing p4 checkout..."1287p4_sync("...")1288 self.check()12891290 commits = []1291for line inread_pipe_lines("git rev-list --no-merges%s..%s"% (self.origin, self.master)):1292 commits.append(line.strip())1293 commits.reverse()12941295if self.preserveUser or(gitConfig("git-p4.skipUserNameCheck") =="true"):1296 self.checkAuthorship =False1297else:1298 self.checkAuthorship =True12991300if self.preserveUser:1301 self.checkValidP4Users(commits)13021303whilelen(commits) >0:1304 commit = commits[0]1305 commits = commits[1:]1306 self.applyCommit(commit)1307if not self.interactive:1308break13091310iflen(commits) ==0:1311print"All changes applied!"1312chdir(self.oldWorkingDirectory)13131314 sync =P4Sync()1315 sync.run([])13161317 rebase =P4Rebase()1318 rebase.rebase()13191320return True13211322classView(object):1323"""Represent a p4 view ("p4 help views"), and map files in a1324 repo according to the view."""13251326classPath(object):1327"""A depot or client path, possibly containing wildcards.1328 The only one supported is ... at the end, currently.1329 Initialize with the full path, with //depot or //client."""13301331def__init__(self, path, is_depot):1332 self.path = path1333 self.is_depot = is_depot1334 self.find_wildcards()1335# remember the prefix bit, useful for relative mappings1336 m = re.match("(//[^/]+/)", self.path)1337if not m:1338die("Path%sdoes not start with //prefix/"% self.path)1339 prefix = m.group(1)1340if not self.is_depot:1341# strip //client/ on client paths1342 self.path = self.path[len(prefix):]13431344deffind_wildcards(self):1345"""Make sure wildcards are valid, and set up internal1346 variables."""13471348 self.ends_triple_dot =False1349# There are three wildcards allowed in p4 views1350# (see "p4 help views"). This code knows how to1351# handle "..." (only at the end), but cannot deal with1352# "%%n" or "*". Only check the depot_side, as p4 should1353# validate that the client_side matches too.1354if re.search(r'%%[1-9]', self.path):1355die("Can't handle%%n wildcards in view:%s"% self.path)1356if self.path.find("*") >=0:1357die("Can't handle * wildcards in view:%s"% self.path)1358 triple_dot_index = self.path.find("...")1359if triple_dot_index >=0:1360if triple_dot_index !=len(self.path) -3:1361die("Can handle only single ... wildcard, at end:%s"%1362 self.path)1363 self.ends_triple_dot =True13641365defensure_compatible(self, other_path):1366"""Make sure the wildcards agree."""1367if self.ends_triple_dot != other_path.ends_triple_dot:1368die("Both paths must end with ... if either does;\n"+1369"paths:%s %s"% (self.path, other_path.path))13701371defmatch_wildcards(self, test_path):1372"""See if this test_path matches us, and fill in the value1373 of the wildcards if so. Returns a tuple of1374 (True|False, wildcards[]). For now, only the ... at end1375 is supported, so at most one wildcard."""1376if self.ends_triple_dot:1377 dotless = self.path[:-3]1378if test_path.startswith(dotless):1379 wildcard = test_path[len(dotless):]1380return(True, [ wildcard ])1381else:1382if test_path == self.path:1383return(True, [])1384return(False, [])13851386defmatch(self, test_path):1387"""Just return if it matches; don't bother with the wildcards."""1388 b, _ = self.match_wildcards(test_path)1389return b13901391deffill_in_wildcards(self, wildcards):1392"""Return the relative path, with the wildcards filled in1393 if there are any."""1394if self.ends_triple_dot:1395return self.path[:-3] + wildcards[0]1396else:1397return self.path13981399classMapping(object):1400def__init__(self, depot_side, client_side, overlay, exclude):1401# depot_side is without the trailing /... if it had one1402 self.depot_side = View.Path(depot_side, is_depot=True)1403 self.client_side = View.Path(client_side, is_depot=False)1404 self.overlay = overlay # started with "+"1405 self.exclude = exclude # started with "-"1406assert not(self.overlay and self.exclude)1407 self.depot_side.ensure_compatible(self.client_side)14081409def__str__(self):1410 c =" "1411if self.overlay:1412 c ="+"1413if self.exclude:1414 c ="-"1415return"View.Mapping:%s%s->%s"% \1416(c, self.depot_side.path, self.client_side.path)14171418defmap_depot_to_client(self, depot_path):1419"""Calculate the client path if using this mapping on the1420 given depot path; does not consider the effect of other1421 mappings in a view. Even excluded mappings are returned."""1422 matches, wildcards = self.depot_side.match_wildcards(depot_path)1423if not matches:1424return""1425 client_path = self.client_side.fill_in_wildcards(wildcards)1426return client_path14271428#1429# View methods1430#1431def__init__(self):1432 self.mappings = []14331434defappend(self, view_line):1435"""Parse a view line, splitting it into depot and client1436 sides. Append to self.mappings, preserving order."""14371438# Split the view line into exactly two words. P4 enforces1439# structure on these lines that simplifies this quite a bit.1440#1441# Either or both words may be double-quoted.1442# Single quotes do not matter.1443# Double-quote marks cannot occur inside the words.1444# A + or - prefix is also inside the quotes.1445# There are no quotes unless they contain a space.1446# The line is already white-space stripped.1447# The two words are separated by a single space.1448#1449if view_line[0] =='"':1450# First word is double quoted. Find its end.1451 close_quote_index = view_line.find('"',1)1452if close_quote_index <=0:1453die("No first-word closing quote found:%s"% view_line)1454 depot_side = view_line[1:close_quote_index]1455# skip closing quote and space1456 rhs_index = close_quote_index +1+11457else:1458 space_index = view_line.find(" ")1459if space_index <=0:1460die("No word-splitting space found:%s"% view_line)1461 depot_side = view_line[0:space_index]1462 rhs_index = space_index +114631464if view_line[rhs_index] =='"':1465# Second word is double quoted. Make sure there is a1466# double quote at the end too.1467if not view_line.endswith('"'):1468die("View line with rhs quote should end with one:%s"%1469 view_line)1470# skip the quotes1471 client_side = view_line[rhs_index+1:-1]1472else:1473 client_side = view_line[rhs_index:]14741475# prefix + means overlay on previous mapping1476 overlay =False1477if depot_side.startswith("+"):1478 overlay =True1479 depot_side = depot_side[1:]14801481# prefix - means exclude this path1482 exclude =False1483if depot_side.startswith("-"):1484 exclude =True1485 depot_side = depot_side[1:]14861487 m = View.Mapping(depot_side, client_side, overlay, exclude)1488 self.mappings.append(m)14891490defmap_in_client(self, depot_path):1491"""Return the relative location in the client where this1492 depot file should live. Returns "" if the file should1493 not be mapped in the client."""14941495 paths_filled = []1496 client_path =""14971498# look at later entries first1499for m in self.mappings[::-1]:15001501# see where will this path end up in the client1502 p = m.map_depot_to_client(depot_path)15031504if p =="":1505# Depot path does not belong in client. Must remember1506# this, as previous items should not cause files to1507# exist in this path either. Remember that the list is1508# being walked from the end, which has higher precedence.1509# Overlap mappings do not exclude previous mappings.1510if not m.overlay:1511 paths_filled.append(m.client_side)15121513else:1514# This mapping matched; no need to search any further.1515# But, the mapping could be rejected if the client path1516# has already been claimed by an earlier mapping (i.e.1517# one later in the list, which we are walking backwards).1518 already_mapped_in_client =False1519for f in paths_filled:1520# this is View.Path.match1521if f.match(p):1522 already_mapped_in_client =True1523break1524if not already_mapped_in_client:1525# Include this file, unless it is from a line that1526# explicitly said to exclude it.1527if not m.exclude:1528 client_path = p15291530# a match, even if rejected, always stops the search1531break15321533return client_path15341535classP4Sync(Command, P4UserMap):1536 delete_actions = ("delete","move/delete","purge")15371538def__init__(self):1539 Command.__init__(self)1540 P4UserMap.__init__(self)1541 self.options = [1542 optparse.make_option("--branch", dest="branch"),1543 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1544 optparse.make_option("--changesfile", dest="changesFile"),1545 optparse.make_option("--silent", dest="silent", action="store_true"),1546 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1547 optparse.make_option("--verbose", dest="verbose", action="store_true"),1548 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1549help="Import into refs/heads/ , not refs/remotes"),1550 optparse.make_option("--max-changes", dest="maxChanges"),1551 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',1552help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),1553 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',1554help="Only sync files that are included in the Perforce Client Spec")1555]1556 self.description ="""Imports from Perforce into a git repository.\n1557 example:1558 //depot/my/project/ -- to import the current head1559 //depot/my/project/@all -- to import everything1560 //depot/my/project/@1,6 -- to import only from revision 1 to 615611562 (a ... is not needed in the path p4 specification, it's added implicitly)"""15631564 self.usage +=" //depot/path[@revRange]"1565 self.silent =False1566 self.createdBranches =set()1567 self.committedChanges =set()1568 self.branch =""1569 self.detectBranches =False1570 self.detectLabels =False1571 self.changesFile =""1572 self.syncWithOrigin =True1573 self.verbose =False1574 self.importIntoRemotes =True1575 self.maxChanges =""1576 self.isWindows = (platform.system() =="Windows")1577 self.keepRepoPath =False1578 self.depotPaths =None1579 self.p4BranchesInGit = []1580 self.cloneExclude = []1581 self.useClientSpec =False1582 self.useClientSpec_from_options =False1583 self.clientSpecDirs =None1584 self.tempBranches = []1585 self.tempBranchLocation ="git-p4-tmp"15861587ifgitConfig("git-p4.syncFromOrigin") =="false":1588 self.syncWithOrigin =False15891590#1591# P4 wildcards are not allowed in filenames. P4 complains1592# if you simply add them, but you can force it with "-f", in1593# which case it translates them into %xx encoding internally.1594# Search for and fix just these four characters. Do % last so1595# that fixing it does not inadvertently create new %-escapes.1596#1597defwildcard_decode(self, path):1598# Cannot have * in a filename in windows; untested as to1599# what p4 would do in such a case.1600if not self.isWindows:1601 path = path.replace("%2A","*")1602 path = path.replace("%23","#") \1603.replace("%40","@") \1604.replace("%25","%")1605return path16061607# Force a checkpoint in fast-import and wait for it to finish1608defcheckpoint(self):1609 self.gitStream.write("checkpoint\n\n")1610 self.gitStream.write("progress checkpoint\n\n")1611 out = self.gitOutput.readline()1612if self.verbose:1613print"checkpoint finished: "+ out16141615defextractFilesFromCommit(self, commit):1616 self.cloneExclude = [re.sub(r"\.\.\.$","", path)1617for path in self.cloneExclude]1618 files = []1619 fnum =01620while commit.has_key("depotFile%s"% fnum):1621 path = commit["depotFile%s"% fnum]16221623if[p for p in self.cloneExclude1624ifp4PathStartsWith(path, p)]:1625 found =False1626else:1627 found = [p for p in self.depotPaths1628ifp4PathStartsWith(path, p)]1629if not found:1630 fnum = fnum +11631continue16321633file= {}1634file["path"] = path1635file["rev"] = commit["rev%s"% fnum]1636file["action"] = commit["action%s"% fnum]1637file["type"] = commit["type%s"% fnum]1638 files.append(file)1639 fnum = fnum +11640return files16411642defstripRepoPath(self, path, prefixes):1643if self.useClientSpec:1644return self.clientSpecDirs.map_in_client(path)16451646if self.keepRepoPath:1647 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]16481649for p in prefixes:1650ifp4PathStartsWith(path, p):1651 path = path[len(p):]16521653return path16541655defsplitFilesIntoBranches(self, commit):1656 branches = {}1657 fnum =01658while commit.has_key("depotFile%s"% fnum):1659 path = commit["depotFile%s"% fnum]1660 found = [p for p in self.depotPaths1661ifp4PathStartsWith(path, p)]1662if not found:1663 fnum = fnum +11664continue16651666file= {}1667file["path"] = path1668file["rev"] = commit["rev%s"% fnum]1669file["action"] = commit["action%s"% fnum]1670file["type"] = commit["type%s"% fnum]1671 fnum = fnum +116721673 relPath = self.stripRepoPath(path, self.depotPaths)16741675for branch in self.knownBranches.keys():16761677# add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.21678if relPath.startswith(branch +"/"):1679if branch not in branches:1680 branches[branch] = []1681 branches[branch].append(file)1682break16831684return branches16851686# output one file from the P4 stream1687# - helper for streamP4Files16881689defstreamOneP4File(self,file, contents):1690 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)1691 relPath = self.wildcard_decode(relPath)1692if verbose:1693 sys.stderr.write("%s\n"% relPath)16941695(type_base, type_mods) =split_p4_type(file["type"])16961697 git_mode ="100644"1698if"x"in type_mods:1699 git_mode ="100755"1700if type_base =="symlink":1701 git_mode ="120000"1702# p4 print on a symlink contains "target\n"; remove the newline1703 data =''.join(contents)1704 contents = [data[:-1]]17051706if type_base =="utf16":1707# p4 delivers different text in the python output to -G1708# than it does when using "print -o", or normal p4 client1709# operations. utf16 is converted to ascii or utf8, perhaps.1710# But ascii text saved as -t utf16 is completely mangled.1711# Invoke print -o to get the real contents.1712 text =p4_read_pipe(['print','-q','-o','-',file['depotFile']])1713 contents = [ text ]17141715if type_base =="apple":1716# Apple filetype files will be streamed as a concatenation of1717# its appledouble header and the contents. This is useless1718# on both macs and non-macs. If using "print -q -o xx", it1719# will create "xx" with the data, and "%xx" with the header.1720# This is also not very useful.1721#1722# Ideally, someday, this script can learn how to generate1723# appledouble files directly and import those to git, but1724# non-mac machines can never find a use for apple filetype.1725print"\nIgnoring apple filetype file%s"%file['depotFile']1726return17271728# Perhaps windows wants unicode, utf16 newlines translated too;1729# but this is not doing it.1730if self.isWindows and type_base =="text":1731 mangled = []1732for data in contents:1733 data = data.replace("\r\n","\n")1734 mangled.append(data)1735 contents = mangled17361737# Note that we do not try to de-mangle keywords on utf16 files,1738# even though in theory somebody may want that.1739 pattern =p4_keywords_regexp_for_type(type_base, type_mods)1740if pattern:1741 regexp = re.compile(pattern, re.VERBOSE)1742 text =''.join(contents)1743 text = regexp.sub(r'$\1$', text)1744 contents = [ text ]17451746 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))17471748# total length...1749 length =01750for d in contents:1751 length = length +len(d)17521753 self.gitStream.write("data%d\n"% length)1754for d in contents:1755 self.gitStream.write(d)1756 self.gitStream.write("\n")17571758defstreamOneP4Deletion(self,file):1759 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)1760if verbose:1761 sys.stderr.write("delete%s\n"% relPath)1762 self.gitStream.write("D%s\n"% relPath)17631764# handle another chunk of streaming data1765defstreamP4FilesCb(self, marshalled):17661767if marshalled.has_key('depotFile')and self.stream_have_file_info:1768# start of a new file - output the old one first1769 self.streamOneP4File(self.stream_file, self.stream_contents)1770 self.stream_file = {}1771 self.stream_contents = []1772 self.stream_have_file_info =False17731774# pick up the new file information... for the1775# 'data' field we need to append to our array1776for k in marshalled.keys():1777if k =='data':1778 self.stream_contents.append(marshalled['data'])1779else:1780 self.stream_file[k] = marshalled[k]17811782 self.stream_have_file_info =True17831784# Stream directly from "p4 files" into "git fast-import"1785defstreamP4Files(self, files):1786 filesForCommit = []1787 filesToRead = []1788 filesToDelete = []17891790for f in files:1791# if using a client spec, only add the files that have1792# a path in the client1793if self.clientSpecDirs:1794if self.clientSpecDirs.map_in_client(f['path']) =="":1795continue17961797 filesForCommit.append(f)1798if f['action']in self.delete_actions:1799 filesToDelete.append(f)1800else:1801 filesToRead.append(f)18021803# deleted files...1804for f in filesToDelete:1805 self.streamOneP4Deletion(f)18061807iflen(filesToRead) >0:1808 self.stream_file = {}1809 self.stream_contents = []1810 self.stream_have_file_info =False18111812# curry self argument1813defstreamP4FilesCbSelf(entry):1814 self.streamP4FilesCb(entry)18151816 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]18171818p4CmdList(["-x","-","print"],1819 stdin=fileArgs,1820 cb=streamP4FilesCbSelf)18211822# do the last chunk1823if self.stream_file.has_key('depotFile'):1824 self.streamOneP4File(self.stream_file, self.stream_contents)18251826defmake_email(self, userid):1827if userid in self.users:1828return self.users[userid]1829else:1830return"%s<a@b>"% userid18311832defcommit(self, details, files, branch, branchPrefixes, parent =""):1833 epoch = details["time"]1834 author = details["user"]1835 self.branchPrefixes = branchPrefixes18361837if self.verbose:1838print"commit into%s"% branch18391840# start with reading files; if that fails, we should not1841# create a commit.1842 new_files = []1843for f in files:1844if[p for p in branchPrefixes ifp4PathStartsWith(f['path'], p)]:1845 new_files.append(f)1846else:1847 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])18481849 self.gitStream.write("commit%s\n"% branch)1850# gitStream.write("mark :%s\n" % details["change"])1851 self.committedChanges.add(int(details["change"]))1852 committer =""1853if author not in self.users:1854 self.getUserMapFromPerforceServer()1855 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)18561857 self.gitStream.write("committer%s\n"% committer)18581859 self.gitStream.write("data <<EOT\n")1860 self.gitStream.write(details["desc"])1861 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"1862% (','.join(branchPrefixes), details["change"]))1863iflen(details['options']) >0:1864 self.gitStream.write(": options =%s"% details['options'])1865 self.gitStream.write("]\nEOT\n\n")18661867iflen(parent) >0:1868if self.verbose:1869print"parent%s"% parent1870 self.gitStream.write("from%s\n"% parent)18711872 self.streamP4Files(new_files)1873 self.gitStream.write("\n")18741875 change =int(details["change"])18761877if self.labels.has_key(change):1878 label = self.labels[change]1879 labelDetails = label[0]1880 labelRevisions = label[1]1881if self.verbose:1882print"Change%sis labelled%s"% (change, labelDetails)18831884 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)1885for p in branchPrefixes])18861887iflen(files) ==len(labelRevisions):18881889 cleanedFiles = {}1890for info in files:1891if info["action"]in self.delete_actions:1892continue1893 cleanedFiles[info["depotFile"]] = info["rev"]18941895if cleanedFiles == labelRevisions:1896 self.gitStream.write("tag tag_%s\n"% labelDetails["label"])1897 self.gitStream.write("from%s\n"% branch)18981899 owner = labelDetails["Owner"]19001901# Try to use the owner of the p4 label, or failing that,1902# the current p4 user id.1903if owner:1904 email = self.make_email(owner)1905else:1906 email = self.make_email(self.p4UserId())1907 tagger ="%s %s %s"% (email, epoch, self.tz)19081909 self.gitStream.write("tagger%s\n"% tagger)19101911 description = labelDetails["Description"]1912 self.gitStream.write("data%d\n"%len(description))1913 self.gitStream.write(description)1914 self.gitStream.write("\n")19151916else:1917if not self.silent:1918print("Tag%sdoes not match with change%s: files do not match."1919% (labelDetails["label"], change))19201921else:1922if not self.silent:1923print("Tag%sdoes not match with change%s: file count is different."1924% (labelDetails["label"], change))19251926defgetLabels(self):1927 self.labels = {}19281929 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])1930iflen(l) >0and not self.silent:1931print"Finding files belonging to labels in%s"% `self.depotPaths`19321933for output in l:1934 label = output["label"]1935 revisions = {}1936 newestChange =01937if self.verbose:1938print"Querying files for label%s"% label1939forfileinp4CmdList(["files"] +1940["%s...@%s"% (p, label)1941for p in self.depotPaths]):1942 revisions[file["depotFile"]] =file["rev"]1943 change =int(file["change"])1944if change > newestChange:1945 newestChange = change19461947 self.labels[newestChange] = [output, revisions]19481949if self.verbose:1950print"Label changes:%s"% self.labels.keys()19511952defguessProjectName(self):1953for p in self.depotPaths:1954if p.endswith("/"):1955 p = p[:-1]1956 p = p[p.strip().rfind("/") +1:]1957if not p.endswith("/"):1958 p +="/"1959return p19601961defgetBranchMapping(self):1962 lostAndFoundBranches =set()19631964 user =gitConfig("git-p4.branchUser")1965iflen(user) >0:1966 command ="branches -u%s"% user1967else:1968 command ="branches"19691970for info inp4CmdList(command):1971 details =p4Cmd(["branch","-o", info["branch"]])1972 viewIdx =01973while details.has_key("View%s"% viewIdx):1974 paths = details["View%s"% viewIdx].split(" ")1975 viewIdx = viewIdx +11976# require standard //depot/foo/... //depot/bar/... mapping1977iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):1978continue1979 source = paths[0]1980 destination = paths[1]1981## HACK1982ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):1983 source = source[len(self.depotPaths[0]):-4]1984 destination = destination[len(self.depotPaths[0]):-4]19851986if destination in self.knownBranches:1987if not self.silent:1988print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)1989print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)1990continue19911992 self.knownBranches[destination] = source19931994 lostAndFoundBranches.discard(destination)19951996if source not in self.knownBranches:1997 lostAndFoundBranches.add(source)19981999# Perforce does not strictly require branches to be defined, so we also2000# check git config for a branch list.2001#2002# Example of branch definition in git config file:2003# [git-p4]2004# branchList=main:branchA2005# branchList=main:branchB2006# branchList=branchA:branchC2007 configBranches =gitConfigList("git-p4.branchList")2008for branch in configBranches:2009if branch:2010(source, destination) = branch.split(":")2011 self.knownBranches[destination] = source20122013 lostAndFoundBranches.discard(destination)20142015if source not in self.knownBranches:2016 lostAndFoundBranches.add(source)201720182019for branch in lostAndFoundBranches:2020 self.knownBranches[branch] = branch20212022defgetBranchMappingFromGitBranches(self):2023 branches =p4BranchesInGit(self.importIntoRemotes)2024for branch in branches.keys():2025if branch =="master":2026 branch ="main"2027else:2028 branch = branch[len(self.projectName):]2029 self.knownBranches[branch] = branch20302031deflistExistingP4GitBranches(self):2032# branches holds mapping from name to commit2033 branches =p4BranchesInGit(self.importIntoRemotes)2034 self.p4BranchesInGit = branches.keys()2035for branch in branches.keys():2036 self.initialParents[self.refPrefix + branch] = branches[branch]20372038defupdateOptionDict(self, d):2039 option_keys = {}2040if self.keepRepoPath:2041 option_keys['keepRepoPath'] =120422043 d["options"] =' '.join(sorted(option_keys.keys()))20442045defreadOptions(self, d):2046 self.keepRepoPath = (d.has_key('options')2047and('keepRepoPath'in d['options']))20482049defgitRefForBranch(self, branch):2050if branch =="main":2051return self.refPrefix +"master"20522053iflen(branch) <=0:2054return branch20552056return self.refPrefix + self.projectName + branch20572058defgitCommitByP4Change(self, ref, change):2059if self.verbose:2060print"looking in ref "+ ref +" for change%susing bisect..."% change20612062 earliestCommit =""2063 latestCommit =parseRevision(ref)20642065while True:2066if self.verbose:2067print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2068 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2069iflen(next) ==0:2070if self.verbose:2071print"argh"2072return""2073 log =extractLogMessageFromGitCommit(next)2074 settings =extractSettingsGitLog(log)2075 currentChange =int(settings['change'])2076if self.verbose:2077print"current change%s"% currentChange20782079if currentChange == change:2080if self.verbose:2081print"found%s"% next2082return next20832084if currentChange < change:2085 earliestCommit ="^%s"% next2086else:2087 latestCommit ="%s"% next20882089return""20902091defimportNewBranch(self, branch, maxChange):2092# make fast-import flush all changes to disk and update the refs using the checkpoint2093# command so that we can try to find the branch parent in the git history2094 self.gitStream.write("checkpoint\n\n");2095 self.gitStream.flush();2096 branchPrefix = self.depotPaths[0] + branch +"/"2097range="@1,%s"% maxChange2098#print "prefix" + branchPrefix2099 changes =p4ChangesForPaths([branchPrefix],range)2100iflen(changes) <=0:2101return False2102 firstChange = changes[0]2103#print "first change in branch: %s" % firstChange2104 sourceBranch = self.knownBranches[branch]2105 sourceDepotPath = self.depotPaths[0] + sourceBranch2106 sourceRef = self.gitRefForBranch(sourceBranch)2107#print "source " + sourceBranch21082109 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2110#print "branch parent: %s" % branchParentChange2111 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2112iflen(gitParent) >0:2113 self.initialParents[self.gitRefForBranch(branch)] = gitParent2114#print "parent git commit: %s" % gitParent21152116 self.importChanges(changes)2117return True21182119defsearchParent(self, parent, branch, target):2120 parentFound =False2121for blob inread_pipe_lines(["git","rev-list","--reverse","--no-merges", parent]):2122 blob = blob.strip()2123iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2124 parentFound =True2125if self.verbose:2126print"Found parent of%sin commit%s"% (branch, blob)2127break2128if parentFound:2129return blob2130else:2131return None21322133defimportChanges(self, changes):2134 cnt =12135for change in changes:2136 description =p4Cmd(["describe",str(change)])2137 self.updateOptionDict(description)21382139if not self.silent:2140 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2141 sys.stdout.flush()2142 cnt = cnt +121432144try:2145if self.detectBranches:2146 branches = self.splitFilesIntoBranches(description)2147for branch in branches.keys():2148## HACK --hwn2149 branchPrefix = self.depotPaths[0] + branch +"/"21502151 parent =""21522153 filesForCommit = branches[branch]21542155if self.verbose:2156print"branch is%s"% branch21572158 self.updatedBranches.add(branch)21592160if branch not in self.createdBranches:2161 self.createdBranches.add(branch)2162 parent = self.knownBranches[branch]2163if parent == branch:2164 parent =""2165else:2166 fullBranch = self.projectName + branch2167if fullBranch not in self.p4BranchesInGit:2168if not self.silent:2169print("\nImporting new branch%s"% fullBranch);2170if self.importNewBranch(branch, change -1):2171 parent =""2172 self.p4BranchesInGit.append(fullBranch)2173if not self.silent:2174print("\nResuming with change%s"% change);21752176if self.verbose:2177print"parent determined through known branches:%s"% parent21782179 branch = self.gitRefForBranch(branch)2180 parent = self.gitRefForBranch(parent)21812182if self.verbose:2183print"looking for initial parent for%s; current parent is%s"% (branch, parent)21842185iflen(parent) ==0and branch in self.initialParents:2186 parent = self.initialParents[branch]2187del self.initialParents[branch]21882189 blob =None2190iflen(parent) >0:2191 tempBranch = os.path.join(self.tempBranchLocation,"%d"% (change))2192if self.verbose:2193print"Creating temporary branch: "+ tempBranch2194 self.commit(description, filesForCommit, tempBranch, [branchPrefix])2195 self.tempBranches.append(tempBranch)2196 self.checkpoint()2197 blob = self.searchParent(parent, branch, tempBranch)2198if blob:2199 self.commit(description, filesForCommit, branch, [branchPrefix], blob)2200else:2201if self.verbose:2202print"Parent of%snot found. Committing into head of%s"% (branch, parent)2203 self.commit(description, filesForCommit, branch, [branchPrefix], parent)2204else:2205 files = self.extractFilesFromCommit(description)2206 self.commit(description, files, self.branch, self.depotPaths,2207 self.initialParent)2208 self.initialParent =""2209exceptIOError:2210print self.gitError.read()2211 sys.exit(1)22122213defimportHeadRevision(self, revision):2214print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)22152216 details = {}2217 details["user"] ="git perforce import user"2218 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2219% (' '.join(self.depotPaths), revision))2220 details["change"] = revision2221 newestRevision =022222223 fileCnt =02224 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]22252226for info inp4CmdList(["files"] + fileArgs):22272228if'code'in info and info['code'] =='error':2229 sys.stderr.write("p4 returned an error:%s\n"2230% info['data'])2231if info['data'].find("must refer to client") >=0:2232 sys.stderr.write("This particular p4 error is misleading.\n")2233 sys.stderr.write("Perhaps the depot path was misspelled.\n");2234 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2235 sys.exit(1)2236if'p4ExitCode'in info:2237 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2238 sys.exit(1)223922402241 change =int(info["change"])2242if change > newestRevision:2243 newestRevision = change22442245if info["action"]in self.delete_actions:2246# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2247#fileCnt = fileCnt + 12248continue22492250for prop in["depotFile","rev","action","type"]:2251 details["%s%s"% (prop, fileCnt)] = info[prop]22522253 fileCnt = fileCnt +122542255 details["change"] = newestRevision22562257# Use time from top-most change so that all git-p4 clones of2258# the same p4 repo have the same commit SHA1s.2259 res =p4CmdList("describe -s%d"% newestRevision)2260 newestTime =None2261for r in res:2262if r.has_key('time'):2263 newestTime =int(r['time'])2264if newestTime is None:2265die("\"describe -s\"on newest change%ddid not give a time")2266 details["time"] = newestTime22672268 self.updateOptionDict(details)2269try:2270 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)2271exceptIOError:2272print"IO error with git fast-import. Is your git version recent enough?"2273print self.gitError.read()227422752276defrun(self, args):2277 self.depotPaths = []2278 self.changeRange =""2279 self.initialParent =""2280 self.previousDepotPaths = []22812282# map from branch depot path to parent branch2283 self.knownBranches = {}2284 self.initialParents = {}2285 self.hasOrigin =originP4BranchesExist()2286if not self.syncWithOrigin:2287 self.hasOrigin =False22882289if self.importIntoRemotes:2290 self.refPrefix ="refs/remotes/p4/"2291else:2292 self.refPrefix ="refs/heads/p4/"22932294if self.syncWithOrigin and self.hasOrigin:2295if not self.silent:2296print"Syncing with origin first by calling git fetch origin"2297system("git fetch origin")22982299iflen(self.branch) ==0:2300 self.branch = self.refPrefix +"master"2301ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2302system("git update-ref%srefs/heads/p4"% self.branch)2303system("git branch -D p4");2304# create it /after/ importing, when master exists2305if notgitBranchExists(self.refPrefix +"HEAD")and self.importIntoRemotes andgitBranchExists(self.branch):2306system("git symbolic-ref%sHEAD%s"% (self.refPrefix, self.branch))23072308# accept either the command-line option, or the configuration variable2309if self.useClientSpec:2310# will use this after clone to set the variable2311 self.useClientSpec_from_options =True2312else:2313ifgitConfig("git-p4.useclientspec","--bool") =="true":2314 self.useClientSpec =True2315if self.useClientSpec:2316 self.clientSpecDirs =getClientSpec()23172318# TODO: should always look at previous commits,2319# merge with previous imports, if possible.2320if args == []:2321if self.hasOrigin:2322createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)2323 self.listExistingP4GitBranches()23242325iflen(self.p4BranchesInGit) >1:2326if not self.silent:2327print"Importing from/into multiple branches"2328 self.detectBranches =True23292330if self.verbose:2331print"branches:%s"% self.p4BranchesInGit23322333 p4Change =02334for branch in self.p4BranchesInGit:2335 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)23362337 settings =extractSettingsGitLog(logMsg)23382339 self.readOptions(settings)2340if(settings.has_key('depot-paths')2341and settings.has_key('change')):2342 change =int(settings['change']) +12343 p4Change =max(p4Change, change)23442345 depotPaths =sorted(settings['depot-paths'])2346if self.previousDepotPaths == []:2347 self.previousDepotPaths = depotPaths2348else:2349 paths = []2350for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2351 prev_list = prev.split("/")2352 cur_list = cur.split("/")2353for i inrange(0,min(len(cur_list),len(prev_list))):2354if cur_list[i] <> prev_list[i]:2355 i = i -12356break23572358 paths.append("/".join(cur_list[:i +1]))23592360 self.previousDepotPaths = paths23612362if p4Change >0:2363 self.depotPaths =sorted(self.previousDepotPaths)2364 self.changeRange ="@%s,#head"% p4Change2365if not self.detectBranches:2366 self.initialParent =parseRevision(self.branch)2367if not self.silent and not self.detectBranches:2368print"Performing incremental import into%sgit branch"% self.branch23692370if not self.branch.startswith("refs/"):2371 self.branch ="refs/heads/"+ self.branch23722373iflen(args) ==0and self.depotPaths:2374if not self.silent:2375print"Depot paths:%s"%' '.join(self.depotPaths)2376else:2377if self.depotPaths and self.depotPaths != args:2378print("previous import used depot path%sand now%swas specified. "2379"This doesn't work!"% (' '.join(self.depotPaths),2380' '.join(args)))2381 sys.exit(1)23822383 self.depotPaths =sorted(args)23842385 revision =""2386 self.users = {}23872388# Make sure no revision specifiers are used when --changesfile2389# is specified.2390 bad_changesfile =False2391iflen(self.changesFile) >0:2392for p in self.depotPaths:2393if p.find("@") >=0or p.find("#") >=0:2394 bad_changesfile =True2395break2396if bad_changesfile:2397die("Option --changesfile is incompatible with revision specifiers")23982399 newPaths = []2400for p in self.depotPaths:2401if p.find("@") != -1:2402 atIdx = p.index("@")2403 self.changeRange = p[atIdx:]2404if self.changeRange =="@all":2405 self.changeRange =""2406elif','not in self.changeRange:2407 revision = self.changeRange2408 self.changeRange =""2409 p = p[:atIdx]2410elif p.find("#") != -1:2411 hashIdx = p.index("#")2412 revision = p[hashIdx:]2413 p = p[:hashIdx]2414elif self.previousDepotPaths == []:2415# pay attention to changesfile, if given, else import2416# the entire p4 tree at the head revision2417iflen(self.changesFile) ==0:2418 revision ="#head"24192420 p = re.sub("\.\.\.$","", p)2421if not p.endswith("/"):2422 p +="/"24232424 newPaths.append(p)24252426 self.depotPaths = newPaths242724282429 self.loadUserMapFromCache()2430 self.labels = {}2431if self.detectLabels:2432 self.getLabels();24332434if self.detectBranches:2435## FIXME - what's a P4 projectName ?2436 self.projectName = self.guessProjectName()24372438if self.hasOrigin:2439 self.getBranchMappingFromGitBranches()2440else:2441 self.getBranchMapping()2442if self.verbose:2443print"p4-git branches:%s"% self.p4BranchesInGit2444print"initial parents:%s"% self.initialParents2445for b in self.p4BranchesInGit:2446if b !="master":24472448## FIXME2449 b = b[len(self.projectName):]2450 self.createdBranches.add(b)24512452 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))24532454 importProcess = subprocess.Popen(["git","fast-import"],2455 stdin=subprocess.PIPE, stdout=subprocess.PIPE,2456 stderr=subprocess.PIPE);2457 self.gitOutput = importProcess.stdout2458 self.gitStream = importProcess.stdin2459 self.gitError = importProcess.stderr24602461if revision:2462 self.importHeadRevision(revision)2463else:2464 changes = []24652466iflen(self.changesFile) >0:2467 output =open(self.changesFile).readlines()2468 changeSet =set()2469for line in output:2470 changeSet.add(int(line))24712472for change in changeSet:2473 changes.append(change)24742475 changes.sort()2476else:2477# catch "git-p4 sync" with no new branches, in a repo that2478# does not have any existing git-p4 branches2479iflen(args) ==0and not self.p4BranchesInGit:2480die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.");2481if self.verbose:2482print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),2483 self.changeRange)2484 changes =p4ChangesForPaths(self.depotPaths, self.changeRange)24852486iflen(self.maxChanges) >0:2487 changes = changes[:min(int(self.maxChanges),len(changes))]24882489iflen(changes) ==0:2490if not self.silent:2491print"No changes to import!"2492return True24932494if not self.silent and not self.detectBranches:2495print"Import destination:%s"% self.branch24962497 self.updatedBranches =set()24982499 self.importChanges(changes)25002501if not self.silent:2502print""2503iflen(self.updatedBranches) >0:2504 sys.stdout.write("Updated branches: ")2505for b in self.updatedBranches:2506 sys.stdout.write("%s"% b)2507 sys.stdout.write("\n")25082509 self.gitStream.close()2510if importProcess.wait() !=0:2511die("fast-import failed:%s"% self.gitError.read())2512 self.gitOutput.close()2513 self.gitError.close()25142515# Cleanup temporary branches created during import2516if self.tempBranches != []:2517for branch in self.tempBranches:2518read_pipe("git update-ref -d%s"% branch)2519 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))25202521return True25222523classP4Rebase(Command):2524def__init__(self):2525 Command.__init__(self)2526 self.options = [ ]2527 self.description = ("Fetches the latest revision from perforce and "2528+"rebases the current work (branch) against it")2529 self.verbose =False25302531defrun(self, args):2532 sync =P4Sync()2533 sync.run([])25342535return self.rebase()25362537defrebase(self):2538if os.system("git update-index --refresh") !=0:2539die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");2540iflen(read_pipe("git diff-index HEAD --")) >0:2541die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");25422543[upstream, settings] =findUpstreamBranchPoint()2544iflen(upstream) ==0:2545die("Cannot find upstream branchpoint for rebase")25462547# the branchpoint may be p4/foo~3, so strip off the parent2548 upstream = re.sub("~[0-9]+$","", upstream)25492550print"Rebasing the current branch onto%s"% upstream2551 oldHead =read_pipe("git rev-parse HEAD").strip()2552system("git rebase%s"% upstream)2553system("git diff-tree --stat --summary -M%sHEAD"% oldHead)2554return True25552556classP4Clone(P4Sync):2557def__init__(self):2558 P4Sync.__init__(self)2559 self.description ="Creates a new git repository and imports from Perforce into it"2560 self.usage ="usage: %prog [options] //depot/path[@revRange]"2561 self.options += [2562 optparse.make_option("--destination", dest="cloneDestination",2563 action='store', default=None,2564help="where to leave result of the clone"),2565 optparse.make_option("-/", dest="cloneExclude",2566 action="append",type="string",2567help="exclude depot path"),2568 optparse.make_option("--bare", dest="cloneBare",2569 action="store_true", default=False),2570]2571 self.cloneDestination =None2572 self.needsGit =False2573 self.cloneBare =False25742575# This is required for the "append" cloneExclude action2576defensure_value(self, attr, value):2577if nothasattr(self, attr)orgetattr(self, attr)is None:2578setattr(self, attr, value)2579returngetattr(self, attr)25802581defdefaultDestination(self, args):2582## TODO: use common prefix of args?2583 depotPath = args[0]2584 depotDir = re.sub("(@[^@]*)$","", depotPath)2585 depotDir = re.sub("(#[^#]*)$","", depotDir)2586 depotDir = re.sub(r"\.\.\.$","", depotDir)2587 depotDir = re.sub(r"/$","", depotDir)2588return os.path.split(depotDir)[1]25892590defrun(self, args):2591iflen(args) <1:2592return False25932594if self.keepRepoPath and not self.cloneDestination:2595 sys.stderr.write("Must specify destination for --keep-path\n")2596 sys.exit(1)25972598 depotPaths = args25992600if not self.cloneDestination andlen(depotPaths) >1:2601 self.cloneDestination = depotPaths[-1]2602 depotPaths = depotPaths[:-1]26032604 self.cloneExclude = ["/"+p for p in self.cloneExclude]2605for p in depotPaths:2606if not p.startswith("//"):2607return False26082609if not self.cloneDestination:2610 self.cloneDestination = self.defaultDestination(args)26112612print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)26132614if not os.path.exists(self.cloneDestination):2615 os.makedirs(self.cloneDestination)2616chdir(self.cloneDestination)26172618 init_cmd = ["git","init"]2619if self.cloneBare:2620 init_cmd.append("--bare")2621 subprocess.check_call(init_cmd)26222623if not P4Sync.run(self, depotPaths):2624return False2625if self.branch !="master":2626if self.importIntoRemotes:2627 masterbranch ="refs/remotes/p4/master"2628else:2629 masterbranch ="refs/heads/p4/master"2630ifgitBranchExists(masterbranch):2631system("git branch master%s"% masterbranch)2632if not self.cloneBare:2633system("git checkout -f")2634else:2635print"Could not detect main branch. No checkout/master branch created."26362637# auto-set this variable if invoked with --use-client-spec2638if self.useClientSpec_from_options:2639system("git config --bool git-p4.useclientspec true")26402641return True26422643classP4Branches(Command):2644def__init__(self):2645 Command.__init__(self)2646 self.options = [ ]2647 self.description = ("Shows the git branches that hold imports and their "2648+"corresponding perforce depot paths")2649 self.verbose =False26502651defrun(self, args):2652iforiginP4BranchesExist():2653createOrUpdateBranchesFromOrigin()26542655 cmdline ="git rev-parse --symbolic "2656 cmdline +=" --remotes"26572658for line inread_pipe_lines(cmdline):2659 line = line.strip()26602661if not line.startswith('p4/')or line =="p4/HEAD":2662continue2663 branch = line26642665 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)2666 settings =extractSettingsGitLog(log)26672668print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])2669return True26702671classHelpFormatter(optparse.IndentedHelpFormatter):2672def__init__(self):2673 optparse.IndentedHelpFormatter.__init__(self)26742675defformat_description(self, description):2676if description:2677return description +"\n"2678else:2679return""26802681defprintUsage(commands):2682print"usage:%s<command> [options]"% sys.argv[0]2683print""2684print"valid commands:%s"%", ".join(commands)2685print""2686print"Try%s<command> --help for command specific help."% sys.argv[0]2687print""26882689commands = {2690"debug": P4Debug,2691"submit": P4Submit,2692"commit": P4Submit,2693"sync": P4Sync,2694"rebase": P4Rebase,2695"clone": P4Clone,2696"rollback": P4RollBack,2697"branches": P4Branches2698}269927002701defmain():2702iflen(sys.argv[1:]) ==0:2703printUsage(commands.keys())2704 sys.exit(2)27052706 cmd =""2707 cmdName = sys.argv[1]2708try:2709 klass = commands[cmdName]2710 cmd =klass()2711exceptKeyError:2712print"unknown command%s"% cmdName2713print""2714printUsage(commands.keys())2715 sys.exit(2)27162717 options = cmd.options2718 cmd.gitdir = os.environ.get("GIT_DIR",None)27192720 args = sys.argv[2:]27212722iflen(options) >0:2723if cmd.needsGit:2724 options.append(optparse.make_option("--git-dir", dest="gitdir"))27252726 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),2727 options,2728 description = cmd.description,2729 formatter =HelpFormatter())27302731(cmd, args) = parser.parse_args(sys.argv[2:], cmd);2732global verbose2733 verbose = cmd.verbose2734if cmd.needsGit:2735if cmd.gitdir ==None:2736 cmd.gitdir = os.path.abspath(".git")2737if notisValidGitDir(cmd.gitdir):2738 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()2739if os.path.exists(cmd.gitdir):2740 cdup =read_pipe("git rev-parse --show-cdup").strip()2741iflen(cdup) >0:2742chdir(cdup);27432744if notisValidGitDir(cmd.gitdir):2745ifisValidGitDir(cmd.gitdir +"/.git"):2746 cmd.gitdir +="/.git"2747else:2748die("fatal: cannot locate git repository at%s"% cmd.gitdir)27492750 os.environ["GIT_DIR"] = cmd.gitdir27512752if not cmd.run(args):2753 parser.print_help()2754 sys.exit(2)275527562757if __name__ =='__main__':2758main()