1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10 11import optparse, sys, os, marshal, subprocess, shelve 12import tempfile, getopt, os.path, time, platform 13import re, shutil 14 15verbose =False 16 17# Only labels/tags matching this will be imported/exported 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 19 20defp4_build_cmd(cmd): 21"""Build a suitable p4 command line. 22 23 This consolidates building and returning a p4 command line into one 24 location. It means that hooking into the environment, or other configuration 25 can be done more easily. 26 """ 27 real_cmd = ["p4"] 28 29 user =gitConfig("git-p4.user") 30iflen(user) >0: 31 real_cmd += ["-u",user] 32 33 password =gitConfig("git-p4.password") 34iflen(password) >0: 35 real_cmd += ["-P", password] 36 37 port =gitConfig("git-p4.port") 38iflen(port) >0: 39 real_cmd += ["-p", port] 40 41 host =gitConfig("git-p4.host") 42iflen(host) >0: 43 real_cmd += ["-H", host] 44 45 client =gitConfig("git-p4.client") 46iflen(client) >0: 47 real_cmd += ["-c", client] 48 49 50ifisinstance(cmd,basestring): 51 real_cmd =' '.join(real_cmd) +' '+ cmd 52else: 53 real_cmd += cmd 54return real_cmd 55 56defchdir(dir): 57# P4 uses the PWD environment variable rather than getcwd(). Since we're 58# not using the shell, we have to set it ourselves. This path could 59# be relative, so go there first, then figure out where we ended up. 60 os.chdir(dir) 61 os.environ['PWD'] = os.getcwd() 62 63defdie(msg): 64if verbose: 65raiseException(msg) 66else: 67 sys.stderr.write(msg +"\n") 68 sys.exit(1) 69 70defwrite_pipe(c, stdin): 71if verbose: 72 sys.stderr.write('Writing pipe:%s\n'%str(c)) 73 74 expand =isinstance(c,basestring) 75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 76 pipe = p.stdin 77 val = pipe.write(stdin) 78 pipe.close() 79if p.wait(): 80die('Command failed:%s'%str(c)) 81 82return val 83 84defp4_write_pipe(c, stdin): 85 real_cmd =p4_build_cmd(c) 86returnwrite_pipe(real_cmd, stdin) 87 88defread_pipe(c, ignore_error=False): 89if verbose: 90 sys.stderr.write('Reading pipe:%s\n'%str(c)) 91 92 expand =isinstance(c,basestring) 93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 94 pipe = p.stdout 95 val = pipe.read() 96if p.wait()and not ignore_error: 97die('Command failed:%s'%str(c)) 98 99return val 100 101defp4_read_pipe(c, ignore_error=False): 102 real_cmd =p4_build_cmd(c) 103returnread_pipe(real_cmd, ignore_error) 104 105defread_pipe_lines(c): 106if verbose: 107 sys.stderr.write('Reading pipe:%s\n'%str(c)) 108 109 expand =isinstance(c, basestring) 110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 111 pipe = p.stdout 112 val = pipe.readlines() 113if pipe.close()or p.wait(): 114die('Command failed:%s'%str(c)) 115 116return val 117 118defp4_read_pipe_lines(c): 119"""Specifically invoke p4 on the command supplied. """ 120 real_cmd =p4_build_cmd(c) 121returnread_pipe_lines(real_cmd) 122 123defsystem(cmd): 124 expand =isinstance(cmd,basestring) 125if verbose: 126 sys.stderr.write("executing%s\n"%str(cmd)) 127 subprocess.check_call(cmd, shell=expand) 128 129defp4_system(cmd): 130"""Specifically invoke p4 as the system command. """ 131 real_cmd =p4_build_cmd(cmd) 132 expand =isinstance(real_cmd, basestring) 133 subprocess.check_call(real_cmd, shell=expand) 134 135defp4_integrate(src, dest): 136p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 137 138defp4_sync(f, *options): 139p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 140 141defp4_add(f): 142# forcibly add file names with wildcards 143ifwildcard_present(f): 144p4_system(["add","-f", f]) 145else: 146p4_system(["add", f]) 147 148defp4_delete(f): 149p4_system(["delete",wildcard_encode(f)]) 150 151defp4_edit(f): 152p4_system(["edit",wildcard_encode(f)]) 153 154defp4_revert(f): 155p4_system(["revert",wildcard_encode(f)]) 156 157defp4_reopen(type, f): 158p4_system(["reopen","-t",type,wildcard_encode(f)]) 159 160# 161# Canonicalize the p4 type and return a tuple of the 162# base type, plus any modifiers. See "p4 help filetypes" 163# for a list and explanation. 164# 165defsplit_p4_type(p4type): 166 167 p4_filetypes_historical = { 168"ctempobj":"binary+Sw", 169"ctext":"text+C", 170"cxtext":"text+Cx", 171"ktext":"text+k", 172"kxtext":"text+kx", 173"ltext":"text+F", 174"tempobj":"binary+FSw", 175"ubinary":"binary+F", 176"uresource":"resource+F", 177"uxbinary":"binary+Fx", 178"xbinary":"binary+x", 179"xltext":"text+Fx", 180"xtempobj":"binary+Swx", 181"xtext":"text+x", 182"xunicode":"unicode+x", 183"xutf16":"utf16+x", 184} 185if p4type in p4_filetypes_historical: 186 p4type = p4_filetypes_historical[p4type] 187 mods ="" 188 s = p4type.split("+") 189 base = s[0] 190 mods ="" 191iflen(s) >1: 192 mods = s[1] 193return(base, mods) 194 195# 196# return the raw p4 type of a file (text, text+ko, etc) 197# 198defp4_type(file): 199 results =p4CmdList(["fstat","-T","headType",file]) 200return results[0]['headType'] 201 202# 203# Given a type base and modifier, return a regexp matching 204# the keywords that can be expanded in the file 205# 206defp4_keywords_regexp_for_type(base, type_mods): 207if base in("text","unicode","binary"): 208 kwords =None 209if"ko"in type_mods: 210 kwords ='Id|Header' 211elif"k"in type_mods: 212 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 213else: 214return None 215 pattern = r""" 216 \$ # Starts with a dollar, followed by... 217 (%s) # one of the keywords, followed by... 218 (:[^$]+)? # possibly an old expansion, followed by... 219 \$ # another dollar 220 """% kwords 221return pattern 222else: 223return None 224 225# 226# Given a file, return a regexp matching the possible 227# RCS keywords that will be expanded, or None for files 228# with kw expansion turned off. 229# 230defp4_keywords_regexp_for_file(file): 231if not os.path.exists(file): 232return None 233else: 234(type_base, type_mods) =split_p4_type(p4_type(file)) 235returnp4_keywords_regexp_for_type(type_base, type_mods) 236 237defsetP4ExecBit(file, mode): 238# Reopens an already open file and changes the execute bit to match 239# the execute bit setting in the passed in mode. 240 241 p4Type ="+x" 242 243if notisModeExec(mode): 244 p4Type =getP4OpenedType(file) 245 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 246 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 247if p4Type[-1] =="+": 248 p4Type = p4Type[0:-1] 249 250p4_reopen(p4Type,file) 251 252defgetP4OpenedType(file): 253# Returns the perforce file type for the given file. 254 255 result =p4_read_pipe(["opened",wildcard_encode(file)]) 256 match = re.match(".*\((.+)\)\r?$", result) 257if match: 258return match.group(1) 259else: 260die("Could not determine file type for%s(result: '%s')"% (file, result)) 261 262# Return the set of all p4 labels 263defgetP4Labels(depotPaths): 264 labels =set() 265ifisinstance(depotPaths,basestring): 266 depotPaths = [depotPaths] 267 268for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 269 label = l['label'] 270 labels.add(label) 271 272return labels 273 274# Return the set of all git tags 275defgetGitTags(): 276 gitTags =set() 277for line inread_pipe_lines(["git","tag"]): 278 tag = line.strip() 279 gitTags.add(tag) 280return gitTags 281 282defdiffTreePattern(): 283# This is a simple generator for the diff tree regex pattern. This could be 284# a class variable if this and parseDiffTreeEntry were a part of a class. 285 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 286while True: 287yield pattern 288 289defparseDiffTreeEntry(entry): 290"""Parses a single diff tree entry into its component elements. 291 292 See git-diff-tree(1) manpage for details about the format of the diff 293 output. This method returns a dictionary with the following elements: 294 295 src_mode - The mode of the source file 296 dst_mode - The mode of the destination file 297 src_sha1 - The sha1 for the source file 298 dst_sha1 - The sha1 fr the destination file 299 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 300 status_score - The score for the status (applicable for 'C' and 'R' 301 statuses). This is None if there is no score. 302 src - The path for the source file. 303 dst - The path for the destination file. This is only present for 304 copy or renames. If it is not present, this is None. 305 306 If the pattern is not matched, None is returned.""" 307 308 match =diffTreePattern().next().match(entry) 309if match: 310return{ 311'src_mode': match.group(1), 312'dst_mode': match.group(2), 313'src_sha1': match.group(3), 314'dst_sha1': match.group(4), 315'status': match.group(5), 316'status_score': match.group(6), 317'src': match.group(7), 318'dst': match.group(10) 319} 320return None 321 322defisModeExec(mode): 323# Returns True if the given git mode represents an executable file, 324# otherwise False. 325return mode[-3:] =="755" 326 327defisModeExecChanged(src_mode, dst_mode): 328returnisModeExec(src_mode) !=isModeExec(dst_mode) 329 330defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 331 332ifisinstance(cmd,basestring): 333 cmd ="-G "+ cmd 334 expand =True 335else: 336 cmd = ["-G"] + cmd 337 expand =False 338 339 cmd =p4_build_cmd(cmd) 340if verbose: 341 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 342 343# Use a temporary file to avoid deadlocks without 344# subprocess.communicate(), which would put another copy 345# of stdout into memory. 346 stdin_file =None 347if stdin is not None: 348 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 349ifisinstance(stdin,basestring): 350 stdin_file.write(stdin) 351else: 352for i in stdin: 353 stdin_file.write(i +'\n') 354 stdin_file.flush() 355 stdin_file.seek(0) 356 357 p4 = subprocess.Popen(cmd, 358 shell=expand, 359 stdin=stdin_file, 360 stdout=subprocess.PIPE) 361 362 result = [] 363try: 364while True: 365 entry = marshal.load(p4.stdout) 366if cb is not None: 367cb(entry) 368else: 369 result.append(entry) 370exceptEOFError: 371pass 372 exitCode = p4.wait() 373if exitCode !=0: 374 entry = {} 375 entry["p4ExitCode"] = exitCode 376 result.append(entry) 377 378return result 379 380defp4Cmd(cmd): 381list=p4CmdList(cmd) 382 result = {} 383for entry inlist: 384 result.update(entry) 385return result; 386 387defp4Where(depotPath): 388if not depotPath.endswith("/"): 389 depotPath +="/" 390 depotPath = depotPath +"..." 391 outputList =p4CmdList(["where", depotPath]) 392 output =None 393for entry in outputList: 394if"depotFile"in entry: 395if entry["depotFile"] == depotPath: 396 output = entry 397break 398elif"data"in entry: 399 data = entry.get("data") 400 space = data.find(" ") 401if data[:space] == depotPath: 402 output = entry 403break 404if output ==None: 405return"" 406if output["code"] =="error": 407return"" 408 clientPath ="" 409if"path"in output: 410 clientPath = output.get("path") 411elif"data"in output: 412 data = output.get("data") 413 lastSpace = data.rfind(" ") 414 clientPath = data[lastSpace +1:] 415 416if clientPath.endswith("..."): 417 clientPath = clientPath[:-3] 418return clientPath 419 420defcurrentGitBranch(): 421returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 422 423defisValidGitDir(path): 424if(os.path.exists(path +"/HEAD") 425and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 426return True; 427return False 428 429defparseRevision(ref): 430returnread_pipe("git rev-parse%s"% ref).strip() 431 432defbranchExists(ref): 433 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 434 ignore_error=True) 435returnlen(rev) >0 436 437defextractLogMessageFromGitCommit(commit): 438 logMessage ="" 439 440## fixme: title is first line of commit, not 1st paragraph. 441 foundTitle =False 442for log inread_pipe_lines("git cat-file commit%s"% commit): 443if not foundTitle: 444iflen(log) ==1: 445 foundTitle =True 446continue 447 448 logMessage += log 449return logMessage 450 451defextractSettingsGitLog(log): 452 values = {} 453for line in log.split("\n"): 454 line = line.strip() 455 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 456if not m: 457continue 458 459 assignments = m.group(1).split(':') 460for a in assignments: 461 vals = a.split('=') 462 key = vals[0].strip() 463 val = ('='.join(vals[1:])).strip() 464if val.endswith('\"')and val.startswith('"'): 465 val = val[1:-1] 466 467 values[key] = val 468 469 paths = values.get("depot-paths") 470if not paths: 471 paths = values.get("depot-path") 472if paths: 473 values['depot-paths'] = paths.split(',') 474return values 475 476defgitBranchExists(branch): 477 proc = subprocess.Popen(["git","rev-parse", branch], 478 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 479return proc.wait() ==0; 480 481_gitConfig = {} 482defgitConfig(key, args =None):# set args to "--bool", for instance 483if not _gitConfig.has_key(key): 484 argsFilter ="" 485if args !=None: 486 argsFilter ="%s"% args 487 cmd ="git config%s%s"% (argsFilter, key) 488 _gitConfig[key] =read_pipe(cmd, ignore_error=True).strip() 489return _gitConfig[key] 490 491defgitConfigList(key): 492if not _gitConfig.has_key(key): 493 _gitConfig[key] =read_pipe("git config --get-all%s"% key, ignore_error=True).strip().split(os.linesep) 494return _gitConfig[key] 495 496defp4BranchesInGit(branchesAreInRemotes =True): 497 branches = {} 498 499 cmdline ="git rev-parse --symbolic " 500if branchesAreInRemotes: 501 cmdline +=" --remotes" 502else: 503 cmdline +=" --branches" 504 505for line inread_pipe_lines(cmdline): 506 line = line.strip() 507 508## only import to p4/ 509if not line.startswith('p4/')or line =="p4/HEAD": 510continue 511 branch = line 512 513# strip off p4 514 branch = re.sub("^p4/","", line) 515 516 branches[branch] =parseRevision(line) 517return branches 518 519deffindUpstreamBranchPoint(head ="HEAD"): 520 branches =p4BranchesInGit() 521# map from depot-path to branch name 522 branchByDepotPath = {} 523for branch in branches.keys(): 524 tip = branches[branch] 525 log =extractLogMessageFromGitCommit(tip) 526 settings =extractSettingsGitLog(log) 527if settings.has_key("depot-paths"): 528 paths =",".join(settings["depot-paths"]) 529 branchByDepotPath[paths] ="remotes/p4/"+ branch 530 531 settings =None 532 parent =0 533while parent <65535: 534 commit = head +"~%s"% parent 535 log =extractLogMessageFromGitCommit(commit) 536 settings =extractSettingsGitLog(log) 537if settings.has_key("depot-paths"): 538 paths =",".join(settings["depot-paths"]) 539if branchByDepotPath.has_key(paths): 540return[branchByDepotPath[paths], settings] 541 542 parent = parent +1 543 544return["", settings] 545 546defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 547if not silent: 548print("Creating/updating branch(es) in%sbased on origin branch(es)" 549% localRefPrefix) 550 551 originPrefix ="origin/p4/" 552 553for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 554 line = line.strip() 555if(not line.startswith(originPrefix))or line.endswith("HEAD"): 556continue 557 558 headName = line[len(originPrefix):] 559 remoteHead = localRefPrefix + headName 560 originHead = line 561 562 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 563if(not original.has_key('depot-paths') 564or not original.has_key('change')): 565continue 566 567 update =False 568if notgitBranchExists(remoteHead): 569if verbose: 570print"creating%s"% remoteHead 571 update =True 572else: 573 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 574if settings.has_key('change') >0: 575if settings['depot-paths'] == original['depot-paths']: 576 originP4Change =int(original['change']) 577 p4Change =int(settings['change']) 578if originP4Change > p4Change: 579print("%s(%s) is newer than%s(%s). " 580"Updating p4 branch from origin." 581% (originHead, originP4Change, 582 remoteHead, p4Change)) 583 update =True 584else: 585print("Ignoring:%swas imported from%swhile " 586"%swas imported from%s" 587% (originHead,','.join(original['depot-paths']), 588 remoteHead,','.join(settings['depot-paths']))) 589 590if update: 591system("git update-ref%s %s"% (remoteHead, originHead)) 592 593deforiginP4BranchesExist(): 594returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 595 596defp4ChangesForPaths(depotPaths, changeRange): 597assert depotPaths 598 cmd = ['changes'] 599for p in depotPaths: 600 cmd += ["%s...%s"% (p, changeRange)] 601 output =p4_read_pipe_lines(cmd) 602 603 changes = {} 604for line in output: 605 changeNum =int(line.split(" ")[1]) 606 changes[changeNum] =True 607 608 changelist = changes.keys() 609 changelist.sort() 610return changelist 611 612defp4PathStartsWith(path, prefix): 613# This method tries to remedy a potential mixed-case issue: 614# 615# If UserA adds //depot/DirA/file1 616# and UserB adds //depot/dira/file2 617# 618# we may or may not have a problem. If you have core.ignorecase=true, 619# we treat DirA and dira as the same directory 620 ignorecase =gitConfig("core.ignorecase","--bool") =="true" 621if ignorecase: 622return path.lower().startswith(prefix.lower()) 623return path.startswith(prefix) 624 625defgetClientSpec(): 626"""Look at the p4 client spec, create a View() object that contains 627 all the mappings, and return it.""" 628 629 specList =p4CmdList("client -o") 630iflen(specList) !=1: 631die('Output from "client -o" is%dlines, expecting 1'% 632len(specList)) 633 634# dictionary of all client parameters 635 entry = specList[0] 636 637# just the keys that start with "View" 638 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 639 640# hold this new View 641 view =View() 642 643# append the lines, in order, to the view 644for view_num inrange(len(view_keys)): 645 k ="View%d"% view_num 646if k not in view_keys: 647die("Expected view key%smissing"% k) 648 view.append(entry[k]) 649 650return view 651 652defgetClientRoot(): 653"""Grab the client directory.""" 654 655 output =p4CmdList("client -o") 656iflen(output) !=1: 657die('Output from "client -o" is%dlines, expecting 1'%len(output)) 658 659 entry = output[0] 660if"Root"not in entry: 661die('Client has no "Root"') 662 663return entry["Root"] 664 665# 666# P4 wildcards are not allowed in filenames. P4 complains 667# if you simply add them, but you can force it with "-f", in 668# which case it translates them into %xx encoding internally. 669# 670defwildcard_decode(path): 671# Search for and fix just these four characters. Do % last so 672# that fixing it does not inadvertently create new %-escapes. 673# Cannot have * in a filename in windows; untested as to 674# what p4 would do in such a case. 675if not platform.system() =="Windows": 676 path = path.replace("%2A","*") 677 path = path.replace("%23","#") \ 678.replace("%40","@") \ 679.replace("%25","%") 680return path 681 682defwildcard_encode(path): 683# do % first to avoid double-encoding the %s introduced here 684 path = path.replace("%","%25") \ 685.replace("*","%2A") \ 686.replace("#","%23") \ 687.replace("@","%40") 688return path 689 690defwildcard_present(path): 691return path.translate(None,"*#@%") != path 692 693class Command: 694def__init__(self): 695 self.usage ="usage: %prog [options]" 696 self.needsGit =True 697 self.verbose =False 698 699class P4UserMap: 700def__init__(self): 701 self.userMapFromPerforceServer =False 702 self.myP4UserId =None 703 704defp4UserId(self): 705if self.myP4UserId: 706return self.myP4UserId 707 708 results =p4CmdList("user -o") 709for r in results: 710if r.has_key('User'): 711 self.myP4UserId = r['User'] 712return r['User'] 713die("Could not find your p4 user id") 714 715defp4UserIsMe(self, p4User): 716# return True if the given p4 user is actually me 717 me = self.p4UserId() 718if not p4User or p4User != me: 719return False 720else: 721return True 722 723defgetUserCacheFilename(self): 724 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 725return home +"/.gitp4-usercache.txt" 726 727defgetUserMapFromPerforceServer(self): 728if self.userMapFromPerforceServer: 729return 730 self.users = {} 731 self.emails = {} 732 733for output inp4CmdList("users"): 734if not output.has_key("User"): 735continue 736 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 737 self.emails[output["Email"]] = output["User"] 738 739 740 s ='' 741for(key, val)in self.users.items(): 742 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 743 744open(self.getUserCacheFilename(),"wb").write(s) 745 self.userMapFromPerforceServer =True 746 747defloadUserMapFromCache(self): 748 self.users = {} 749 self.userMapFromPerforceServer =False 750try: 751 cache =open(self.getUserCacheFilename(),"rb") 752 lines = cache.readlines() 753 cache.close() 754for line in lines: 755 entry = line.strip().split("\t") 756 self.users[entry[0]] = entry[1] 757exceptIOError: 758 self.getUserMapFromPerforceServer() 759 760classP4Debug(Command): 761def__init__(self): 762 Command.__init__(self) 763 self.options = [] 764 self.description ="A tool to debug the output of p4 -G." 765 self.needsGit =False 766 767defrun(self, args): 768 j =0 769for output inp4CmdList(args): 770print'Element:%d'% j 771 j +=1 772print output 773return True 774 775classP4RollBack(Command): 776def__init__(self): 777 Command.__init__(self) 778 self.options = [ 779 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") 780] 781 self.description ="A tool to debug the multi-branch import. Don't use :)" 782 self.rollbackLocalBranches =False 783 784defrun(self, args): 785iflen(args) !=1: 786return False 787 maxChange =int(args[0]) 788 789if"p4ExitCode"inp4Cmd("changes -m 1"): 790die("Problems executing p4"); 791 792if self.rollbackLocalBranches: 793 refPrefix ="refs/heads/" 794 lines =read_pipe_lines("git rev-parse --symbolic --branches") 795else: 796 refPrefix ="refs/remotes/" 797 lines =read_pipe_lines("git rev-parse --symbolic --remotes") 798 799for line in lines: 800if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"): 801 line = line.strip() 802 ref = refPrefix + line 803 log =extractLogMessageFromGitCommit(ref) 804 settings =extractSettingsGitLog(log) 805 806 depotPaths = settings['depot-paths'] 807 change = settings['change'] 808 809 changed =False 810 811iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange) 812for p in depotPaths]))) ==0: 813print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange) 814system("git update-ref -d%s`git rev-parse%s`"% (ref, ref)) 815continue 816 817while change andint(change) > maxChange: 818 changed =True 819if self.verbose: 820print"%sis at%s; rewinding towards%s"% (ref, change, maxChange) 821system("git update-ref%s\"%s^\""% (ref, ref)) 822 log =extractLogMessageFromGitCommit(ref) 823 settings =extractSettingsGitLog(log) 824 825 826 depotPaths = settings['depot-paths'] 827 change = settings['change'] 828 829if changed: 830print"%srewound to%s"% (ref, change) 831 832return True 833 834classP4Submit(Command, P4UserMap): 835def__init__(self): 836 Command.__init__(self) 837 P4UserMap.__init__(self) 838 self.options = [ 839 optparse.make_option("--origin", dest="origin"), 840 optparse.make_option("-M", dest="detectRenames", action="store_true"), 841# preserve the user, requires relevant p4 permissions 842 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), 843 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"), 844] 845 self.description ="Submit changes from git to the perforce depot." 846 self.usage +=" [name of git branch to submit into perforce depot]" 847 self.interactive =True 848 self.origin ="" 849 self.detectRenames =False 850 self.preserveUser =gitConfig("git-p4.preserveUser").lower() =="true" 851 self.isWindows = (platform.system() =="Windows") 852 self.exportLabels =False 853 854defcheck(self): 855iflen(p4CmdList("opened ...")) >0: 856die("You have files opened with perforce! Close them before starting the sync.") 857 858# replaces everything between 'Description:' and the next P4 submit template field with the 859# commit message 860defprepareLogMessage(self, template, message): 861 result ="" 862 863 inDescriptionSection =False 864 865for line in template.split("\n"): 866if line.startswith("#"): 867 result += line +"\n" 868continue 869 870if inDescriptionSection: 871if line.startswith("Files:")or line.startswith("Jobs:"): 872 inDescriptionSection =False 873else: 874continue 875else: 876if line.startswith("Description:"): 877 inDescriptionSection =True 878 line +="\n" 879for messageLine in message.split("\n"): 880 line +="\t"+ messageLine +"\n" 881 882 result += line +"\n" 883 884return result 885 886defpatchRCSKeywords(self,file, pattern): 887# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern 888(handle, outFileName) = tempfile.mkstemp(dir='.') 889try: 890 outFile = os.fdopen(handle,"w+") 891 inFile =open(file,"r") 892 regexp = re.compile(pattern, re.VERBOSE) 893for line in inFile.readlines(): 894 line = regexp.sub(r'$\1$', line) 895 outFile.write(line) 896 inFile.close() 897 outFile.close() 898# Forcibly overwrite the original file 899 os.unlink(file) 900 shutil.move(outFileName,file) 901except: 902# cleanup our temporary file 903 os.unlink(outFileName) 904print"Failed to strip RCS keywords in%s"%file 905raise 906 907print"Patched up RCS keywords in%s"%file 908 909defp4UserForCommit(self,id): 910# Return the tuple (perforce user,git email) for a given git commit id 911 self.getUserMapFromPerforceServer() 912 gitEmail =read_pipe("git log --max-count=1 --format='%%ae'%s"%id) 913 gitEmail = gitEmail.strip() 914if not self.emails.has_key(gitEmail): 915return(None,gitEmail) 916else: 917return(self.emails[gitEmail],gitEmail) 918 919defcheckValidP4Users(self,commits): 920# check if any git authors cannot be mapped to p4 users 921foridin commits: 922(user,email) = self.p4UserForCommit(id) 923if not user: 924 msg ="Cannot find p4 user for email%sin commit%s."% (email,id) 925ifgitConfig('git-p4.allowMissingP4Users').lower() =="true": 926print"%s"% msg 927else: 928die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg) 929 930deflastP4Changelist(self): 931# Get back the last changelist number submitted in this client spec. This 932# then gets used to patch up the username in the change. If the same 933# client spec is being used by multiple processes then this might go 934# wrong. 935 results =p4CmdList("client -o")# find the current client 936 client =None 937for r in results: 938if r.has_key('Client'): 939 client = r['Client'] 940break 941if not client: 942die("could not get client spec") 943 results =p4CmdList(["changes","-c", client,"-m","1"]) 944for r in results: 945if r.has_key('change'): 946return r['change'] 947die("Could not get changelist number for last submit - cannot patch up user details") 948 949defmodifyChangelistUser(self, changelist, newUser): 950# fixup the user field of a changelist after it has been submitted. 951 changes =p4CmdList("change -o%s"% changelist) 952iflen(changes) !=1: 953die("Bad output from p4 change modifying%sto user%s"% 954(changelist, newUser)) 955 956 c = changes[0] 957if c['User'] == newUser:return# nothing to do 958 c['User'] = newUser 959input= marshal.dumps(c) 960 961 result =p4CmdList("change -f -i", stdin=input) 962for r in result: 963if r.has_key('code'): 964if r['code'] =='error': 965die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data'])) 966if r.has_key('data'): 967print("Updated user field for changelist%sto%s"% (changelist, newUser)) 968return 969die("Could not modify user field of changelist%sto%s"% (changelist, newUser)) 970 971defcanChangeChangelists(self): 972# check to see if we have p4 admin or super-user permissions, either of 973# which are required to modify changelists. 974 results =p4CmdList(["protects", self.depotPath]) 975for r in results: 976if r.has_key('perm'): 977if r['perm'] =='admin': 978return1 979if r['perm'] =='super': 980return1 981return0 982 983defprepareSubmitTemplate(self): 984# remove lines in the Files section that show changes to files outside the depot path we're committing into 985 template ="" 986 inFilesSection =False 987for line inp4_read_pipe_lines(['change','-o']): 988if line.endswith("\r\n"): 989 line = line[:-2] +"\n" 990if inFilesSection: 991if line.startswith("\t"): 992# path starts and ends with a tab 993 path = line[1:] 994 lastTab = path.rfind("\t") 995if lastTab != -1: 996 path = path[:lastTab] 997if notp4PathStartsWith(path, self.depotPath): 998continue 999else:1000 inFilesSection =False1001else:1002if line.startswith("Files:"):1003 inFilesSection =True10041005 template += line10061007return template10081009defedit_template(self, template_file):1010"""Invoke the editor to let the user change the submission1011 message. Return true if okay to continue with the submit."""10121013# if configured to skip the editing part, just submit1014ifgitConfig("git-p4.skipSubmitEdit") =="true":1015return True10161017# look at the modification time, to check later if the user saved1018# the file1019 mtime = os.stat(template_file).st_mtime10201021# invoke the editor1022if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1023 editor = os.environ.get("P4EDITOR")1024else:1025 editor =read_pipe("git var GIT_EDITOR").strip()1026system(editor +" "+ template_file)10271028# If the file was not saved, prompt to see if this patch should1029# be skipped. But skip this verification step if configured so.1030ifgitConfig("git-p4.skipSubmitEditCheck") =="true":1031return True10321033# modification time updated means user saved the file1034if os.stat(template_file).st_mtime > mtime:1035return True10361037while True:1038 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1039if response =='y':1040return True1041if response =='n':1042return False10431044defapplyCommit(self,id):1045print"Applying%s"% (read_pipe("git log --max-count=1 --pretty=oneline%s"%id))10461047(p4User, gitEmail) = self.p4UserForCommit(id)104810491050 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1051 filesToAdd =set()1052 filesToDelete =set()1053 editedFiles =set()1054 pureRenameCopy =set()1055 filesToChangeExecBit = {}10561057for line in diff:1058 diff =parseDiffTreeEntry(line)1059 modifier = diff['status']1060 path = diff['src']1061if modifier =="M":1062p4_edit(path)1063ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1064 filesToChangeExecBit[path] = diff['dst_mode']1065 editedFiles.add(path)1066elif modifier =="A":1067 filesToAdd.add(path)1068 filesToChangeExecBit[path] = diff['dst_mode']1069if path in filesToDelete:1070 filesToDelete.remove(path)1071elif modifier =="D":1072 filesToDelete.add(path)1073if path in filesToAdd:1074 filesToAdd.remove(path)1075elif modifier =="C":1076 src, dest = diff['src'], diff['dst']1077p4_integrate(src, dest)1078 pureRenameCopy.add(dest)1079if diff['src_sha1'] != diff['dst_sha1']:1080p4_edit(dest)1081 pureRenameCopy.discard(dest)1082ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1083p4_edit(dest)1084 pureRenameCopy.discard(dest)1085 filesToChangeExecBit[dest] = diff['dst_mode']1086 os.unlink(dest)1087 editedFiles.add(dest)1088elif modifier =="R":1089 src, dest = diff['src'], diff['dst']1090p4_integrate(src, dest)1091if diff['src_sha1'] != diff['dst_sha1']:1092p4_edit(dest)1093else:1094 pureRenameCopy.add(dest)1095ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1096p4_edit(dest)1097 filesToChangeExecBit[dest] = diff['dst_mode']1098 os.unlink(dest)1099 editedFiles.add(dest)1100 filesToDelete.add(src)1101else:1102die("unknown modifier%sfor%s"% (modifier, path))11031104 diffcmd ="git format-patch -k --stdout\"%s^\"..\"%s\""% (id,id)1105 patchcmd = diffcmd +" | git apply "1106 tryPatchCmd = patchcmd +"--check -"1107 applyPatchCmd = patchcmd +"--check --apply -"1108 patch_succeeded =True11091110if os.system(tryPatchCmd) !=0:1111 fixed_rcs_keywords =False1112 patch_succeeded =False1113print"Unfortunately applying the change failed!"11141115# Patch failed, maybe it's just RCS keyword woes. Look through1116# the patch to see if that's possible.1117ifgitConfig("git-p4.attemptRCSCleanup","--bool") =="true":1118file=None1119 pattern =None1120 kwfiles = {}1121forfilein editedFiles | filesToDelete:1122# did this file's delta contain RCS keywords?1123 pattern =p4_keywords_regexp_for_file(file)11241125if pattern:1126# this file is a possibility...look for RCS keywords.1127 regexp = re.compile(pattern, re.VERBOSE)1128for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1129if regexp.search(line):1130if verbose:1131print"got keyword match on%sin%sin%s"% (pattern, line,file)1132 kwfiles[file] = pattern1133break11341135forfilein kwfiles:1136if verbose:1137print"zapping%swith%s"% (line,pattern)1138 self.patchRCSKeywords(file, kwfiles[file])1139 fixed_rcs_keywords =True11401141if fixed_rcs_keywords:1142print"Retrying the patch with RCS keywords cleaned up"1143if os.system(tryPatchCmd) ==0:1144 patch_succeeded =True11451146if not patch_succeeded:1147print"What do you want to do?"1148 response ="x"1149while response !="s"and response !="a"and response !="w":1150 response =raw_input("[s]kip this patch / [a]pply the patch forcibly "1151"and with .rej files / [w]rite the patch to a file (patch.txt) ")1152if response =="s":1153print"Skipping! Good luck with the next patches..."1154for f in editedFiles:1155p4_revert(f)1156for f in filesToAdd:1157 os.remove(f)1158return1159elif response =="a":1160 os.system(applyPatchCmd)1161iflen(filesToAdd) >0:1162print"You may also want to call p4 add on the following files:"1163print" ".join(filesToAdd)1164iflen(filesToDelete):1165print"The following files should be scheduled for deletion with p4 delete:"1166print" ".join(filesToDelete)1167die("Please resolve and submit the conflict manually and "1168+"continue afterwards with git p4 submit --continue")1169elif response =="w":1170system(diffcmd +" > patch.txt")1171print"Patch saved to patch.txt in%s!"% self.clientPath1172die("Please resolve and submit the conflict manually and "1173"continue afterwards with git p4 submit --continue")11741175system(applyPatchCmd)11761177for f in filesToAdd:1178p4_add(f)1179for f in filesToDelete:1180p4_revert(f)1181p4_delete(f)11821183# Set/clear executable bits1184for f in filesToChangeExecBit.keys():1185 mode = filesToChangeExecBit[f]1186setP4ExecBit(f, mode)11871188 logMessage =extractLogMessageFromGitCommit(id)1189 logMessage = logMessage.strip()11901191 template = self.prepareSubmitTemplate()11921193if self.interactive:1194 submitTemplate = self.prepareLogMessage(template, logMessage)11951196if self.preserveUser:1197 submitTemplate = submitTemplate + ("\n######## Actual user%s, modified after commit\n"% p4User)11981199if os.environ.has_key("P4DIFF"):1200del(os.environ["P4DIFF"])1201 diff =""1202for editedFile in editedFiles:1203 diff +=p4_read_pipe(['diff','-du',1204wildcard_encode(editedFile)])12051206 newdiff =""1207for newFile in filesToAdd:1208 newdiff +="==== new file ====\n"1209 newdiff +="--- /dev/null\n"1210 newdiff +="+++%s\n"% newFile1211 f =open(newFile,"r")1212for line in f.readlines():1213 newdiff +="+"+ line1214 f.close()12151216if self.checkAuthorship and not self.p4UserIsMe(p4User):1217 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1218 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1219 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"12201221 separatorLine ="######## everything below this line is just the diff #######\n"12221223(handle, fileName) = tempfile.mkstemp()1224 tmpFile = os.fdopen(handle,"w+")1225if self.isWindows:1226 submitTemplate = submitTemplate.replace("\n","\r\n")1227 separatorLine = separatorLine.replace("\n","\r\n")1228 newdiff = newdiff.replace("\n","\r\n")1229 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)1230 tmpFile.close()12311232if self.edit_template(fileName):1233# read the edited message and submit1234 tmpFile =open(fileName,"rb")1235 message = tmpFile.read()1236 tmpFile.close()1237 submitTemplate = message[:message.index(separatorLine)]1238if self.isWindows:1239 submitTemplate = submitTemplate.replace("\r\n","\n")1240p4_write_pipe(['submit','-i'], submitTemplate)12411242if self.preserveUser:1243if p4User:1244# Get last changelist number. Cannot easily get it from1245# the submit command output as the output is1246# unmarshalled.1247 changelist = self.lastP4Changelist()1248 self.modifyChangelistUser(changelist, p4User)12491250# The rename/copy happened by applying a patch that created a1251# new file. This leaves it writable, which confuses p4.1252for f in pureRenameCopy:1253p4_sync(f,"-f")12541255else:1256# skip this patch1257print"Submission cancelled, undoing p4 changes."1258for f in editedFiles:1259p4_revert(f)1260for f in filesToAdd:1261p4_revert(f)1262 os.remove(f)12631264 os.remove(fileName)1265else:1266 fileName ="submit.txt"1267file=open(fileName,"w+")1268file.write(self.prepareLogMessage(template, logMessage))1269file.close()1270print("Perforce submit template written as%s. "1271+"Please review/edit and then use p4 submit -i <%sto submit directly!"1272% (fileName, fileName))12731274# Export git tags as p4 labels. Create a p4 label and then tag1275# with that.1276defexportGitTags(self, gitTags):1277 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1278iflen(validLabelRegexp) ==0:1279 validLabelRegexp = defaultLabelRegexp1280 m = re.compile(validLabelRegexp)12811282for name in gitTags:12831284if not m.match(name):1285if verbose:1286print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1287continue12881289# Get the p4 commit this corresponds to1290 logMessage =extractLogMessageFromGitCommit(name)1291 values =extractSettingsGitLog(logMessage)12921293if not values.has_key('change'):1294# a tag pointing to something not sent to p4; ignore1295if verbose:1296print"git tag%sdoes not give a p4 commit"% name1297continue1298else:1299 changelist = values['change']13001301# Get the tag details.1302 inHeader =True1303 isAnnotated =False1304 body = []1305for l inread_pipe_lines(["git","cat-file","-p", name]):1306 l = l.strip()1307if inHeader:1308if re.match(r'tag\s+', l):1309 isAnnotated =True1310elif re.match(r'\s*$', l):1311 inHeader =False1312continue1313else:1314 body.append(l)13151316if not isAnnotated:1317 body = ["lightweight tag imported by git p4\n"]13181319# Create the label - use the same view as the client spec we are using1320 clientSpec =getClientSpec()13211322 labelTemplate ="Label:%s\n"% name1323 labelTemplate +="Description:\n"1324for b in body:1325 labelTemplate +="\t"+ b +"\n"1326 labelTemplate +="View:\n"1327for mapping in clientSpec.mappings:1328 labelTemplate +="\t%s\n"% mapping.depot_side.path13291330p4_write_pipe(["label","-i"], labelTemplate)13311332# Use the label1333p4_system(["tag","-l", name] +1334["%s@%s"% (mapping.depot_side.path, changelist)for mapping in clientSpec.mappings])13351336if verbose:1337print"created p4 label for tag%s"% name13381339defrun(self, args):1340iflen(args) ==0:1341 self.master =currentGitBranch()1342iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1343die("Detecting current git branch failed!")1344eliflen(args) ==1:1345 self.master = args[0]1346if notbranchExists(self.master):1347die("Branch%sdoes not exist"% self.master)1348else:1349return False13501351 allowSubmit =gitConfig("git-p4.allowSubmit")1352iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1353die("%sis not in git-p4.allowSubmit"% self.master)13541355[upstream, settings] =findUpstreamBranchPoint()1356 self.depotPath = settings['depot-paths'][0]1357iflen(self.origin) ==0:1358 self.origin = upstream13591360if self.preserveUser:1361if not self.canChangeChangelists():1362die("Cannot preserve user names without p4 super-user or admin permissions")13631364if self.verbose:1365print"Origin branch is "+ self.origin13661367iflen(self.depotPath) ==0:1368print"Internal error: cannot locate perforce depot path from existing branches"1369 sys.exit(128)13701371 self.useClientSpec =False1372ifgitConfig("git-p4.useclientspec","--bool") =="true":1373 self.useClientSpec =True1374if self.useClientSpec:1375 self.clientSpecDirs =getClientSpec()13761377if self.useClientSpec:1378# all files are relative to the client spec1379 self.clientPath =getClientRoot()1380else:1381 self.clientPath =p4Where(self.depotPath)13821383if self.clientPath =="":1384die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)13851386print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1387 self.oldWorkingDirectory = os.getcwd()13881389# ensure the clientPath exists1390 new_client_dir =False1391if not os.path.exists(self.clientPath):1392 new_client_dir =True1393 os.makedirs(self.clientPath)13941395chdir(self.clientPath)1396print"Synchronizing p4 checkout..."1397if new_client_dir:1398# old one was destroyed, and maybe nobody told p41399p4_sync("...","-f")1400else:1401p4_sync("...")1402 self.check()14031404 commits = []1405for line inread_pipe_lines("git rev-list --no-merges%s..%s"% (self.origin, self.master)):1406 commits.append(line.strip())1407 commits.reverse()14081409if self.preserveUser or(gitConfig("git-p4.skipUserNameCheck") =="true"):1410 self.checkAuthorship =False1411else:1412 self.checkAuthorship =True14131414if self.preserveUser:1415 self.checkValidP4Users(commits)14161417#1418# Build up a set of options to be passed to diff when1419# submitting each commit to p4.1420#1421if self.detectRenames:1422# command-line -M arg1423 self.diffOpts ="-M"1424else:1425# If not explicitly set check the config variable1426 detectRenames =gitConfig("git-p4.detectRenames")14271428if detectRenames.lower() =="false"or detectRenames =="":1429 self.diffOpts =""1430elif detectRenames.lower() =="true":1431 self.diffOpts ="-M"1432else:1433 self.diffOpts ="-M%s"% detectRenames14341435# no command-line arg for -C or --find-copies-harder, just1436# config variables1437 detectCopies =gitConfig("git-p4.detectCopies")1438if detectCopies.lower() =="false"or detectCopies =="":1439pass1440elif detectCopies.lower() =="true":1441 self.diffOpts +=" -C"1442else:1443 self.diffOpts +=" -C%s"% detectCopies14441445ifgitConfig("git-p4.detectCopiesHarder","--bool") =="true":1446 self.diffOpts +=" --find-copies-harder"14471448whilelen(commits) >0:1449 commit = commits[0]1450 commits = commits[1:]1451 self.applyCommit(commit)1452if not self.interactive:1453break14541455iflen(commits) ==0:1456print"All changes applied!"1457chdir(self.oldWorkingDirectory)14581459 sync =P4Sync()1460 sync.run([])14611462 rebase =P4Rebase()1463 rebase.rebase()14641465ifgitConfig("git-p4.exportLabels","--bool") =="true":1466 self.exportLabels =True14671468if self.exportLabels:1469 p4Labels =getP4Labels(self.depotPath)1470 gitTags =getGitTags()14711472 missingGitTags = gitTags - p4Labels1473 self.exportGitTags(missingGitTags)14741475return True14761477classView(object):1478"""Represent a p4 view ("p4 help views"), and map files in a1479 repo according to the view."""14801481classPath(object):1482"""A depot or client path, possibly containing wildcards.1483 The only one supported is ... at the end, currently.1484 Initialize with the full path, with //depot or //client."""14851486def__init__(self, path, is_depot):1487 self.path = path1488 self.is_depot = is_depot1489 self.find_wildcards()1490# remember the prefix bit, useful for relative mappings1491 m = re.match("(//[^/]+/)", self.path)1492if not m:1493die("Path%sdoes not start with //prefix/"% self.path)1494 prefix = m.group(1)1495if not self.is_depot:1496# strip //client/ on client paths1497 self.path = self.path[len(prefix):]14981499deffind_wildcards(self):1500"""Make sure wildcards are valid, and set up internal1501 variables."""15021503 self.ends_triple_dot =False1504# There are three wildcards allowed in p4 views1505# (see "p4 help views"). This code knows how to1506# handle "..." (only at the end), but cannot deal with1507# "%%n" or "*". Only check the depot_side, as p4 should1508# validate that the client_side matches too.1509if re.search(r'%%[1-9]', self.path):1510die("Can't handle%%n wildcards in view:%s"% self.path)1511if self.path.find("*") >=0:1512die("Can't handle * wildcards in view:%s"% self.path)1513 triple_dot_index = self.path.find("...")1514if triple_dot_index >=0:1515if triple_dot_index !=len(self.path) -3:1516die("Can handle only single ... wildcard, at end:%s"%1517 self.path)1518 self.ends_triple_dot =True15191520defensure_compatible(self, other_path):1521"""Make sure the wildcards agree."""1522if self.ends_triple_dot != other_path.ends_triple_dot:1523die("Both paths must end with ... if either does;\n"+1524"paths:%s %s"% (self.path, other_path.path))15251526defmatch_wildcards(self, test_path):1527"""See if this test_path matches us, and fill in the value1528 of the wildcards if so. Returns a tuple of1529 (True|False, wildcards[]). For now, only the ... at end1530 is supported, so at most one wildcard."""1531if self.ends_triple_dot:1532 dotless = self.path[:-3]1533if test_path.startswith(dotless):1534 wildcard = test_path[len(dotless):]1535return(True, [ wildcard ])1536else:1537if test_path == self.path:1538return(True, [])1539return(False, [])15401541defmatch(self, test_path):1542"""Just return if it matches; don't bother with the wildcards."""1543 b, _ = self.match_wildcards(test_path)1544return b15451546deffill_in_wildcards(self, wildcards):1547"""Return the relative path, with the wildcards filled in1548 if there are any."""1549if self.ends_triple_dot:1550return self.path[:-3] + wildcards[0]1551else:1552return self.path15531554classMapping(object):1555def__init__(self, depot_side, client_side, overlay, exclude):1556# depot_side is without the trailing /... if it had one1557 self.depot_side = View.Path(depot_side, is_depot=True)1558 self.client_side = View.Path(client_side, is_depot=False)1559 self.overlay = overlay # started with "+"1560 self.exclude = exclude # started with "-"1561assert not(self.overlay and self.exclude)1562 self.depot_side.ensure_compatible(self.client_side)15631564def__str__(self):1565 c =" "1566if self.overlay:1567 c ="+"1568if self.exclude:1569 c ="-"1570return"View.Mapping:%s%s->%s"% \1571(c, self.depot_side.path, self.client_side.path)15721573defmap_depot_to_client(self, depot_path):1574"""Calculate the client path if using this mapping on the1575 given depot path; does not consider the effect of other1576 mappings in a view. Even excluded mappings are returned."""1577 matches, wildcards = self.depot_side.match_wildcards(depot_path)1578if not matches:1579return""1580 client_path = self.client_side.fill_in_wildcards(wildcards)1581return client_path15821583#1584# View methods1585#1586def__init__(self):1587 self.mappings = []15881589defappend(self, view_line):1590"""Parse a view line, splitting it into depot and client1591 sides. Append to self.mappings, preserving order."""15921593# Split the view line into exactly two words. P4 enforces1594# structure on these lines that simplifies this quite a bit.1595#1596# Either or both words may be double-quoted.1597# Single quotes do not matter.1598# Double-quote marks cannot occur inside the words.1599# A + or - prefix is also inside the quotes.1600# There are no quotes unless they contain a space.1601# The line is already white-space stripped.1602# The two words are separated by a single space.1603#1604if view_line[0] =='"':1605# First word is double quoted. Find its end.1606 close_quote_index = view_line.find('"',1)1607if close_quote_index <=0:1608die("No first-word closing quote found:%s"% view_line)1609 depot_side = view_line[1:close_quote_index]1610# skip closing quote and space1611 rhs_index = close_quote_index +1+11612else:1613 space_index = view_line.find(" ")1614if space_index <=0:1615die("No word-splitting space found:%s"% view_line)1616 depot_side = view_line[0:space_index]1617 rhs_index = space_index +116181619if view_line[rhs_index] =='"':1620# Second word is double quoted. Make sure there is a1621# double quote at the end too.1622if not view_line.endswith('"'):1623die("View line with rhs quote should end with one:%s"%1624 view_line)1625# skip the quotes1626 client_side = view_line[rhs_index+1:-1]1627else:1628 client_side = view_line[rhs_index:]16291630# prefix + means overlay on previous mapping1631 overlay =False1632if depot_side.startswith("+"):1633 overlay =True1634 depot_side = depot_side[1:]16351636# prefix - means exclude this path1637 exclude =False1638if depot_side.startswith("-"):1639 exclude =True1640 depot_side = depot_side[1:]16411642 m = View.Mapping(depot_side, client_side, overlay, exclude)1643 self.mappings.append(m)16441645defmap_in_client(self, depot_path):1646"""Return the relative location in the client where this1647 depot file should live. Returns "" if the file should1648 not be mapped in the client."""16491650 paths_filled = []1651 client_path =""16521653# look at later entries first1654for m in self.mappings[::-1]:16551656# see where will this path end up in the client1657 p = m.map_depot_to_client(depot_path)16581659if p =="":1660# Depot path does not belong in client. Must remember1661# this, as previous items should not cause files to1662# exist in this path either. Remember that the list is1663# being walked from the end, which has higher precedence.1664# Overlap mappings do not exclude previous mappings.1665if not m.overlay:1666 paths_filled.append(m.client_side)16671668else:1669# This mapping matched; no need to search any further.1670# But, the mapping could be rejected if the client path1671# has already been claimed by an earlier mapping (i.e.1672# one later in the list, which we are walking backwards).1673 already_mapped_in_client =False1674for f in paths_filled:1675# this is View.Path.match1676if f.match(p):1677 already_mapped_in_client =True1678break1679if not already_mapped_in_client:1680# Include this file, unless it is from a line that1681# explicitly said to exclude it.1682if not m.exclude:1683 client_path = p16841685# a match, even if rejected, always stops the search1686break16871688return client_path16891690classP4Sync(Command, P4UserMap):1691 delete_actions = ("delete","move/delete","purge")16921693def__init__(self):1694 Command.__init__(self)1695 P4UserMap.__init__(self)1696 self.options = [1697 optparse.make_option("--branch", dest="branch"),1698 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1699 optparse.make_option("--changesfile", dest="changesFile"),1700 optparse.make_option("--silent", dest="silent", action="store_true"),1701 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1702 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),1703 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1704help="Import into refs/heads/ , not refs/remotes"),1705 optparse.make_option("--max-changes", dest="maxChanges"),1706 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',1707help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),1708 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',1709help="Only sync files that are included in the Perforce Client Spec")1710]1711 self.description ="""Imports from Perforce into a git repository.\n1712 example:1713 //depot/my/project/ -- to import the current head1714 //depot/my/project/@all -- to import everything1715 //depot/my/project/@1,6 -- to import only from revision 1 to 617161717 (a ... is not needed in the path p4 specification, it's added implicitly)"""17181719 self.usage +=" //depot/path[@revRange]"1720 self.silent =False1721 self.createdBranches =set()1722 self.committedChanges =set()1723 self.branch =""1724 self.detectBranches =False1725 self.detectLabels =False1726 self.importLabels =False1727 self.changesFile =""1728 self.syncWithOrigin =True1729 self.importIntoRemotes =True1730 self.maxChanges =""1731 self.isWindows = (platform.system() =="Windows")1732 self.keepRepoPath =False1733 self.depotPaths =None1734 self.p4BranchesInGit = []1735 self.cloneExclude = []1736 self.useClientSpec =False1737 self.useClientSpec_from_options =False1738 self.clientSpecDirs =None1739 self.tempBranches = []1740 self.tempBranchLocation ="git-p4-tmp"17411742ifgitConfig("git-p4.syncFromOrigin") =="false":1743 self.syncWithOrigin =False17441745# Force a checkpoint in fast-import and wait for it to finish1746defcheckpoint(self):1747 self.gitStream.write("checkpoint\n\n")1748 self.gitStream.write("progress checkpoint\n\n")1749 out = self.gitOutput.readline()1750if self.verbose:1751print"checkpoint finished: "+ out17521753defextractFilesFromCommit(self, commit):1754 self.cloneExclude = [re.sub(r"\.\.\.$","", path)1755for path in self.cloneExclude]1756 files = []1757 fnum =01758while commit.has_key("depotFile%s"% fnum):1759 path = commit["depotFile%s"% fnum]17601761if[p for p in self.cloneExclude1762ifp4PathStartsWith(path, p)]:1763 found =False1764else:1765 found = [p for p in self.depotPaths1766ifp4PathStartsWith(path, p)]1767if not found:1768 fnum = fnum +11769continue17701771file= {}1772file["path"] = path1773file["rev"] = commit["rev%s"% fnum]1774file["action"] = commit["action%s"% fnum]1775file["type"] = commit["type%s"% fnum]1776 files.append(file)1777 fnum = fnum +11778return files17791780defstripRepoPath(self, path, prefixes):1781if self.useClientSpec:1782return self.clientSpecDirs.map_in_client(path)17831784if self.keepRepoPath:1785 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]17861787for p in prefixes:1788ifp4PathStartsWith(path, p):1789 path = path[len(p):]17901791return path17921793defsplitFilesIntoBranches(self, commit):1794 branches = {}1795 fnum =01796while commit.has_key("depotFile%s"% fnum):1797 path = commit["depotFile%s"% fnum]1798 found = [p for p in self.depotPaths1799ifp4PathStartsWith(path, p)]1800if not found:1801 fnum = fnum +11802continue18031804file= {}1805file["path"] = path1806file["rev"] = commit["rev%s"% fnum]1807file["action"] = commit["action%s"% fnum]1808file["type"] = commit["type%s"% fnum]1809 fnum = fnum +118101811 relPath = self.stripRepoPath(path, self.depotPaths)1812 relPath =wildcard_decode(relPath)18131814for branch in self.knownBranches.keys():18151816# add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.21817if relPath.startswith(branch +"/"):1818if branch not in branches:1819 branches[branch] = []1820 branches[branch].append(file)1821break18221823return branches18241825# output one file from the P4 stream1826# - helper for streamP4Files18271828defstreamOneP4File(self,file, contents):1829 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)1830 relPath =wildcard_decode(relPath)1831if verbose:1832 sys.stderr.write("%s\n"% relPath)18331834(type_base, type_mods) =split_p4_type(file["type"])18351836 git_mode ="100644"1837if"x"in type_mods:1838 git_mode ="100755"1839if type_base =="symlink":1840 git_mode ="120000"1841# p4 print on a symlink contains "target\n"; remove the newline1842 data =''.join(contents)1843 contents = [data[:-1]]18441845if type_base =="utf16":1846# p4 delivers different text in the python output to -G1847# than it does when using "print -o", or normal p4 client1848# operations. utf16 is converted to ascii or utf8, perhaps.1849# But ascii text saved as -t utf16 is completely mangled.1850# Invoke print -o to get the real contents.1851 text =p4_read_pipe(['print','-q','-o','-',file['depotFile']])1852 contents = [ text ]18531854if type_base =="apple":1855# Apple filetype files will be streamed as a concatenation of1856# its appledouble header and the contents. This is useless1857# on both macs and non-macs. If using "print -q -o xx", it1858# will create "xx" with the data, and "%xx" with the header.1859# This is also not very useful.1860#1861# Ideally, someday, this script can learn how to generate1862# appledouble files directly and import those to git, but1863# non-mac machines can never find a use for apple filetype.1864print"\nIgnoring apple filetype file%s"%file['depotFile']1865return18661867# Perhaps windows wants unicode, utf16 newlines translated too;1868# but this is not doing it.1869if self.isWindows and type_base =="text":1870 mangled = []1871for data in contents:1872 data = data.replace("\r\n","\n")1873 mangled.append(data)1874 contents = mangled18751876# Note that we do not try to de-mangle keywords on utf16 files,1877# even though in theory somebody may want that.1878 pattern =p4_keywords_regexp_for_type(type_base, type_mods)1879if pattern:1880 regexp = re.compile(pattern, re.VERBOSE)1881 text =''.join(contents)1882 text = regexp.sub(r'$\1$', text)1883 contents = [ text ]18841885 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))18861887# total length...1888 length =01889for d in contents:1890 length = length +len(d)18911892 self.gitStream.write("data%d\n"% length)1893for d in contents:1894 self.gitStream.write(d)1895 self.gitStream.write("\n")18961897defstreamOneP4Deletion(self,file):1898 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)1899 relPath =wildcard_decode(relPath)1900if verbose:1901 sys.stderr.write("delete%s\n"% relPath)1902 self.gitStream.write("D%s\n"% relPath)19031904# handle another chunk of streaming data1905defstreamP4FilesCb(self, marshalled):19061907if marshalled.has_key('depotFile')and self.stream_have_file_info:1908# start of a new file - output the old one first1909 self.streamOneP4File(self.stream_file, self.stream_contents)1910 self.stream_file = {}1911 self.stream_contents = []1912 self.stream_have_file_info =False19131914# pick up the new file information... for the1915# 'data' field we need to append to our array1916for k in marshalled.keys():1917if k =='data':1918 self.stream_contents.append(marshalled['data'])1919else:1920 self.stream_file[k] = marshalled[k]19211922 self.stream_have_file_info =True19231924# Stream directly from "p4 files" into "git fast-import"1925defstreamP4Files(self, files):1926 filesForCommit = []1927 filesToRead = []1928 filesToDelete = []19291930for f in files:1931# if using a client spec, only add the files that have1932# a path in the client1933if self.clientSpecDirs:1934if self.clientSpecDirs.map_in_client(f['path']) =="":1935continue19361937 filesForCommit.append(f)1938if f['action']in self.delete_actions:1939 filesToDelete.append(f)1940else:1941 filesToRead.append(f)19421943# deleted files...1944for f in filesToDelete:1945 self.streamOneP4Deletion(f)19461947iflen(filesToRead) >0:1948 self.stream_file = {}1949 self.stream_contents = []1950 self.stream_have_file_info =False19511952# curry self argument1953defstreamP4FilesCbSelf(entry):1954 self.streamP4FilesCb(entry)19551956 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]19571958p4CmdList(["-x","-","print"],1959 stdin=fileArgs,1960 cb=streamP4FilesCbSelf)19611962# do the last chunk1963if self.stream_file.has_key('depotFile'):1964 self.streamOneP4File(self.stream_file, self.stream_contents)19651966defmake_email(self, userid):1967if userid in self.users:1968return self.users[userid]1969else:1970return"%s<a@b>"% userid19711972# Stream a p4 tag1973defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):1974if verbose:1975print"writing tag%sfor commit%s"% (labelName, commit)1976 gitStream.write("tag%s\n"% labelName)1977 gitStream.write("from%s\n"% commit)19781979if labelDetails.has_key('Owner'):1980 owner = labelDetails["Owner"]1981else:1982 owner =None19831984# Try to use the owner of the p4 label, or failing that,1985# the current p4 user id.1986if owner:1987 email = self.make_email(owner)1988else:1989 email = self.make_email(self.p4UserId())1990 tagger ="%s %s %s"% (email, epoch, self.tz)19911992 gitStream.write("tagger%s\n"% tagger)19931994print"labelDetails=",labelDetails1995if labelDetails.has_key('Description'):1996 description = labelDetails['Description']1997else:1998 description ='Label from git p4'19992000 gitStream.write("data%d\n"%len(description))2001 gitStream.write(description)2002 gitStream.write("\n")20032004defcommit(self, details, files, branch, branchPrefixes, parent =""):2005 epoch = details["time"]2006 author = details["user"]2007 self.branchPrefixes = branchPrefixes20082009if self.verbose:2010print"commit into%s"% branch20112012# start with reading files; if that fails, we should not2013# create a commit.2014 new_files = []2015for f in files:2016if[p for p in branchPrefixes ifp4PathStartsWith(f['path'], p)]:2017 new_files.append(f)2018else:2019 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])20202021 self.gitStream.write("commit%s\n"% branch)2022# gitStream.write("mark :%s\n" % details["change"])2023 self.committedChanges.add(int(details["change"]))2024 committer =""2025if author not in self.users:2026 self.getUserMapFromPerforceServer()2027 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)20282029 self.gitStream.write("committer%s\n"% committer)20302031 self.gitStream.write("data <<EOT\n")2032 self.gitStream.write(details["desc"])2033 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"2034% (','.join(branchPrefixes), details["change"]))2035iflen(details['options']) >0:2036 self.gitStream.write(": options =%s"% details['options'])2037 self.gitStream.write("]\nEOT\n\n")20382039iflen(parent) >0:2040if self.verbose:2041print"parent%s"% parent2042 self.gitStream.write("from%s\n"% parent)20432044 self.streamP4Files(new_files)2045 self.gitStream.write("\n")20462047 change =int(details["change"])20482049if self.labels.has_key(change):2050 label = self.labels[change]2051 labelDetails = label[0]2052 labelRevisions = label[1]2053if self.verbose:2054print"Change%sis labelled%s"% (change, labelDetails)20552056 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2057for p in branchPrefixes])20582059iflen(files) ==len(labelRevisions):20602061 cleanedFiles = {}2062for info in files:2063if info["action"]in self.delete_actions:2064continue2065 cleanedFiles[info["depotFile"]] = info["rev"]20662067if cleanedFiles == labelRevisions:2068 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)20692070else:2071if not self.silent:2072print("Tag%sdoes not match with change%s: files do not match."2073% (labelDetails["label"], change))20742075else:2076if not self.silent:2077print("Tag%sdoes not match with change%s: file count is different."2078% (labelDetails["label"], change))20792080# Build a dictionary of changelists and labels, for "detect-labels" option.2081defgetLabels(self):2082 self.labels = {}20832084 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2085iflen(l) >0and not self.silent:2086print"Finding files belonging to labels in%s"% `self.depotPaths`20872088for output in l:2089 label = output["label"]2090 revisions = {}2091 newestChange =02092if self.verbose:2093print"Querying files for label%s"% label2094forfileinp4CmdList(["files"] +2095["%s...@%s"% (p, label)2096for p in self.depotPaths]):2097 revisions[file["depotFile"]] =file["rev"]2098 change =int(file["change"])2099if change > newestChange:2100 newestChange = change21012102 self.labels[newestChange] = [output, revisions]21032104if self.verbose:2105print"Label changes:%s"% self.labels.keys()21062107# Import p4 labels as git tags. A direct mapping does not2108# exist, so assume that if all the files are at the same revision2109# then we can use that, or it's something more complicated we should2110# just ignore.2111defimportP4Labels(self, stream, p4Labels):2112if verbose:2113print"import p4 labels: "+' '.join(p4Labels)21142115 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2116 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2117iflen(validLabelRegexp) ==0:2118 validLabelRegexp = defaultLabelRegexp2119 m = re.compile(validLabelRegexp)21202121for name in p4Labels:2122 commitFound =False21232124if not m.match(name):2125if verbose:2126print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2127continue21282129if name in ignoredP4Labels:2130continue21312132 labelDetails =p4CmdList(['label',"-o", name])[0]21332134# get the most recent changelist for each file in this label2135 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2136for p in self.depotPaths])21372138if change.has_key('change'):2139# find the corresponding git commit; take the oldest commit2140 changelist =int(change['change'])2141 gitCommit =read_pipe(["git","rev-list","--max-count=1",2142"--reverse",":/\[git-p4:.*change =%d\]"% changelist])2143iflen(gitCommit) ==0:2144print"could not find git commit for changelist%d"% changelist2145else:2146 gitCommit = gitCommit.strip()2147 commitFound =True2148# Convert from p4 time format2149try:2150 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2151exceptValueError:2152print"Could not convert label time%s"% labelDetail['Update']2153 tmwhen =121542155 when =int(time.mktime(tmwhen))2156 self.streamTag(stream, name, labelDetails, gitCommit, when)2157if verbose:2158print"p4 label%smapped to git commit%s"% (name, gitCommit)2159else:2160if verbose:2161print"Label%shas no changelists - possibly deleted?"% name21622163if not commitFound:2164# We can't import this label; don't try again as it will get very2165# expensive repeatedly fetching all the files for labels that will2166# never be imported. If the label is moved in the future, the2167# ignore will need to be removed manually.2168system(["git","config","--add","git-p4.ignoredP4Labels", name])21692170defguessProjectName(self):2171for p in self.depotPaths:2172if p.endswith("/"):2173 p = p[:-1]2174 p = p[p.strip().rfind("/") +1:]2175if not p.endswith("/"):2176 p +="/"2177return p21782179defgetBranchMapping(self):2180 lostAndFoundBranches =set()21812182 user =gitConfig("git-p4.branchUser")2183iflen(user) >0:2184 command ="branches -u%s"% user2185else:2186 command ="branches"21872188for info inp4CmdList(command):2189 details =p4Cmd(["branch","-o", info["branch"]])2190 viewIdx =02191while details.has_key("View%s"% viewIdx):2192 paths = details["View%s"% viewIdx].split(" ")2193 viewIdx = viewIdx +12194# require standard //depot/foo/... //depot/bar/... mapping2195iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2196continue2197 source = paths[0]2198 destination = paths[1]2199## HACK2200ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2201 source = source[len(self.depotPaths[0]):-4]2202 destination = destination[len(self.depotPaths[0]):-4]22032204if destination in self.knownBranches:2205if not self.silent:2206print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2207print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2208continue22092210 self.knownBranches[destination] = source22112212 lostAndFoundBranches.discard(destination)22132214if source not in self.knownBranches:2215 lostAndFoundBranches.add(source)22162217# Perforce does not strictly require branches to be defined, so we also2218# check git config for a branch list.2219#2220# Example of branch definition in git config file:2221# [git-p4]2222# branchList=main:branchA2223# branchList=main:branchB2224# branchList=branchA:branchC2225 configBranches =gitConfigList("git-p4.branchList")2226for branch in configBranches:2227if branch:2228(source, destination) = branch.split(":")2229 self.knownBranches[destination] = source22302231 lostAndFoundBranches.discard(destination)22322233if source not in self.knownBranches:2234 lostAndFoundBranches.add(source)223522362237for branch in lostAndFoundBranches:2238 self.knownBranches[branch] = branch22392240defgetBranchMappingFromGitBranches(self):2241 branches =p4BranchesInGit(self.importIntoRemotes)2242for branch in branches.keys():2243if branch =="master":2244 branch ="main"2245else:2246 branch = branch[len(self.projectName):]2247 self.knownBranches[branch] = branch22482249deflistExistingP4GitBranches(self):2250# branches holds mapping from name to commit2251 branches =p4BranchesInGit(self.importIntoRemotes)2252 self.p4BranchesInGit = branches.keys()2253for branch in branches.keys():2254 self.initialParents[self.refPrefix + branch] = branches[branch]22552256defupdateOptionDict(self, d):2257 option_keys = {}2258if self.keepRepoPath:2259 option_keys['keepRepoPath'] =122602261 d["options"] =' '.join(sorted(option_keys.keys()))22622263defreadOptions(self, d):2264 self.keepRepoPath = (d.has_key('options')2265and('keepRepoPath'in d['options']))22662267defgitRefForBranch(self, branch):2268if branch =="main":2269return self.refPrefix +"master"22702271iflen(branch) <=0:2272return branch22732274return self.refPrefix + self.projectName + branch22752276defgitCommitByP4Change(self, ref, change):2277if self.verbose:2278print"looking in ref "+ ref +" for change%susing bisect..."% change22792280 earliestCommit =""2281 latestCommit =parseRevision(ref)22822283while True:2284if self.verbose:2285print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2286 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2287iflen(next) ==0:2288if self.verbose:2289print"argh"2290return""2291 log =extractLogMessageFromGitCommit(next)2292 settings =extractSettingsGitLog(log)2293 currentChange =int(settings['change'])2294if self.verbose:2295print"current change%s"% currentChange22962297if currentChange == change:2298if self.verbose:2299print"found%s"% next2300return next23012302if currentChange < change:2303 earliestCommit ="^%s"% next2304else:2305 latestCommit ="%s"% next23062307return""23082309defimportNewBranch(self, branch, maxChange):2310# make fast-import flush all changes to disk and update the refs using the checkpoint2311# command so that we can try to find the branch parent in the git history2312 self.gitStream.write("checkpoint\n\n");2313 self.gitStream.flush();2314 branchPrefix = self.depotPaths[0] + branch +"/"2315range="@1,%s"% maxChange2316#print "prefix" + branchPrefix2317 changes =p4ChangesForPaths([branchPrefix],range)2318iflen(changes) <=0:2319return False2320 firstChange = changes[0]2321#print "first change in branch: %s" % firstChange2322 sourceBranch = self.knownBranches[branch]2323 sourceDepotPath = self.depotPaths[0] + sourceBranch2324 sourceRef = self.gitRefForBranch(sourceBranch)2325#print "source " + sourceBranch23262327 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2328#print "branch parent: %s" % branchParentChange2329 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2330iflen(gitParent) >0:2331 self.initialParents[self.gitRefForBranch(branch)] = gitParent2332#print "parent git commit: %s" % gitParent23332334 self.importChanges(changes)2335return True23362337defsearchParent(self, parent, branch, target):2338 parentFound =False2339for blob inread_pipe_lines(["git","rev-list","--reverse","--no-merges", parent]):2340 blob = blob.strip()2341iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2342 parentFound =True2343if self.verbose:2344print"Found parent of%sin commit%s"% (branch, blob)2345break2346if parentFound:2347return blob2348else:2349return None23502351defimportChanges(self, changes):2352 cnt =12353for change in changes:2354 description =p4Cmd(["describe",str(change)])2355 self.updateOptionDict(description)23562357if not self.silent:2358 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2359 sys.stdout.flush()2360 cnt = cnt +123612362try:2363if self.detectBranches:2364 branches = self.splitFilesIntoBranches(description)2365for branch in branches.keys():2366## HACK --hwn2367 branchPrefix = self.depotPaths[0] + branch +"/"23682369 parent =""23702371 filesForCommit = branches[branch]23722373if self.verbose:2374print"branch is%s"% branch23752376 self.updatedBranches.add(branch)23772378if branch not in self.createdBranches:2379 self.createdBranches.add(branch)2380 parent = self.knownBranches[branch]2381if parent == branch:2382 parent =""2383else:2384 fullBranch = self.projectName + branch2385if fullBranch not in self.p4BranchesInGit:2386if not self.silent:2387print("\nImporting new branch%s"% fullBranch);2388if self.importNewBranch(branch, change -1):2389 parent =""2390 self.p4BranchesInGit.append(fullBranch)2391if not self.silent:2392print("\nResuming with change%s"% change);23932394if self.verbose:2395print"parent determined through known branches:%s"% parent23962397 branch = self.gitRefForBranch(branch)2398 parent = self.gitRefForBranch(parent)23992400if self.verbose:2401print"looking for initial parent for%s; current parent is%s"% (branch, parent)24022403iflen(parent) ==0and branch in self.initialParents:2404 parent = self.initialParents[branch]2405del self.initialParents[branch]24062407 blob =None2408iflen(parent) >0:2409 tempBranch = os.path.join(self.tempBranchLocation,"%d"% (change))2410if self.verbose:2411print"Creating temporary branch: "+ tempBranch2412 self.commit(description, filesForCommit, tempBranch, [branchPrefix])2413 self.tempBranches.append(tempBranch)2414 self.checkpoint()2415 blob = self.searchParent(parent, branch, tempBranch)2416if blob:2417 self.commit(description, filesForCommit, branch, [branchPrefix], blob)2418else:2419if self.verbose:2420print"Parent of%snot found. Committing into head of%s"% (branch, parent)2421 self.commit(description, filesForCommit, branch, [branchPrefix], parent)2422else:2423 files = self.extractFilesFromCommit(description)2424 self.commit(description, files, self.branch, self.depotPaths,2425 self.initialParent)2426 self.initialParent =""2427exceptIOError:2428print self.gitError.read()2429 sys.exit(1)24302431defimportHeadRevision(self, revision):2432print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)24332434 details = {}2435 details["user"] ="git perforce import user"2436 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2437% (' '.join(self.depotPaths), revision))2438 details["change"] = revision2439 newestRevision =024402441 fileCnt =02442 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]24432444for info inp4CmdList(["files"] + fileArgs):24452446if'code'in info and info['code'] =='error':2447 sys.stderr.write("p4 returned an error:%s\n"2448% info['data'])2449if info['data'].find("must refer to client") >=0:2450 sys.stderr.write("This particular p4 error is misleading.\n")2451 sys.stderr.write("Perhaps the depot path was misspelled.\n");2452 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2453 sys.exit(1)2454if'p4ExitCode'in info:2455 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2456 sys.exit(1)245724582459 change =int(info["change"])2460if change > newestRevision:2461 newestRevision = change24622463if info["action"]in self.delete_actions:2464# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2465#fileCnt = fileCnt + 12466continue24672468for prop in["depotFile","rev","action","type"]:2469 details["%s%s"% (prop, fileCnt)] = info[prop]24702471 fileCnt = fileCnt +124722473 details["change"] = newestRevision24742475# Use time from top-most change so that all git p4 clones of2476# the same p4 repo have the same commit SHA1s.2477 res =p4CmdList("describe -s%d"% newestRevision)2478 newestTime =None2479for r in res:2480if r.has_key('time'):2481 newestTime =int(r['time'])2482if newestTime is None:2483die("\"describe -s\"on newest change%ddid not give a time")2484 details["time"] = newestTime24852486 self.updateOptionDict(details)2487try:2488 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)2489exceptIOError:2490print"IO error with git fast-import. Is your git version recent enough?"2491print self.gitError.read()249224932494defrun(self, args):2495 self.depotPaths = []2496 self.changeRange =""2497 self.initialParent =""2498 self.previousDepotPaths = []24992500# map from branch depot path to parent branch2501 self.knownBranches = {}2502 self.initialParents = {}2503 self.hasOrigin =originP4BranchesExist()2504if not self.syncWithOrigin:2505 self.hasOrigin =False25062507if self.importIntoRemotes:2508 self.refPrefix ="refs/remotes/p4/"2509else:2510 self.refPrefix ="refs/heads/p4/"25112512if self.syncWithOrigin and self.hasOrigin:2513if not self.silent:2514print"Syncing with origin first by calling git fetch origin"2515system("git fetch origin")25162517iflen(self.branch) ==0:2518 self.branch = self.refPrefix +"master"2519ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2520system("git update-ref%srefs/heads/p4"% self.branch)2521system("git branch -D p4");2522# create it /after/ importing, when master exists2523if notgitBranchExists(self.refPrefix +"HEAD")and self.importIntoRemotes andgitBranchExists(self.branch):2524system("git symbolic-ref%sHEAD%s"% (self.refPrefix, self.branch))25252526# accept either the command-line option, or the configuration variable2527if self.useClientSpec:2528# will use this after clone to set the variable2529 self.useClientSpec_from_options =True2530else:2531ifgitConfig("git-p4.useclientspec","--bool") =="true":2532 self.useClientSpec =True2533if self.useClientSpec:2534 self.clientSpecDirs =getClientSpec()25352536# TODO: should always look at previous commits,2537# merge with previous imports, if possible.2538if args == []:2539if self.hasOrigin:2540createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)2541 self.listExistingP4GitBranches()25422543iflen(self.p4BranchesInGit) >1:2544if not self.silent:2545print"Importing from/into multiple branches"2546 self.detectBranches =True25472548if self.verbose:2549print"branches:%s"% self.p4BranchesInGit25502551 p4Change =02552for branch in self.p4BranchesInGit:2553 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)25542555 settings =extractSettingsGitLog(logMsg)25562557 self.readOptions(settings)2558if(settings.has_key('depot-paths')2559and settings.has_key('change')):2560 change =int(settings['change']) +12561 p4Change =max(p4Change, change)25622563 depotPaths =sorted(settings['depot-paths'])2564if self.previousDepotPaths == []:2565 self.previousDepotPaths = depotPaths2566else:2567 paths = []2568for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2569 prev_list = prev.split("/")2570 cur_list = cur.split("/")2571for i inrange(0,min(len(cur_list),len(prev_list))):2572if cur_list[i] <> prev_list[i]:2573 i = i -12574break25752576 paths.append("/".join(cur_list[:i +1]))25772578 self.previousDepotPaths = paths25792580if p4Change >0:2581 self.depotPaths =sorted(self.previousDepotPaths)2582 self.changeRange ="@%s,#head"% p4Change2583if not self.detectBranches:2584 self.initialParent =parseRevision(self.branch)2585if not self.silent and not self.detectBranches:2586print"Performing incremental import into%sgit branch"% self.branch25872588if not self.branch.startswith("refs/"):2589 self.branch ="refs/heads/"+ self.branch25902591iflen(args) ==0and self.depotPaths:2592if not self.silent:2593print"Depot paths:%s"%' '.join(self.depotPaths)2594else:2595if self.depotPaths and self.depotPaths != args:2596print("previous import used depot path%sand now%swas specified. "2597"This doesn't work!"% (' '.join(self.depotPaths),2598' '.join(args)))2599 sys.exit(1)26002601 self.depotPaths =sorted(args)26022603 revision =""2604 self.users = {}26052606# Make sure no revision specifiers are used when --changesfile2607# is specified.2608 bad_changesfile =False2609iflen(self.changesFile) >0:2610for p in self.depotPaths:2611if p.find("@") >=0or p.find("#") >=0:2612 bad_changesfile =True2613break2614if bad_changesfile:2615die("Option --changesfile is incompatible with revision specifiers")26162617 newPaths = []2618for p in self.depotPaths:2619if p.find("@") != -1:2620 atIdx = p.index("@")2621 self.changeRange = p[atIdx:]2622if self.changeRange =="@all":2623 self.changeRange =""2624elif','not in self.changeRange:2625 revision = self.changeRange2626 self.changeRange =""2627 p = p[:atIdx]2628elif p.find("#") != -1:2629 hashIdx = p.index("#")2630 revision = p[hashIdx:]2631 p = p[:hashIdx]2632elif self.previousDepotPaths == []:2633# pay attention to changesfile, if given, else import2634# the entire p4 tree at the head revision2635iflen(self.changesFile) ==0:2636 revision ="#head"26372638 p = re.sub("\.\.\.$","", p)2639if not p.endswith("/"):2640 p +="/"26412642 newPaths.append(p)26432644 self.depotPaths = newPaths26452646 self.loadUserMapFromCache()2647 self.labels = {}2648if self.detectLabels:2649 self.getLabels();26502651if self.detectBranches:2652## FIXME - what's a P4 projectName ?2653 self.projectName = self.guessProjectName()26542655if self.hasOrigin:2656 self.getBranchMappingFromGitBranches()2657else:2658 self.getBranchMapping()2659if self.verbose:2660print"p4-git branches:%s"% self.p4BranchesInGit2661print"initial parents:%s"% self.initialParents2662for b in self.p4BranchesInGit:2663if b !="master":26642665## FIXME2666 b = b[len(self.projectName):]2667 self.createdBranches.add(b)26682669 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))26702671 importProcess = subprocess.Popen(["git","fast-import"],2672 stdin=subprocess.PIPE, stdout=subprocess.PIPE,2673 stderr=subprocess.PIPE);2674 self.gitOutput = importProcess.stdout2675 self.gitStream = importProcess.stdin2676 self.gitError = importProcess.stderr26772678if revision:2679 self.importHeadRevision(revision)2680else:2681 changes = []26822683iflen(self.changesFile) >0:2684 output =open(self.changesFile).readlines()2685 changeSet =set()2686for line in output:2687 changeSet.add(int(line))26882689for change in changeSet:2690 changes.append(change)26912692 changes.sort()2693else:2694# catch "git p4 sync" with no new branches, in a repo that2695# does not have any existing p4 branches2696iflen(args) ==0and not self.p4BranchesInGit:2697die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.");2698if self.verbose:2699print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),2700 self.changeRange)2701 changes =p4ChangesForPaths(self.depotPaths, self.changeRange)27022703iflen(self.maxChanges) >0:2704 changes = changes[:min(int(self.maxChanges),len(changes))]27052706iflen(changes) ==0:2707if not self.silent:2708print"No changes to import!"2709else:2710if not self.silent and not self.detectBranches:2711print"Import destination:%s"% self.branch27122713 self.updatedBranches =set()27142715 self.importChanges(changes)27162717if not self.silent:2718print""2719iflen(self.updatedBranches) >0:2720 sys.stdout.write("Updated branches: ")2721for b in self.updatedBranches:2722 sys.stdout.write("%s"% b)2723 sys.stdout.write("\n")27242725ifgitConfig("git-p4.importLabels","--bool") =="true":2726 self.importLabels =True27272728if self.importLabels:2729 p4Labels =getP4Labels(self.depotPaths)2730 gitTags =getGitTags()27312732 missingP4Labels = p4Labels - gitTags2733 self.importP4Labels(self.gitStream, missingP4Labels)27342735 self.gitStream.close()2736if importProcess.wait() !=0:2737die("fast-import failed:%s"% self.gitError.read())2738 self.gitOutput.close()2739 self.gitError.close()27402741# Cleanup temporary branches created during import2742if self.tempBranches != []:2743for branch in self.tempBranches:2744read_pipe("git update-ref -d%s"% branch)2745 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))27462747return True27482749classP4Rebase(Command):2750def__init__(self):2751 Command.__init__(self)2752 self.options = [2753 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2754]2755 self.importLabels =False2756 self.description = ("Fetches the latest revision from perforce and "2757+"rebases the current work (branch) against it")27582759defrun(self, args):2760 sync =P4Sync()2761 sync.importLabels = self.importLabels2762 sync.run([])27632764return self.rebase()27652766defrebase(self):2767if os.system("git update-index --refresh") !=0:2768die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");2769iflen(read_pipe("git diff-index HEAD --")) >0:2770die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");27712772[upstream, settings] =findUpstreamBranchPoint()2773iflen(upstream) ==0:2774die("Cannot find upstream branchpoint for rebase")27752776# the branchpoint may be p4/foo~3, so strip off the parent2777 upstream = re.sub("~[0-9]+$","", upstream)27782779print"Rebasing the current branch onto%s"% upstream2780 oldHead =read_pipe("git rev-parse HEAD").strip()2781system("git rebase%s"% upstream)2782system("git diff-tree --stat --summary -M%sHEAD"% oldHead)2783return True27842785classP4Clone(P4Sync):2786def__init__(self):2787 P4Sync.__init__(self)2788 self.description ="Creates a new git repository and imports from Perforce into it"2789 self.usage ="usage: %prog [options] //depot/path[@revRange]"2790 self.options += [2791 optparse.make_option("--destination", dest="cloneDestination",2792 action='store', default=None,2793help="where to leave result of the clone"),2794 optparse.make_option("-/", dest="cloneExclude",2795 action="append",type="string",2796help="exclude depot path"),2797 optparse.make_option("--bare", dest="cloneBare",2798 action="store_true", default=False),2799]2800 self.cloneDestination =None2801 self.needsGit =False2802 self.cloneBare =False28032804# This is required for the "append" cloneExclude action2805defensure_value(self, attr, value):2806if nothasattr(self, attr)orgetattr(self, attr)is None:2807setattr(self, attr, value)2808returngetattr(self, attr)28092810defdefaultDestination(self, args):2811## TODO: use common prefix of args?2812 depotPath = args[0]2813 depotDir = re.sub("(@[^@]*)$","", depotPath)2814 depotDir = re.sub("(#[^#]*)$","", depotDir)2815 depotDir = re.sub(r"\.\.\.$","", depotDir)2816 depotDir = re.sub(r"/$","", depotDir)2817return os.path.split(depotDir)[1]28182819defrun(self, args):2820iflen(args) <1:2821return False28222823if self.keepRepoPath and not self.cloneDestination:2824 sys.stderr.write("Must specify destination for --keep-path\n")2825 sys.exit(1)28262827 depotPaths = args28282829if not self.cloneDestination andlen(depotPaths) >1:2830 self.cloneDestination = depotPaths[-1]2831 depotPaths = depotPaths[:-1]28322833 self.cloneExclude = ["/"+p for p in self.cloneExclude]2834for p in depotPaths:2835if not p.startswith("//"):2836return False28372838if not self.cloneDestination:2839 self.cloneDestination = self.defaultDestination(args)28402841print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)28422843if not os.path.exists(self.cloneDestination):2844 os.makedirs(self.cloneDestination)2845chdir(self.cloneDestination)28462847 init_cmd = ["git","init"]2848if self.cloneBare:2849 init_cmd.append("--bare")2850 subprocess.check_call(init_cmd)28512852if not P4Sync.run(self, depotPaths):2853return False2854if self.branch !="master":2855if self.importIntoRemotes:2856 masterbranch ="refs/remotes/p4/master"2857else:2858 masterbranch ="refs/heads/p4/master"2859ifgitBranchExists(masterbranch):2860system("git branch master%s"% masterbranch)2861if not self.cloneBare:2862system("git checkout -f")2863else:2864print"Could not detect main branch. No checkout/master branch created."28652866# auto-set this variable if invoked with --use-client-spec2867if self.useClientSpec_from_options:2868system("git config --bool git-p4.useclientspec true")28692870return True28712872classP4Branches(Command):2873def__init__(self):2874 Command.__init__(self)2875 self.options = [ ]2876 self.description = ("Shows the git branches that hold imports and their "2877+"corresponding perforce depot paths")2878 self.verbose =False28792880defrun(self, args):2881iforiginP4BranchesExist():2882createOrUpdateBranchesFromOrigin()28832884 cmdline ="git rev-parse --symbolic "2885 cmdline +=" --remotes"28862887for line inread_pipe_lines(cmdline):2888 line = line.strip()28892890if not line.startswith('p4/')or line =="p4/HEAD":2891continue2892 branch = line28932894 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)2895 settings =extractSettingsGitLog(log)28962897print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])2898return True28992900classHelpFormatter(optparse.IndentedHelpFormatter):2901def__init__(self):2902 optparse.IndentedHelpFormatter.__init__(self)29032904defformat_description(self, description):2905if description:2906return description +"\n"2907else:2908return""29092910defprintUsage(commands):2911print"usage:%s<command> [options]"% sys.argv[0]2912print""2913print"valid commands:%s"%", ".join(commands)2914print""2915print"Try%s<command> --help for command specific help."% sys.argv[0]2916print""29172918commands = {2919"debug": P4Debug,2920"submit": P4Submit,2921"commit": P4Submit,2922"sync": P4Sync,2923"rebase": P4Rebase,2924"clone": P4Clone,2925"rollback": P4RollBack,2926"branches": P4Branches2927}292829292930defmain():2931iflen(sys.argv[1:]) ==0:2932printUsage(commands.keys())2933 sys.exit(2)29342935 cmd =""2936 cmdName = sys.argv[1]2937try:2938 klass = commands[cmdName]2939 cmd =klass()2940exceptKeyError:2941print"unknown command%s"% cmdName2942print""2943printUsage(commands.keys())2944 sys.exit(2)29452946 options = cmd.options2947 cmd.gitdir = os.environ.get("GIT_DIR",None)29482949 args = sys.argv[2:]29502951 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))2952if cmd.needsGit:2953 options.append(optparse.make_option("--git-dir", dest="gitdir"))29542955 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),2956 options,2957 description = cmd.description,2958 formatter =HelpFormatter())29592960(cmd, args) = parser.parse_args(sys.argv[2:], cmd);2961global verbose2962 verbose = cmd.verbose2963if cmd.needsGit:2964if cmd.gitdir ==None:2965 cmd.gitdir = os.path.abspath(".git")2966if notisValidGitDir(cmd.gitdir):2967 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()2968if os.path.exists(cmd.gitdir):2969 cdup =read_pipe("git rev-parse --show-cdup").strip()2970iflen(cdup) >0:2971chdir(cdup);29722973if notisValidGitDir(cmd.gitdir):2974ifisValidGitDir(cmd.gitdir +"/.git"):2975 cmd.gitdir +="/.git"2976else:2977die("fatal: cannot locate git repository at%s"% cmd.gitdir)29782979 os.environ["GIT_DIR"] = cmd.gitdir29802981if not cmd.run(args):2982 parser.print_help()2983 sys.exit(2)298429852986if __name__ =='__main__':2987main()