1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10 11import optparse, sys, os, marshal, subprocess, shelve 12import tempfile, getopt, os.path, time, platform 13import re, shutil 14 15verbose =False 16 17# Only labels/tags matching this will be imported/exported 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 19 20defp4_build_cmd(cmd): 21"""Build a suitable p4 command line. 22 23 This consolidates building and returning a p4 command line into one 24 location. It means that hooking into the environment, or other configuration 25 can be done more easily. 26 """ 27 real_cmd = ["p4"] 28 29 user =gitConfig("git-p4.user") 30iflen(user) >0: 31 real_cmd += ["-u",user] 32 33 password =gitConfig("git-p4.password") 34iflen(password) >0: 35 real_cmd += ["-P", password] 36 37 port =gitConfig("git-p4.port") 38iflen(port) >0: 39 real_cmd += ["-p", port] 40 41 host =gitConfig("git-p4.host") 42iflen(host) >0: 43 real_cmd += ["-H", host] 44 45 client =gitConfig("git-p4.client") 46iflen(client) >0: 47 real_cmd += ["-c", client] 48 49 50ifisinstance(cmd,basestring): 51 real_cmd =' '.join(real_cmd) +' '+ cmd 52else: 53 real_cmd += cmd 54return real_cmd 55 56defchdir(dir): 57# P4 uses the PWD environment variable rather than getcwd(). Since we're 58# not using the shell, we have to set it ourselves. This path could 59# be relative, so go there first, then figure out where we ended up. 60 os.chdir(dir) 61 os.environ['PWD'] = os.getcwd() 62 63defdie(msg): 64if verbose: 65raiseException(msg) 66else: 67 sys.stderr.write(msg +"\n") 68 sys.exit(1) 69 70defwrite_pipe(c, stdin): 71if verbose: 72 sys.stderr.write('Writing pipe:%s\n'%str(c)) 73 74 expand =isinstance(c,basestring) 75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 76 pipe = p.stdin 77 val = pipe.write(stdin) 78 pipe.close() 79if p.wait(): 80die('Command failed:%s'%str(c)) 81 82return val 83 84defp4_write_pipe(c, stdin): 85 real_cmd =p4_build_cmd(c) 86returnwrite_pipe(real_cmd, stdin) 87 88defread_pipe(c, ignore_error=False): 89if verbose: 90 sys.stderr.write('Reading pipe:%s\n'%str(c)) 91 92 expand =isinstance(c,basestring) 93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 94 pipe = p.stdout 95 val = pipe.read() 96if p.wait()and not ignore_error: 97die('Command failed:%s'%str(c)) 98 99return val 100 101defp4_read_pipe(c, ignore_error=False): 102 real_cmd =p4_build_cmd(c) 103returnread_pipe(real_cmd, ignore_error) 104 105defread_pipe_lines(c): 106if verbose: 107 sys.stderr.write('Reading pipe:%s\n'%str(c)) 108 109 expand =isinstance(c, basestring) 110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 111 pipe = p.stdout 112 val = pipe.readlines() 113if pipe.close()or p.wait(): 114die('Command failed:%s'%str(c)) 115 116return val 117 118defp4_read_pipe_lines(c): 119"""Specifically invoke p4 on the command supplied. """ 120 real_cmd =p4_build_cmd(c) 121returnread_pipe_lines(real_cmd) 122 123defp4_has_command(cmd): 124"""Ask p4 for help on this command. If it returns an error, the 125 command does not exist in this version of p4.""" 126 real_cmd =p4_build_cmd(["help", cmd]) 127 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 128 stderr=subprocess.PIPE) 129 p.communicate() 130return p.returncode ==0 131 132defsystem(cmd): 133 expand =isinstance(cmd,basestring) 134if verbose: 135 sys.stderr.write("executing%s\n"%str(cmd)) 136 subprocess.check_call(cmd, shell=expand) 137 138defp4_system(cmd): 139"""Specifically invoke p4 as the system command. """ 140 real_cmd =p4_build_cmd(cmd) 141 expand =isinstance(real_cmd, basestring) 142 subprocess.check_call(real_cmd, shell=expand) 143 144defp4_integrate(src, dest): 145p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 146 147defp4_sync(f, *options): 148p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 149 150defp4_add(f): 151# forcibly add file names with wildcards 152ifwildcard_present(f): 153p4_system(["add","-f", f]) 154else: 155p4_system(["add", f]) 156 157defp4_delete(f): 158p4_system(["delete",wildcard_encode(f)]) 159 160defp4_edit(f): 161p4_system(["edit",wildcard_encode(f)]) 162 163defp4_revert(f): 164p4_system(["revert",wildcard_encode(f)]) 165 166defp4_reopen(type, f): 167p4_system(["reopen","-t",type,wildcard_encode(f)]) 168 169defp4_move(src, dest): 170p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 171 172# 173# Canonicalize the p4 type and return a tuple of the 174# base type, plus any modifiers. See "p4 help filetypes" 175# for a list and explanation. 176# 177defsplit_p4_type(p4type): 178 179 p4_filetypes_historical = { 180"ctempobj":"binary+Sw", 181"ctext":"text+C", 182"cxtext":"text+Cx", 183"ktext":"text+k", 184"kxtext":"text+kx", 185"ltext":"text+F", 186"tempobj":"binary+FSw", 187"ubinary":"binary+F", 188"uresource":"resource+F", 189"uxbinary":"binary+Fx", 190"xbinary":"binary+x", 191"xltext":"text+Fx", 192"xtempobj":"binary+Swx", 193"xtext":"text+x", 194"xunicode":"unicode+x", 195"xutf16":"utf16+x", 196} 197if p4type in p4_filetypes_historical: 198 p4type = p4_filetypes_historical[p4type] 199 mods ="" 200 s = p4type.split("+") 201 base = s[0] 202 mods ="" 203iflen(s) >1: 204 mods = s[1] 205return(base, mods) 206 207# 208# return the raw p4 type of a file (text, text+ko, etc) 209# 210defp4_type(file): 211 results =p4CmdList(["fstat","-T","headType",file]) 212return results[0]['headType'] 213 214# 215# Given a type base and modifier, return a regexp matching 216# the keywords that can be expanded in the file 217# 218defp4_keywords_regexp_for_type(base, type_mods): 219if base in("text","unicode","binary"): 220 kwords =None 221if"ko"in type_mods: 222 kwords ='Id|Header' 223elif"k"in type_mods: 224 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 225else: 226return None 227 pattern = r""" 228 \$ # Starts with a dollar, followed by... 229 (%s) # one of the keywords, followed by... 230 (:[^$]+)? # possibly an old expansion, followed by... 231 \$ # another dollar 232 """% kwords 233return pattern 234else: 235return None 236 237# 238# Given a file, return a regexp matching the possible 239# RCS keywords that will be expanded, or None for files 240# with kw expansion turned off. 241# 242defp4_keywords_regexp_for_file(file): 243if not os.path.exists(file): 244return None 245else: 246(type_base, type_mods) =split_p4_type(p4_type(file)) 247returnp4_keywords_regexp_for_type(type_base, type_mods) 248 249defsetP4ExecBit(file, mode): 250# Reopens an already open file and changes the execute bit to match 251# the execute bit setting in the passed in mode. 252 253 p4Type ="+x" 254 255if notisModeExec(mode): 256 p4Type =getP4OpenedType(file) 257 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 258 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 259if p4Type[-1] =="+": 260 p4Type = p4Type[0:-1] 261 262p4_reopen(p4Type,file) 263 264defgetP4OpenedType(file): 265# Returns the perforce file type for the given file. 266 267 result =p4_read_pipe(["opened",wildcard_encode(file)]) 268 match = re.match(".*\((.+)\)\r?$", result) 269if match: 270return match.group(1) 271else: 272die("Could not determine file type for%s(result: '%s')"% (file, result)) 273 274# Return the set of all p4 labels 275defgetP4Labels(depotPaths): 276 labels =set() 277ifisinstance(depotPaths,basestring): 278 depotPaths = [depotPaths] 279 280for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 281 label = l['label'] 282 labels.add(label) 283 284return labels 285 286# Return the set of all git tags 287defgetGitTags(): 288 gitTags =set() 289for line inread_pipe_lines(["git","tag"]): 290 tag = line.strip() 291 gitTags.add(tag) 292return gitTags 293 294defdiffTreePattern(): 295# This is a simple generator for the diff tree regex pattern. This could be 296# a class variable if this and parseDiffTreeEntry were a part of a class. 297 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 298while True: 299yield pattern 300 301defparseDiffTreeEntry(entry): 302"""Parses a single diff tree entry into its component elements. 303 304 See git-diff-tree(1) manpage for details about the format of the diff 305 output. This method returns a dictionary with the following elements: 306 307 src_mode - The mode of the source file 308 dst_mode - The mode of the destination file 309 src_sha1 - The sha1 for the source file 310 dst_sha1 - The sha1 fr the destination file 311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 312 status_score - The score for the status (applicable for 'C' and 'R' 313 statuses). This is None if there is no score. 314 src - The path for the source file. 315 dst - The path for the destination file. This is only present for 316 copy or renames. If it is not present, this is None. 317 318 If the pattern is not matched, None is returned.""" 319 320 match =diffTreePattern().next().match(entry) 321if match: 322return{ 323'src_mode': match.group(1), 324'dst_mode': match.group(2), 325'src_sha1': match.group(3), 326'dst_sha1': match.group(4), 327'status': match.group(5), 328'status_score': match.group(6), 329'src': match.group(7), 330'dst': match.group(10) 331} 332return None 333 334defisModeExec(mode): 335# Returns True if the given git mode represents an executable file, 336# otherwise False. 337return mode[-3:] =="755" 338 339defisModeExecChanged(src_mode, dst_mode): 340returnisModeExec(src_mode) !=isModeExec(dst_mode) 341 342defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 343 344ifisinstance(cmd,basestring): 345 cmd ="-G "+ cmd 346 expand =True 347else: 348 cmd = ["-G"] + cmd 349 expand =False 350 351 cmd =p4_build_cmd(cmd) 352if verbose: 353 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 354 355# Use a temporary file to avoid deadlocks without 356# subprocess.communicate(), which would put another copy 357# of stdout into memory. 358 stdin_file =None 359if stdin is not None: 360 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 361ifisinstance(stdin,basestring): 362 stdin_file.write(stdin) 363else: 364for i in stdin: 365 stdin_file.write(i +'\n') 366 stdin_file.flush() 367 stdin_file.seek(0) 368 369 p4 = subprocess.Popen(cmd, 370 shell=expand, 371 stdin=stdin_file, 372 stdout=subprocess.PIPE) 373 374 result = [] 375try: 376while True: 377 entry = marshal.load(p4.stdout) 378if cb is not None: 379cb(entry) 380else: 381 result.append(entry) 382exceptEOFError: 383pass 384 exitCode = p4.wait() 385if exitCode !=0: 386 entry = {} 387 entry["p4ExitCode"] = exitCode 388 result.append(entry) 389 390return result 391 392defp4Cmd(cmd): 393list=p4CmdList(cmd) 394 result = {} 395for entry inlist: 396 result.update(entry) 397return result; 398 399defp4Where(depotPath): 400if not depotPath.endswith("/"): 401 depotPath +="/" 402 depotPath = depotPath +"..." 403 outputList =p4CmdList(["where", depotPath]) 404 output =None 405for entry in outputList: 406if"depotFile"in entry: 407if entry["depotFile"] == depotPath: 408 output = entry 409break 410elif"data"in entry: 411 data = entry.get("data") 412 space = data.find(" ") 413if data[:space] == depotPath: 414 output = entry 415break 416if output ==None: 417return"" 418if output["code"] =="error": 419return"" 420 clientPath ="" 421if"path"in output: 422 clientPath = output.get("path") 423elif"data"in output: 424 data = output.get("data") 425 lastSpace = data.rfind(" ") 426 clientPath = data[lastSpace +1:] 427 428if clientPath.endswith("..."): 429 clientPath = clientPath[:-3] 430return clientPath 431 432defcurrentGitBranch(): 433returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 434 435defisValidGitDir(path): 436if(os.path.exists(path +"/HEAD") 437and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 438return True; 439return False 440 441defparseRevision(ref): 442returnread_pipe("git rev-parse%s"% ref).strip() 443 444defbranchExists(ref): 445 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 446 ignore_error=True) 447returnlen(rev) >0 448 449defextractLogMessageFromGitCommit(commit): 450 logMessage ="" 451 452## fixme: title is first line of commit, not 1st paragraph. 453 foundTitle =False 454for log inread_pipe_lines("git cat-file commit%s"% commit): 455if not foundTitle: 456iflen(log) ==1: 457 foundTitle =True 458continue 459 460 logMessage += log 461return logMessage 462 463defextractSettingsGitLog(log): 464 values = {} 465for line in log.split("\n"): 466 line = line.strip() 467 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 468if not m: 469continue 470 471 assignments = m.group(1).split(':') 472for a in assignments: 473 vals = a.split('=') 474 key = vals[0].strip() 475 val = ('='.join(vals[1:])).strip() 476if val.endswith('\"')and val.startswith('"'): 477 val = val[1:-1] 478 479 values[key] = val 480 481 paths = values.get("depot-paths") 482if not paths: 483 paths = values.get("depot-path") 484if paths: 485 values['depot-paths'] = paths.split(',') 486return values 487 488defgitBranchExists(branch): 489 proc = subprocess.Popen(["git","rev-parse", branch], 490 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 491return proc.wait() ==0; 492 493_gitConfig = {} 494defgitConfig(key, args =None):# set args to "--bool", for instance 495if not _gitConfig.has_key(key): 496 argsFilter ="" 497if args !=None: 498 argsFilter ="%s"% args 499 cmd ="git config%s%s"% (argsFilter, key) 500 _gitConfig[key] =read_pipe(cmd, ignore_error=True).strip() 501return _gitConfig[key] 502 503defgitConfigList(key): 504if not _gitConfig.has_key(key): 505 _gitConfig[key] =read_pipe("git config --get-all%s"% key, ignore_error=True).strip().split(os.linesep) 506return _gitConfig[key] 507 508defp4BranchesInGit(branchesAreInRemotes =True): 509 branches = {} 510 511 cmdline ="git rev-parse --symbolic " 512if branchesAreInRemotes: 513 cmdline +=" --remotes" 514else: 515 cmdline +=" --branches" 516 517for line inread_pipe_lines(cmdline): 518 line = line.strip() 519 520## only import to p4/ 521if not line.startswith('p4/')or line =="p4/HEAD": 522continue 523 branch = line 524 525# strip off p4 526 branch = re.sub("^p4/","", line) 527 528 branches[branch] =parseRevision(line) 529return branches 530 531deffindUpstreamBranchPoint(head ="HEAD"): 532 branches =p4BranchesInGit() 533# map from depot-path to branch name 534 branchByDepotPath = {} 535for branch in branches.keys(): 536 tip = branches[branch] 537 log =extractLogMessageFromGitCommit(tip) 538 settings =extractSettingsGitLog(log) 539if settings.has_key("depot-paths"): 540 paths =",".join(settings["depot-paths"]) 541 branchByDepotPath[paths] ="remotes/p4/"+ branch 542 543 settings =None 544 parent =0 545while parent <65535: 546 commit = head +"~%s"% parent 547 log =extractLogMessageFromGitCommit(commit) 548 settings =extractSettingsGitLog(log) 549if settings.has_key("depot-paths"): 550 paths =",".join(settings["depot-paths"]) 551if branchByDepotPath.has_key(paths): 552return[branchByDepotPath[paths], settings] 553 554 parent = parent +1 555 556return["", settings] 557 558defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 559if not silent: 560print("Creating/updating branch(es) in%sbased on origin branch(es)" 561% localRefPrefix) 562 563 originPrefix ="origin/p4/" 564 565for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 566 line = line.strip() 567if(not line.startswith(originPrefix))or line.endswith("HEAD"): 568continue 569 570 headName = line[len(originPrefix):] 571 remoteHead = localRefPrefix + headName 572 originHead = line 573 574 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 575if(not original.has_key('depot-paths') 576or not original.has_key('change')): 577continue 578 579 update =False 580if notgitBranchExists(remoteHead): 581if verbose: 582print"creating%s"% remoteHead 583 update =True 584else: 585 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 586if settings.has_key('change') >0: 587if settings['depot-paths'] == original['depot-paths']: 588 originP4Change =int(original['change']) 589 p4Change =int(settings['change']) 590if originP4Change > p4Change: 591print("%s(%s) is newer than%s(%s). " 592"Updating p4 branch from origin." 593% (originHead, originP4Change, 594 remoteHead, p4Change)) 595 update =True 596else: 597print("Ignoring:%swas imported from%swhile " 598"%swas imported from%s" 599% (originHead,','.join(original['depot-paths']), 600 remoteHead,','.join(settings['depot-paths']))) 601 602if update: 603system("git update-ref%s %s"% (remoteHead, originHead)) 604 605deforiginP4BranchesExist(): 606returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 607 608defp4ChangesForPaths(depotPaths, changeRange): 609assert depotPaths 610 cmd = ['changes'] 611for p in depotPaths: 612 cmd += ["%s...%s"% (p, changeRange)] 613 output =p4_read_pipe_lines(cmd) 614 615 changes = {} 616for line in output: 617 changeNum =int(line.split(" ")[1]) 618 changes[changeNum] =True 619 620 changelist = changes.keys() 621 changelist.sort() 622return changelist 623 624defp4PathStartsWith(path, prefix): 625# This method tries to remedy a potential mixed-case issue: 626# 627# If UserA adds //depot/DirA/file1 628# and UserB adds //depot/dira/file2 629# 630# we may or may not have a problem. If you have core.ignorecase=true, 631# we treat DirA and dira as the same directory 632 ignorecase =gitConfig("core.ignorecase","--bool") =="true" 633if ignorecase: 634return path.lower().startswith(prefix.lower()) 635return path.startswith(prefix) 636 637defgetClientSpec(): 638"""Look at the p4 client spec, create a View() object that contains 639 all the mappings, and return it.""" 640 641 specList =p4CmdList("client -o") 642iflen(specList) !=1: 643die('Output from "client -o" is%dlines, expecting 1'% 644len(specList)) 645 646# dictionary of all client parameters 647 entry = specList[0] 648 649# just the keys that start with "View" 650 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 651 652# hold this new View 653 view =View() 654 655# append the lines, in order, to the view 656for view_num inrange(len(view_keys)): 657 k ="View%d"% view_num 658if k not in view_keys: 659die("Expected view key%smissing"% k) 660 view.append(entry[k]) 661 662return view 663 664defgetClientRoot(): 665"""Grab the client directory.""" 666 667 output =p4CmdList("client -o") 668iflen(output) !=1: 669die('Output from "client -o" is%dlines, expecting 1'%len(output)) 670 671 entry = output[0] 672if"Root"not in entry: 673die('Client has no "Root"') 674 675return entry["Root"] 676 677# 678# P4 wildcards are not allowed in filenames. P4 complains 679# if you simply add them, but you can force it with "-f", in 680# which case it translates them into %xx encoding internally. 681# 682defwildcard_decode(path): 683# Search for and fix just these four characters. Do % last so 684# that fixing it does not inadvertently create new %-escapes. 685# Cannot have * in a filename in windows; untested as to 686# what p4 would do in such a case. 687if not platform.system() =="Windows": 688 path = path.replace("%2A","*") 689 path = path.replace("%23","#") \ 690.replace("%40","@") \ 691.replace("%25","%") 692return path 693 694defwildcard_encode(path): 695# do % first to avoid double-encoding the %s introduced here 696 path = path.replace("%","%25") \ 697.replace("*","%2A") \ 698.replace("#","%23") \ 699.replace("@","%40") 700return path 701 702defwildcard_present(path): 703return path.translate(None,"*#@%") != path 704 705class Command: 706def__init__(self): 707 self.usage ="usage: %prog [options]" 708 self.needsGit =True 709 self.verbose =False 710 711class P4UserMap: 712def__init__(self): 713 self.userMapFromPerforceServer =False 714 self.myP4UserId =None 715 716defp4UserId(self): 717if self.myP4UserId: 718return self.myP4UserId 719 720 results =p4CmdList("user -o") 721for r in results: 722if r.has_key('User'): 723 self.myP4UserId = r['User'] 724return r['User'] 725die("Could not find your p4 user id") 726 727defp4UserIsMe(self, p4User): 728# return True if the given p4 user is actually me 729 me = self.p4UserId() 730if not p4User or p4User != me: 731return False 732else: 733return True 734 735defgetUserCacheFilename(self): 736 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 737return home +"/.gitp4-usercache.txt" 738 739defgetUserMapFromPerforceServer(self): 740if self.userMapFromPerforceServer: 741return 742 self.users = {} 743 self.emails = {} 744 745for output inp4CmdList("users"): 746if not output.has_key("User"): 747continue 748 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 749 self.emails[output["Email"]] = output["User"] 750 751 752 s ='' 753for(key, val)in self.users.items(): 754 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 755 756open(self.getUserCacheFilename(),"wb").write(s) 757 self.userMapFromPerforceServer =True 758 759defloadUserMapFromCache(self): 760 self.users = {} 761 self.userMapFromPerforceServer =False 762try: 763 cache =open(self.getUserCacheFilename(),"rb") 764 lines = cache.readlines() 765 cache.close() 766for line in lines: 767 entry = line.strip().split("\t") 768 self.users[entry[0]] = entry[1] 769exceptIOError: 770 self.getUserMapFromPerforceServer() 771 772classP4Debug(Command): 773def__init__(self): 774 Command.__init__(self) 775 self.options = [] 776 self.description ="A tool to debug the output of p4 -G." 777 self.needsGit =False 778 779defrun(self, args): 780 j =0 781for output inp4CmdList(args): 782print'Element:%d'% j 783 j +=1 784print output 785return True 786 787classP4RollBack(Command): 788def__init__(self): 789 Command.__init__(self) 790 self.options = [ 791 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") 792] 793 self.description ="A tool to debug the multi-branch import. Don't use :)" 794 self.rollbackLocalBranches =False 795 796defrun(self, args): 797iflen(args) !=1: 798return False 799 maxChange =int(args[0]) 800 801if"p4ExitCode"inp4Cmd("changes -m 1"): 802die("Problems executing p4"); 803 804if self.rollbackLocalBranches: 805 refPrefix ="refs/heads/" 806 lines =read_pipe_lines("git rev-parse --symbolic --branches") 807else: 808 refPrefix ="refs/remotes/" 809 lines =read_pipe_lines("git rev-parse --symbolic --remotes") 810 811for line in lines: 812if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"): 813 line = line.strip() 814 ref = refPrefix + line 815 log =extractLogMessageFromGitCommit(ref) 816 settings =extractSettingsGitLog(log) 817 818 depotPaths = settings['depot-paths'] 819 change = settings['change'] 820 821 changed =False 822 823iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange) 824for p in depotPaths]))) ==0: 825print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange) 826system("git update-ref -d%s`git rev-parse%s`"% (ref, ref)) 827continue 828 829while change andint(change) > maxChange: 830 changed =True 831if self.verbose: 832print"%sis at%s; rewinding towards%s"% (ref, change, maxChange) 833system("git update-ref%s\"%s^\""% (ref, ref)) 834 log =extractLogMessageFromGitCommit(ref) 835 settings =extractSettingsGitLog(log) 836 837 838 depotPaths = settings['depot-paths'] 839 change = settings['change'] 840 841if changed: 842print"%srewound to%s"% (ref, change) 843 844return True 845 846classP4Submit(Command, P4UserMap): 847def__init__(self): 848 Command.__init__(self) 849 P4UserMap.__init__(self) 850 self.options = [ 851 optparse.make_option("--origin", dest="origin"), 852 optparse.make_option("-M", dest="detectRenames", action="store_true"), 853# preserve the user, requires relevant p4 permissions 854 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), 855 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"), 856] 857 self.description ="Submit changes from git to the perforce depot." 858 self.usage +=" [name of git branch to submit into perforce depot]" 859 self.origin ="" 860 self.detectRenames =False 861 self.preserveUser =gitConfig("git-p4.preserveUser").lower() =="true" 862 self.isWindows = (platform.system() =="Windows") 863 self.exportLabels =False 864 self.p4HasMoveCommand =p4_has_command("move") 865 866defcheck(self): 867iflen(p4CmdList("opened ...")) >0: 868die("You have files opened with perforce! Close them before starting the sync.") 869 870defseparate_jobs_from_description(self, message): 871"""Extract and return a possible Jobs field in the commit 872 message. It goes into a separate section in the p4 change 873 specification. 874 875 A jobs line starts with "Jobs:" and looks like a new field 876 in a form. Values are white-space separated on the same 877 line or on following lines that start with a tab. 878 879 This does not parse and extract the full git commit message 880 like a p4 form. It just sees the Jobs: line as a marker 881 to pass everything from then on directly into the p4 form, 882 but outside the description section. 883 884 Return a tuple (stripped log message, jobs string).""" 885 886 m = re.search(r'^Jobs:', message, re.MULTILINE) 887if m is None: 888return(message,None) 889 890 jobtext = message[m.start():] 891 stripped_message = message[:m.start()].rstrip() 892return(stripped_message, jobtext) 893 894defprepareLogMessage(self, template, message, jobs): 895"""Edits the template returned from "p4 change -o" to insert 896 the message in the Description field, and the jobs text in 897 the Jobs field.""" 898 result ="" 899 900 inDescriptionSection =False 901 902for line in template.split("\n"): 903if line.startswith("#"): 904 result += line +"\n" 905continue 906 907if inDescriptionSection: 908if line.startswith("Files:")or line.startswith("Jobs:"): 909 inDescriptionSection =False 910# insert Jobs section 911if jobs: 912 result += jobs +"\n" 913else: 914continue 915else: 916if line.startswith("Description:"): 917 inDescriptionSection =True 918 line +="\n" 919for messageLine in message.split("\n"): 920 line +="\t"+ messageLine +"\n" 921 922 result += line +"\n" 923 924return result 925 926defpatchRCSKeywords(self,file, pattern): 927# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern 928(handle, outFileName) = tempfile.mkstemp(dir='.') 929try: 930 outFile = os.fdopen(handle,"w+") 931 inFile =open(file,"r") 932 regexp = re.compile(pattern, re.VERBOSE) 933for line in inFile.readlines(): 934 line = regexp.sub(r'$\1$', line) 935 outFile.write(line) 936 inFile.close() 937 outFile.close() 938# Forcibly overwrite the original file 939 os.unlink(file) 940 shutil.move(outFileName,file) 941except: 942# cleanup our temporary file 943 os.unlink(outFileName) 944print"Failed to strip RCS keywords in%s"%file 945raise 946 947print"Patched up RCS keywords in%s"%file 948 949defp4UserForCommit(self,id): 950# Return the tuple (perforce user,git email) for a given git commit id 951 self.getUserMapFromPerforceServer() 952 gitEmail =read_pipe("git log --max-count=1 --format='%%ae'%s"%id) 953 gitEmail = gitEmail.strip() 954if not self.emails.has_key(gitEmail): 955return(None,gitEmail) 956else: 957return(self.emails[gitEmail],gitEmail) 958 959defcheckValidP4Users(self,commits): 960# check if any git authors cannot be mapped to p4 users 961foridin commits: 962(user,email) = self.p4UserForCommit(id) 963if not user: 964 msg ="Cannot find p4 user for email%sin commit%s."% (email,id) 965ifgitConfig('git-p4.allowMissingP4Users').lower() =="true": 966print"%s"% msg 967else: 968die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg) 969 970deflastP4Changelist(self): 971# Get back the last changelist number submitted in this client spec. This 972# then gets used to patch up the username in the change. If the same 973# client spec is being used by multiple processes then this might go 974# wrong. 975 results =p4CmdList("client -o")# find the current client 976 client =None 977for r in results: 978if r.has_key('Client'): 979 client = r['Client'] 980break 981if not client: 982die("could not get client spec") 983 results =p4CmdList(["changes","-c", client,"-m","1"]) 984for r in results: 985if r.has_key('change'): 986return r['change'] 987die("Could not get changelist number for last submit - cannot patch up user details") 988 989defmodifyChangelistUser(self, changelist, newUser): 990# fixup the user field of a changelist after it has been submitted. 991 changes =p4CmdList("change -o%s"% changelist) 992iflen(changes) !=1: 993die("Bad output from p4 change modifying%sto user%s"% 994(changelist, newUser)) 995 996 c = changes[0] 997if c['User'] == newUser:return# nothing to do 998 c['User'] = newUser 999input= marshal.dumps(c)10001001 result =p4CmdList("change -f -i", stdin=input)1002for r in result:1003if r.has_key('code'):1004if r['code'] =='error':1005die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1006if r.has_key('data'):1007print("Updated user field for changelist%sto%s"% (changelist, newUser))1008return1009die("Could not modify user field of changelist%sto%s"% (changelist, newUser))10101011defcanChangeChangelists(self):1012# check to see if we have p4 admin or super-user permissions, either of1013# which are required to modify changelists.1014 results =p4CmdList(["protects", self.depotPath])1015for r in results:1016if r.has_key('perm'):1017if r['perm'] =='admin':1018return11019if r['perm'] =='super':1020return11021return010221023defprepareSubmitTemplate(self):1024"""Run "p4 change -o" to grab a change specification template.1025 This does not use "p4 -G", as it is nice to keep the submission1026 template in original order, since a human might edit it.10271028 Remove lines in the Files section that show changes to files1029 outside the depot path we're committing into."""10301031 template =""1032 inFilesSection =False1033for line inp4_read_pipe_lines(['change','-o']):1034if line.endswith("\r\n"):1035 line = line[:-2] +"\n"1036if inFilesSection:1037if line.startswith("\t"):1038# path starts and ends with a tab1039 path = line[1:]1040 lastTab = path.rfind("\t")1041if lastTab != -1:1042 path = path[:lastTab]1043if notp4PathStartsWith(path, self.depotPath):1044continue1045else:1046 inFilesSection =False1047else:1048if line.startswith("Files:"):1049 inFilesSection =True10501051 template += line10521053return template10541055defedit_template(self, template_file):1056"""Invoke the editor to let the user change the submission1057 message. Return true if okay to continue with the submit."""10581059# if configured to skip the editing part, just submit1060ifgitConfig("git-p4.skipSubmitEdit") =="true":1061return True10621063# look at the modification time, to check later if the user saved1064# the file1065 mtime = os.stat(template_file).st_mtime10661067# invoke the editor1068if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1069 editor = os.environ.get("P4EDITOR")1070else:1071 editor =read_pipe("git var GIT_EDITOR").strip()1072system(editor +" "+ template_file)10731074# If the file was not saved, prompt to see if this patch should1075# be skipped. But skip this verification step if configured so.1076ifgitConfig("git-p4.skipSubmitEditCheck") =="true":1077return True10781079# modification time updated means user saved the file1080if os.stat(template_file).st_mtime > mtime:1081return True10821083while True:1084 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1085if response =='y':1086return True1087if response =='n':1088return False10891090defapplyCommit(self,id):1091print"Applying%s"% (read_pipe("git log --max-count=1 --pretty=oneline%s"%id))10921093(p4User, gitEmail) = self.p4UserForCommit(id)10941095 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1096 filesToAdd =set()1097 filesToDelete =set()1098 editedFiles =set()1099 pureRenameCopy =set()1100 filesToChangeExecBit = {}11011102for line in diff:1103 diff =parseDiffTreeEntry(line)1104 modifier = diff['status']1105 path = diff['src']1106if modifier =="M":1107p4_edit(path)1108ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1109 filesToChangeExecBit[path] = diff['dst_mode']1110 editedFiles.add(path)1111elif modifier =="A":1112 filesToAdd.add(path)1113 filesToChangeExecBit[path] = diff['dst_mode']1114if path in filesToDelete:1115 filesToDelete.remove(path)1116elif modifier =="D":1117 filesToDelete.add(path)1118if path in filesToAdd:1119 filesToAdd.remove(path)1120elif modifier =="C":1121 src, dest = diff['src'], diff['dst']1122p4_integrate(src, dest)1123 pureRenameCopy.add(dest)1124if diff['src_sha1'] != diff['dst_sha1']:1125p4_edit(dest)1126 pureRenameCopy.discard(dest)1127ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1128p4_edit(dest)1129 pureRenameCopy.discard(dest)1130 filesToChangeExecBit[dest] = diff['dst_mode']1131 os.unlink(dest)1132 editedFiles.add(dest)1133elif modifier =="R":1134 src, dest = diff['src'], diff['dst']1135if self.p4HasMoveCommand:1136p4_edit(src)# src must be open before move1137p4_move(src, dest)# opens for (move/delete, move/add)1138else:1139p4_integrate(src, dest)1140if diff['src_sha1'] != diff['dst_sha1']:1141p4_edit(dest)1142else:1143 pureRenameCopy.add(dest)1144ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1145if not self.p4HasMoveCommand:1146p4_edit(dest)# with move: already open, writable1147 filesToChangeExecBit[dest] = diff['dst_mode']1148if not self.p4HasMoveCommand:1149 os.unlink(dest)1150 filesToDelete.add(src)1151 editedFiles.add(dest)1152else:1153die("unknown modifier%sfor%s"% (modifier, path))11541155 diffcmd ="git format-patch -k --stdout\"%s^\"..\"%s\""% (id,id)1156 patchcmd = diffcmd +" | git apply "1157 tryPatchCmd = patchcmd +"--check -"1158 applyPatchCmd = patchcmd +"--check --apply -"1159 patch_succeeded =True11601161if os.system(tryPatchCmd) !=0:1162 fixed_rcs_keywords =False1163 patch_succeeded =False1164print"Unfortunately applying the change failed!"11651166# Patch failed, maybe it's just RCS keyword woes. Look through1167# the patch to see if that's possible.1168ifgitConfig("git-p4.attemptRCSCleanup","--bool") =="true":1169file=None1170 pattern =None1171 kwfiles = {}1172forfilein editedFiles | filesToDelete:1173# did this file's delta contain RCS keywords?1174 pattern =p4_keywords_regexp_for_file(file)11751176if pattern:1177# this file is a possibility...look for RCS keywords.1178 regexp = re.compile(pattern, re.VERBOSE)1179for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1180if regexp.search(line):1181if verbose:1182print"got keyword match on%sin%sin%s"% (pattern, line,file)1183 kwfiles[file] = pattern1184break11851186forfilein kwfiles:1187if verbose:1188print"zapping%swith%s"% (line,pattern)1189 self.patchRCSKeywords(file, kwfiles[file])1190 fixed_rcs_keywords =True11911192if fixed_rcs_keywords:1193print"Retrying the patch with RCS keywords cleaned up"1194if os.system(tryPatchCmd) ==0:1195 patch_succeeded =True11961197if not patch_succeeded:1198print"What do you want to do?"1199 response ="x"1200while response !="s"and response !="a"and response !="w":1201 response =raw_input("[s]kip this patch / [a]pply the patch forcibly "1202"and with .rej files / [w]rite the patch to a file (patch.txt) ")1203if response =="s":1204print"Skipping! Good luck with the next patches..."1205for f in editedFiles:1206p4_revert(f)1207for f in filesToAdd:1208 os.remove(f)1209return1210elif response =="a":1211 os.system(applyPatchCmd)1212iflen(filesToAdd) >0:1213print"You may also want to call p4 add on the following files:"1214print" ".join(filesToAdd)1215iflen(filesToDelete):1216print"The following files should be scheduled for deletion with p4 delete:"1217print" ".join(filesToDelete)1218die("Please resolve and submit the conflict manually and "1219+"continue afterwards with git p4 submit --continue")1220elif response =="w":1221system(diffcmd +" > patch.txt")1222print"Patch saved to patch.txt in%s!"% self.clientPath1223die("Please resolve and submit the conflict manually and "1224"continue afterwards with git p4 submit --continue")12251226system(applyPatchCmd)12271228for f in filesToAdd:1229p4_add(f)1230for f in filesToDelete:1231p4_revert(f)1232p4_delete(f)12331234# Set/clear executable bits1235for f in filesToChangeExecBit.keys():1236 mode = filesToChangeExecBit[f]1237setP4ExecBit(f, mode)12381239 logMessage =extractLogMessageFromGitCommit(id)1240 logMessage = logMessage.strip()1241(logMessage, jobs) = self.separate_jobs_from_description(logMessage)12421243 template = self.prepareSubmitTemplate()1244 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)12451246if self.preserveUser:1247 submitTemplate = submitTemplate + ("\n######## Actual user%s, modified after commit\n"% p4User)12481249if os.environ.has_key("P4DIFF"):1250del(os.environ["P4DIFF"])1251 diff =""1252for editedFile in editedFiles:1253 diff +=p4_read_pipe(['diff','-du',1254wildcard_encode(editedFile)])12551256 newdiff =""1257for newFile in filesToAdd:1258 newdiff +="==== new file ====\n"1259 newdiff +="--- /dev/null\n"1260 newdiff +="+++%s\n"% newFile1261 f =open(newFile,"r")1262for line in f.readlines():1263 newdiff +="+"+ line1264 f.close()12651266if self.checkAuthorship and not self.p4UserIsMe(p4User):1267 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1268 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1269 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"12701271 separatorLine ="######## everything below this line is just the diff #######\n"12721273(handle, fileName) = tempfile.mkstemp()1274 tmpFile = os.fdopen(handle,"w+")1275if self.isWindows:1276 submitTemplate = submitTemplate.replace("\n","\r\n")1277 separatorLine = separatorLine.replace("\n","\r\n")1278 newdiff = newdiff.replace("\n","\r\n")1279 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)1280 tmpFile.close()12811282if self.edit_template(fileName):1283# read the edited message and submit1284 tmpFile =open(fileName,"rb")1285 message = tmpFile.read()1286 tmpFile.close()1287 submitTemplate = message[:message.index(separatorLine)]1288if self.isWindows:1289 submitTemplate = submitTemplate.replace("\r\n","\n")1290p4_write_pipe(['submit','-i'], submitTemplate)12911292if self.preserveUser:1293if p4User:1294# Get last changelist number. Cannot easily get it from1295# the submit command output as the output is1296# unmarshalled.1297 changelist = self.lastP4Changelist()1298 self.modifyChangelistUser(changelist, p4User)12991300# The rename/copy happened by applying a patch that created a1301# new file. This leaves it writable, which confuses p4.1302for f in pureRenameCopy:1303p4_sync(f,"-f")13041305else:1306# skip this patch1307print"Submission cancelled, undoing p4 changes."1308for f in editedFiles:1309p4_revert(f)1310for f in filesToAdd:1311p4_revert(f)1312 os.remove(f)13131314 os.remove(fileName)13151316# Export git tags as p4 labels. Create a p4 label and then tag1317# with that.1318defexportGitTags(self, gitTags):1319 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1320iflen(validLabelRegexp) ==0:1321 validLabelRegexp = defaultLabelRegexp1322 m = re.compile(validLabelRegexp)13231324for name in gitTags:13251326if not m.match(name):1327if verbose:1328print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1329continue13301331# Get the p4 commit this corresponds to1332 logMessage =extractLogMessageFromGitCommit(name)1333 values =extractSettingsGitLog(logMessage)13341335if not values.has_key('change'):1336# a tag pointing to something not sent to p4; ignore1337if verbose:1338print"git tag%sdoes not give a p4 commit"% name1339continue1340else:1341 changelist = values['change']13421343# Get the tag details.1344 inHeader =True1345 isAnnotated =False1346 body = []1347for l inread_pipe_lines(["git","cat-file","-p", name]):1348 l = l.strip()1349if inHeader:1350if re.match(r'tag\s+', l):1351 isAnnotated =True1352elif re.match(r'\s*$', l):1353 inHeader =False1354continue1355else:1356 body.append(l)13571358if not isAnnotated:1359 body = ["lightweight tag imported by git p4\n"]13601361# Create the label - use the same view as the client spec we are using1362 clientSpec =getClientSpec()13631364 labelTemplate ="Label:%s\n"% name1365 labelTemplate +="Description:\n"1366for b in body:1367 labelTemplate +="\t"+ b +"\n"1368 labelTemplate +="View:\n"1369for mapping in clientSpec.mappings:1370 labelTemplate +="\t%s\n"% mapping.depot_side.path13711372p4_write_pipe(["label","-i"], labelTemplate)13731374# Use the label1375p4_system(["tag","-l", name] +1376["%s@%s"% (mapping.depot_side.path, changelist)for mapping in clientSpec.mappings])13771378if verbose:1379print"created p4 label for tag%s"% name13801381defrun(self, args):1382iflen(args) ==0:1383 self.master =currentGitBranch()1384iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1385die("Detecting current git branch failed!")1386eliflen(args) ==1:1387 self.master = args[0]1388if notbranchExists(self.master):1389die("Branch%sdoes not exist"% self.master)1390else:1391return False13921393 allowSubmit =gitConfig("git-p4.allowSubmit")1394iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1395die("%sis not in git-p4.allowSubmit"% self.master)13961397[upstream, settings] =findUpstreamBranchPoint()1398 self.depotPath = settings['depot-paths'][0]1399iflen(self.origin) ==0:1400 self.origin = upstream14011402if self.preserveUser:1403if not self.canChangeChangelists():1404die("Cannot preserve user names without p4 super-user or admin permissions")14051406if self.verbose:1407print"Origin branch is "+ self.origin14081409iflen(self.depotPath) ==0:1410print"Internal error: cannot locate perforce depot path from existing branches"1411 sys.exit(128)14121413 self.useClientSpec =False1414ifgitConfig("git-p4.useclientspec","--bool") =="true":1415 self.useClientSpec =True1416if self.useClientSpec:1417 self.clientSpecDirs =getClientSpec()14181419if self.useClientSpec:1420# all files are relative to the client spec1421 self.clientPath =getClientRoot()1422else:1423 self.clientPath =p4Where(self.depotPath)14241425if self.clientPath =="":1426die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)14271428print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1429 self.oldWorkingDirectory = os.getcwd()14301431# ensure the clientPath exists1432 new_client_dir =False1433if not os.path.exists(self.clientPath):1434 new_client_dir =True1435 os.makedirs(self.clientPath)14361437chdir(self.clientPath)1438print"Synchronizing p4 checkout..."1439if new_client_dir:1440# old one was destroyed, and maybe nobody told p41441p4_sync("...","-f")1442else:1443p4_sync("...")1444 self.check()14451446 commits = []1447for line inread_pipe_lines("git rev-list --no-merges%s..%s"% (self.origin, self.master)):1448 commits.append(line.strip())1449 commits.reverse()14501451if self.preserveUser or(gitConfig("git-p4.skipUserNameCheck") =="true"):1452 self.checkAuthorship =False1453else:1454 self.checkAuthorship =True14551456if self.preserveUser:1457 self.checkValidP4Users(commits)14581459#1460# Build up a set of options to be passed to diff when1461# submitting each commit to p4.1462#1463if self.detectRenames:1464# command-line -M arg1465 self.diffOpts ="-M"1466else:1467# If not explicitly set check the config variable1468 detectRenames =gitConfig("git-p4.detectRenames")14691470if detectRenames.lower() =="false"or detectRenames =="":1471 self.diffOpts =""1472elif detectRenames.lower() =="true":1473 self.diffOpts ="-M"1474else:1475 self.diffOpts ="-M%s"% detectRenames14761477# no command-line arg for -C or --find-copies-harder, just1478# config variables1479 detectCopies =gitConfig("git-p4.detectCopies")1480if detectCopies.lower() =="false"or detectCopies =="":1481pass1482elif detectCopies.lower() =="true":1483 self.diffOpts +=" -C"1484else:1485 self.diffOpts +=" -C%s"% detectCopies14861487ifgitConfig("git-p4.detectCopiesHarder","--bool") =="true":1488 self.diffOpts +=" --find-copies-harder"14891490whilelen(commits) >0:1491 commit = commits[0]1492 commits = commits[1:]1493 self.applyCommit(commit)14941495iflen(commits) ==0:1496print"All changes applied!"1497chdir(self.oldWorkingDirectory)14981499 sync =P4Sync()1500 sync.run([])15011502 rebase =P4Rebase()1503 rebase.rebase()15041505ifgitConfig("git-p4.exportLabels","--bool") =="true":1506 self.exportLabels =True15071508if self.exportLabels:1509 p4Labels =getP4Labels(self.depotPath)1510 gitTags =getGitTags()15111512 missingGitTags = gitTags - p4Labels1513 self.exportGitTags(missingGitTags)15141515return True15161517classView(object):1518"""Represent a p4 view ("p4 help views"), and map files in a1519 repo according to the view."""15201521classPath(object):1522"""A depot or client path, possibly containing wildcards.1523 The only one supported is ... at the end, currently.1524 Initialize with the full path, with //depot or //client."""15251526def__init__(self, path, is_depot):1527 self.path = path1528 self.is_depot = is_depot1529 self.find_wildcards()1530# remember the prefix bit, useful for relative mappings1531 m = re.match("(//[^/]+/)", self.path)1532if not m:1533die("Path%sdoes not start with //prefix/"% self.path)1534 prefix = m.group(1)1535if not self.is_depot:1536# strip //client/ on client paths1537 self.path = self.path[len(prefix):]15381539deffind_wildcards(self):1540"""Make sure wildcards are valid, and set up internal1541 variables."""15421543 self.ends_triple_dot =False1544# There are three wildcards allowed in p4 views1545# (see "p4 help views"). This code knows how to1546# handle "..." (only at the end), but cannot deal with1547# "%%n" or "*". Only check the depot_side, as p4 should1548# validate that the client_side matches too.1549if re.search(r'%%[1-9]', self.path):1550die("Can't handle%%n wildcards in view:%s"% self.path)1551if self.path.find("*") >=0:1552die("Can't handle * wildcards in view:%s"% self.path)1553 triple_dot_index = self.path.find("...")1554if triple_dot_index >=0:1555if triple_dot_index !=len(self.path) -3:1556die("Can handle only single ... wildcard, at end:%s"%1557 self.path)1558 self.ends_triple_dot =True15591560defensure_compatible(self, other_path):1561"""Make sure the wildcards agree."""1562if self.ends_triple_dot != other_path.ends_triple_dot:1563die("Both paths must end with ... if either does;\n"+1564"paths:%s %s"% (self.path, other_path.path))15651566defmatch_wildcards(self, test_path):1567"""See if this test_path matches us, and fill in the value1568 of the wildcards if so. Returns a tuple of1569 (True|False, wildcards[]). For now, only the ... at end1570 is supported, so at most one wildcard."""1571if self.ends_triple_dot:1572 dotless = self.path[:-3]1573if test_path.startswith(dotless):1574 wildcard = test_path[len(dotless):]1575return(True, [ wildcard ])1576else:1577if test_path == self.path:1578return(True, [])1579return(False, [])15801581defmatch(self, test_path):1582"""Just return if it matches; don't bother with the wildcards."""1583 b, _ = self.match_wildcards(test_path)1584return b15851586deffill_in_wildcards(self, wildcards):1587"""Return the relative path, with the wildcards filled in1588 if there are any."""1589if self.ends_triple_dot:1590return self.path[:-3] + wildcards[0]1591else:1592return self.path15931594classMapping(object):1595def__init__(self, depot_side, client_side, overlay, exclude):1596# depot_side is without the trailing /... if it had one1597 self.depot_side = View.Path(depot_side, is_depot=True)1598 self.client_side = View.Path(client_side, is_depot=False)1599 self.overlay = overlay # started with "+"1600 self.exclude = exclude # started with "-"1601assert not(self.overlay and self.exclude)1602 self.depot_side.ensure_compatible(self.client_side)16031604def__str__(self):1605 c =" "1606if self.overlay:1607 c ="+"1608if self.exclude:1609 c ="-"1610return"View.Mapping:%s%s->%s"% \1611(c, self.depot_side.path, self.client_side.path)16121613defmap_depot_to_client(self, depot_path):1614"""Calculate the client path if using this mapping on the1615 given depot path; does not consider the effect of other1616 mappings in a view. Even excluded mappings are returned."""1617 matches, wildcards = self.depot_side.match_wildcards(depot_path)1618if not matches:1619return""1620 client_path = self.client_side.fill_in_wildcards(wildcards)1621return client_path16221623#1624# View methods1625#1626def__init__(self):1627 self.mappings = []16281629defappend(self, view_line):1630"""Parse a view line, splitting it into depot and client1631 sides. Append to self.mappings, preserving order."""16321633# Split the view line into exactly two words. P4 enforces1634# structure on these lines that simplifies this quite a bit.1635#1636# Either or both words may be double-quoted.1637# Single quotes do not matter.1638# Double-quote marks cannot occur inside the words.1639# A + or - prefix is also inside the quotes.1640# There are no quotes unless they contain a space.1641# The line is already white-space stripped.1642# The two words are separated by a single space.1643#1644if view_line[0] =='"':1645# First word is double quoted. Find its end.1646 close_quote_index = view_line.find('"',1)1647if close_quote_index <=0:1648die("No first-word closing quote found:%s"% view_line)1649 depot_side = view_line[1:close_quote_index]1650# skip closing quote and space1651 rhs_index = close_quote_index +1+11652else:1653 space_index = view_line.find(" ")1654if space_index <=0:1655die("No word-splitting space found:%s"% view_line)1656 depot_side = view_line[0:space_index]1657 rhs_index = space_index +116581659if view_line[rhs_index] =='"':1660# Second word is double quoted. Make sure there is a1661# double quote at the end too.1662if not view_line.endswith('"'):1663die("View line with rhs quote should end with one:%s"%1664 view_line)1665# skip the quotes1666 client_side = view_line[rhs_index+1:-1]1667else:1668 client_side = view_line[rhs_index:]16691670# prefix + means overlay on previous mapping1671 overlay =False1672if depot_side.startswith("+"):1673 overlay =True1674 depot_side = depot_side[1:]16751676# prefix - means exclude this path1677 exclude =False1678if depot_side.startswith("-"):1679 exclude =True1680 depot_side = depot_side[1:]16811682 m = View.Mapping(depot_side, client_side, overlay, exclude)1683 self.mappings.append(m)16841685defmap_in_client(self, depot_path):1686"""Return the relative location in the client where this1687 depot file should live. Returns "" if the file should1688 not be mapped in the client."""16891690 paths_filled = []1691 client_path =""16921693# look at later entries first1694for m in self.mappings[::-1]:16951696# see where will this path end up in the client1697 p = m.map_depot_to_client(depot_path)16981699if p =="":1700# Depot path does not belong in client. Must remember1701# this, as previous items should not cause files to1702# exist in this path either. Remember that the list is1703# being walked from the end, which has higher precedence.1704# Overlap mappings do not exclude previous mappings.1705if not m.overlay:1706 paths_filled.append(m.client_side)17071708else:1709# This mapping matched; no need to search any further.1710# But, the mapping could be rejected if the client path1711# has already been claimed by an earlier mapping (i.e.1712# one later in the list, which we are walking backwards).1713 already_mapped_in_client =False1714for f in paths_filled:1715# this is View.Path.match1716if f.match(p):1717 already_mapped_in_client =True1718break1719if not already_mapped_in_client:1720# Include this file, unless it is from a line that1721# explicitly said to exclude it.1722if not m.exclude:1723 client_path = p17241725# a match, even if rejected, always stops the search1726break17271728return client_path17291730classP4Sync(Command, P4UserMap):1731 delete_actions = ("delete","move/delete","purge")17321733def__init__(self):1734 Command.__init__(self)1735 P4UserMap.__init__(self)1736 self.options = [1737 optparse.make_option("--branch", dest="branch"),1738 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1739 optparse.make_option("--changesfile", dest="changesFile"),1740 optparse.make_option("--silent", dest="silent", action="store_true"),1741 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1742 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),1743 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1744help="Import into refs/heads/ , not refs/remotes"),1745 optparse.make_option("--max-changes", dest="maxChanges"),1746 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',1747help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),1748 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',1749help="Only sync files that are included in the Perforce Client Spec")1750]1751 self.description ="""Imports from Perforce into a git repository.\n1752 example:1753 //depot/my/project/ -- to import the current head1754 //depot/my/project/@all -- to import everything1755 //depot/my/project/@1,6 -- to import only from revision 1 to 617561757 (a ... is not needed in the path p4 specification, it's added implicitly)"""17581759 self.usage +=" //depot/path[@revRange]"1760 self.silent =False1761 self.createdBranches =set()1762 self.committedChanges =set()1763 self.branch =""1764 self.detectBranches =False1765 self.detectLabels =False1766 self.importLabels =False1767 self.changesFile =""1768 self.syncWithOrigin =True1769 self.importIntoRemotes =True1770 self.maxChanges =""1771 self.isWindows = (platform.system() =="Windows")1772 self.keepRepoPath =False1773 self.depotPaths =None1774 self.p4BranchesInGit = []1775 self.cloneExclude = []1776 self.useClientSpec =False1777 self.useClientSpec_from_options =False1778 self.clientSpecDirs =None1779 self.tempBranches = []1780 self.tempBranchLocation ="git-p4-tmp"17811782ifgitConfig("git-p4.syncFromOrigin") =="false":1783 self.syncWithOrigin =False17841785# Force a checkpoint in fast-import and wait for it to finish1786defcheckpoint(self):1787 self.gitStream.write("checkpoint\n\n")1788 self.gitStream.write("progress checkpoint\n\n")1789 out = self.gitOutput.readline()1790if self.verbose:1791print"checkpoint finished: "+ out17921793defextractFilesFromCommit(self, commit):1794 self.cloneExclude = [re.sub(r"\.\.\.$","", path)1795for path in self.cloneExclude]1796 files = []1797 fnum =01798while commit.has_key("depotFile%s"% fnum):1799 path = commit["depotFile%s"% fnum]18001801if[p for p in self.cloneExclude1802ifp4PathStartsWith(path, p)]:1803 found =False1804else:1805 found = [p for p in self.depotPaths1806ifp4PathStartsWith(path, p)]1807if not found:1808 fnum = fnum +11809continue18101811file= {}1812file["path"] = path1813file["rev"] = commit["rev%s"% fnum]1814file["action"] = commit["action%s"% fnum]1815file["type"] = commit["type%s"% fnum]1816 files.append(file)1817 fnum = fnum +11818return files18191820defstripRepoPath(self, path, prefixes):1821if self.useClientSpec:1822 path = self.clientSpecDirs.map_in_client(path)18231824else:1825if self.keepRepoPath:1826 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]18271828for p in prefixes:1829ifp4PathStartsWith(path, p):1830 path = path[len(p):]18311832 path =wildcard_decode(path)1833return path18341835defsplitFilesIntoBranches(self, commit):1836 branches = {}1837 fnum =01838while commit.has_key("depotFile%s"% fnum):1839 path = commit["depotFile%s"% fnum]1840 found = [p for p in self.depotPaths1841ifp4PathStartsWith(path, p)]1842if not found:1843 fnum = fnum +11844continue18451846file= {}1847file["path"] = path1848file["rev"] = commit["rev%s"% fnum]1849file["action"] = commit["action%s"% fnum]1850file["type"] = commit["type%s"% fnum]1851 fnum = fnum +118521853 relPath = self.stripRepoPath(path, self.depotPaths)18541855for branch in self.knownBranches.keys():18561857# add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.21858if relPath.startswith(branch +"/"):1859if branch not in branches:1860 branches[branch] = []1861 branches[branch].append(file)1862break18631864return branches18651866# output one file from the P4 stream1867# - helper for streamP4Files18681869defstreamOneP4File(self,file, contents):1870 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)1871if verbose:1872 sys.stderr.write("%s\n"% relPath)18731874(type_base, type_mods) =split_p4_type(file["type"])18751876 git_mode ="100644"1877if"x"in type_mods:1878 git_mode ="100755"1879if type_base =="symlink":1880 git_mode ="120000"1881# p4 print on a symlink contains "target\n"; remove the newline1882 data =''.join(contents)1883 contents = [data[:-1]]18841885if type_base =="utf16":1886# p4 delivers different text in the python output to -G1887# than it does when using "print -o", or normal p4 client1888# operations. utf16 is converted to ascii or utf8, perhaps.1889# But ascii text saved as -t utf16 is completely mangled.1890# Invoke print -o to get the real contents.1891 text =p4_read_pipe(['print','-q','-o','-',file['depotFile']])1892 contents = [ text ]18931894if type_base =="apple":1895# Apple filetype files will be streamed as a concatenation of1896# its appledouble header and the contents. This is useless1897# on both macs and non-macs. If using "print -q -o xx", it1898# will create "xx" with the data, and "%xx" with the header.1899# This is also not very useful.1900#1901# Ideally, someday, this script can learn how to generate1902# appledouble files directly and import those to git, but1903# non-mac machines can never find a use for apple filetype.1904print"\nIgnoring apple filetype file%s"%file['depotFile']1905return19061907# Perhaps windows wants unicode, utf16 newlines translated too;1908# but this is not doing it.1909if self.isWindows and type_base =="text":1910 mangled = []1911for data in contents:1912 data = data.replace("\r\n","\n")1913 mangled.append(data)1914 contents = mangled19151916# Note that we do not try to de-mangle keywords on utf16 files,1917# even though in theory somebody may want that.1918 pattern =p4_keywords_regexp_for_type(type_base, type_mods)1919if pattern:1920 regexp = re.compile(pattern, re.VERBOSE)1921 text =''.join(contents)1922 text = regexp.sub(r'$\1$', text)1923 contents = [ text ]19241925 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))19261927# total length...1928 length =01929for d in contents:1930 length = length +len(d)19311932 self.gitStream.write("data%d\n"% length)1933for d in contents:1934 self.gitStream.write(d)1935 self.gitStream.write("\n")19361937defstreamOneP4Deletion(self,file):1938 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)1939if verbose:1940 sys.stderr.write("delete%s\n"% relPath)1941 self.gitStream.write("D%s\n"% relPath)19421943# handle another chunk of streaming data1944defstreamP4FilesCb(self, marshalled):19451946if marshalled.has_key('depotFile')and self.stream_have_file_info:1947# start of a new file - output the old one first1948 self.streamOneP4File(self.stream_file, self.stream_contents)1949 self.stream_file = {}1950 self.stream_contents = []1951 self.stream_have_file_info =False19521953# pick up the new file information... for the1954# 'data' field we need to append to our array1955for k in marshalled.keys():1956if k =='data':1957 self.stream_contents.append(marshalled['data'])1958else:1959 self.stream_file[k] = marshalled[k]19601961 self.stream_have_file_info =True19621963# Stream directly from "p4 files" into "git fast-import"1964defstreamP4Files(self, files):1965 filesForCommit = []1966 filesToRead = []1967 filesToDelete = []19681969for f in files:1970# if using a client spec, only add the files that have1971# a path in the client1972if self.clientSpecDirs:1973if self.clientSpecDirs.map_in_client(f['path']) =="":1974continue19751976 filesForCommit.append(f)1977if f['action']in self.delete_actions:1978 filesToDelete.append(f)1979else:1980 filesToRead.append(f)19811982# deleted files...1983for f in filesToDelete:1984 self.streamOneP4Deletion(f)19851986iflen(filesToRead) >0:1987 self.stream_file = {}1988 self.stream_contents = []1989 self.stream_have_file_info =False19901991# curry self argument1992defstreamP4FilesCbSelf(entry):1993 self.streamP4FilesCb(entry)19941995 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]19961997p4CmdList(["-x","-","print"],1998 stdin=fileArgs,1999 cb=streamP4FilesCbSelf)20002001# do the last chunk2002if self.stream_file.has_key('depotFile'):2003 self.streamOneP4File(self.stream_file, self.stream_contents)20042005defmake_email(self, userid):2006if userid in self.users:2007return self.users[userid]2008else:2009return"%s<a@b>"% userid20102011# Stream a p4 tag2012defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2013if verbose:2014print"writing tag%sfor commit%s"% (labelName, commit)2015 gitStream.write("tag%s\n"% labelName)2016 gitStream.write("from%s\n"% commit)20172018if labelDetails.has_key('Owner'):2019 owner = labelDetails["Owner"]2020else:2021 owner =None20222023# Try to use the owner of the p4 label, or failing that,2024# the current p4 user id.2025if owner:2026 email = self.make_email(owner)2027else:2028 email = self.make_email(self.p4UserId())2029 tagger ="%s %s %s"% (email, epoch, self.tz)20302031 gitStream.write("tagger%s\n"% tagger)20322033print"labelDetails=",labelDetails2034if labelDetails.has_key('Description'):2035 description = labelDetails['Description']2036else:2037 description ='Label from git p4'20382039 gitStream.write("data%d\n"%len(description))2040 gitStream.write(description)2041 gitStream.write("\n")20422043defcommit(self, details, files, branch, parent =""):2044 epoch = details["time"]2045 author = details["user"]20462047if self.verbose:2048print"commit into%s"% branch20492050# start with reading files; if that fails, we should not2051# create a commit.2052 new_files = []2053for f in files:2054if[p for p in self.branchPrefixes ifp4PathStartsWith(f['path'], p)]:2055 new_files.append(f)2056else:2057 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])20582059 self.gitStream.write("commit%s\n"% branch)2060# gitStream.write("mark :%s\n" % details["change"])2061 self.committedChanges.add(int(details["change"]))2062 committer =""2063if author not in self.users:2064 self.getUserMapFromPerforceServer()2065 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)20662067 self.gitStream.write("committer%s\n"% committer)20682069 self.gitStream.write("data <<EOT\n")2070 self.gitStream.write(details["desc"])2071 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2072(','.join(self.branchPrefixes), details["change"]))2073iflen(details['options']) >0:2074 self.gitStream.write(": options =%s"% details['options'])2075 self.gitStream.write("]\nEOT\n\n")20762077iflen(parent) >0:2078if self.verbose:2079print"parent%s"% parent2080 self.gitStream.write("from%s\n"% parent)20812082 self.streamP4Files(new_files)2083 self.gitStream.write("\n")20842085 change =int(details["change"])20862087if self.labels.has_key(change):2088 label = self.labels[change]2089 labelDetails = label[0]2090 labelRevisions = label[1]2091if self.verbose:2092print"Change%sis labelled%s"% (change, labelDetails)20932094 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2095for p in self.branchPrefixes])20962097iflen(files) ==len(labelRevisions):20982099 cleanedFiles = {}2100for info in files:2101if info["action"]in self.delete_actions:2102continue2103 cleanedFiles[info["depotFile"]] = info["rev"]21042105if cleanedFiles == labelRevisions:2106 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)21072108else:2109if not self.silent:2110print("Tag%sdoes not match with change%s: files do not match."2111% (labelDetails["label"], change))21122113else:2114if not self.silent:2115print("Tag%sdoes not match with change%s: file count is different."2116% (labelDetails["label"], change))21172118# Build a dictionary of changelists and labels, for "detect-labels" option.2119defgetLabels(self):2120 self.labels = {}21212122 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2123iflen(l) >0and not self.silent:2124print"Finding files belonging to labels in%s"% `self.depotPaths`21252126for output in l:2127 label = output["label"]2128 revisions = {}2129 newestChange =02130if self.verbose:2131print"Querying files for label%s"% label2132forfileinp4CmdList(["files"] +2133["%s...@%s"% (p, label)2134for p in self.depotPaths]):2135 revisions[file["depotFile"]] =file["rev"]2136 change =int(file["change"])2137if change > newestChange:2138 newestChange = change21392140 self.labels[newestChange] = [output, revisions]21412142if self.verbose:2143print"Label changes:%s"% self.labels.keys()21442145# Import p4 labels as git tags. A direct mapping does not2146# exist, so assume that if all the files are at the same revision2147# then we can use that, or it's something more complicated we should2148# just ignore.2149defimportP4Labels(self, stream, p4Labels):2150if verbose:2151print"import p4 labels: "+' '.join(p4Labels)21522153 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2154 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2155iflen(validLabelRegexp) ==0:2156 validLabelRegexp = defaultLabelRegexp2157 m = re.compile(validLabelRegexp)21582159for name in p4Labels:2160 commitFound =False21612162if not m.match(name):2163if verbose:2164print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2165continue21662167if name in ignoredP4Labels:2168continue21692170 labelDetails =p4CmdList(['label',"-o", name])[0]21712172# get the most recent changelist for each file in this label2173 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2174for p in self.depotPaths])21752176if change.has_key('change'):2177# find the corresponding git commit; take the oldest commit2178 changelist =int(change['change'])2179 gitCommit =read_pipe(["git","rev-list","--max-count=1",2180"--reverse",":/\[git-p4:.*change =%d\]"% changelist])2181iflen(gitCommit) ==0:2182print"could not find git commit for changelist%d"% changelist2183else:2184 gitCommit = gitCommit.strip()2185 commitFound =True2186# Convert from p4 time format2187try:2188 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2189exceptValueError:2190print"Could not convert label time%s"% labelDetail['Update']2191 tmwhen =121922193 when =int(time.mktime(tmwhen))2194 self.streamTag(stream, name, labelDetails, gitCommit, when)2195if verbose:2196print"p4 label%smapped to git commit%s"% (name, gitCommit)2197else:2198if verbose:2199print"Label%shas no changelists - possibly deleted?"% name22002201if not commitFound:2202# We can't import this label; don't try again as it will get very2203# expensive repeatedly fetching all the files for labels that will2204# never be imported. If the label is moved in the future, the2205# ignore will need to be removed manually.2206system(["git","config","--add","git-p4.ignoredP4Labels", name])22072208defguessProjectName(self):2209for p in self.depotPaths:2210if p.endswith("/"):2211 p = p[:-1]2212 p = p[p.strip().rfind("/") +1:]2213if not p.endswith("/"):2214 p +="/"2215return p22162217defgetBranchMapping(self):2218 lostAndFoundBranches =set()22192220 user =gitConfig("git-p4.branchUser")2221iflen(user) >0:2222 command ="branches -u%s"% user2223else:2224 command ="branches"22252226for info inp4CmdList(command):2227 details =p4Cmd(["branch","-o", info["branch"]])2228 viewIdx =02229while details.has_key("View%s"% viewIdx):2230 paths = details["View%s"% viewIdx].split(" ")2231 viewIdx = viewIdx +12232# require standard //depot/foo/... //depot/bar/... mapping2233iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2234continue2235 source = paths[0]2236 destination = paths[1]2237## HACK2238ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2239 source = source[len(self.depotPaths[0]):-4]2240 destination = destination[len(self.depotPaths[0]):-4]22412242if destination in self.knownBranches:2243if not self.silent:2244print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2245print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2246continue22472248 self.knownBranches[destination] = source22492250 lostAndFoundBranches.discard(destination)22512252if source not in self.knownBranches:2253 lostAndFoundBranches.add(source)22542255# Perforce does not strictly require branches to be defined, so we also2256# check git config for a branch list.2257#2258# Example of branch definition in git config file:2259# [git-p4]2260# branchList=main:branchA2261# branchList=main:branchB2262# branchList=branchA:branchC2263 configBranches =gitConfigList("git-p4.branchList")2264for branch in configBranches:2265if branch:2266(source, destination) = branch.split(":")2267 self.knownBranches[destination] = source22682269 lostAndFoundBranches.discard(destination)22702271if source not in self.knownBranches:2272 lostAndFoundBranches.add(source)227322742275for branch in lostAndFoundBranches:2276 self.knownBranches[branch] = branch22772278defgetBranchMappingFromGitBranches(self):2279 branches =p4BranchesInGit(self.importIntoRemotes)2280for branch in branches.keys():2281if branch =="master":2282 branch ="main"2283else:2284 branch = branch[len(self.projectName):]2285 self.knownBranches[branch] = branch22862287deflistExistingP4GitBranches(self):2288# branches holds mapping from name to commit2289 branches =p4BranchesInGit(self.importIntoRemotes)2290 self.p4BranchesInGit = branches.keys()2291for branch in branches.keys():2292 self.initialParents[self.refPrefix + branch] = branches[branch]22932294defupdateOptionDict(self, d):2295 option_keys = {}2296if self.keepRepoPath:2297 option_keys['keepRepoPath'] =122982299 d["options"] =' '.join(sorted(option_keys.keys()))23002301defreadOptions(self, d):2302 self.keepRepoPath = (d.has_key('options')2303and('keepRepoPath'in d['options']))23042305defgitRefForBranch(self, branch):2306if branch =="main":2307return self.refPrefix +"master"23082309iflen(branch) <=0:2310return branch23112312return self.refPrefix + self.projectName + branch23132314defgitCommitByP4Change(self, ref, change):2315if self.verbose:2316print"looking in ref "+ ref +" for change%susing bisect..."% change23172318 earliestCommit =""2319 latestCommit =parseRevision(ref)23202321while True:2322if self.verbose:2323print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2324 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2325iflen(next) ==0:2326if self.verbose:2327print"argh"2328return""2329 log =extractLogMessageFromGitCommit(next)2330 settings =extractSettingsGitLog(log)2331 currentChange =int(settings['change'])2332if self.verbose:2333print"current change%s"% currentChange23342335if currentChange == change:2336if self.verbose:2337print"found%s"% next2338return next23392340if currentChange < change:2341 earliestCommit ="^%s"% next2342else:2343 latestCommit ="%s"% next23442345return""23462347defimportNewBranch(self, branch, maxChange):2348# make fast-import flush all changes to disk and update the refs using the checkpoint2349# command so that we can try to find the branch parent in the git history2350 self.gitStream.write("checkpoint\n\n");2351 self.gitStream.flush();2352 branchPrefix = self.depotPaths[0] + branch +"/"2353range="@1,%s"% maxChange2354#print "prefix" + branchPrefix2355 changes =p4ChangesForPaths([branchPrefix],range)2356iflen(changes) <=0:2357return False2358 firstChange = changes[0]2359#print "first change in branch: %s" % firstChange2360 sourceBranch = self.knownBranches[branch]2361 sourceDepotPath = self.depotPaths[0] + sourceBranch2362 sourceRef = self.gitRefForBranch(sourceBranch)2363#print "source " + sourceBranch23642365 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2366#print "branch parent: %s" % branchParentChange2367 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2368iflen(gitParent) >0:2369 self.initialParents[self.gitRefForBranch(branch)] = gitParent2370#print "parent git commit: %s" % gitParent23712372 self.importChanges(changes)2373return True23742375defsearchParent(self, parent, branch, target):2376 parentFound =False2377for blob inread_pipe_lines(["git","rev-list","--reverse","--no-merges", parent]):2378 blob = blob.strip()2379iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2380 parentFound =True2381if self.verbose:2382print"Found parent of%sin commit%s"% (branch, blob)2383break2384if parentFound:2385return blob2386else:2387return None23882389defimportChanges(self, changes):2390 cnt =12391for change in changes:2392 description =p4Cmd(["describe",str(change)])2393 self.updateOptionDict(description)23942395if not self.silent:2396 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2397 sys.stdout.flush()2398 cnt = cnt +123992400try:2401if self.detectBranches:2402 branches = self.splitFilesIntoBranches(description)2403for branch in branches.keys():2404## HACK --hwn2405 branchPrefix = self.depotPaths[0] + branch +"/"2406 self.branchPrefixes = [ branchPrefix ]24072408 parent =""24092410 filesForCommit = branches[branch]24112412if self.verbose:2413print"branch is%s"% branch24142415 self.updatedBranches.add(branch)24162417if branch not in self.createdBranches:2418 self.createdBranches.add(branch)2419 parent = self.knownBranches[branch]2420if parent == branch:2421 parent =""2422else:2423 fullBranch = self.projectName + branch2424if fullBranch not in self.p4BranchesInGit:2425if not self.silent:2426print("\nImporting new branch%s"% fullBranch);2427if self.importNewBranch(branch, change -1):2428 parent =""2429 self.p4BranchesInGit.append(fullBranch)2430if not self.silent:2431print("\nResuming with change%s"% change);24322433if self.verbose:2434print"parent determined through known branches:%s"% parent24352436 branch = self.gitRefForBranch(branch)2437 parent = self.gitRefForBranch(parent)24382439if self.verbose:2440print"looking for initial parent for%s; current parent is%s"% (branch, parent)24412442iflen(parent) ==0and branch in self.initialParents:2443 parent = self.initialParents[branch]2444del self.initialParents[branch]24452446 blob =None2447iflen(parent) >0:2448 tempBranch = os.path.join(self.tempBranchLocation,"%d"% (change))2449if self.verbose:2450print"Creating temporary branch: "+ tempBranch2451 self.commit(description, filesForCommit, tempBranch)2452 self.tempBranches.append(tempBranch)2453 self.checkpoint()2454 blob = self.searchParent(parent, branch, tempBranch)2455if blob:2456 self.commit(description, filesForCommit, branch, blob)2457else:2458if self.verbose:2459print"Parent of%snot found. Committing into head of%s"% (branch, parent)2460 self.commit(description, filesForCommit, branch, parent)2461else:2462 files = self.extractFilesFromCommit(description)2463 self.commit(description, files, self.branch,2464 self.initialParent)2465 self.initialParent =""2466exceptIOError:2467print self.gitError.read()2468 sys.exit(1)24692470defimportHeadRevision(self, revision):2471print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)24722473 details = {}2474 details["user"] ="git perforce import user"2475 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2476% (' '.join(self.depotPaths), revision))2477 details["change"] = revision2478 newestRevision =024792480 fileCnt =02481 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]24822483for info inp4CmdList(["files"] + fileArgs):24842485if'code'in info and info['code'] =='error':2486 sys.stderr.write("p4 returned an error:%s\n"2487% info['data'])2488if info['data'].find("must refer to client") >=0:2489 sys.stderr.write("This particular p4 error is misleading.\n")2490 sys.stderr.write("Perhaps the depot path was misspelled.\n");2491 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2492 sys.exit(1)2493if'p4ExitCode'in info:2494 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2495 sys.exit(1)249624972498 change =int(info["change"])2499if change > newestRevision:2500 newestRevision = change25012502if info["action"]in self.delete_actions:2503# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2504#fileCnt = fileCnt + 12505continue25062507for prop in["depotFile","rev","action","type"]:2508 details["%s%s"% (prop, fileCnt)] = info[prop]25092510 fileCnt = fileCnt +125112512 details["change"] = newestRevision25132514# Use time from top-most change so that all git p4 clones of2515# the same p4 repo have the same commit SHA1s.2516 res =p4CmdList("describe -s%d"% newestRevision)2517 newestTime =None2518for r in res:2519if r.has_key('time'):2520 newestTime =int(r['time'])2521if newestTime is None:2522die("\"describe -s\"on newest change%ddid not give a time")2523 details["time"] = newestTime25242525 self.updateOptionDict(details)2526try:2527 self.commit(details, self.extractFilesFromCommit(details), self.branch)2528exceptIOError:2529print"IO error with git fast-import. Is your git version recent enough?"2530print self.gitError.read()253125322533defrun(self, args):2534 self.depotPaths = []2535 self.changeRange =""2536 self.initialParent =""2537 self.previousDepotPaths = []25382539# map from branch depot path to parent branch2540 self.knownBranches = {}2541 self.initialParents = {}2542 self.hasOrigin =originP4BranchesExist()2543if not self.syncWithOrigin:2544 self.hasOrigin =False25452546if self.importIntoRemotes:2547 self.refPrefix ="refs/remotes/p4/"2548else:2549 self.refPrefix ="refs/heads/p4/"25502551if self.syncWithOrigin and self.hasOrigin:2552if not self.silent:2553print"Syncing with origin first by calling git fetch origin"2554system("git fetch origin")25552556iflen(self.branch) ==0:2557 self.branch = self.refPrefix +"master"2558ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2559system("git update-ref%srefs/heads/p4"% self.branch)2560system("git branch -D p4");2561# create it /after/ importing, when master exists2562if notgitBranchExists(self.refPrefix +"HEAD")and self.importIntoRemotes andgitBranchExists(self.branch):2563system("git symbolic-ref%sHEAD%s"% (self.refPrefix, self.branch))25642565# accept either the command-line option, or the configuration variable2566if self.useClientSpec:2567# will use this after clone to set the variable2568 self.useClientSpec_from_options =True2569else:2570ifgitConfig("git-p4.useclientspec","--bool") =="true":2571 self.useClientSpec =True2572if self.useClientSpec:2573 self.clientSpecDirs =getClientSpec()25742575# TODO: should always look at previous commits,2576# merge with previous imports, if possible.2577if args == []:2578if self.hasOrigin:2579createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)2580 self.listExistingP4GitBranches()25812582iflen(self.p4BranchesInGit) >1:2583if not self.silent:2584print"Importing from/into multiple branches"2585 self.detectBranches =True25862587if self.verbose:2588print"branches:%s"% self.p4BranchesInGit25892590 p4Change =02591for branch in self.p4BranchesInGit:2592 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)25932594 settings =extractSettingsGitLog(logMsg)25952596 self.readOptions(settings)2597if(settings.has_key('depot-paths')2598and settings.has_key('change')):2599 change =int(settings['change']) +12600 p4Change =max(p4Change, change)26012602 depotPaths =sorted(settings['depot-paths'])2603if self.previousDepotPaths == []:2604 self.previousDepotPaths = depotPaths2605else:2606 paths = []2607for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2608 prev_list = prev.split("/")2609 cur_list = cur.split("/")2610for i inrange(0,min(len(cur_list),len(prev_list))):2611if cur_list[i] <> prev_list[i]:2612 i = i -12613break26142615 paths.append("/".join(cur_list[:i +1]))26162617 self.previousDepotPaths = paths26182619if p4Change >0:2620 self.depotPaths =sorted(self.previousDepotPaths)2621 self.changeRange ="@%s,#head"% p4Change2622if not self.detectBranches:2623 self.initialParent =parseRevision(self.branch)2624if not self.silent and not self.detectBranches:2625print"Performing incremental import into%sgit branch"% self.branch26262627if not self.branch.startswith("refs/"):2628 self.branch ="refs/heads/"+ self.branch26292630iflen(args) ==0and self.depotPaths:2631if not self.silent:2632print"Depot paths:%s"%' '.join(self.depotPaths)2633else:2634if self.depotPaths and self.depotPaths != args:2635print("previous import used depot path%sand now%swas specified. "2636"This doesn't work!"% (' '.join(self.depotPaths),2637' '.join(args)))2638 sys.exit(1)26392640 self.depotPaths =sorted(args)26412642 revision =""2643 self.users = {}26442645# Make sure no revision specifiers are used when --changesfile2646# is specified.2647 bad_changesfile =False2648iflen(self.changesFile) >0:2649for p in self.depotPaths:2650if p.find("@") >=0or p.find("#") >=0:2651 bad_changesfile =True2652break2653if bad_changesfile:2654die("Option --changesfile is incompatible with revision specifiers")26552656 newPaths = []2657for p in self.depotPaths:2658if p.find("@") != -1:2659 atIdx = p.index("@")2660 self.changeRange = p[atIdx:]2661if self.changeRange =="@all":2662 self.changeRange =""2663elif','not in self.changeRange:2664 revision = self.changeRange2665 self.changeRange =""2666 p = p[:atIdx]2667elif p.find("#") != -1:2668 hashIdx = p.index("#")2669 revision = p[hashIdx:]2670 p = p[:hashIdx]2671elif self.previousDepotPaths == []:2672# pay attention to changesfile, if given, else import2673# the entire p4 tree at the head revision2674iflen(self.changesFile) ==0:2675 revision ="#head"26762677 p = re.sub("\.\.\.$","", p)2678if not p.endswith("/"):2679 p +="/"26802681 newPaths.append(p)26822683 self.depotPaths = newPaths26842685# --detect-branches may change this for each branch2686 self.branchPrefixes = self.depotPaths26872688 self.loadUserMapFromCache()2689 self.labels = {}2690if self.detectLabels:2691 self.getLabels();26922693if self.detectBranches:2694## FIXME - what's a P4 projectName ?2695 self.projectName = self.guessProjectName()26962697if self.hasOrigin:2698 self.getBranchMappingFromGitBranches()2699else:2700 self.getBranchMapping()2701if self.verbose:2702print"p4-git branches:%s"% self.p4BranchesInGit2703print"initial parents:%s"% self.initialParents2704for b in self.p4BranchesInGit:2705if b !="master":27062707## FIXME2708 b = b[len(self.projectName):]2709 self.createdBranches.add(b)27102711 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))27122713 importProcess = subprocess.Popen(["git","fast-import"],2714 stdin=subprocess.PIPE, stdout=subprocess.PIPE,2715 stderr=subprocess.PIPE);2716 self.gitOutput = importProcess.stdout2717 self.gitStream = importProcess.stdin2718 self.gitError = importProcess.stderr27192720if revision:2721 self.importHeadRevision(revision)2722else:2723 changes = []27242725iflen(self.changesFile) >0:2726 output =open(self.changesFile).readlines()2727 changeSet =set()2728for line in output:2729 changeSet.add(int(line))27302731for change in changeSet:2732 changes.append(change)27332734 changes.sort()2735else:2736# catch "git p4 sync" with no new branches, in a repo that2737# does not have any existing p4 branches2738iflen(args) ==0and not self.p4BranchesInGit:2739die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.");2740if self.verbose:2741print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),2742 self.changeRange)2743 changes =p4ChangesForPaths(self.depotPaths, self.changeRange)27442745iflen(self.maxChanges) >0:2746 changes = changes[:min(int(self.maxChanges),len(changes))]27472748iflen(changes) ==0:2749if not self.silent:2750print"No changes to import!"2751else:2752if not self.silent and not self.detectBranches:2753print"Import destination:%s"% self.branch27542755 self.updatedBranches =set()27562757 self.importChanges(changes)27582759if not self.silent:2760print""2761iflen(self.updatedBranches) >0:2762 sys.stdout.write("Updated branches: ")2763for b in self.updatedBranches:2764 sys.stdout.write("%s"% b)2765 sys.stdout.write("\n")27662767ifgitConfig("git-p4.importLabels","--bool") =="true":2768 self.importLabels =True27692770if self.importLabels:2771 p4Labels =getP4Labels(self.depotPaths)2772 gitTags =getGitTags()27732774 missingP4Labels = p4Labels - gitTags2775 self.importP4Labels(self.gitStream, missingP4Labels)27762777 self.gitStream.close()2778if importProcess.wait() !=0:2779die("fast-import failed:%s"% self.gitError.read())2780 self.gitOutput.close()2781 self.gitError.close()27822783# Cleanup temporary branches created during import2784if self.tempBranches != []:2785for branch in self.tempBranches:2786read_pipe("git update-ref -d%s"% branch)2787 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))27882789return True27902791classP4Rebase(Command):2792def__init__(self):2793 Command.__init__(self)2794 self.options = [2795 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2796]2797 self.importLabels =False2798 self.description = ("Fetches the latest revision from perforce and "2799+"rebases the current work (branch) against it")28002801defrun(self, args):2802 sync =P4Sync()2803 sync.importLabels = self.importLabels2804 sync.run([])28052806return self.rebase()28072808defrebase(self):2809if os.system("git update-index --refresh") !=0:2810die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");2811iflen(read_pipe("git diff-index HEAD --")) >0:2812die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");28132814[upstream, settings] =findUpstreamBranchPoint()2815iflen(upstream) ==0:2816die("Cannot find upstream branchpoint for rebase")28172818# the branchpoint may be p4/foo~3, so strip off the parent2819 upstream = re.sub("~[0-9]+$","", upstream)28202821print"Rebasing the current branch onto%s"% upstream2822 oldHead =read_pipe("git rev-parse HEAD").strip()2823system("git rebase%s"% upstream)2824system("git diff-tree --stat --summary -M%sHEAD"% oldHead)2825return True28262827classP4Clone(P4Sync):2828def__init__(self):2829 P4Sync.__init__(self)2830 self.description ="Creates a new git repository and imports from Perforce into it"2831 self.usage ="usage: %prog [options] //depot/path[@revRange]"2832 self.options += [2833 optparse.make_option("--destination", dest="cloneDestination",2834 action='store', default=None,2835help="where to leave result of the clone"),2836 optparse.make_option("-/", dest="cloneExclude",2837 action="append",type="string",2838help="exclude depot path"),2839 optparse.make_option("--bare", dest="cloneBare",2840 action="store_true", default=False),2841]2842 self.cloneDestination =None2843 self.needsGit =False2844 self.cloneBare =False28452846# This is required for the "append" cloneExclude action2847defensure_value(self, attr, value):2848if nothasattr(self, attr)orgetattr(self, attr)is None:2849setattr(self, attr, value)2850returngetattr(self, attr)28512852defdefaultDestination(self, args):2853## TODO: use common prefix of args?2854 depotPath = args[0]2855 depotDir = re.sub("(@[^@]*)$","", depotPath)2856 depotDir = re.sub("(#[^#]*)$","", depotDir)2857 depotDir = re.sub(r"\.\.\.$","", depotDir)2858 depotDir = re.sub(r"/$","", depotDir)2859return os.path.split(depotDir)[1]28602861defrun(self, args):2862iflen(args) <1:2863return False28642865if self.keepRepoPath and not self.cloneDestination:2866 sys.stderr.write("Must specify destination for --keep-path\n")2867 sys.exit(1)28682869 depotPaths = args28702871if not self.cloneDestination andlen(depotPaths) >1:2872 self.cloneDestination = depotPaths[-1]2873 depotPaths = depotPaths[:-1]28742875 self.cloneExclude = ["/"+p for p in self.cloneExclude]2876for p in depotPaths:2877if not p.startswith("//"):2878return False28792880if not self.cloneDestination:2881 self.cloneDestination = self.defaultDestination(args)28822883print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)28842885if not os.path.exists(self.cloneDestination):2886 os.makedirs(self.cloneDestination)2887chdir(self.cloneDestination)28882889 init_cmd = ["git","init"]2890if self.cloneBare:2891 init_cmd.append("--bare")2892 subprocess.check_call(init_cmd)28932894if not P4Sync.run(self, depotPaths):2895return False2896if self.branch !="master":2897if self.importIntoRemotes:2898 masterbranch ="refs/remotes/p4/master"2899else:2900 masterbranch ="refs/heads/p4/master"2901ifgitBranchExists(masterbranch):2902system("git branch master%s"% masterbranch)2903if not self.cloneBare:2904system("git checkout -f")2905else:2906print"Could not detect main branch. No checkout/master branch created."29072908# auto-set this variable if invoked with --use-client-spec2909if self.useClientSpec_from_options:2910system("git config --bool git-p4.useclientspec true")29112912return True29132914classP4Branches(Command):2915def__init__(self):2916 Command.__init__(self)2917 self.options = [ ]2918 self.description = ("Shows the git branches that hold imports and their "2919+"corresponding perforce depot paths")2920 self.verbose =False29212922defrun(self, args):2923iforiginP4BranchesExist():2924createOrUpdateBranchesFromOrigin()29252926 cmdline ="git rev-parse --symbolic "2927 cmdline +=" --remotes"29282929for line inread_pipe_lines(cmdline):2930 line = line.strip()29312932if not line.startswith('p4/')or line =="p4/HEAD":2933continue2934 branch = line29352936 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)2937 settings =extractSettingsGitLog(log)29382939print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])2940return True29412942classHelpFormatter(optparse.IndentedHelpFormatter):2943def__init__(self):2944 optparse.IndentedHelpFormatter.__init__(self)29452946defformat_description(self, description):2947if description:2948return description +"\n"2949else:2950return""29512952defprintUsage(commands):2953print"usage:%s<command> [options]"% sys.argv[0]2954print""2955print"valid commands:%s"%", ".join(commands)2956print""2957print"Try%s<command> --help for command specific help."% sys.argv[0]2958print""29592960commands = {2961"debug": P4Debug,2962"submit": P4Submit,2963"commit": P4Submit,2964"sync": P4Sync,2965"rebase": P4Rebase,2966"clone": P4Clone,2967"rollback": P4RollBack,2968"branches": P4Branches2969}297029712972defmain():2973iflen(sys.argv[1:]) ==0:2974printUsage(commands.keys())2975 sys.exit(2)29762977 cmd =""2978 cmdName = sys.argv[1]2979try:2980 klass = commands[cmdName]2981 cmd =klass()2982exceptKeyError:2983print"unknown command%s"% cmdName2984print""2985printUsage(commands.keys())2986 sys.exit(2)29872988 options = cmd.options2989 cmd.gitdir = os.environ.get("GIT_DIR",None)29902991 args = sys.argv[2:]29922993 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))2994if cmd.needsGit:2995 options.append(optparse.make_option("--git-dir", dest="gitdir"))29962997 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),2998 options,2999 description = cmd.description,3000 formatter =HelpFormatter())30013002(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3003global verbose3004 verbose = cmd.verbose3005if cmd.needsGit:3006if cmd.gitdir ==None:3007 cmd.gitdir = os.path.abspath(".git")3008if notisValidGitDir(cmd.gitdir):3009 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3010if os.path.exists(cmd.gitdir):3011 cdup =read_pipe("git rev-parse --show-cdup").strip()3012iflen(cdup) >0:3013chdir(cdup);30143015if notisValidGitDir(cmd.gitdir):3016ifisValidGitDir(cmd.gitdir +"/.git"):3017 cmd.gitdir +="/.git"3018else:3019die("fatal: cannot locate git repository at%s"% cmd.gitdir)30203021 os.environ["GIT_DIR"] = cmd.gitdir30223023if not cmd.run(args):3024 parser.print_help()3025 sys.exit(2)302630273028if __name__ =='__main__':3029main()