1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10 11import optparse, sys, os, marshal, subprocess, shelve 12import tempfile, getopt, os.path, time, platform 13import re, shutil 14 15verbose =False 16 17# Only labels/tags matching this will be imported/exported 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 19 20defp4_build_cmd(cmd): 21"""Build a suitable p4 command line. 22 23 This consolidates building and returning a p4 command line into one 24 location. It means that hooking into the environment, or other configuration 25 can be done more easily. 26 """ 27 real_cmd = ["p4"] 28 29 user =gitConfig("git-p4.user") 30iflen(user) >0: 31 real_cmd += ["-u",user] 32 33 password =gitConfig("git-p4.password") 34iflen(password) >0: 35 real_cmd += ["-P", password] 36 37 port =gitConfig("git-p4.port") 38iflen(port) >0: 39 real_cmd += ["-p", port] 40 41 host =gitConfig("git-p4.host") 42iflen(host) >0: 43 real_cmd += ["-H", host] 44 45 client =gitConfig("git-p4.client") 46iflen(client) >0: 47 real_cmd += ["-c", client] 48 49 50ifisinstance(cmd,basestring): 51 real_cmd =' '.join(real_cmd) +' '+ cmd 52else: 53 real_cmd += cmd 54return real_cmd 55 56defchdir(dir): 57# P4 uses the PWD environment variable rather than getcwd(). Since we're 58# not using the shell, we have to set it ourselves. This path could 59# be relative, so go there first, then figure out where we ended up. 60 os.chdir(dir) 61 os.environ['PWD'] = os.getcwd() 62 63defdie(msg): 64if verbose: 65raiseException(msg) 66else: 67 sys.stderr.write(msg +"\n") 68 sys.exit(1) 69 70defwrite_pipe(c, stdin): 71if verbose: 72 sys.stderr.write('Writing pipe:%s\n'%str(c)) 73 74 expand =isinstance(c,basestring) 75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 76 pipe = p.stdin 77 val = pipe.write(stdin) 78 pipe.close() 79if p.wait(): 80die('Command failed:%s'%str(c)) 81 82return val 83 84defp4_write_pipe(c, stdin): 85 real_cmd =p4_build_cmd(c) 86returnwrite_pipe(real_cmd, stdin) 87 88defread_pipe(c, ignore_error=False): 89if verbose: 90 sys.stderr.write('Reading pipe:%s\n'%str(c)) 91 92 expand =isinstance(c,basestring) 93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 94 pipe = p.stdout 95 val = pipe.read() 96if p.wait()and not ignore_error: 97die('Command failed:%s'%str(c)) 98 99return val 100 101defp4_read_pipe(c, ignore_error=False): 102 real_cmd =p4_build_cmd(c) 103returnread_pipe(real_cmd, ignore_error) 104 105defread_pipe_lines(c): 106if verbose: 107 sys.stderr.write('Reading pipe:%s\n'%str(c)) 108 109 expand =isinstance(c, basestring) 110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 111 pipe = p.stdout 112 val = pipe.readlines() 113if pipe.close()or p.wait(): 114die('Command failed:%s'%str(c)) 115 116return val 117 118defp4_read_pipe_lines(c): 119"""Specifically invoke p4 on the command supplied. """ 120 real_cmd =p4_build_cmd(c) 121returnread_pipe_lines(real_cmd) 122 123defp4_has_command(cmd): 124"""Ask p4 for help on this command. If it returns an error, the 125 command does not exist in this version of p4.""" 126 real_cmd =p4_build_cmd(["help", cmd]) 127 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 128 stderr=subprocess.PIPE) 129 p.communicate() 130return p.returncode ==0 131 132defsystem(cmd): 133 expand =isinstance(cmd,basestring) 134if verbose: 135 sys.stderr.write("executing%s\n"%str(cmd)) 136 subprocess.check_call(cmd, shell=expand) 137 138defp4_system(cmd): 139"""Specifically invoke p4 as the system command. """ 140 real_cmd =p4_build_cmd(cmd) 141 expand =isinstance(real_cmd, basestring) 142 subprocess.check_call(real_cmd, shell=expand) 143 144defp4_integrate(src, dest): 145p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 146 147defp4_sync(f, *options): 148p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 149 150defp4_add(f): 151# forcibly add file names with wildcards 152ifwildcard_present(f): 153p4_system(["add","-f", f]) 154else: 155p4_system(["add", f]) 156 157defp4_delete(f): 158p4_system(["delete",wildcard_encode(f)]) 159 160defp4_edit(f): 161p4_system(["edit",wildcard_encode(f)]) 162 163defp4_revert(f): 164p4_system(["revert",wildcard_encode(f)]) 165 166defp4_reopen(type, f): 167p4_system(["reopen","-t",type,wildcard_encode(f)]) 168 169defp4_move(src, dest): 170p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 171 172# 173# Canonicalize the p4 type and return a tuple of the 174# base type, plus any modifiers. See "p4 help filetypes" 175# for a list and explanation. 176# 177defsplit_p4_type(p4type): 178 179 p4_filetypes_historical = { 180"ctempobj":"binary+Sw", 181"ctext":"text+C", 182"cxtext":"text+Cx", 183"ktext":"text+k", 184"kxtext":"text+kx", 185"ltext":"text+F", 186"tempobj":"binary+FSw", 187"ubinary":"binary+F", 188"uresource":"resource+F", 189"uxbinary":"binary+Fx", 190"xbinary":"binary+x", 191"xltext":"text+Fx", 192"xtempobj":"binary+Swx", 193"xtext":"text+x", 194"xunicode":"unicode+x", 195"xutf16":"utf16+x", 196} 197if p4type in p4_filetypes_historical: 198 p4type = p4_filetypes_historical[p4type] 199 mods ="" 200 s = p4type.split("+") 201 base = s[0] 202 mods ="" 203iflen(s) >1: 204 mods = s[1] 205return(base, mods) 206 207# 208# return the raw p4 type of a file (text, text+ko, etc) 209# 210defp4_type(file): 211 results =p4CmdList(["fstat","-T","headType",file]) 212return results[0]['headType'] 213 214# 215# Given a type base and modifier, return a regexp matching 216# the keywords that can be expanded in the file 217# 218defp4_keywords_regexp_for_type(base, type_mods): 219if base in("text","unicode","binary"): 220 kwords =None 221if"ko"in type_mods: 222 kwords ='Id|Header' 223elif"k"in type_mods: 224 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 225else: 226return None 227 pattern = r""" 228 \$ # Starts with a dollar, followed by... 229 (%s) # one of the keywords, followed by... 230 (:[^$]+)? # possibly an old expansion, followed by... 231 \$ # another dollar 232 """% kwords 233return pattern 234else: 235return None 236 237# 238# Given a file, return a regexp matching the possible 239# RCS keywords that will be expanded, or None for files 240# with kw expansion turned off. 241# 242defp4_keywords_regexp_for_file(file): 243if not os.path.exists(file): 244return None 245else: 246(type_base, type_mods) =split_p4_type(p4_type(file)) 247returnp4_keywords_regexp_for_type(type_base, type_mods) 248 249defsetP4ExecBit(file, mode): 250# Reopens an already open file and changes the execute bit to match 251# the execute bit setting in the passed in mode. 252 253 p4Type ="+x" 254 255if notisModeExec(mode): 256 p4Type =getP4OpenedType(file) 257 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 258 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 259if p4Type[-1] =="+": 260 p4Type = p4Type[0:-1] 261 262p4_reopen(p4Type,file) 263 264defgetP4OpenedType(file): 265# Returns the perforce file type for the given file. 266 267 result =p4_read_pipe(["opened",wildcard_encode(file)]) 268 match = re.match(".*\((.+)\)\r?$", result) 269if match: 270return match.group(1) 271else: 272die("Could not determine file type for%s(result: '%s')"% (file, result)) 273 274# Return the set of all p4 labels 275defgetP4Labels(depotPaths): 276 labels =set() 277ifisinstance(depotPaths,basestring): 278 depotPaths = [depotPaths] 279 280for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 281 label = l['label'] 282 labels.add(label) 283 284return labels 285 286# Return the set of all git tags 287defgetGitTags(): 288 gitTags =set() 289for line inread_pipe_lines(["git","tag"]): 290 tag = line.strip() 291 gitTags.add(tag) 292return gitTags 293 294defdiffTreePattern(): 295# This is a simple generator for the diff tree regex pattern. This could be 296# a class variable if this and parseDiffTreeEntry were a part of a class. 297 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 298while True: 299yield pattern 300 301defparseDiffTreeEntry(entry): 302"""Parses a single diff tree entry into its component elements. 303 304 See git-diff-tree(1) manpage for details about the format of the diff 305 output. This method returns a dictionary with the following elements: 306 307 src_mode - The mode of the source file 308 dst_mode - The mode of the destination file 309 src_sha1 - The sha1 for the source file 310 dst_sha1 - The sha1 fr the destination file 311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 312 status_score - The score for the status (applicable for 'C' and 'R' 313 statuses). This is None if there is no score. 314 src - The path for the source file. 315 dst - The path for the destination file. This is only present for 316 copy or renames. If it is not present, this is None. 317 318 If the pattern is not matched, None is returned.""" 319 320 match =diffTreePattern().next().match(entry) 321if match: 322return{ 323'src_mode': match.group(1), 324'dst_mode': match.group(2), 325'src_sha1': match.group(3), 326'dst_sha1': match.group(4), 327'status': match.group(5), 328'status_score': match.group(6), 329'src': match.group(7), 330'dst': match.group(10) 331} 332return None 333 334defisModeExec(mode): 335# Returns True if the given git mode represents an executable file, 336# otherwise False. 337return mode[-3:] =="755" 338 339defisModeExecChanged(src_mode, dst_mode): 340returnisModeExec(src_mode) !=isModeExec(dst_mode) 341 342defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 343 344ifisinstance(cmd,basestring): 345 cmd ="-G "+ cmd 346 expand =True 347else: 348 cmd = ["-G"] + cmd 349 expand =False 350 351 cmd =p4_build_cmd(cmd) 352if verbose: 353 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 354 355# Use a temporary file to avoid deadlocks without 356# subprocess.communicate(), which would put another copy 357# of stdout into memory. 358 stdin_file =None 359if stdin is not None: 360 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 361ifisinstance(stdin,basestring): 362 stdin_file.write(stdin) 363else: 364for i in stdin: 365 stdin_file.write(i +'\n') 366 stdin_file.flush() 367 stdin_file.seek(0) 368 369 p4 = subprocess.Popen(cmd, 370 shell=expand, 371 stdin=stdin_file, 372 stdout=subprocess.PIPE) 373 374 result = [] 375try: 376while True: 377 entry = marshal.load(p4.stdout) 378if cb is not None: 379cb(entry) 380else: 381 result.append(entry) 382exceptEOFError: 383pass 384 exitCode = p4.wait() 385if exitCode !=0: 386 entry = {} 387 entry["p4ExitCode"] = exitCode 388 result.append(entry) 389 390return result 391 392defp4Cmd(cmd): 393list=p4CmdList(cmd) 394 result = {} 395for entry inlist: 396 result.update(entry) 397return result; 398 399defp4Where(depotPath): 400if not depotPath.endswith("/"): 401 depotPath +="/" 402 depotPath = depotPath +"..." 403 outputList =p4CmdList(["where", depotPath]) 404 output =None 405for entry in outputList: 406if"depotFile"in entry: 407if entry["depotFile"] == depotPath: 408 output = entry 409break 410elif"data"in entry: 411 data = entry.get("data") 412 space = data.find(" ") 413if data[:space] == depotPath: 414 output = entry 415break 416if output ==None: 417return"" 418if output["code"] =="error": 419return"" 420 clientPath ="" 421if"path"in output: 422 clientPath = output.get("path") 423elif"data"in output: 424 data = output.get("data") 425 lastSpace = data.rfind(" ") 426 clientPath = data[lastSpace +1:] 427 428if clientPath.endswith("..."): 429 clientPath = clientPath[:-3] 430return clientPath 431 432defcurrentGitBranch(): 433returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 434 435defisValidGitDir(path): 436if(os.path.exists(path +"/HEAD") 437and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 438return True; 439return False 440 441defparseRevision(ref): 442returnread_pipe("git rev-parse%s"% ref).strip() 443 444defbranchExists(ref): 445 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 446 ignore_error=True) 447returnlen(rev) >0 448 449defextractLogMessageFromGitCommit(commit): 450 logMessage ="" 451 452## fixme: title is first line of commit, not 1st paragraph. 453 foundTitle =False 454for log inread_pipe_lines("git cat-file commit%s"% commit): 455if not foundTitle: 456iflen(log) ==1: 457 foundTitle =True 458continue 459 460 logMessage += log 461return logMessage 462 463defextractSettingsGitLog(log): 464 values = {} 465for line in log.split("\n"): 466 line = line.strip() 467 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 468if not m: 469continue 470 471 assignments = m.group(1).split(':') 472for a in assignments: 473 vals = a.split('=') 474 key = vals[0].strip() 475 val = ('='.join(vals[1:])).strip() 476if val.endswith('\"')and val.startswith('"'): 477 val = val[1:-1] 478 479 values[key] = val 480 481 paths = values.get("depot-paths") 482if not paths: 483 paths = values.get("depot-path") 484if paths: 485 values['depot-paths'] = paths.split(',') 486return values 487 488defgitBranchExists(branch): 489 proc = subprocess.Popen(["git","rev-parse", branch], 490 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 491return proc.wait() ==0; 492 493_gitConfig = {} 494defgitConfig(key, args =None):# set args to "--bool", for instance 495if not _gitConfig.has_key(key): 496 argsFilter ="" 497if args !=None: 498 argsFilter ="%s"% args 499 cmd ="git config%s%s"% (argsFilter, key) 500 _gitConfig[key] =read_pipe(cmd, ignore_error=True).strip() 501return _gitConfig[key] 502 503defgitConfigList(key): 504if not _gitConfig.has_key(key): 505 _gitConfig[key] =read_pipe("git config --get-all%s"% key, ignore_error=True).strip().split(os.linesep) 506return _gitConfig[key] 507 508defp4BranchesInGit(branchesAreInRemotes =True): 509 branches = {} 510 511 cmdline ="git rev-parse --symbolic " 512if branchesAreInRemotes: 513 cmdline +=" --remotes" 514else: 515 cmdline +=" --branches" 516 517for line inread_pipe_lines(cmdline): 518 line = line.strip() 519 520## only import to p4/ 521if not line.startswith('p4/')or line =="p4/HEAD": 522continue 523 branch = line 524 525# strip off p4 526 branch = re.sub("^p4/","", line) 527 528 branches[branch] =parseRevision(line) 529return branches 530 531deffindUpstreamBranchPoint(head ="HEAD"): 532 branches =p4BranchesInGit() 533# map from depot-path to branch name 534 branchByDepotPath = {} 535for branch in branches.keys(): 536 tip = branches[branch] 537 log =extractLogMessageFromGitCommit(tip) 538 settings =extractSettingsGitLog(log) 539if settings.has_key("depot-paths"): 540 paths =",".join(settings["depot-paths"]) 541 branchByDepotPath[paths] ="remotes/p4/"+ branch 542 543 settings =None 544 parent =0 545while parent <65535: 546 commit = head +"~%s"% parent 547 log =extractLogMessageFromGitCommit(commit) 548 settings =extractSettingsGitLog(log) 549if settings.has_key("depot-paths"): 550 paths =",".join(settings["depot-paths"]) 551if branchByDepotPath.has_key(paths): 552return[branchByDepotPath[paths], settings] 553 554 parent = parent +1 555 556return["", settings] 557 558defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 559if not silent: 560print("Creating/updating branch(es) in%sbased on origin branch(es)" 561% localRefPrefix) 562 563 originPrefix ="origin/p4/" 564 565for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 566 line = line.strip() 567if(not line.startswith(originPrefix))or line.endswith("HEAD"): 568continue 569 570 headName = line[len(originPrefix):] 571 remoteHead = localRefPrefix + headName 572 originHead = line 573 574 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 575if(not original.has_key('depot-paths') 576or not original.has_key('change')): 577continue 578 579 update =False 580if notgitBranchExists(remoteHead): 581if verbose: 582print"creating%s"% remoteHead 583 update =True 584else: 585 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 586if settings.has_key('change') >0: 587if settings['depot-paths'] == original['depot-paths']: 588 originP4Change =int(original['change']) 589 p4Change =int(settings['change']) 590if originP4Change > p4Change: 591print("%s(%s) is newer than%s(%s). " 592"Updating p4 branch from origin." 593% (originHead, originP4Change, 594 remoteHead, p4Change)) 595 update =True 596else: 597print("Ignoring:%swas imported from%swhile " 598"%swas imported from%s" 599% (originHead,','.join(original['depot-paths']), 600 remoteHead,','.join(settings['depot-paths']))) 601 602if update: 603system("git update-ref%s %s"% (remoteHead, originHead)) 604 605deforiginP4BranchesExist(): 606returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 607 608defp4ChangesForPaths(depotPaths, changeRange): 609assert depotPaths 610 cmd = ['changes'] 611for p in depotPaths: 612 cmd += ["%s...%s"% (p, changeRange)] 613 output =p4_read_pipe_lines(cmd) 614 615 changes = {} 616for line in output: 617 changeNum =int(line.split(" ")[1]) 618 changes[changeNum] =True 619 620 changelist = changes.keys() 621 changelist.sort() 622return changelist 623 624defp4PathStartsWith(path, prefix): 625# This method tries to remedy a potential mixed-case issue: 626# 627# If UserA adds //depot/DirA/file1 628# and UserB adds //depot/dira/file2 629# 630# we may or may not have a problem. If you have core.ignorecase=true, 631# we treat DirA and dira as the same directory 632 ignorecase =gitConfig("core.ignorecase","--bool") =="true" 633if ignorecase: 634return path.lower().startswith(prefix.lower()) 635return path.startswith(prefix) 636 637defgetClientSpec(): 638"""Look at the p4 client spec, create a View() object that contains 639 all the mappings, and return it.""" 640 641 specList =p4CmdList("client -o") 642iflen(specList) !=1: 643die('Output from "client -o" is%dlines, expecting 1'% 644len(specList)) 645 646# dictionary of all client parameters 647 entry = specList[0] 648 649# just the keys that start with "View" 650 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 651 652# hold this new View 653 view =View() 654 655# append the lines, in order, to the view 656for view_num inrange(len(view_keys)): 657 k ="View%d"% view_num 658if k not in view_keys: 659die("Expected view key%smissing"% k) 660 view.append(entry[k]) 661 662return view 663 664defgetClientRoot(): 665"""Grab the client directory.""" 666 667 output =p4CmdList("client -o") 668iflen(output) !=1: 669die('Output from "client -o" is%dlines, expecting 1'%len(output)) 670 671 entry = output[0] 672if"Root"not in entry: 673die('Client has no "Root"') 674 675return entry["Root"] 676 677# 678# P4 wildcards are not allowed in filenames. P4 complains 679# if you simply add them, but you can force it with "-f", in 680# which case it translates them into %xx encoding internally. 681# 682defwildcard_decode(path): 683# Search for and fix just these four characters. Do % last so 684# that fixing it does not inadvertently create new %-escapes. 685# Cannot have * in a filename in windows; untested as to 686# what p4 would do in such a case. 687if not platform.system() =="Windows": 688 path = path.replace("%2A","*") 689 path = path.replace("%23","#") \ 690.replace("%40","@") \ 691.replace("%25","%") 692return path 693 694defwildcard_encode(path): 695# do % first to avoid double-encoding the %s introduced here 696 path = path.replace("%","%25") \ 697.replace("*","%2A") \ 698.replace("#","%23") \ 699.replace("@","%40") 700return path 701 702defwildcard_present(path): 703return path.translate(None,"*#@%") != path 704 705class Command: 706def__init__(self): 707 self.usage ="usage: %prog [options]" 708 self.needsGit =True 709 self.verbose =False 710 711class P4UserMap: 712def__init__(self): 713 self.userMapFromPerforceServer =False 714 self.myP4UserId =None 715 716defp4UserId(self): 717if self.myP4UserId: 718return self.myP4UserId 719 720 results =p4CmdList("user -o") 721for r in results: 722if r.has_key('User'): 723 self.myP4UserId = r['User'] 724return r['User'] 725die("Could not find your p4 user id") 726 727defp4UserIsMe(self, p4User): 728# return True if the given p4 user is actually me 729 me = self.p4UserId() 730if not p4User or p4User != me: 731return False 732else: 733return True 734 735defgetUserCacheFilename(self): 736 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 737return home +"/.gitp4-usercache.txt" 738 739defgetUserMapFromPerforceServer(self): 740if self.userMapFromPerforceServer: 741return 742 self.users = {} 743 self.emails = {} 744 745for output inp4CmdList("users"): 746if not output.has_key("User"): 747continue 748 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 749 self.emails[output["Email"]] = output["User"] 750 751 752 s ='' 753for(key, val)in self.users.items(): 754 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 755 756open(self.getUserCacheFilename(),"wb").write(s) 757 self.userMapFromPerforceServer =True 758 759defloadUserMapFromCache(self): 760 self.users = {} 761 self.userMapFromPerforceServer =False 762try: 763 cache =open(self.getUserCacheFilename(),"rb") 764 lines = cache.readlines() 765 cache.close() 766for line in lines: 767 entry = line.strip().split("\t") 768 self.users[entry[0]] = entry[1] 769exceptIOError: 770 self.getUserMapFromPerforceServer() 771 772classP4Debug(Command): 773def__init__(self): 774 Command.__init__(self) 775 self.options = [] 776 self.description ="A tool to debug the output of p4 -G." 777 self.needsGit =False 778 779defrun(self, args): 780 j =0 781for output inp4CmdList(args): 782print'Element:%d'% j 783 j +=1 784print output 785return True 786 787classP4RollBack(Command): 788def__init__(self): 789 Command.__init__(self) 790 self.options = [ 791 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") 792] 793 self.description ="A tool to debug the multi-branch import. Don't use :)" 794 self.rollbackLocalBranches =False 795 796defrun(self, args): 797iflen(args) !=1: 798return False 799 maxChange =int(args[0]) 800 801if"p4ExitCode"inp4Cmd("changes -m 1"): 802die("Problems executing p4"); 803 804if self.rollbackLocalBranches: 805 refPrefix ="refs/heads/" 806 lines =read_pipe_lines("git rev-parse --symbolic --branches") 807else: 808 refPrefix ="refs/remotes/" 809 lines =read_pipe_lines("git rev-parse --symbolic --remotes") 810 811for line in lines: 812if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"): 813 line = line.strip() 814 ref = refPrefix + line 815 log =extractLogMessageFromGitCommit(ref) 816 settings =extractSettingsGitLog(log) 817 818 depotPaths = settings['depot-paths'] 819 change = settings['change'] 820 821 changed =False 822 823iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange) 824for p in depotPaths]))) ==0: 825print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange) 826system("git update-ref -d%s`git rev-parse%s`"% (ref, ref)) 827continue 828 829while change andint(change) > maxChange: 830 changed =True 831if self.verbose: 832print"%sis at%s; rewinding towards%s"% (ref, change, maxChange) 833system("git update-ref%s\"%s^\""% (ref, ref)) 834 log =extractLogMessageFromGitCommit(ref) 835 settings =extractSettingsGitLog(log) 836 837 838 depotPaths = settings['depot-paths'] 839 change = settings['change'] 840 841if changed: 842print"%srewound to%s"% (ref, change) 843 844return True 845 846classP4Submit(Command, P4UserMap): 847def__init__(self): 848 Command.__init__(self) 849 P4UserMap.__init__(self) 850 self.options = [ 851 optparse.make_option("--origin", dest="origin"), 852 optparse.make_option("-M", dest="detectRenames", action="store_true"), 853# preserve the user, requires relevant p4 permissions 854 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), 855 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"), 856] 857 self.description ="Submit changes from git to the perforce depot." 858 self.usage +=" [name of git branch to submit into perforce depot]" 859 self.origin ="" 860 self.detectRenames =False 861 self.preserveUser =gitConfig("git-p4.preserveUser").lower() =="true" 862 self.isWindows = (platform.system() =="Windows") 863 self.exportLabels =False 864 self.p4HasMoveCommand =p4_has_command("move") 865 866defcheck(self): 867iflen(p4CmdList("opened ...")) >0: 868die("You have files opened with perforce! Close them before starting the sync.") 869 870defseparate_jobs_from_description(self, message): 871"""Extract and return a possible Jobs field in the commit 872 message. It goes into a separate section in the p4 change 873 specification. 874 875 A jobs line starts with "Jobs:" and looks like a new field 876 in a form. Values are white-space separated on the same 877 line or on following lines that start with a tab. 878 879 This does not parse and extract the full git commit message 880 like a p4 form. It just sees the Jobs: line as a marker 881 to pass everything from then on directly into the p4 form, 882 but outside the description section. 883 884 Return a tuple (stripped log message, jobs string).""" 885 886 m = re.search(r'^Jobs:', message, re.MULTILINE) 887if m is None: 888return(message,None) 889 890 jobtext = message[m.start():] 891 stripped_message = message[:m.start()].rstrip() 892return(stripped_message, jobtext) 893 894defprepareLogMessage(self, template, message, jobs): 895"""Edits the template returned from "p4 change -o" to insert 896 the message in the Description field, and the jobs text in 897 the Jobs field.""" 898 result ="" 899 900 inDescriptionSection =False 901 902for line in template.split("\n"): 903if line.startswith("#"): 904 result += line +"\n" 905continue 906 907if inDescriptionSection: 908if line.startswith("Files:")or line.startswith("Jobs:"): 909 inDescriptionSection =False 910# insert Jobs section 911if jobs: 912 result += jobs +"\n" 913else: 914continue 915else: 916if line.startswith("Description:"): 917 inDescriptionSection =True 918 line +="\n" 919for messageLine in message.split("\n"): 920 line +="\t"+ messageLine +"\n" 921 922 result += line +"\n" 923 924return result 925 926defpatchRCSKeywords(self,file, pattern): 927# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern 928(handle, outFileName) = tempfile.mkstemp(dir='.') 929try: 930 outFile = os.fdopen(handle,"w+") 931 inFile =open(file,"r") 932 regexp = re.compile(pattern, re.VERBOSE) 933for line in inFile.readlines(): 934 line = regexp.sub(r'$\1$', line) 935 outFile.write(line) 936 inFile.close() 937 outFile.close() 938# Forcibly overwrite the original file 939 os.unlink(file) 940 shutil.move(outFileName,file) 941except: 942# cleanup our temporary file 943 os.unlink(outFileName) 944print"Failed to strip RCS keywords in%s"%file 945raise 946 947print"Patched up RCS keywords in%s"%file 948 949defp4UserForCommit(self,id): 950# Return the tuple (perforce user,git email) for a given git commit id 951 self.getUserMapFromPerforceServer() 952 gitEmail =read_pipe("git log --max-count=1 --format='%%ae'%s"%id) 953 gitEmail = gitEmail.strip() 954if not self.emails.has_key(gitEmail): 955return(None,gitEmail) 956else: 957return(self.emails[gitEmail],gitEmail) 958 959defcheckValidP4Users(self,commits): 960# check if any git authors cannot be mapped to p4 users 961foridin commits: 962(user,email) = self.p4UserForCommit(id) 963if not user: 964 msg ="Cannot find p4 user for email%sin commit%s."% (email,id) 965ifgitConfig('git-p4.allowMissingP4Users').lower() =="true": 966print"%s"% msg 967else: 968die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg) 969 970deflastP4Changelist(self): 971# Get back the last changelist number submitted in this client spec. This 972# then gets used to patch up the username in the change. If the same 973# client spec is being used by multiple processes then this might go 974# wrong. 975 results =p4CmdList("client -o")# find the current client 976 client =None 977for r in results: 978if r.has_key('Client'): 979 client = r['Client'] 980break 981if not client: 982die("could not get client spec") 983 results =p4CmdList(["changes","-c", client,"-m","1"]) 984for r in results: 985if r.has_key('change'): 986return r['change'] 987die("Could not get changelist number for last submit - cannot patch up user details") 988 989defmodifyChangelistUser(self, changelist, newUser): 990# fixup the user field of a changelist after it has been submitted. 991 changes =p4CmdList("change -o%s"% changelist) 992iflen(changes) !=1: 993die("Bad output from p4 change modifying%sto user%s"% 994(changelist, newUser)) 995 996 c = changes[0] 997if c['User'] == newUser:return# nothing to do 998 c['User'] = newUser 999input= marshal.dumps(c)10001001 result =p4CmdList("change -f -i", stdin=input)1002for r in result:1003if r.has_key('code'):1004if r['code'] =='error':1005die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1006if r.has_key('data'):1007print("Updated user field for changelist%sto%s"% (changelist, newUser))1008return1009die("Could not modify user field of changelist%sto%s"% (changelist, newUser))10101011defcanChangeChangelists(self):1012# check to see if we have p4 admin or super-user permissions, either of1013# which are required to modify changelists.1014 results =p4CmdList(["protects", self.depotPath])1015for r in results:1016if r.has_key('perm'):1017if r['perm'] =='admin':1018return11019if r['perm'] =='super':1020return11021return010221023defprepareSubmitTemplate(self):1024"""Run "p4 change -o" to grab a change specification template.1025 This does not use "p4 -G", as it is nice to keep the submission1026 template in original order, since a human might edit it.10271028 Remove lines in the Files section that show changes to files1029 outside the depot path we're committing into."""10301031 template =""1032 inFilesSection =False1033for line inp4_read_pipe_lines(['change','-o']):1034if line.endswith("\r\n"):1035 line = line[:-2] +"\n"1036if inFilesSection:1037if line.startswith("\t"):1038# path starts and ends with a tab1039 path = line[1:]1040 lastTab = path.rfind("\t")1041if lastTab != -1:1042 path = path[:lastTab]1043if notp4PathStartsWith(path, self.depotPath):1044continue1045else:1046 inFilesSection =False1047else:1048if line.startswith("Files:"):1049 inFilesSection =True10501051 template += line10521053return template10541055defedit_template(self, template_file):1056"""Invoke the editor to let the user change the submission1057 message. Return true if okay to continue with the submit."""10581059# if configured to skip the editing part, just submit1060ifgitConfig("git-p4.skipSubmitEdit") =="true":1061return True10621063# look at the modification time, to check later if the user saved1064# the file1065 mtime = os.stat(template_file).st_mtime10661067# invoke the editor1068if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1069 editor = os.environ.get("P4EDITOR")1070else:1071 editor =read_pipe("git var GIT_EDITOR").strip()1072system(editor +" "+ template_file)10731074# If the file was not saved, prompt to see if this patch should1075# be skipped. But skip this verification step if configured so.1076ifgitConfig("git-p4.skipSubmitEditCheck") =="true":1077return True10781079# modification time updated means user saved the file1080if os.stat(template_file).st_mtime > mtime:1081return True10821083while True:1084 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1085if response =='y':1086return True1087if response =='n':1088return False10891090defapplyCommit(self,id):1091"""Apply one commit, return True if it succeeded."""10921093print"Applying",read_pipe(["git","show","-s",1094"--format=format:%h%s",id])10951096(p4User, gitEmail) = self.p4UserForCommit(id)10971098 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1099 filesToAdd =set()1100 filesToDelete =set()1101 editedFiles =set()1102 pureRenameCopy =set()1103 filesToChangeExecBit = {}11041105for line in diff:1106 diff =parseDiffTreeEntry(line)1107 modifier = diff['status']1108 path = diff['src']1109if modifier =="M":1110p4_edit(path)1111ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1112 filesToChangeExecBit[path] = diff['dst_mode']1113 editedFiles.add(path)1114elif modifier =="A":1115 filesToAdd.add(path)1116 filesToChangeExecBit[path] = diff['dst_mode']1117if path in filesToDelete:1118 filesToDelete.remove(path)1119elif modifier =="D":1120 filesToDelete.add(path)1121if path in filesToAdd:1122 filesToAdd.remove(path)1123elif modifier =="C":1124 src, dest = diff['src'], diff['dst']1125p4_integrate(src, dest)1126 pureRenameCopy.add(dest)1127if diff['src_sha1'] != diff['dst_sha1']:1128p4_edit(dest)1129 pureRenameCopy.discard(dest)1130ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1131p4_edit(dest)1132 pureRenameCopy.discard(dest)1133 filesToChangeExecBit[dest] = diff['dst_mode']1134 os.unlink(dest)1135 editedFiles.add(dest)1136elif modifier =="R":1137 src, dest = diff['src'], diff['dst']1138if self.p4HasMoveCommand:1139p4_edit(src)# src must be open before move1140p4_move(src, dest)# opens for (move/delete, move/add)1141else:1142p4_integrate(src, dest)1143if diff['src_sha1'] != diff['dst_sha1']:1144p4_edit(dest)1145else:1146 pureRenameCopy.add(dest)1147ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1148if not self.p4HasMoveCommand:1149p4_edit(dest)# with move: already open, writable1150 filesToChangeExecBit[dest] = diff['dst_mode']1151if not self.p4HasMoveCommand:1152 os.unlink(dest)1153 filesToDelete.add(src)1154 editedFiles.add(dest)1155else:1156die("unknown modifier%sfor%s"% (modifier, path))11571158 diffcmd ="git format-patch -k --stdout\"%s^\"..\"%s\""% (id,id)1159 patchcmd = diffcmd +" | git apply "1160 tryPatchCmd = patchcmd +"--check -"1161 applyPatchCmd = patchcmd +"--check --apply -"1162 patch_succeeded =True11631164if os.system(tryPatchCmd) !=0:1165 fixed_rcs_keywords =False1166 patch_succeeded =False1167print"Unfortunately applying the change failed!"11681169# Patch failed, maybe it's just RCS keyword woes. Look through1170# the patch to see if that's possible.1171ifgitConfig("git-p4.attemptRCSCleanup","--bool") =="true":1172file=None1173 pattern =None1174 kwfiles = {}1175forfilein editedFiles | filesToDelete:1176# did this file's delta contain RCS keywords?1177 pattern =p4_keywords_regexp_for_file(file)11781179if pattern:1180# this file is a possibility...look for RCS keywords.1181 regexp = re.compile(pattern, re.VERBOSE)1182for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1183if regexp.search(line):1184if verbose:1185print"got keyword match on%sin%sin%s"% (pattern, line,file)1186 kwfiles[file] = pattern1187break11881189forfilein kwfiles:1190if verbose:1191print"zapping%swith%s"% (line,pattern)1192 self.patchRCSKeywords(file, kwfiles[file])1193 fixed_rcs_keywords =True11941195if fixed_rcs_keywords:1196print"Retrying the patch with RCS keywords cleaned up"1197if os.system(tryPatchCmd) ==0:1198 patch_succeeded =True11991200if not patch_succeeded:1201print"What do you want to do?"1202 response ="x"1203while response !="s":1204 response =raw_input("[s]kip this patch ")1205if response =="s":1206print"Skipping! Good luck with the next patches..."1207for f in editedFiles:1208p4_revert(f)1209for f in filesToAdd:1210 os.remove(f)1211return False12121213system(applyPatchCmd)12141215for f in filesToAdd:1216p4_add(f)1217for f in filesToDelete:1218p4_revert(f)1219p4_delete(f)12201221# Set/clear executable bits1222for f in filesToChangeExecBit.keys():1223 mode = filesToChangeExecBit[f]1224setP4ExecBit(f, mode)12251226 logMessage =extractLogMessageFromGitCommit(id)1227 logMessage = logMessage.strip()1228(logMessage, jobs) = self.separate_jobs_from_description(logMessage)12291230 template = self.prepareSubmitTemplate()1231 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)12321233if self.preserveUser:1234 submitTemplate = submitTemplate + ("\n######## Actual user%s, modified after commit\n"% p4User)12351236if os.environ.has_key("P4DIFF"):1237del(os.environ["P4DIFF"])1238 diff =""1239for editedFile in editedFiles:1240 diff +=p4_read_pipe(['diff','-du',1241wildcard_encode(editedFile)])12421243 newdiff =""1244for newFile in filesToAdd:1245 newdiff +="==== new file ====\n"1246 newdiff +="--- /dev/null\n"1247 newdiff +="+++%s\n"% newFile1248 f =open(newFile,"r")1249for line in f.readlines():1250 newdiff +="+"+ line1251 f.close()12521253if self.checkAuthorship and not self.p4UserIsMe(p4User):1254 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1255 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1256 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"12571258 separatorLine ="######## everything below this line is just the diff #######\n"12591260(handle, fileName) = tempfile.mkstemp()1261 tmpFile = os.fdopen(handle,"w+")1262if self.isWindows:1263 submitTemplate = submitTemplate.replace("\n","\r\n")1264 separatorLine = separatorLine.replace("\n","\r\n")1265 newdiff = newdiff.replace("\n","\r\n")1266 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)1267 tmpFile.close()12681269if self.edit_template(fileName):1270# read the edited message and submit1271 tmpFile =open(fileName,"rb")1272 message = tmpFile.read()1273 tmpFile.close()1274 submitTemplate = message[:message.index(separatorLine)]1275if self.isWindows:1276 submitTemplate = submitTemplate.replace("\r\n","\n")1277p4_write_pipe(['submit','-i'], submitTemplate)12781279if self.preserveUser:1280if p4User:1281# Get last changelist number. Cannot easily get it from1282# the submit command output as the output is1283# unmarshalled.1284 changelist = self.lastP4Changelist()1285 self.modifyChangelistUser(changelist, p4User)12861287# The rename/copy happened by applying a patch that created a1288# new file. This leaves it writable, which confuses p4.1289for f in pureRenameCopy:1290p4_sync(f,"-f")12911292else:1293# skip this patch1294print"Submission cancelled, undoing p4 changes."1295for f in editedFiles:1296p4_revert(f)1297for f in filesToAdd:1298p4_revert(f)1299 os.remove(f)13001301 os.remove(fileName)1302return True# success13031304# Export git tags as p4 labels. Create a p4 label and then tag1305# with that.1306defexportGitTags(self, gitTags):1307 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1308iflen(validLabelRegexp) ==0:1309 validLabelRegexp = defaultLabelRegexp1310 m = re.compile(validLabelRegexp)13111312for name in gitTags:13131314if not m.match(name):1315if verbose:1316print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1317continue13181319# Get the p4 commit this corresponds to1320 logMessage =extractLogMessageFromGitCommit(name)1321 values =extractSettingsGitLog(logMessage)13221323if not values.has_key('change'):1324# a tag pointing to something not sent to p4; ignore1325if verbose:1326print"git tag%sdoes not give a p4 commit"% name1327continue1328else:1329 changelist = values['change']13301331# Get the tag details.1332 inHeader =True1333 isAnnotated =False1334 body = []1335for l inread_pipe_lines(["git","cat-file","-p", name]):1336 l = l.strip()1337if inHeader:1338if re.match(r'tag\s+', l):1339 isAnnotated =True1340elif re.match(r'\s*$', l):1341 inHeader =False1342continue1343else:1344 body.append(l)13451346if not isAnnotated:1347 body = ["lightweight tag imported by git p4\n"]13481349# Create the label - use the same view as the client spec we are using1350 clientSpec =getClientSpec()13511352 labelTemplate ="Label:%s\n"% name1353 labelTemplate +="Description:\n"1354for b in body:1355 labelTemplate +="\t"+ b +"\n"1356 labelTemplate +="View:\n"1357for mapping in clientSpec.mappings:1358 labelTemplate +="\t%s\n"% mapping.depot_side.path13591360p4_write_pipe(["label","-i"], labelTemplate)13611362# Use the label1363p4_system(["tag","-l", name] +1364["%s@%s"% (mapping.depot_side.path, changelist)for mapping in clientSpec.mappings])13651366if verbose:1367print"created p4 label for tag%s"% name13681369defrun(self, args):1370iflen(args) ==0:1371 self.master =currentGitBranch()1372iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1373die("Detecting current git branch failed!")1374eliflen(args) ==1:1375 self.master = args[0]1376if notbranchExists(self.master):1377die("Branch%sdoes not exist"% self.master)1378else:1379return False13801381 allowSubmit =gitConfig("git-p4.allowSubmit")1382iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1383die("%sis not in git-p4.allowSubmit"% self.master)13841385[upstream, settings] =findUpstreamBranchPoint()1386 self.depotPath = settings['depot-paths'][0]1387iflen(self.origin) ==0:1388 self.origin = upstream13891390if self.preserveUser:1391if not self.canChangeChangelists():1392die("Cannot preserve user names without p4 super-user or admin permissions")13931394if self.verbose:1395print"Origin branch is "+ self.origin13961397iflen(self.depotPath) ==0:1398print"Internal error: cannot locate perforce depot path from existing branches"1399 sys.exit(128)14001401 self.useClientSpec =False1402ifgitConfig("git-p4.useclientspec","--bool") =="true":1403 self.useClientSpec =True1404if self.useClientSpec:1405 self.clientSpecDirs =getClientSpec()14061407if self.useClientSpec:1408# all files are relative to the client spec1409 self.clientPath =getClientRoot()1410else:1411 self.clientPath =p4Where(self.depotPath)14121413if self.clientPath =="":1414die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)14151416print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1417 self.oldWorkingDirectory = os.getcwd()14181419# ensure the clientPath exists1420 new_client_dir =False1421if not os.path.exists(self.clientPath):1422 new_client_dir =True1423 os.makedirs(self.clientPath)14241425chdir(self.clientPath)1426print"Synchronizing p4 checkout..."1427if new_client_dir:1428# old one was destroyed, and maybe nobody told p41429p4_sync("...","-f")1430else:1431p4_sync("...")1432 self.check()14331434 commits = []1435for line inread_pipe_lines("git rev-list --no-merges%s..%s"% (self.origin, self.master)):1436 commits.append(line.strip())1437 commits.reverse()14381439if self.preserveUser or(gitConfig("git-p4.skipUserNameCheck") =="true"):1440 self.checkAuthorship =False1441else:1442 self.checkAuthorship =True14431444if self.preserveUser:1445 self.checkValidP4Users(commits)14461447#1448# Build up a set of options to be passed to diff when1449# submitting each commit to p4.1450#1451if self.detectRenames:1452# command-line -M arg1453 self.diffOpts ="-M"1454else:1455# If not explicitly set check the config variable1456 detectRenames =gitConfig("git-p4.detectRenames")14571458if detectRenames.lower() =="false"or detectRenames =="":1459 self.diffOpts =""1460elif detectRenames.lower() =="true":1461 self.diffOpts ="-M"1462else:1463 self.diffOpts ="-M%s"% detectRenames14641465# no command-line arg for -C or --find-copies-harder, just1466# config variables1467 detectCopies =gitConfig("git-p4.detectCopies")1468if detectCopies.lower() =="false"or detectCopies =="":1469pass1470elif detectCopies.lower() =="true":1471 self.diffOpts +=" -C"1472else:1473 self.diffOpts +=" -C%s"% detectCopies14741475ifgitConfig("git-p4.detectCopiesHarder","--bool") =="true":1476 self.diffOpts +=" --find-copies-harder"14771478 applied = []1479for commit in commits:1480 ok = self.applyCommit(commit)1481if ok:1482 applied.append(commit)14831484chdir(self.oldWorkingDirectory)14851486iflen(commits) ==len(applied):1487print"All commits applied!"14881489 sync =P4Sync()1490 sync.run([])14911492 rebase =P4Rebase()1493 rebase.rebase()14941495else:1496iflen(applied) ==0:1497print"No commits applied."1498else:1499print"Applied only the commits marked with '*':"1500for c in commits:1501if c in applied:1502 star ="*"1503else:1504 star =" "1505print star,read_pipe(["git","show","-s",1506"--format=format:%h%s", c])1507print"You will have to do 'git p4 sync' and rebase."15081509ifgitConfig("git-p4.exportLabels","--bool") =="true":1510 self.exportLabels =True15111512if self.exportLabels:1513 p4Labels =getP4Labels(self.depotPath)1514 gitTags =getGitTags()15151516 missingGitTags = gitTags - p4Labels1517 self.exportGitTags(missingGitTags)15181519# exit with error unless everything applied perfecly1520iflen(commits) !=len(applied):1521 sys.exit(1)15221523return True15241525classView(object):1526"""Represent a p4 view ("p4 help views"), and map files in a1527 repo according to the view."""15281529classPath(object):1530"""A depot or client path, possibly containing wildcards.1531 The only one supported is ... at the end, currently.1532 Initialize with the full path, with //depot or //client."""15331534def__init__(self, path, is_depot):1535 self.path = path1536 self.is_depot = is_depot1537 self.find_wildcards()1538# remember the prefix bit, useful for relative mappings1539 m = re.match("(//[^/]+/)", self.path)1540if not m:1541die("Path%sdoes not start with //prefix/"% self.path)1542 prefix = m.group(1)1543if not self.is_depot:1544# strip //client/ on client paths1545 self.path = self.path[len(prefix):]15461547deffind_wildcards(self):1548"""Make sure wildcards are valid, and set up internal1549 variables."""15501551 self.ends_triple_dot =False1552# There are three wildcards allowed in p4 views1553# (see "p4 help views"). This code knows how to1554# handle "..." (only at the end), but cannot deal with1555# "%%n" or "*". Only check the depot_side, as p4 should1556# validate that the client_side matches too.1557if re.search(r'%%[1-9]', self.path):1558die("Can't handle%%n wildcards in view:%s"% self.path)1559if self.path.find("*") >=0:1560die("Can't handle * wildcards in view:%s"% self.path)1561 triple_dot_index = self.path.find("...")1562if triple_dot_index >=0:1563if triple_dot_index !=len(self.path) -3:1564die("Can handle only single ... wildcard, at end:%s"%1565 self.path)1566 self.ends_triple_dot =True15671568defensure_compatible(self, other_path):1569"""Make sure the wildcards agree."""1570if self.ends_triple_dot != other_path.ends_triple_dot:1571die("Both paths must end with ... if either does;\n"+1572"paths:%s %s"% (self.path, other_path.path))15731574defmatch_wildcards(self, test_path):1575"""See if this test_path matches us, and fill in the value1576 of the wildcards if so. Returns a tuple of1577 (True|False, wildcards[]). For now, only the ... at end1578 is supported, so at most one wildcard."""1579if self.ends_triple_dot:1580 dotless = self.path[:-3]1581if test_path.startswith(dotless):1582 wildcard = test_path[len(dotless):]1583return(True, [ wildcard ])1584else:1585if test_path == self.path:1586return(True, [])1587return(False, [])15881589defmatch(self, test_path):1590"""Just return if it matches; don't bother with the wildcards."""1591 b, _ = self.match_wildcards(test_path)1592return b15931594deffill_in_wildcards(self, wildcards):1595"""Return the relative path, with the wildcards filled in1596 if there are any."""1597if self.ends_triple_dot:1598return self.path[:-3] + wildcards[0]1599else:1600return self.path16011602classMapping(object):1603def__init__(self, depot_side, client_side, overlay, exclude):1604# depot_side is without the trailing /... if it had one1605 self.depot_side = View.Path(depot_side, is_depot=True)1606 self.client_side = View.Path(client_side, is_depot=False)1607 self.overlay = overlay # started with "+"1608 self.exclude = exclude # started with "-"1609assert not(self.overlay and self.exclude)1610 self.depot_side.ensure_compatible(self.client_side)16111612def__str__(self):1613 c =" "1614if self.overlay:1615 c ="+"1616if self.exclude:1617 c ="-"1618return"View.Mapping:%s%s->%s"% \1619(c, self.depot_side.path, self.client_side.path)16201621defmap_depot_to_client(self, depot_path):1622"""Calculate the client path if using this mapping on the1623 given depot path; does not consider the effect of other1624 mappings in a view. Even excluded mappings are returned."""1625 matches, wildcards = self.depot_side.match_wildcards(depot_path)1626if not matches:1627return""1628 client_path = self.client_side.fill_in_wildcards(wildcards)1629return client_path16301631#1632# View methods1633#1634def__init__(self):1635 self.mappings = []16361637defappend(self, view_line):1638"""Parse a view line, splitting it into depot and client1639 sides. Append to self.mappings, preserving order."""16401641# Split the view line into exactly two words. P4 enforces1642# structure on these lines that simplifies this quite a bit.1643#1644# Either or both words may be double-quoted.1645# Single quotes do not matter.1646# Double-quote marks cannot occur inside the words.1647# A + or - prefix is also inside the quotes.1648# There are no quotes unless they contain a space.1649# The line is already white-space stripped.1650# The two words are separated by a single space.1651#1652if view_line[0] =='"':1653# First word is double quoted. Find its end.1654 close_quote_index = view_line.find('"',1)1655if close_quote_index <=0:1656die("No first-word closing quote found:%s"% view_line)1657 depot_side = view_line[1:close_quote_index]1658# skip closing quote and space1659 rhs_index = close_quote_index +1+11660else:1661 space_index = view_line.find(" ")1662if space_index <=0:1663die("No word-splitting space found:%s"% view_line)1664 depot_side = view_line[0:space_index]1665 rhs_index = space_index +116661667if view_line[rhs_index] =='"':1668# Second word is double quoted. Make sure there is a1669# double quote at the end too.1670if not view_line.endswith('"'):1671die("View line with rhs quote should end with one:%s"%1672 view_line)1673# skip the quotes1674 client_side = view_line[rhs_index+1:-1]1675else:1676 client_side = view_line[rhs_index:]16771678# prefix + means overlay on previous mapping1679 overlay =False1680if depot_side.startswith("+"):1681 overlay =True1682 depot_side = depot_side[1:]16831684# prefix - means exclude this path1685 exclude =False1686if depot_side.startswith("-"):1687 exclude =True1688 depot_side = depot_side[1:]16891690 m = View.Mapping(depot_side, client_side, overlay, exclude)1691 self.mappings.append(m)16921693defmap_in_client(self, depot_path):1694"""Return the relative location in the client where this1695 depot file should live. Returns "" if the file should1696 not be mapped in the client."""16971698 paths_filled = []1699 client_path =""17001701# look at later entries first1702for m in self.mappings[::-1]:17031704# see where will this path end up in the client1705 p = m.map_depot_to_client(depot_path)17061707if p =="":1708# Depot path does not belong in client. Must remember1709# this, as previous items should not cause files to1710# exist in this path either. Remember that the list is1711# being walked from the end, which has higher precedence.1712# Overlap mappings do not exclude previous mappings.1713if not m.overlay:1714 paths_filled.append(m.client_side)17151716else:1717# This mapping matched; no need to search any further.1718# But, the mapping could be rejected if the client path1719# has already been claimed by an earlier mapping (i.e.1720# one later in the list, which we are walking backwards).1721 already_mapped_in_client =False1722for f in paths_filled:1723# this is View.Path.match1724if f.match(p):1725 already_mapped_in_client =True1726break1727if not already_mapped_in_client:1728# Include this file, unless it is from a line that1729# explicitly said to exclude it.1730if not m.exclude:1731 client_path = p17321733# a match, even if rejected, always stops the search1734break17351736return client_path17371738classP4Sync(Command, P4UserMap):1739 delete_actions = ("delete","move/delete","purge")17401741def__init__(self):1742 Command.__init__(self)1743 P4UserMap.__init__(self)1744 self.options = [1745 optparse.make_option("--branch", dest="branch"),1746 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1747 optparse.make_option("--changesfile", dest="changesFile"),1748 optparse.make_option("--silent", dest="silent", action="store_true"),1749 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1750 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),1751 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1752help="Import into refs/heads/ , not refs/remotes"),1753 optparse.make_option("--max-changes", dest="maxChanges"),1754 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',1755help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),1756 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',1757help="Only sync files that are included in the Perforce Client Spec")1758]1759 self.description ="""Imports from Perforce into a git repository.\n1760 example:1761 //depot/my/project/ -- to import the current head1762 //depot/my/project/@all -- to import everything1763 //depot/my/project/@1,6 -- to import only from revision 1 to 617641765 (a ... is not needed in the path p4 specification, it's added implicitly)"""17661767 self.usage +=" //depot/path[@revRange]"1768 self.silent =False1769 self.createdBranches =set()1770 self.committedChanges =set()1771 self.branch =""1772 self.detectBranches =False1773 self.detectLabels =False1774 self.importLabels =False1775 self.changesFile =""1776 self.syncWithOrigin =True1777 self.importIntoRemotes =True1778 self.maxChanges =""1779 self.isWindows = (platform.system() =="Windows")1780 self.keepRepoPath =False1781 self.depotPaths =None1782 self.p4BranchesInGit = []1783 self.cloneExclude = []1784 self.useClientSpec =False1785 self.useClientSpec_from_options =False1786 self.clientSpecDirs =None1787 self.tempBranches = []1788 self.tempBranchLocation ="git-p4-tmp"17891790ifgitConfig("git-p4.syncFromOrigin") =="false":1791 self.syncWithOrigin =False17921793# Force a checkpoint in fast-import and wait for it to finish1794defcheckpoint(self):1795 self.gitStream.write("checkpoint\n\n")1796 self.gitStream.write("progress checkpoint\n\n")1797 out = self.gitOutput.readline()1798if self.verbose:1799print"checkpoint finished: "+ out18001801defextractFilesFromCommit(self, commit):1802 self.cloneExclude = [re.sub(r"\.\.\.$","", path)1803for path in self.cloneExclude]1804 files = []1805 fnum =01806while commit.has_key("depotFile%s"% fnum):1807 path = commit["depotFile%s"% fnum]18081809if[p for p in self.cloneExclude1810ifp4PathStartsWith(path, p)]:1811 found =False1812else:1813 found = [p for p in self.depotPaths1814ifp4PathStartsWith(path, p)]1815if not found:1816 fnum = fnum +11817continue18181819file= {}1820file["path"] = path1821file["rev"] = commit["rev%s"% fnum]1822file["action"] = commit["action%s"% fnum]1823file["type"] = commit["type%s"% fnum]1824 files.append(file)1825 fnum = fnum +11826return files18271828defstripRepoPath(self, path, prefixes):1829if self.useClientSpec:1830return self.clientSpecDirs.map_in_client(path)18311832if self.keepRepoPath:1833 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]18341835for p in prefixes:1836ifp4PathStartsWith(path, p):1837 path = path[len(p):]18381839return path18401841defsplitFilesIntoBranches(self, commit):1842 branches = {}1843 fnum =01844while commit.has_key("depotFile%s"% fnum):1845 path = commit["depotFile%s"% fnum]1846 found = [p for p in self.depotPaths1847ifp4PathStartsWith(path, p)]1848if not found:1849 fnum = fnum +11850continue18511852file= {}1853file["path"] = path1854file["rev"] = commit["rev%s"% fnum]1855file["action"] = commit["action%s"% fnum]1856file["type"] = commit["type%s"% fnum]1857 fnum = fnum +118581859 relPath = self.stripRepoPath(path, self.depotPaths)1860 relPath =wildcard_decode(relPath)18611862for branch in self.knownBranches.keys():18631864# add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.21865if relPath.startswith(branch +"/"):1866if branch not in branches:1867 branches[branch] = []1868 branches[branch].append(file)1869break18701871return branches18721873# output one file from the P4 stream1874# - helper for streamP4Files18751876defstreamOneP4File(self,file, contents):1877 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)1878 relPath =wildcard_decode(relPath)1879if verbose:1880 sys.stderr.write("%s\n"% relPath)18811882(type_base, type_mods) =split_p4_type(file["type"])18831884 git_mode ="100644"1885if"x"in type_mods:1886 git_mode ="100755"1887if type_base =="symlink":1888 git_mode ="120000"1889# p4 print on a symlink contains "target\n"; remove the newline1890 data =''.join(contents)1891 contents = [data[:-1]]18921893if type_base =="utf16":1894# p4 delivers different text in the python output to -G1895# than it does when using "print -o", or normal p4 client1896# operations. utf16 is converted to ascii or utf8, perhaps.1897# But ascii text saved as -t utf16 is completely mangled.1898# Invoke print -o to get the real contents.1899 text =p4_read_pipe(['print','-q','-o','-',file['depotFile']])1900 contents = [ text ]19011902if type_base =="apple":1903# Apple filetype files will be streamed as a concatenation of1904# its appledouble header and the contents. This is useless1905# on both macs and non-macs. If using "print -q -o xx", it1906# will create "xx" with the data, and "%xx" with the header.1907# This is also not very useful.1908#1909# Ideally, someday, this script can learn how to generate1910# appledouble files directly and import those to git, but1911# non-mac machines can never find a use for apple filetype.1912print"\nIgnoring apple filetype file%s"%file['depotFile']1913return19141915# Perhaps windows wants unicode, utf16 newlines translated too;1916# but this is not doing it.1917if self.isWindows and type_base =="text":1918 mangled = []1919for data in contents:1920 data = data.replace("\r\n","\n")1921 mangled.append(data)1922 contents = mangled19231924# Note that we do not try to de-mangle keywords on utf16 files,1925# even though in theory somebody may want that.1926 pattern =p4_keywords_regexp_for_type(type_base, type_mods)1927if pattern:1928 regexp = re.compile(pattern, re.VERBOSE)1929 text =''.join(contents)1930 text = regexp.sub(r'$\1$', text)1931 contents = [ text ]19321933 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))19341935# total length...1936 length =01937for d in contents:1938 length = length +len(d)19391940 self.gitStream.write("data%d\n"% length)1941for d in contents:1942 self.gitStream.write(d)1943 self.gitStream.write("\n")19441945defstreamOneP4Deletion(self,file):1946 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)1947 relPath =wildcard_decode(relPath)1948if verbose:1949 sys.stderr.write("delete%s\n"% relPath)1950 self.gitStream.write("D%s\n"% relPath)19511952# handle another chunk of streaming data1953defstreamP4FilesCb(self, marshalled):19541955if marshalled.has_key('depotFile')and self.stream_have_file_info:1956# start of a new file - output the old one first1957 self.streamOneP4File(self.stream_file, self.stream_contents)1958 self.stream_file = {}1959 self.stream_contents = []1960 self.stream_have_file_info =False19611962# pick up the new file information... for the1963# 'data' field we need to append to our array1964for k in marshalled.keys():1965if k =='data':1966 self.stream_contents.append(marshalled['data'])1967else:1968 self.stream_file[k] = marshalled[k]19691970 self.stream_have_file_info =True19711972# Stream directly from "p4 files" into "git fast-import"1973defstreamP4Files(self, files):1974 filesForCommit = []1975 filesToRead = []1976 filesToDelete = []19771978for f in files:1979# if using a client spec, only add the files that have1980# a path in the client1981if self.clientSpecDirs:1982if self.clientSpecDirs.map_in_client(f['path']) =="":1983continue19841985 filesForCommit.append(f)1986if f['action']in self.delete_actions:1987 filesToDelete.append(f)1988else:1989 filesToRead.append(f)19901991# deleted files...1992for f in filesToDelete:1993 self.streamOneP4Deletion(f)19941995iflen(filesToRead) >0:1996 self.stream_file = {}1997 self.stream_contents = []1998 self.stream_have_file_info =False19992000# curry self argument2001defstreamP4FilesCbSelf(entry):2002 self.streamP4FilesCb(entry)20032004 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]20052006p4CmdList(["-x","-","print"],2007 stdin=fileArgs,2008 cb=streamP4FilesCbSelf)20092010# do the last chunk2011if self.stream_file.has_key('depotFile'):2012 self.streamOneP4File(self.stream_file, self.stream_contents)20132014defmake_email(self, userid):2015if userid in self.users:2016return self.users[userid]2017else:2018return"%s<a@b>"% userid20192020# Stream a p4 tag2021defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2022if verbose:2023print"writing tag%sfor commit%s"% (labelName, commit)2024 gitStream.write("tag%s\n"% labelName)2025 gitStream.write("from%s\n"% commit)20262027if labelDetails.has_key('Owner'):2028 owner = labelDetails["Owner"]2029else:2030 owner =None20312032# Try to use the owner of the p4 label, or failing that,2033# the current p4 user id.2034if owner:2035 email = self.make_email(owner)2036else:2037 email = self.make_email(self.p4UserId())2038 tagger ="%s %s %s"% (email, epoch, self.tz)20392040 gitStream.write("tagger%s\n"% tagger)20412042print"labelDetails=",labelDetails2043if labelDetails.has_key('Description'):2044 description = labelDetails['Description']2045else:2046 description ='Label from git p4'20472048 gitStream.write("data%d\n"%len(description))2049 gitStream.write(description)2050 gitStream.write("\n")20512052defcommit(self, details, files, branch, branchPrefixes, parent =""):2053 epoch = details["time"]2054 author = details["user"]2055 self.branchPrefixes = branchPrefixes20562057if self.verbose:2058print"commit into%s"% branch20592060# start with reading files; if that fails, we should not2061# create a commit.2062 new_files = []2063for f in files:2064if[p for p in branchPrefixes ifp4PathStartsWith(f['path'], p)]:2065 new_files.append(f)2066else:2067 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])20682069 self.gitStream.write("commit%s\n"% branch)2070# gitStream.write("mark :%s\n" % details["change"])2071 self.committedChanges.add(int(details["change"]))2072 committer =""2073if author not in self.users:2074 self.getUserMapFromPerforceServer()2075 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)20762077 self.gitStream.write("committer%s\n"% committer)20782079 self.gitStream.write("data <<EOT\n")2080 self.gitStream.write(details["desc"])2081 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"2082% (','.join(branchPrefixes), details["change"]))2083iflen(details['options']) >0:2084 self.gitStream.write(": options =%s"% details['options'])2085 self.gitStream.write("]\nEOT\n\n")20862087iflen(parent) >0:2088if self.verbose:2089print"parent%s"% parent2090 self.gitStream.write("from%s\n"% parent)20912092 self.streamP4Files(new_files)2093 self.gitStream.write("\n")20942095 change =int(details["change"])20962097if self.labels.has_key(change):2098 label = self.labels[change]2099 labelDetails = label[0]2100 labelRevisions = label[1]2101if self.verbose:2102print"Change%sis labelled%s"% (change, labelDetails)21032104 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2105for p in branchPrefixes])21062107iflen(files) ==len(labelRevisions):21082109 cleanedFiles = {}2110for info in files:2111if info["action"]in self.delete_actions:2112continue2113 cleanedFiles[info["depotFile"]] = info["rev"]21142115if cleanedFiles == labelRevisions:2116 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)21172118else:2119if not self.silent:2120print("Tag%sdoes not match with change%s: files do not match."2121% (labelDetails["label"], change))21222123else:2124if not self.silent:2125print("Tag%sdoes not match with change%s: file count is different."2126% (labelDetails["label"], change))21272128# Build a dictionary of changelists and labels, for "detect-labels" option.2129defgetLabels(self):2130 self.labels = {}21312132 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2133iflen(l) >0and not self.silent:2134print"Finding files belonging to labels in%s"% `self.depotPaths`21352136for output in l:2137 label = output["label"]2138 revisions = {}2139 newestChange =02140if self.verbose:2141print"Querying files for label%s"% label2142forfileinp4CmdList(["files"] +2143["%s...@%s"% (p, label)2144for p in self.depotPaths]):2145 revisions[file["depotFile"]] =file["rev"]2146 change =int(file["change"])2147if change > newestChange:2148 newestChange = change21492150 self.labels[newestChange] = [output, revisions]21512152if self.verbose:2153print"Label changes:%s"% self.labels.keys()21542155# Import p4 labels as git tags. A direct mapping does not2156# exist, so assume that if all the files are at the same revision2157# then we can use that, or it's something more complicated we should2158# just ignore.2159defimportP4Labels(self, stream, p4Labels):2160if verbose:2161print"import p4 labels: "+' '.join(p4Labels)21622163 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2164 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2165iflen(validLabelRegexp) ==0:2166 validLabelRegexp = defaultLabelRegexp2167 m = re.compile(validLabelRegexp)21682169for name in p4Labels:2170 commitFound =False21712172if not m.match(name):2173if verbose:2174print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2175continue21762177if name in ignoredP4Labels:2178continue21792180 labelDetails =p4CmdList(['label',"-o", name])[0]21812182# get the most recent changelist for each file in this label2183 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2184for p in self.depotPaths])21852186if change.has_key('change'):2187# find the corresponding git commit; take the oldest commit2188 changelist =int(change['change'])2189 gitCommit =read_pipe(["git","rev-list","--max-count=1",2190"--reverse",":/\[git-p4:.*change =%d\]"% changelist])2191iflen(gitCommit) ==0:2192print"could not find git commit for changelist%d"% changelist2193else:2194 gitCommit = gitCommit.strip()2195 commitFound =True2196# Convert from p4 time format2197try:2198 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2199exceptValueError:2200print"Could not convert label time%s"% labelDetail['Update']2201 tmwhen =122022203 when =int(time.mktime(tmwhen))2204 self.streamTag(stream, name, labelDetails, gitCommit, when)2205if verbose:2206print"p4 label%smapped to git commit%s"% (name, gitCommit)2207else:2208if verbose:2209print"Label%shas no changelists - possibly deleted?"% name22102211if not commitFound:2212# We can't import this label; don't try again as it will get very2213# expensive repeatedly fetching all the files for labels that will2214# never be imported. If the label is moved in the future, the2215# ignore will need to be removed manually.2216system(["git","config","--add","git-p4.ignoredP4Labels", name])22172218defguessProjectName(self):2219for p in self.depotPaths:2220if p.endswith("/"):2221 p = p[:-1]2222 p = p[p.strip().rfind("/") +1:]2223if not p.endswith("/"):2224 p +="/"2225return p22262227defgetBranchMapping(self):2228 lostAndFoundBranches =set()22292230 user =gitConfig("git-p4.branchUser")2231iflen(user) >0:2232 command ="branches -u%s"% user2233else:2234 command ="branches"22352236for info inp4CmdList(command):2237 details =p4Cmd(["branch","-o", info["branch"]])2238 viewIdx =02239while details.has_key("View%s"% viewIdx):2240 paths = details["View%s"% viewIdx].split(" ")2241 viewIdx = viewIdx +12242# require standard //depot/foo/... //depot/bar/... mapping2243iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2244continue2245 source = paths[0]2246 destination = paths[1]2247## HACK2248ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2249 source = source[len(self.depotPaths[0]):-4]2250 destination = destination[len(self.depotPaths[0]):-4]22512252if destination in self.knownBranches:2253if not self.silent:2254print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2255print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2256continue22572258 self.knownBranches[destination] = source22592260 lostAndFoundBranches.discard(destination)22612262if source not in self.knownBranches:2263 lostAndFoundBranches.add(source)22642265# Perforce does not strictly require branches to be defined, so we also2266# check git config for a branch list.2267#2268# Example of branch definition in git config file:2269# [git-p4]2270# branchList=main:branchA2271# branchList=main:branchB2272# branchList=branchA:branchC2273 configBranches =gitConfigList("git-p4.branchList")2274for branch in configBranches:2275if branch:2276(source, destination) = branch.split(":")2277 self.knownBranches[destination] = source22782279 lostAndFoundBranches.discard(destination)22802281if source not in self.knownBranches:2282 lostAndFoundBranches.add(source)228322842285for branch in lostAndFoundBranches:2286 self.knownBranches[branch] = branch22872288defgetBranchMappingFromGitBranches(self):2289 branches =p4BranchesInGit(self.importIntoRemotes)2290for branch in branches.keys():2291if branch =="master":2292 branch ="main"2293else:2294 branch = branch[len(self.projectName):]2295 self.knownBranches[branch] = branch22962297deflistExistingP4GitBranches(self):2298# branches holds mapping from name to commit2299 branches =p4BranchesInGit(self.importIntoRemotes)2300 self.p4BranchesInGit = branches.keys()2301for branch in branches.keys():2302 self.initialParents[self.refPrefix + branch] = branches[branch]23032304defupdateOptionDict(self, d):2305 option_keys = {}2306if self.keepRepoPath:2307 option_keys['keepRepoPath'] =123082309 d["options"] =' '.join(sorted(option_keys.keys()))23102311defreadOptions(self, d):2312 self.keepRepoPath = (d.has_key('options')2313and('keepRepoPath'in d['options']))23142315defgitRefForBranch(self, branch):2316if branch =="main":2317return self.refPrefix +"master"23182319iflen(branch) <=0:2320return branch23212322return self.refPrefix + self.projectName + branch23232324defgitCommitByP4Change(self, ref, change):2325if self.verbose:2326print"looking in ref "+ ref +" for change%susing bisect..."% change23272328 earliestCommit =""2329 latestCommit =parseRevision(ref)23302331while True:2332if self.verbose:2333print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2334 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2335iflen(next) ==0:2336if self.verbose:2337print"argh"2338return""2339 log =extractLogMessageFromGitCommit(next)2340 settings =extractSettingsGitLog(log)2341 currentChange =int(settings['change'])2342if self.verbose:2343print"current change%s"% currentChange23442345if currentChange == change:2346if self.verbose:2347print"found%s"% next2348return next23492350if currentChange < change:2351 earliestCommit ="^%s"% next2352else:2353 latestCommit ="%s"% next23542355return""23562357defimportNewBranch(self, branch, maxChange):2358# make fast-import flush all changes to disk and update the refs using the checkpoint2359# command so that we can try to find the branch parent in the git history2360 self.gitStream.write("checkpoint\n\n");2361 self.gitStream.flush();2362 branchPrefix = self.depotPaths[0] + branch +"/"2363range="@1,%s"% maxChange2364#print "prefix" + branchPrefix2365 changes =p4ChangesForPaths([branchPrefix],range)2366iflen(changes) <=0:2367return False2368 firstChange = changes[0]2369#print "first change in branch: %s" % firstChange2370 sourceBranch = self.knownBranches[branch]2371 sourceDepotPath = self.depotPaths[0] + sourceBranch2372 sourceRef = self.gitRefForBranch(sourceBranch)2373#print "source " + sourceBranch23742375 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2376#print "branch parent: %s" % branchParentChange2377 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2378iflen(gitParent) >0:2379 self.initialParents[self.gitRefForBranch(branch)] = gitParent2380#print "parent git commit: %s" % gitParent23812382 self.importChanges(changes)2383return True23842385defsearchParent(self, parent, branch, target):2386 parentFound =False2387for blob inread_pipe_lines(["git","rev-list","--reverse","--no-merges", parent]):2388 blob = blob.strip()2389iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2390 parentFound =True2391if self.verbose:2392print"Found parent of%sin commit%s"% (branch, blob)2393break2394if parentFound:2395return blob2396else:2397return None23982399defimportChanges(self, changes):2400 cnt =12401for change in changes:2402 description =p4Cmd(["describe",str(change)])2403 self.updateOptionDict(description)24042405if not self.silent:2406 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2407 sys.stdout.flush()2408 cnt = cnt +124092410try:2411if self.detectBranches:2412 branches = self.splitFilesIntoBranches(description)2413for branch in branches.keys():2414## HACK --hwn2415 branchPrefix = self.depotPaths[0] + branch +"/"24162417 parent =""24182419 filesForCommit = branches[branch]24202421if self.verbose:2422print"branch is%s"% branch24232424 self.updatedBranches.add(branch)24252426if branch not in self.createdBranches:2427 self.createdBranches.add(branch)2428 parent = self.knownBranches[branch]2429if parent == branch:2430 parent =""2431else:2432 fullBranch = self.projectName + branch2433if fullBranch not in self.p4BranchesInGit:2434if not self.silent:2435print("\nImporting new branch%s"% fullBranch);2436if self.importNewBranch(branch, change -1):2437 parent =""2438 self.p4BranchesInGit.append(fullBranch)2439if not self.silent:2440print("\nResuming with change%s"% change);24412442if self.verbose:2443print"parent determined through known branches:%s"% parent24442445 branch = self.gitRefForBranch(branch)2446 parent = self.gitRefForBranch(parent)24472448if self.verbose:2449print"looking for initial parent for%s; current parent is%s"% (branch, parent)24502451iflen(parent) ==0and branch in self.initialParents:2452 parent = self.initialParents[branch]2453del self.initialParents[branch]24542455 blob =None2456iflen(parent) >0:2457 tempBranch = os.path.join(self.tempBranchLocation,"%d"% (change))2458if self.verbose:2459print"Creating temporary branch: "+ tempBranch2460 self.commit(description, filesForCommit, tempBranch, [branchPrefix])2461 self.tempBranches.append(tempBranch)2462 self.checkpoint()2463 blob = self.searchParent(parent, branch, tempBranch)2464if blob:2465 self.commit(description, filesForCommit, branch, [branchPrefix], blob)2466else:2467if self.verbose:2468print"Parent of%snot found. Committing into head of%s"% (branch, parent)2469 self.commit(description, filesForCommit, branch, [branchPrefix], parent)2470else:2471 files = self.extractFilesFromCommit(description)2472 self.commit(description, files, self.branch, self.depotPaths,2473 self.initialParent)2474 self.initialParent =""2475exceptIOError:2476print self.gitError.read()2477 sys.exit(1)24782479defimportHeadRevision(self, revision):2480print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)24812482 details = {}2483 details["user"] ="git perforce import user"2484 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2485% (' '.join(self.depotPaths), revision))2486 details["change"] = revision2487 newestRevision =024882489 fileCnt =02490 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]24912492for info inp4CmdList(["files"] + fileArgs):24932494if'code'in info and info['code'] =='error':2495 sys.stderr.write("p4 returned an error:%s\n"2496% info['data'])2497if info['data'].find("must refer to client") >=0:2498 sys.stderr.write("This particular p4 error is misleading.\n")2499 sys.stderr.write("Perhaps the depot path was misspelled.\n");2500 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2501 sys.exit(1)2502if'p4ExitCode'in info:2503 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2504 sys.exit(1)250525062507 change =int(info["change"])2508if change > newestRevision:2509 newestRevision = change25102511if info["action"]in self.delete_actions:2512# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2513#fileCnt = fileCnt + 12514continue25152516for prop in["depotFile","rev","action","type"]:2517 details["%s%s"% (prop, fileCnt)] = info[prop]25182519 fileCnt = fileCnt +125202521 details["change"] = newestRevision25222523# Use time from top-most change so that all git p4 clones of2524# the same p4 repo have the same commit SHA1s.2525 res =p4CmdList("describe -s%d"% newestRevision)2526 newestTime =None2527for r in res:2528if r.has_key('time'):2529 newestTime =int(r['time'])2530if newestTime is None:2531die("\"describe -s\"on newest change%ddid not give a time")2532 details["time"] = newestTime25332534 self.updateOptionDict(details)2535try:2536 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)2537exceptIOError:2538print"IO error with git fast-import. Is your git version recent enough?"2539print self.gitError.read()254025412542defrun(self, args):2543 self.depotPaths = []2544 self.changeRange =""2545 self.initialParent =""2546 self.previousDepotPaths = []25472548# map from branch depot path to parent branch2549 self.knownBranches = {}2550 self.initialParents = {}2551 self.hasOrigin =originP4BranchesExist()2552if not self.syncWithOrigin:2553 self.hasOrigin =False25542555if self.importIntoRemotes:2556 self.refPrefix ="refs/remotes/p4/"2557else:2558 self.refPrefix ="refs/heads/p4/"25592560if self.syncWithOrigin and self.hasOrigin:2561if not self.silent:2562print"Syncing with origin first by calling git fetch origin"2563system("git fetch origin")25642565iflen(self.branch) ==0:2566 self.branch = self.refPrefix +"master"2567ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2568system("git update-ref%srefs/heads/p4"% self.branch)2569system("git branch -D p4");2570# create it /after/ importing, when master exists2571if notgitBranchExists(self.refPrefix +"HEAD")and self.importIntoRemotes andgitBranchExists(self.branch):2572system("git symbolic-ref%sHEAD%s"% (self.refPrefix, self.branch))25732574# accept either the command-line option, or the configuration variable2575if self.useClientSpec:2576# will use this after clone to set the variable2577 self.useClientSpec_from_options =True2578else:2579ifgitConfig("git-p4.useclientspec","--bool") =="true":2580 self.useClientSpec =True2581if self.useClientSpec:2582 self.clientSpecDirs =getClientSpec()25832584# TODO: should always look at previous commits,2585# merge with previous imports, if possible.2586if args == []:2587if self.hasOrigin:2588createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)2589 self.listExistingP4GitBranches()25902591iflen(self.p4BranchesInGit) >1:2592if not self.silent:2593print"Importing from/into multiple branches"2594 self.detectBranches =True25952596if self.verbose:2597print"branches:%s"% self.p4BranchesInGit25982599 p4Change =02600for branch in self.p4BranchesInGit:2601 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)26022603 settings =extractSettingsGitLog(logMsg)26042605 self.readOptions(settings)2606if(settings.has_key('depot-paths')2607and settings.has_key('change')):2608 change =int(settings['change']) +12609 p4Change =max(p4Change, change)26102611 depotPaths =sorted(settings['depot-paths'])2612if self.previousDepotPaths == []:2613 self.previousDepotPaths = depotPaths2614else:2615 paths = []2616for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2617 prev_list = prev.split("/")2618 cur_list = cur.split("/")2619for i inrange(0,min(len(cur_list),len(prev_list))):2620if cur_list[i] <> prev_list[i]:2621 i = i -12622break26232624 paths.append("/".join(cur_list[:i +1]))26252626 self.previousDepotPaths = paths26272628if p4Change >0:2629 self.depotPaths =sorted(self.previousDepotPaths)2630 self.changeRange ="@%s,#head"% p4Change2631if not self.detectBranches:2632 self.initialParent =parseRevision(self.branch)2633if not self.silent and not self.detectBranches:2634print"Performing incremental import into%sgit branch"% self.branch26352636if not self.branch.startswith("refs/"):2637 self.branch ="refs/heads/"+ self.branch26382639iflen(args) ==0and self.depotPaths:2640if not self.silent:2641print"Depot paths:%s"%' '.join(self.depotPaths)2642else:2643if self.depotPaths and self.depotPaths != args:2644print("previous import used depot path%sand now%swas specified. "2645"This doesn't work!"% (' '.join(self.depotPaths),2646' '.join(args)))2647 sys.exit(1)26482649 self.depotPaths =sorted(args)26502651 revision =""2652 self.users = {}26532654# Make sure no revision specifiers are used when --changesfile2655# is specified.2656 bad_changesfile =False2657iflen(self.changesFile) >0:2658for p in self.depotPaths:2659if p.find("@") >=0or p.find("#") >=0:2660 bad_changesfile =True2661break2662if bad_changesfile:2663die("Option --changesfile is incompatible with revision specifiers")26642665 newPaths = []2666for p in self.depotPaths:2667if p.find("@") != -1:2668 atIdx = p.index("@")2669 self.changeRange = p[atIdx:]2670if self.changeRange =="@all":2671 self.changeRange =""2672elif','not in self.changeRange:2673 revision = self.changeRange2674 self.changeRange =""2675 p = p[:atIdx]2676elif p.find("#") != -1:2677 hashIdx = p.index("#")2678 revision = p[hashIdx:]2679 p = p[:hashIdx]2680elif self.previousDepotPaths == []:2681# pay attention to changesfile, if given, else import2682# the entire p4 tree at the head revision2683iflen(self.changesFile) ==0:2684 revision ="#head"26852686 p = re.sub("\.\.\.$","", p)2687if not p.endswith("/"):2688 p +="/"26892690 newPaths.append(p)26912692 self.depotPaths = newPaths26932694 self.loadUserMapFromCache()2695 self.labels = {}2696if self.detectLabels:2697 self.getLabels();26982699if self.detectBranches:2700## FIXME - what's a P4 projectName ?2701 self.projectName = self.guessProjectName()27022703if self.hasOrigin:2704 self.getBranchMappingFromGitBranches()2705else:2706 self.getBranchMapping()2707if self.verbose:2708print"p4-git branches:%s"% self.p4BranchesInGit2709print"initial parents:%s"% self.initialParents2710for b in self.p4BranchesInGit:2711if b !="master":27122713## FIXME2714 b = b[len(self.projectName):]2715 self.createdBranches.add(b)27162717 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))27182719 importProcess = subprocess.Popen(["git","fast-import"],2720 stdin=subprocess.PIPE, stdout=subprocess.PIPE,2721 stderr=subprocess.PIPE);2722 self.gitOutput = importProcess.stdout2723 self.gitStream = importProcess.stdin2724 self.gitError = importProcess.stderr27252726if revision:2727 self.importHeadRevision(revision)2728else:2729 changes = []27302731iflen(self.changesFile) >0:2732 output =open(self.changesFile).readlines()2733 changeSet =set()2734for line in output:2735 changeSet.add(int(line))27362737for change in changeSet:2738 changes.append(change)27392740 changes.sort()2741else:2742# catch "git p4 sync" with no new branches, in a repo that2743# does not have any existing p4 branches2744iflen(args) ==0and not self.p4BranchesInGit:2745die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.");2746if self.verbose:2747print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),2748 self.changeRange)2749 changes =p4ChangesForPaths(self.depotPaths, self.changeRange)27502751iflen(self.maxChanges) >0:2752 changes = changes[:min(int(self.maxChanges),len(changes))]27532754iflen(changes) ==0:2755if not self.silent:2756print"No changes to import!"2757else:2758if not self.silent and not self.detectBranches:2759print"Import destination:%s"% self.branch27602761 self.updatedBranches =set()27622763 self.importChanges(changes)27642765if not self.silent:2766print""2767iflen(self.updatedBranches) >0:2768 sys.stdout.write("Updated branches: ")2769for b in self.updatedBranches:2770 sys.stdout.write("%s"% b)2771 sys.stdout.write("\n")27722773ifgitConfig("git-p4.importLabels","--bool") =="true":2774 self.importLabels =True27752776if self.importLabels:2777 p4Labels =getP4Labels(self.depotPaths)2778 gitTags =getGitTags()27792780 missingP4Labels = p4Labels - gitTags2781 self.importP4Labels(self.gitStream, missingP4Labels)27822783 self.gitStream.close()2784if importProcess.wait() !=0:2785die("fast-import failed:%s"% self.gitError.read())2786 self.gitOutput.close()2787 self.gitError.close()27882789# Cleanup temporary branches created during import2790if self.tempBranches != []:2791for branch in self.tempBranches:2792read_pipe("git update-ref -d%s"% branch)2793 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))27942795return True27962797classP4Rebase(Command):2798def__init__(self):2799 Command.__init__(self)2800 self.options = [2801 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2802]2803 self.importLabels =False2804 self.description = ("Fetches the latest revision from perforce and "2805+"rebases the current work (branch) against it")28062807defrun(self, args):2808 sync =P4Sync()2809 sync.importLabels = self.importLabels2810 sync.run([])28112812return self.rebase()28132814defrebase(self):2815if os.system("git update-index --refresh") !=0:2816die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");2817iflen(read_pipe("git diff-index HEAD --")) >0:2818die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");28192820[upstream, settings] =findUpstreamBranchPoint()2821iflen(upstream) ==0:2822die("Cannot find upstream branchpoint for rebase")28232824# the branchpoint may be p4/foo~3, so strip off the parent2825 upstream = re.sub("~[0-9]+$","", upstream)28262827print"Rebasing the current branch onto%s"% upstream2828 oldHead =read_pipe("git rev-parse HEAD").strip()2829system("git rebase%s"% upstream)2830system("git diff-tree --stat --summary -M%sHEAD"% oldHead)2831return True28322833classP4Clone(P4Sync):2834def__init__(self):2835 P4Sync.__init__(self)2836 self.description ="Creates a new git repository and imports from Perforce into it"2837 self.usage ="usage: %prog [options] //depot/path[@revRange]"2838 self.options += [2839 optparse.make_option("--destination", dest="cloneDestination",2840 action='store', default=None,2841help="where to leave result of the clone"),2842 optparse.make_option("-/", dest="cloneExclude",2843 action="append",type="string",2844help="exclude depot path"),2845 optparse.make_option("--bare", dest="cloneBare",2846 action="store_true", default=False),2847]2848 self.cloneDestination =None2849 self.needsGit =False2850 self.cloneBare =False28512852# This is required for the "append" cloneExclude action2853defensure_value(self, attr, value):2854if nothasattr(self, attr)orgetattr(self, attr)is None:2855setattr(self, attr, value)2856returngetattr(self, attr)28572858defdefaultDestination(self, args):2859## TODO: use common prefix of args?2860 depotPath = args[0]2861 depotDir = re.sub("(@[^@]*)$","", depotPath)2862 depotDir = re.sub("(#[^#]*)$","", depotDir)2863 depotDir = re.sub(r"\.\.\.$","", depotDir)2864 depotDir = re.sub(r"/$","", depotDir)2865return os.path.split(depotDir)[1]28662867defrun(self, args):2868iflen(args) <1:2869return False28702871if self.keepRepoPath and not self.cloneDestination:2872 sys.stderr.write("Must specify destination for --keep-path\n")2873 sys.exit(1)28742875 depotPaths = args28762877if not self.cloneDestination andlen(depotPaths) >1:2878 self.cloneDestination = depotPaths[-1]2879 depotPaths = depotPaths[:-1]28802881 self.cloneExclude = ["/"+p for p in self.cloneExclude]2882for p in depotPaths:2883if not p.startswith("//"):2884return False28852886if not self.cloneDestination:2887 self.cloneDestination = self.defaultDestination(args)28882889print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)28902891if not os.path.exists(self.cloneDestination):2892 os.makedirs(self.cloneDestination)2893chdir(self.cloneDestination)28942895 init_cmd = ["git","init"]2896if self.cloneBare:2897 init_cmd.append("--bare")2898 subprocess.check_call(init_cmd)28992900if not P4Sync.run(self, depotPaths):2901return False2902if self.branch !="master":2903if self.importIntoRemotes:2904 masterbranch ="refs/remotes/p4/master"2905else:2906 masterbranch ="refs/heads/p4/master"2907ifgitBranchExists(masterbranch):2908system("git branch master%s"% masterbranch)2909if not self.cloneBare:2910system("git checkout -f")2911else:2912print"Could not detect main branch. No checkout/master branch created."29132914# auto-set this variable if invoked with --use-client-spec2915if self.useClientSpec_from_options:2916system("git config --bool git-p4.useclientspec true")29172918return True29192920classP4Branches(Command):2921def__init__(self):2922 Command.__init__(self)2923 self.options = [ ]2924 self.description = ("Shows the git branches that hold imports and their "2925+"corresponding perforce depot paths")2926 self.verbose =False29272928defrun(self, args):2929iforiginP4BranchesExist():2930createOrUpdateBranchesFromOrigin()29312932 cmdline ="git rev-parse --symbolic "2933 cmdline +=" --remotes"29342935for line inread_pipe_lines(cmdline):2936 line = line.strip()29372938if not line.startswith('p4/')or line =="p4/HEAD":2939continue2940 branch = line29412942 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)2943 settings =extractSettingsGitLog(log)29442945print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])2946return True29472948classHelpFormatter(optparse.IndentedHelpFormatter):2949def__init__(self):2950 optparse.IndentedHelpFormatter.__init__(self)29512952defformat_description(self, description):2953if description:2954return description +"\n"2955else:2956return""29572958defprintUsage(commands):2959print"usage:%s<command> [options]"% sys.argv[0]2960print""2961print"valid commands:%s"%", ".join(commands)2962print""2963print"Try%s<command> --help for command specific help."% sys.argv[0]2964print""29652966commands = {2967"debug": P4Debug,2968"submit": P4Submit,2969"commit": P4Submit,2970"sync": P4Sync,2971"rebase": P4Rebase,2972"clone": P4Clone,2973"rollback": P4RollBack,2974"branches": P4Branches2975}297629772978defmain():2979iflen(sys.argv[1:]) ==0:2980printUsage(commands.keys())2981 sys.exit(2)29822983 cmd =""2984 cmdName = sys.argv[1]2985try:2986 klass = commands[cmdName]2987 cmd =klass()2988exceptKeyError:2989print"unknown command%s"% cmdName2990print""2991printUsage(commands.keys())2992 sys.exit(2)29932994 options = cmd.options2995 cmd.gitdir = os.environ.get("GIT_DIR",None)29962997 args = sys.argv[2:]29982999 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))3000if cmd.needsGit:3001 options.append(optparse.make_option("--git-dir", dest="gitdir"))30023003 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3004 options,3005 description = cmd.description,3006 formatter =HelpFormatter())30073008(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3009global verbose3010 verbose = cmd.verbose3011if cmd.needsGit:3012if cmd.gitdir ==None:3013 cmd.gitdir = os.path.abspath(".git")3014if notisValidGitDir(cmd.gitdir):3015 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3016if os.path.exists(cmd.gitdir):3017 cdup =read_pipe("git rev-parse --show-cdup").strip()3018iflen(cdup) >0:3019chdir(cdup);30203021if notisValidGitDir(cmd.gitdir):3022ifisValidGitDir(cmd.gitdir +"/.git"):3023 cmd.gitdir +="/.git"3024else:3025die("fatal: cannot locate git repository at%s"% cmd.gitdir)30263027 os.environ["GIT_DIR"] = cmd.gitdir30283029if not cmd.run(args):3030 parser.print_help()3031 sys.exit(2)303230333034if __name__ =='__main__':3035main()