1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10 11import optparse, sys, os, marshal, subprocess, shelve 12import tempfile, getopt, os.path, time, platform 13import re, shutil 14 15verbose =False 16 17# Only labels/tags matching this will be imported/exported 18defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 19 20defp4_build_cmd(cmd): 21"""Build a suitable p4 command line. 22 23 This consolidates building and returning a p4 command line into one 24 location. It means that hooking into the environment, or other configuration 25 can be done more easily. 26 """ 27 real_cmd = ["p4"] 28 29 user =gitConfig("git-p4.user") 30iflen(user) >0: 31 real_cmd += ["-u",user] 32 33 password =gitConfig("git-p4.password") 34iflen(password) >0: 35 real_cmd += ["-P", password] 36 37 port =gitConfig("git-p4.port") 38iflen(port) >0: 39 real_cmd += ["-p", port] 40 41 host =gitConfig("git-p4.host") 42iflen(host) >0: 43 real_cmd += ["-H", host] 44 45 client =gitConfig("git-p4.client") 46iflen(client) >0: 47 real_cmd += ["-c", client] 48 49 50ifisinstance(cmd,basestring): 51 real_cmd =' '.join(real_cmd) +' '+ cmd 52else: 53 real_cmd += cmd 54return real_cmd 55 56defchdir(dir): 57# P4 uses the PWD environment variable rather than getcwd(). Since we're 58# not using the shell, we have to set it ourselves. This path could 59# be relative, so go there first, then figure out where we ended up. 60 os.chdir(dir) 61 os.environ['PWD'] = os.getcwd() 62 63defdie(msg): 64if verbose: 65raiseException(msg) 66else: 67 sys.stderr.write(msg +"\n") 68 sys.exit(1) 69 70defwrite_pipe(c, stdin): 71if verbose: 72 sys.stderr.write('Writing pipe:%s\n'%str(c)) 73 74 expand =isinstance(c,basestring) 75 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 76 pipe = p.stdin 77 val = pipe.write(stdin) 78 pipe.close() 79if p.wait(): 80die('Command failed:%s'%str(c)) 81 82return val 83 84defp4_write_pipe(c, stdin): 85 real_cmd =p4_build_cmd(c) 86returnwrite_pipe(real_cmd, stdin) 87 88defread_pipe(c, ignore_error=False): 89if verbose: 90 sys.stderr.write('Reading pipe:%s\n'%str(c)) 91 92 expand =isinstance(c,basestring) 93 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 94 pipe = p.stdout 95 val = pipe.read() 96if p.wait()and not ignore_error: 97die('Command failed:%s'%str(c)) 98 99return val 100 101defp4_read_pipe(c, ignore_error=False): 102 real_cmd =p4_build_cmd(c) 103returnread_pipe(real_cmd, ignore_error) 104 105defread_pipe_lines(c): 106if verbose: 107 sys.stderr.write('Reading pipe:%s\n'%str(c)) 108 109 expand =isinstance(c, basestring) 110 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 111 pipe = p.stdout 112 val = pipe.readlines() 113if pipe.close()or p.wait(): 114die('Command failed:%s'%str(c)) 115 116return val 117 118defp4_read_pipe_lines(c): 119"""Specifically invoke p4 on the command supplied. """ 120 real_cmd =p4_build_cmd(c) 121returnread_pipe_lines(real_cmd) 122 123defsystem(cmd): 124 expand =isinstance(cmd,basestring) 125if verbose: 126 sys.stderr.write("executing%s\n"%str(cmd)) 127 subprocess.check_call(cmd, shell=expand) 128 129defp4_system(cmd): 130"""Specifically invoke p4 as the system command. """ 131 real_cmd =p4_build_cmd(cmd) 132 expand =isinstance(real_cmd, basestring) 133 subprocess.check_call(real_cmd, shell=expand) 134 135defp4_integrate(src, dest): 136p4_system(["integrate","-Dt", src, dest]) 137 138defp4_sync(path): 139p4_system(["sync", path]) 140 141defp4_add(f): 142p4_system(["add", f]) 143 144defp4_delete(f): 145p4_system(["delete", f]) 146 147defp4_edit(f): 148p4_system(["edit", f]) 149 150defp4_revert(f): 151p4_system(["revert", f]) 152 153defp4_reopen(type,file): 154p4_system(["reopen","-t",type,file]) 155 156# 157# Canonicalize the p4 type and return a tuple of the 158# base type, plus any modifiers. See "p4 help filetypes" 159# for a list and explanation. 160# 161defsplit_p4_type(p4type): 162 163 p4_filetypes_historical = { 164"ctempobj":"binary+Sw", 165"ctext":"text+C", 166"cxtext":"text+Cx", 167"ktext":"text+k", 168"kxtext":"text+kx", 169"ltext":"text+F", 170"tempobj":"binary+FSw", 171"ubinary":"binary+F", 172"uresource":"resource+F", 173"uxbinary":"binary+Fx", 174"xbinary":"binary+x", 175"xltext":"text+Fx", 176"xtempobj":"binary+Swx", 177"xtext":"text+x", 178"xunicode":"unicode+x", 179"xutf16":"utf16+x", 180} 181if p4type in p4_filetypes_historical: 182 p4type = p4_filetypes_historical[p4type] 183 mods ="" 184 s = p4type.split("+") 185 base = s[0] 186 mods ="" 187iflen(s) >1: 188 mods = s[1] 189return(base, mods) 190 191# 192# return the raw p4 type of a file (text, text+ko, etc) 193# 194defp4_type(file): 195 results =p4CmdList(["fstat","-T","headType",file]) 196return results[0]['headType'] 197 198# 199# Given a type base and modifier, return a regexp matching 200# the keywords that can be expanded in the file 201# 202defp4_keywords_regexp_for_type(base, type_mods): 203if base in("text","unicode","binary"): 204 kwords =None 205if"ko"in type_mods: 206 kwords ='Id|Header' 207elif"k"in type_mods: 208 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 209else: 210return None 211 pattern = r""" 212 \$ # Starts with a dollar, followed by... 213 (%s) # one of the keywords, followed by... 214 (:[^$]+)? # possibly an old expansion, followed by... 215 \$ # another dollar 216 """% kwords 217return pattern 218else: 219return None 220 221# 222# Given a file, return a regexp matching the possible 223# RCS keywords that will be expanded, or None for files 224# with kw expansion turned off. 225# 226defp4_keywords_regexp_for_file(file): 227if not os.path.exists(file): 228return None 229else: 230(type_base, type_mods) =split_p4_type(p4_type(file)) 231returnp4_keywords_regexp_for_type(type_base, type_mods) 232 233defsetP4ExecBit(file, mode): 234# Reopens an already open file and changes the execute bit to match 235# the execute bit setting in the passed in mode. 236 237 p4Type ="+x" 238 239if notisModeExec(mode): 240 p4Type =getP4OpenedType(file) 241 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 242 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 243if p4Type[-1] =="+": 244 p4Type = p4Type[0:-1] 245 246p4_reopen(p4Type,file) 247 248defgetP4OpenedType(file): 249# Returns the perforce file type for the given file. 250 251 result =p4_read_pipe(["opened",file]) 252 match = re.match(".*\((.+)\)\r?$", result) 253if match: 254return match.group(1) 255else: 256die("Could not determine file type for%s(result: '%s')"% (file, result)) 257 258# Return the set of all p4 labels 259defgetP4Labels(depotPaths): 260 labels =set() 261ifisinstance(depotPaths,basestring): 262 depotPaths = [depotPaths] 263 264for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 265 label = l['label'] 266 labels.add(label) 267 268return labels 269 270# Return the set of all git tags 271defgetGitTags(): 272 gitTags =set() 273for line inread_pipe_lines(["git","tag"]): 274 tag = line.strip() 275 gitTags.add(tag) 276return gitTags 277 278defdiffTreePattern(): 279# This is a simple generator for the diff tree regex pattern. This could be 280# a class variable if this and parseDiffTreeEntry were a part of a class. 281 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 282while True: 283yield pattern 284 285defparseDiffTreeEntry(entry): 286"""Parses a single diff tree entry into its component elements. 287 288 See git-diff-tree(1) manpage for details about the format of the diff 289 output. This method returns a dictionary with the following elements: 290 291 src_mode - The mode of the source file 292 dst_mode - The mode of the destination file 293 src_sha1 - The sha1 for the source file 294 dst_sha1 - The sha1 fr the destination file 295 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 296 status_score - The score for the status (applicable for 'C' and 'R' 297 statuses). This is None if there is no score. 298 src - The path for the source file. 299 dst - The path for the destination file. This is only present for 300 copy or renames. If it is not present, this is None. 301 302 If the pattern is not matched, None is returned.""" 303 304 match =diffTreePattern().next().match(entry) 305if match: 306return{ 307'src_mode': match.group(1), 308'dst_mode': match.group(2), 309'src_sha1': match.group(3), 310'dst_sha1': match.group(4), 311'status': match.group(5), 312'status_score': match.group(6), 313'src': match.group(7), 314'dst': match.group(10) 315} 316return None 317 318defisModeExec(mode): 319# Returns True if the given git mode represents an executable file, 320# otherwise False. 321return mode[-3:] =="755" 322 323defisModeExecChanged(src_mode, dst_mode): 324returnisModeExec(src_mode) !=isModeExec(dst_mode) 325 326defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 327 328ifisinstance(cmd,basestring): 329 cmd ="-G "+ cmd 330 expand =True 331else: 332 cmd = ["-G"] + cmd 333 expand =False 334 335 cmd =p4_build_cmd(cmd) 336if verbose: 337 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 338 339# Use a temporary file to avoid deadlocks without 340# subprocess.communicate(), which would put another copy 341# of stdout into memory. 342 stdin_file =None 343if stdin is not None: 344 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 345ifisinstance(stdin,basestring): 346 stdin_file.write(stdin) 347else: 348for i in stdin: 349 stdin_file.write(i +'\n') 350 stdin_file.flush() 351 stdin_file.seek(0) 352 353 p4 = subprocess.Popen(cmd, 354 shell=expand, 355 stdin=stdin_file, 356 stdout=subprocess.PIPE) 357 358 result = [] 359try: 360while True: 361 entry = marshal.load(p4.stdout) 362if cb is not None: 363cb(entry) 364else: 365 result.append(entry) 366exceptEOFError: 367pass 368 exitCode = p4.wait() 369if exitCode !=0: 370 entry = {} 371 entry["p4ExitCode"] = exitCode 372 result.append(entry) 373 374return result 375 376defp4Cmd(cmd): 377list=p4CmdList(cmd) 378 result = {} 379for entry inlist: 380 result.update(entry) 381return result; 382 383defp4Where(depotPath): 384if not depotPath.endswith("/"): 385 depotPath +="/" 386 depotPath = depotPath +"..." 387 outputList =p4CmdList(["where", depotPath]) 388 output =None 389for entry in outputList: 390if"depotFile"in entry: 391if entry["depotFile"] == depotPath: 392 output = entry 393break 394elif"data"in entry: 395 data = entry.get("data") 396 space = data.find(" ") 397if data[:space] == depotPath: 398 output = entry 399break 400if output ==None: 401return"" 402if output["code"] =="error": 403return"" 404 clientPath ="" 405if"path"in output: 406 clientPath = output.get("path") 407elif"data"in output: 408 data = output.get("data") 409 lastSpace = data.rfind(" ") 410 clientPath = data[lastSpace +1:] 411 412if clientPath.endswith("..."): 413 clientPath = clientPath[:-3] 414return clientPath 415 416defcurrentGitBranch(): 417returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 418 419defisValidGitDir(path): 420if(os.path.exists(path +"/HEAD") 421and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 422return True; 423return False 424 425defparseRevision(ref): 426returnread_pipe("git rev-parse%s"% ref).strip() 427 428defbranchExists(ref): 429 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 430 ignore_error=True) 431returnlen(rev) >0 432 433defextractLogMessageFromGitCommit(commit): 434 logMessage ="" 435 436## fixme: title is first line of commit, not 1st paragraph. 437 foundTitle =False 438for log inread_pipe_lines("git cat-file commit%s"% commit): 439if not foundTitle: 440iflen(log) ==1: 441 foundTitle =True 442continue 443 444 logMessage += log 445return logMessage 446 447defextractSettingsGitLog(log): 448 values = {} 449for line in log.split("\n"): 450 line = line.strip() 451 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 452if not m: 453continue 454 455 assignments = m.group(1).split(':') 456for a in assignments: 457 vals = a.split('=') 458 key = vals[0].strip() 459 val = ('='.join(vals[1:])).strip() 460if val.endswith('\"')and val.startswith('"'): 461 val = val[1:-1] 462 463 values[key] = val 464 465 paths = values.get("depot-paths") 466if not paths: 467 paths = values.get("depot-path") 468if paths: 469 values['depot-paths'] = paths.split(',') 470return values 471 472defgitBranchExists(branch): 473 proc = subprocess.Popen(["git","rev-parse", branch], 474 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 475return proc.wait() ==0; 476 477_gitConfig = {} 478defgitConfig(key, args =None):# set args to "--bool", for instance 479if not _gitConfig.has_key(key): 480 argsFilter ="" 481if args !=None: 482 argsFilter ="%s"% args 483 cmd ="git config%s%s"% (argsFilter, key) 484 _gitConfig[key] =read_pipe(cmd, ignore_error=True).strip() 485return _gitConfig[key] 486 487defgitConfigList(key): 488if not _gitConfig.has_key(key): 489 _gitConfig[key] =read_pipe("git config --get-all%s"% key, ignore_error=True).strip().split(os.linesep) 490return _gitConfig[key] 491 492defp4BranchesInGit(branchesAreInRemotes =True): 493 branches = {} 494 495 cmdline ="git rev-parse --symbolic " 496if branchesAreInRemotes: 497 cmdline +=" --remotes" 498else: 499 cmdline +=" --branches" 500 501for line inread_pipe_lines(cmdline): 502 line = line.strip() 503 504## only import to p4/ 505if not line.startswith('p4/')or line =="p4/HEAD": 506continue 507 branch = line 508 509# strip off p4 510 branch = re.sub("^p4/","", line) 511 512 branches[branch] =parseRevision(line) 513return branches 514 515deffindUpstreamBranchPoint(head ="HEAD"): 516 branches =p4BranchesInGit() 517# map from depot-path to branch name 518 branchByDepotPath = {} 519for branch in branches.keys(): 520 tip = branches[branch] 521 log =extractLogMessageFromGitCommit(tip) 522 settings =extractSettingsGitLog(log) 523if settings.has_key("depot-paths"): 524 paths =",".join(settings["depot-paths"]) 525 branchByDepotPath[paths] ="remotes/p4/"+ branch 526 527 settings =None 528 parent =0 529while parent <65535: 530 commit = head +"~%s"% parent 531 log =extractLogMessageFromGitCommit(commit) 532 settings =extractSettingsGitLog(log) 533if settings.has_key("depot-paths"): 534 paths =",".join(settings["depot-paths"]) 535if branchByDepotPath.has_key(paths): 536return[branchByDepotPath[paths], settings] 537 538 parent = parent +1 539 540return["", settings] 541 542defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 543if not silent: 544print("Creating/updating branch(es) in%sbased on origin branch(es)" 545% localRefPrefix) 546 547 originPrefix ="origin/p4/" 548 549for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 550 line = line.strip() 551if(not line.startswith(originPrefix))or line.endswith("HEAD"): 552continue 553 554 headName = line[len(originPrefix):] 555 remoteHead = localRefPrefix + headName 556 originHead = line 557 558 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 559if(not original.has_key('depot-paths') 560or not original.has_key('change')): 561continue 562 563 update =False 564if notgitBranchExists(remoteHead): 565if verbose: 566print"creating%s"% remoteHead 567 update =True 568else: 569 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 570if settings.has_key('change') >0: 571if settings['depot-paths'] == original['depot-paths']: 572 originP4Change =int(original['change']) 573 p4Change =int(settings['change']) 574if originP4Change > p4Change: 575print("%s(%s) is newer than%s(%s). " 576"Updating p4 branch from origin." 577% (originHead, originP4Change, 578 remoteHead, p4Change)) 579 update =True 580else: 581print("Ignoring:%swas imported from%swhile " 582"%swas imported from%s" 583% (originHead,','.join(original['depot-paths']), 584 remoteHead,','.join(settings['depot-paths']))) 585 586if update: 587system("git update-ref%s %s"% (remoteHead, originHead)) 588 589deforiginP4BranchesExist(): 590returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 591 592defp4ChangesForPaths(depotPaths, changeRange): 593assert depotPaths 594 cmd = ['changes'] 595for p in depotPaths: 596 cmd += ["%s...%s"% (p, changeRange)] 597 output =p4_read_pipe_lines(cmd) 598 599 changes = {} 600for line in output: 601 changeNum =int(line.split(" ")[1]) 602 changes[changeNum] =True 603 604 changelist = changes.keys() 605 changelist.sort() 606return changelist 607 608defp4PathStartsWith(path, prefix): 609# This method tries to remedy a potential mixed-case issue: 610# 611# If UserA adds //depot/DirA/file1 612# and UserB adds //depot/dira/file2 613# 614# we may or may not have a problem. If you have core.ignorecase=true, 615# we treat DirA and dira as the same directory 616 ignorecase =gitConfig("core.ignorecase","--bool") =="true" 617if ignorecase: 618return path.lower().startswith(prefix.lower()) 619return path.startswith(prefix) 620 621defgetClientSpec(): 622"""Look at the p4 client spec, create a View() object that contains 623 all the mappings, and return it.""" 624 625 specList =p4CmdList("client -o") 626iflen(specList) !=1: 627die('Output from "client -o" is%dlines, expecting 1'% 628len(specList)) 629 630# dictionary of all client parameters 631 entry = specList[0] 632 633# just the keys that start with "View" 634 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 635 636# hold this new View 637 view =View() 638 639# append the lines, in order, to the view 640for view_num inrange(len(view_keys)): 641 k ="View%d"% view_num 642if k not in view_keys: 643die("Expected view key%smissing"% k) 644 view.append(entry[k]) 645 646return view 647 648defgetClientRoot(): 649"""Grab the client directory.""" 650 651 output =p4CmdList("client -o") 652iflen(output) !=1: 653die('Output from "client -o" is%dlines, expecting 1'%len(output)) 654 655 entry = output[0] 656if"Root"not in entry: 657die('Client has no "Root"') 658 659return entry["Root"] 660 661class Command: 662def__init__(self): 663 self.usage ="usage: %prog [options]" 664 self.needsGit =True 665 self.verbose =False 666 667class P4UserMap: 668def__init__(self): 669 self.userMapFromPerforceServer =False 670 self.myP4UserId =None 671 672defp4UserId(self): 673if self.myP4UserId: 674return self.myP4UserId 675 676 results =p4CmdList("user -o") 677for r in results: 678if r.has_key('User'): 679 self.myP4UserId = r['User'] 680return r['User'] 681die("Could not find your p4 user id") 682 683defp4UserIsMe(self, p4User): 684# return True if the given p4 user is actually me 685 me = self.p4UserId() 686if not p4User or p4User != me: 687return False 688else: 689return True 690 691defgetUserCacheFilename(self): 692 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 693return home +"/.gitp4-usercache.txt" 694 695defgetUserMapFromPerforceServer(self): 696if self.userMapFromPerforceServer: 697return 698 self.users = {} 699 self.emails = {} 700 701for output inp4CmdList("users"): 702if not output.has_key("User"): 703continue 704 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 705 self.emails[output["Email"]] = output["User"] 706 707 708 s ='' 709for(key, val)in self.users.items(): 710 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 711 712open(self.getUserCacheFilename(),"wb").write(s) 713 self.userMapFromPerforceServer =True 714 715defloadUserMapFromCache(self): 716 self.users = {} 717 self.userMapFromPerforceServer =False 718try: 719 cache =open(self.getUserCacheFilename(),"rb") 720 lines = cache.readlines() 721 cache.close() 722for line in lines: 723 entry = line.strip().split("\t") 724 self.users[entry[0]] = entry[1] 725exceptIOError: 726 self.getUserMapFromPerforceServer() 727 728classP4Debug(Command): 729def__init__(self): 730 Command.__init__(self) 731 self.options = [] 732 self.description ="A tool to debug the output of p4 -G." 733 self.needsGit =False 734 735defrun(self, args): 736 j =0 737for output inp4CmdList(args): 738print'Element:%d'% j 739 j +=1 740print output 741return True 742 743classP4RollBack(Command): 744def__init__(self): 745 Command.__init__(self) 746 self.options = [ 747 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") 748] 749 self.description ="A tool to debug the multi-branch import. Don't use :)" 750 self.rollbackLocalBranches =False 751 752defrun(self, args): 753iflen(args) !=1: 754return False 755 maxChange =int(args[0]) 756 757if"p4ExitCode"inp4Cmd("changes -m 1"): 758die("Problems executing p4"); 759 760if self.rollbackLocalBranches: 761 refPrefix ="refs/heads/" 762 lines =read_pipe_lines("git rev-parse --symbolic --branches") 763else: 764 refPrefix ="refs/remotes/" 765 lines =read_pipe_lines("git rev-parse --symbolic --remotes") 766 767for line in lines: 768if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"): 769 line = line.strip() 770 ref = refPrefix + line 771 log =extractLogMessageFromGitCommit(ref) 772 settings =extractSettingsGitLog(log) 773 774 depotPaths = settings['depot-paths'] 775 change = settings['change'] 776 777 changed =False 778 779iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange) 780for p in depotPaths]))) ==0: 781print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange) 782system("git update-ref -d%s`git rev-parse%s`"% (ref, ref)) 783continue 784 785while change andint(change) > maxChange: 786 changed =True 787if self.verbose: 788print"%sis at%s; rewinding towards%s"% (ref, change, maxChange) 789system("git update-ref%s\"%s^\""% (ref, ref)) 790 log =extractLogMessageFromGitCommit(ref) 791 settings =extractSettingsGitLog(log) 792 793 794 depotPaths = settings['depot-paths'] 795 change = settings['change'] 796 797if changed: 798print"%srewound to%s"% (ref, change) 799 800return True 801 802classP4Submit(Command, P4UserMap): 803def__init__(self): 804 Command.__init__(self) 805 P4UserMap.__init__(self) 806 self.options = [ 807 optparse.make_option("--origin", dest="origin"), 808 optparse.make_option("-M", dest="detectRenames", action="store_true"), 809# preserve the user, requires relevant p4 permissions 810 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"), 811 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"), 812] 813 self.description ="Submit changes from git to the perforce depot." 814 self.usage +=" [name of git branch to submit into perforce depot]" 815 self.interactive =True 816 self.origin ="" 817 self.detectRenames =False 818 self.preserveUser =gitConfig("git-p4.preserveUser").lower() =="true" 819 self.isWindows = (platform.system() =="Windows") 820 self.exportLabels =False 821 822defcheck(self): 823iflen(p4CmdList("opened ...")) >0: 824die("You have files opened with perforce! Close them before starting the sync.") 825 826# replaces everything between 'Description:' and the next P4 submit template field with the 827# commit message 828defprepareLogMessage(self, template, message): 829 result ="" 830 831 inDescriptionSection =False 832 833for line in template.split("\n"): 834if line.startswith("#"): 835 result += line +"\n" 836continue 837 838if inDescriptionSection: 839if line.startswith("Files:")or line.startswith("Jobs:"): 840 inDescriptionSection =False 841else: 842continue 843else: 844if line.startswith("Description:"): 845 inDescriptionSection =True 846 line +="\n" 847for messageLine in message.split("\n"): 848 line +="\t"+ messageLine +"\n" 849 850 result += line +"\n" 851 852return result 853 854defpatchRCSKeywords(self,file, pattern): 855# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern 856(handle, outFileName) = tempfile.mkstemp(dir='.') 857try: 858 outFile = os.fdopen(handle,"w+") 859 inFile =open(file,"r") 860 regexp = re.compile(pattern, re.VERBOSE) 861for line in inFile.readlines(): 862 line = regexp.sub(r'$\1$', line) 863 outFile.write(line) 864 inFile.close() 865 outFile.close() 866# Forcibly overwrite the original file 867 os.unlink(file) 868 shutil.move(outFileName,file) 869except: 870# cleanup our temporary file 871 os.unlink(outFileName) 872print"Failed to strip RCS keywords in%s"%file 873raise 874 875print"Patched up RCS keywords in%s"%file 876 877defp4UserForCommit(self,id): 878# Return the tuple (perforce user,git email) for a given git commit id 879 self.getUserMapFromPerforceServer() 880 gitEmail =read_pipe("git log --max-count=1 --format='%%ae'%s"%id) 881 gitEmail = gitEmail.strip() 882if not self.emails.has_key(gitEmail): 883return(None,gitEmail) 884else: 885return(self.emails[gitEmail],gitEmail) 886 887defcheckValidP4Users(self,commits): 888# check if any git authors cannot be mapped to p4 users 889foridin commits: 890(user,email) = self.p4UserForCommit(id) 891if not user: 892 msg ="Cannot find p4 user for email%sin commit%s."% (email,id) 893ifgitConfig('git-p4.allowMissingP4Users').lower() =="true": 894print"%s"% msg 895else: 896die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg) 897 898deflastP4Changelist(self): 899# Get back the last changelist number submitted in this client spec. This 900# then gets used to patch up the username in the change. If the same 901# client spec is being used by multiple processes then this might go 902# wrong. 903 results =p4CmdList("client -o")# find the current client 904 client =None 905for r in results: 906if r.has_key('Client'): 907 client = r['Client'] 908break 909if not client: 910die("could not get client spec") 911 results =p4CmdList(["changes","-c", client,"-m","1"]) 912for r in results: 913if r.has_key('change'): 914return r['change'] 915die("Could not get changelist number for last submit - cannot patch up user details") 916 917defmodifyChangelistUser(self, changelist, newUser): 918# fixup the user field of a changelist after it has been submitted. 919 changes =p4CmdList("change -o%s"% changelist) 920iflen(changes) !=1: 921die("Bad output from p4 change modifying%sto user%s"% 922(changelist, newUser)) 923 924 c = changes[0] 925if c['User'] == newUser:return# nothing to do 926 c['User'] = newUser 927input= marshal.dumps(c) 928 929 result =p4CmdList("change -f -i", stdin=input) 930for r in result: 931if r.has_key('code'): 932if r['code'] =='error': 933die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data'])) 934if r.has_key('data'): 935print("Updated user field for changelist%sto%s"% (changelist, newUser)) 936return 937die("Could not modify user field of changelist%sto%s"% (changelist, newUser)) 938 939defcanChangeChangelists(self): 940# check to see if we have p4 admin or super-user permissions, either of 941# which are required to modify changelists. 942 results =p4CmdList(["protects", self.depotPath]) 943for r in results: 944if r.has_key('perm'): 945if r['perm'] =='admin': 946return1 947if r['perm'] =='super': 948return1 949return0 950 951defprepareSubmitTemplate(self): 952# remove lines in the Files section that show changes to files outside the depot path we're committing into 953 template ="" 954 inFilesSection =False 955for line inp4_read_pipe_lines(['change','-o']): 956if line.endswith("\r\n"): 957 line = line[:-2] +"\n" 958if inFilesSection: 959if line.startswith("\t"): 960# path starts and ends with a tab 961 path = line[1:] 962 lastTab = path.rfind("\t") 963if lastTab != -1: 964 path = path[:lastTab] 965if notp4PathStartsWith(path, self.depotPath): 966continue 967else: 968 inFilesSection =False 969else: 970if line.startswith("Files:"): 971 inFilesSection =True 972 973 template += line 974 975return template 976 977defedit_template(self, template_file): 978"""Invoke the editor to let the user change the submission 979 message. Return true if okay to continue with the submit.""" 980 981# if configured to skip the editing part, just submit 982ifgitConfig("git-p4.skipSubmitEdit") =="true": 983return True 984 985# look at the modification time, to check later if the user saved 986# the file 987 mtime = os.stat(template_file).st_mtime 988 989# invoke the editor 990if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""): 991 editor = os.environ.get("P4EDITOR") 992else: 993 editor =read_pipe("git var GIT_EDITOR").strip() 994system(editor +" "+ template_file) 995 996# If the file was not saved, prompt to see if this patch should 997# be skipped. But skip this verification step if configured so. 998ifgitConfig("git-p4.skipSubmitEditCheck") =="true": 999return True10001001# modification time updated means user saved the file1002if os.stat(template_file).st_mtime > mtime:1003return True10041005while True:1006 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1007if response =='y':1008return True1009if response =='n':1010return False10111012defapplyCommit(self,id):1013print"Applying%s"% (read_pipe("git log --max-count=1 --pretty=oneline%s"%id))10141015(p4User, gitEmail) = self.p4UserForCommit(id)10161017if not self.detectRenames:1018# If not explicitly set check the config variable1019 self.detectRenames =gitConfig("git-p4.detectRenames")10201021if self.detectRenames.lower() =="false"or self.detectRenames =="":1022 diffOpts =""1023elif self.detectRenames.lower() =="true":1024 diffOpts ="-M"1025else:1026 diffOpts ="-M%s"% self.detectRenames10271028 detectCopies =gitConfig("git-p4.detectCopies")1029if detectCopies.lower() =="true":1030 diffOpts +=" -C"1031elif detectCopies !=""and detectCopies.lower() !="false":1032 diffOpts +=" -C%s"% detectCopies10331034ifgitConfig("git-p4.detectCopiesHarder","--bool") =="true":1035 diffOpts +=" --find-copies-harder"10361037 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (diffOpts,id,id))1038 filesToAdd =set()1039 filesToDelete =set()1040 editedFiles =set()1041 filesToChangeExecBit = {}10421043for line in diff:1044 diff =parseDiffTreeEntry(line)1045 modifier = diff['status']1046 path = diff['src']1047if modifier =="M":1048p4_edit(path)1049ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1050 filesToChangeExecBit[path] = diff['dst_mode']1051 editedFiles.add(path)1052elif modifier =="A":1053 filesToAdd.add(path)1054 filesToChangeExecBit[path] = diff['dst_mode']1055if path in filesToDelete:1056 filesToDelete.remove(path)1057elif modifier =="D":1058 filesToDelete.add(path)1059if path in filesToAdd:1060 filesToAdd.remove(path)1061elif modifier =="C":1062 src, dest = diff['src'], diff['dst']1063p4_integrate(src, dest)1064if diff['src_sha1'] != diff['dst_sha1']:1065p4_edit(dest)1066ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1067p4_edit(dest)1068 filesToChangeExecBit[dest] = diff['dst_mode']1069 os.unlink(dest)1070 editedFiles.add(dest)1071elif modifier =="R":1072 src, dest = diff['src'], diff['dst']1073p4_integrate(src, dest)1074if diff['src_sha1'] != diff['dst_sha1']:1075p4_edit(dest)1076ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1077p4_edit(dest)1078 filesToChangeExecBit[dest] = diff['dst_mode']1079 os.unlink(dest)1080 editedFiles.add(dest)1081 filesToDelete.add(src)1082else:1083die("unknown modifier%sfor%s"% (modifier, path))10841085 diffcmd ="git format-patch -k --stdout\"%s^\"..\"%s\""% (id,id)1086 patchcmd = diffcmd +" | git apply "1087 tryPatchCmd = patchcmd +"--check -"1088 applyPatchCmd = patchcmd +"--check --apply -"1089 patch_succeeded =True10901091if os.system(tryPatchCmd) !=0:1092 fixed_rcs_keywords =False1093 patch_succeeded =False1094print"Unfortunately applying the change failed!"10951096# Patch failed, maybe it's just RCS keyword woes. Look through1097# the patch to see if that's possible.1098ifgitConfig("git-p4.attemptRCSCleanup","--bool") =="true":1099file=None1100 pattern =None1101 kwfiles = {}1102forfilein editedFiles | filesToDelete:1103# did this file's delta contain RCS keywords?1104 pattern =p4_keywords_regexp_for_file(file)11051106if pattern:1107# this file is a possibility...look for RCS keywords.1108 regexp = re.compile(pattern, re.VERBOSE)1109for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1110if regexp.search(line):1111if verbose:1112print"got keyword match on%sin%sin%s"% (pattern, line,file)1113 kwfiles[file] = pattern1114break11151116forfilein kwfiles:1117if verbose:1118print"zapping%swith%s"% (line,pattern)1119 self.patchRCSKeywords(file, kwfiles[file])1120 fixed_rcs_keywords =True11211122if fixed_rcs_keywords:1123print"Retrying the patch with RCS keywords cleaned up"1124if os.system(tryPatchCmd) ==0:1125 patch_succeeded =True11261127if not patch_succeeded:1128print"What do you want to do?"1129 response ="x"1130while response !="s"and response !="a"and response !="w":1131 response =raw_input("[s]kip this patch / [a]pply the patch forcibly "1132"and with .rej files / [w]rite the patch to a file (patch.txt) ")1133if response =="s":1134print"Skipping! Good luck with the next patches..."1135for f in editedFiles:1136p4_revert(f)1137for f in filesToAdd:1138 os.remove(f)1139return1140elif response =="a":1141 os.system(applyPatchCmd)1142iflen(filesToAdd) >0:1143print"You may also want to call p4 add on the following files:"1144print" ".join(filesToAdd)1145iflen(filesToDelete):1146print"The following files should be scheduled for deletion with p4 delete:"1147print" ".join(filesToDelete)1148die("Please resolve and submit the conflict manually and "1149+"continue afterwards with git p4 submit --continue")1150elif response =="w":1151system(diffcmd +" > patch.txt")1152print"Patch saved to patch.txt in%s!"% self.clientPath1153die("Please resolve and submit the conflict manually and "1154"continue afterwards with git p4 submit --continue")11551156system(applyPatchCmd)11571158for f in filesToAdd:1159p4_add(f)1160for f in filesToDelete:1161p4_revert(f)1162p4_delete(f)11631164# Set/clear executable bits1165for f in filesToChangeExecBit.keys():1166 mode = filesToChangeExecBit[f]1167setP4ExecBit(f, mode)11681169 logMessage =extractLogMessageFromGitCommit(id)1170 logMessage = logMessage.strip()11711172 template = self.prepareSubmitTemplate()11731174if self.interactive:1175 submitTemplate = self.prepareLogMessage(template, logMessage)11761177if self.preserveUser:1178 submitTemplate = submitTemplate + ("\n######## Actual user%s, modified after commit\n"% p4User)11791180if os.environ.has_key("P4DIFF"):1181del(os.environ["P4DIFF"])1182 diff =""1183for editedFile in editedFiles:1184 diff +=p4_read_pipe(['diff','-du', editedFile])11851186 newdiff =""1187for newFile in filesToAdd:1188 newdiff +="==== new file ====\n"1189 newdiff +="--- /dev/null\n"1190 newdiff +="+++%s\n"% newFile1191 f =open(newFile,"r")1192for line in f.readlines():1193 newdiff +="+"+ line1194 f.close()11951196if self.checkAuthorship and not self.p4UserIsMe(p4User):1197 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1198 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1199 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"12001201 separatorLine ="######## everything below this line is just the diff #######\n"12021203(handle, fileName) = tempfile.mkstemp()1204 tmpFile = os.fdopen(handle,"w+")1205if self.isWindows:1206 submitTemplate = submitTemplate.replace("\n","\r\n")1207 separatorLine = separatorLine.replace("\n","\r\n")1208 newdiff = newdiff.replace("\n","\r\n")1209 tmpFile.write(submitTemplate + separatorLine + diff + newdiff)1210 tmpFile.close()12111212if self.edit_template(fileName):1213# read the edited message and submit1214 tmpFile =open(fileName,"rb")1215 message = tmpFile.read()1216 tmpFile.close()1217 submitTemplate = message[:message.index(separatorLine)]1218if self.isWindows:1219 submitTemplate = submitTemplate.replace("\r\n","\n")1220p4_write_pipe(['submit','-i'], submitTemplate)12211222if self.preserveUser:1223if p4User:1224# Get last changelist number. Cannot easily get it from1225# the submit command output as the output is1226# unmarshalled.1227 changelist = self.lastP4Changelist()1228 self.modifyChangelistUser(changelist, p4User)1229else:1230# skip this patch1231print"Submission cancelled, undoing p4 changes."1232for f in editedFiles:1233p4_revert(f)1234for f in filesToAdd:1235p4_revert(f)1236 os.remove(f)12371238 os.remove(fileName)1239else:1240 fileName ="submit.txt"1241file=open(fileName,"w+")1242file.write(self.prepareLogMessage(template, logMessage))1243file.close()1244print("Perforce submit template written as%s. "1245+"Please review/edit and then use p4 submit -i <%sto submit directly!"1246% (fileName, fileName))12471248# Export git tags as p4 labels. Create a p4 label and then tag1249# with that.1250defexportGitTags(self, gitTags):1251 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1252iflen(validLabelRegexp) ==0:1253 validLabelRegexp = defaultLabelRegexp1254 m = re.compile(validLabelRegexp)12551256for name in gitTags:12571258if not m.match(name):1259if verbose:1260print"tag%sdoes not match regexp%s"% (name, validTagRegexp)1261continue12621263# Get the p4 commit this corresponds to1264 logMessage =extractLogMessageFromGitCommit(name)1265 values =extractSettingsGitLog(logMessage)12661267if not values.has_key('change'):1268# a tag pointing to something not sent to p4; ignore1269if verbose:1270print"git tag%sdoes not give a p4 commit"% name1271continue1272else:1273 changelist = values['change']12741275# Get the tag details.1276 inHeader =True1277 isAnnotated =False1278 body = []1279for l inread_pipe_lines(["git","cat-file","-p", name]):1280 l = l.strip()1281if inHeader:1282if re.match(r'tag\s+', l):1283 isAnnotated =True1284elif re.match(r'\s*$', l):1285 inHeader =False1286continue1287else:1288 body.append(l)12891290if not isAnnotated:1291 body = ["lightweight tag imported by git p4\n"]12921293# Create the label - use the same view as the client spec we are using1294 clientSpec =getClientSpec()12951296 labelTemplate ="Label:%s\n"% name1297 labelTemplate +="Description:\n"1298for b in body:1299 labelTemplate +="\t"+ b +"\n"1300 labelTemplate +="View:\n"1301for mapping in clientSpec.mappings:1302 labelTemplate +="\t%s\n"% mapping.depot_side.path13031304p4_write_pipe(["label","-i"], labelTemplate)13051306# Use the label1307p4_system(["tag","-l", name] +1308["%s@%s"% (mapping.depot_side.path, changelist)for mapping in clientSpec.mappings])13091310if verbose:1311print"created p4 label for tag%s"% name13121313defrun(self, args):1314iflen(args) ==0:1315 self.master =currentGitBranch()1316iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1317die("Detecting current git branch failed!")1318eliflen(args) ==1:1319 self.master = args[0]1320if notbranchExists(self.master):1321die("Branch%sdoes not exist"% self.master)1322else:1323return False13241325 allowSubmit =gitConfig("git-p4.allowSubmit")1326iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1327die("%sis not in git-p4.allowSubmit"% self.master)13281329[upstream, settings] =findUpstreamBranchPoint()1330 self.depotPath = settings['depot-paths'][0]1331iflen(self.origin) ==0:1332 self.origin = upstream13331334if self.preserveUser:1335if not self.canChangeChangelists():1336die("Cannot preserve user names without p4 super-user or admin permissions")13371338if self.verbose:1339print"Origin branch is "+ self.origin13401341iflen(self.depotPath) ==0:1342print"Internal error: cannot locate perforce depot path from existing branches"1343 sys.exit(128)13441345 self.useClientSpec =False1346ifgitConfig("git-p4.useclientspec","--bool") =="true":1347 self.useClientSpec =True1348if self.useClientSpec:1349 self.clientSpecDirs =getClientSpec()13501351if self.useClientSpec:1352# all files are relative to the client spec1353 self.clientPath =getClientRoot()1354else:1355 self.clientPath =p4Where(self.depotPath)13561357if self.clientPath =="":1358die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)13591360print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1361 self.oldWorkingDirectory = os.getcwd()13621363# ensure the clientPath exists1364if not os.path.exists(self.clientPath):1365 os.makedirs(self.clientPath)13661367chdir(self.clientPath)1368print"Synchronizing p4 checkout..."1369p4_sync("...")1370 self.check()13711372 commits = []1373for line inread_pipe_lines("git rev-list --no-merges%s..%s"% (self.origin, self.master)):1374 commits.append(line.strip())1375 commits.reverse()13761377if self.preserveUser or(gitConfig("git-p4.skipUserNameCheck") =="true"):1378 self.checkAuthorship =False1379else:1380 self.checkAuthorship =True13811382if self.preserveUser:1383 self.checkValidP4Users(commits)13841385whilelen(commits) >0:1386 commit = commits[0]1387 commits = commits[1:]1388 self.applyCommit(commit)1389if not self.interactive:1390break13911392iflen(commits) ==0:1393print"All changes applied!"1394chdir(self.oldWorkingDirectory)13951396 sync =P4Sync()1397 sync.run([])13981399 rebase =P4Rebase()1400 rebase.rebase()14011402ifgitConfig("git-p4.exportLabels","--bool") =="true":1403 self.exportLabels = true14041405if self.exportLabels:1406 p4Labels =getP4Labels(self.depotPath)1407 gitTags =getGitTags()14081409 missingGitTags = gitTags - p4Labels1410 self.exportGitTags(missingGitTags)14111412return True14131414classView(object):1415"""Represent a p4 view ("p4 help views"), and map files in a1416 repo according to the view."""14171418classPath(object):1419"""A depot or client path, possibly containing wildcards.1420 The only one supported is ... at the end, currently.1421 Initialize with the full path, with //depot or //client."""14221423def__init__(self, path, is_depot):1424 self.path = path1425 self.is_depot = is_depot1426 self.find_wildcards()1427# remember the prefix bit, useful for relative mappings1428 m = re.match("(//[^/]+/)", self.path)1429if not m:1430die("Path%sdoes not start with //prefix/"% self.path)1431 prefix = m.group(1)1432if not self.is_depot:1433# strip //client/ on client paths1434 self.path = self.path[len(prefix):]14351436deffind_wildcards(self):1437"""Make sure wildcards are valid, and set up internal1438 variables."""14391440 self.ends_triple_dot =False1441# There are three wildcards allowed in p4 views1442# (see "p4 help views"). This code knows how to1443# handle "..." (only at the end), but cannot deal with1444# "%%n" or "*". Only check the depot_side, as p4 should1445# validate that the client_side matches too.1446if re.search(r'%%[1-9]', self.path):1447die("Can't handle%%n wildcards in view:%s"% self.path)1448if self.path.find("*") >=0:1449die("Can't handle * wildcards in view:%s"% self.path)1450 triple_dot_index = self.path.find("...")1451if triple_dot_index >=0:1452if triple_dot_index !=len(self.path) -3:1453die("Can handle only single ... wildcard, at end:%s"%1454 self.path)1455 self.ends_triple_dot =True14561457defensure_compatible(self, other_path):1458"""Make sure the wildcards agree."""1459if self.ends_triple_dot != other_path.ends_triple_dot:1460die("Both paths must end with ... if either does;\n"+1461"paths:%s %s"% (self.path, other_path.path))14621463defmatch_wildcards(self, test_path):1464"""See if this test_path matches us, and fill in the value1465 of the wildcards if so. Returns a tuple of1466 (True|False, wildcards[]). For now, only the ... at end1467 is supported, so at most one wildcard."""1468if self.ends_triple_dot:1469 dotless = self.path[:-3]1470if test_path.startswith(dotless):1471 wildcard = test_path[len(dotless):]1472return(True, [ wildcard ])1473else:1474if test_path == self.path:1475return(True, [])1476return(False, [])14771478defmatch(self, test_path):1479"""Just return if it matches; don't bother with the wildcards."""1480 b, _ = self.match_wildcards(test_path)1481return b14821483deffill_in_wildcards(self, wildcards):1484"""Return the relative path, with the wildcards filled in1485 if there are any."""1486if self.ends_triple_dot:1487return self.path[:-3] + wildcards[0]1488else:1489return self.path14901491classMapping(object):1492def__init__(self, depot_side, client_side, overlay, exclude):1493# depot_side is without the trailing /... if it had one1494 self.depot_side = View.Path(depot_side, is_depot=True)1495 self.client_side = View.Path(client_side, is_depot=False)1496 self.overlay = overlay # started with "+"1497 self.exclude = exclude # started with "-"1498assert not(self.overlay and self.exclude)1499 self.depot_side.ensure_compatible(self.client_side)15001501def__str__(self):1502 c =" "1503if self.overlay:1504 c ="+"1505if self.exclude:1506 c ="-"1507return"View.Mapping:%s%s->%s"% \1508(c, self.depot_side.path, self.client_side.path)15091510defmap_depot_to_client(self, depot_path):1511"""Calculate the client path if using this mapping on the1512 given depot path; does not consider the effect of other1513 mappings in a view. Even excluded mappings are returned."""1514 matches, wildcards = self.depot_side.match_wildcards(depot_path)1515if not matches:1516return""1517 client_path = self.client_side.fill_in_wildcards(wildcards)1518return client_path15191520#1521# View methods1522#1523def__init__(self):1524 self.mappings = []15251526defappend(self, view_line):1527"""Parse a view line, splitting it into depot and client1528 sides. Append to self.mappings, preserving order."""15291530# Split the view line into exactly two words. P4 enforces1531# structure on these lines that simplifies this quite a bit.1532#1533# Either or both words may be double-quoted.1534# Single quotes do not matter.1535# Double-quote marks cannot occur inside the words.1536# A + or - prefix is also inside the quotes.1537# There are no quotes unless they contain a space.1538# The line is already white-space stripped.1539# The two words are separated by a single space.1540#1541if view_line[0] =='"':1542# First word is double quoted. Find its end.1543 close_quote_index = view_line.find('"',1)1544if close_quote_index <=0:1545die("No first-word closing quote found:%s"% view_line)1546 depot_side = view_line[1:close_quote_index]1547# skip closing quote and space1548 rhs_index = close_quote_index +1+11549else:1550 space_index = view_line.find(" ")1551if space_index <=0:1552die("No word-splitting space found:%s"% view_line)1553 depot_side = view_line[0:space_index]1554 rhs_index = space_index +115551556if view_line[rhs_index] =='"':1557# Second word is double quoted. Make sure there is a1558# double quote at the end too.1559if not view_line.endswith('"'):1560die("View line with rhs quote should end with one:%s"%1561 view_line)1562# skip the quotes1563 client_side = view_line[rhs_index+1:-1]1564else:1565 client_side = view_line[rhs_index:]15661567# prefix + means overlay on previous mapping1568 overlay =False1569if depot_side.startswith("+"):1570 overlay =True1571 depot_side = depot_side[1:]15721573# prefix - means exclude this path1574 exclude =False1575if depot_side.startswith("-"):1576 exclude =True1577 depot_side = depot_side[1:]15781579 m = View.Mapping(depot_side, client_side, overlay, exclude)1580 self.mappings.append(m)15811582defmap_in_client(self, depot_path):1583"""Return the relative location in the client where this1584 depot file should live. Returns "" if the file should1585 not be mapped in the client."""15861587 paths_filled = []1588 client_path =""15891590# look at later entries first1591for m in self.mappings[::-1]:15921593# see where will this path end up in the client1594 p = m.map_depot_to_client(depot_path)15951596if p =="":1597# Depot path does not belong in client. Must remember1598# this, as previous items should not cause files to1599# exist in this path either. Remember that the list is1600# being walked from the end, which has higher precedence.1601# Overlap mappings do not exclude previous mappings.1602if not m.overlay:1603 paths_filled.append(m.client_side)16041605else:1606# This mapping matched; no need to search any further.1607# But, the mapping could be rejected if the client path1608# has already been claimed by an earlier mapping (i.e.1609# one later in the list, which we are walking backwards).1610 already_mapped_in_client =False1611for f in paths_filled:1612# this is View.Path.match1613if f.match(p):1614 already_mapped_in_client =True1615break1616if not already_mapped_in_client:1617# Include this file, unless it is from a line that1618# explicitly said to exclude it.1619if not m.exclude:1620 client_path = p16211622# a match, even if rejected, always stops the search1623break16241625return client_path16261627classP4Sync(Command, P4UserMap):1628 delete_actions = ("delete","move/delete","purge")16291630def__init__(self):1631 Command.__init__(self)1632 P4UserMap.__init__(self)1633 self.options = [1634 optparse.make_option("--branch", dest="branch"),1635 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1636 optparse.make_option("--changesfile", dest="changesFile"),1637 optparse.make_option("--silent", dest="silent", action="store_true"),1638 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1639 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),1640 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1641help="Import into refs/heads/ , not refs/remotes"),1642 optparse.make_option("--max-changes", dest="maxChanges"),1643 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',1644help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),1645 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',1646help="Only sync files that are included in the Perforce Client Spec")1647]1648 self.description ="""Imports from Perforce into a git repository.\n1649 example:1650 //depot/my/project/ -- to import the current head1651 //depot/my/project/@all -- to import everything1652 //depot/my/project/@1,6 -- to import only from revision 1 to 616531654 (a ... is not needed in the path p4 specification, it's added implicitly)"""16551656 self.usage +=" //depot/path[@revRange]"1657 self.silent =False1658 self.createdBranches =set()1659 self.committedChanges =set()1660 self.branch =""1661 self.detectBranches =False1662 self.detectLabels =False1663 self.importLabels =False1664 self.changesFile =""1665 self.syncWithOrigin =True1666 self.importIntoRemotes =True1667 self.maxChanges =""1668 self.isWindows = (platform.system() =="Windows")1669 self.keepRepoPath =False1670 self.depotPaths =None1671 self.p4BranchesInGit = []1672 self.cloneExclude = []1673 self.useClientSpec =False1674 self.useClientSpec_from_options =False1675 self.clientSpecDirs =None1676 self.tempBranches = []1677 self.tempBranchLocation ="git-p4-tmp"16781679ifgitConfig("git-p4.syncFromOrigin") =="false":1680 self.syncWithOrigin =False16811682#1683# P4 wildcards are not allowed in filenames. P4 complains1684# if you simply add them, but you can force it with "-f", in1685# which case it translates them into %xx encoding internally.1686# Search for and fix just these four characters. Do % last so1687# that fixing it does not inadvertently create new %-escapes.1688#1689defwildcard_decode(self, path):1690# Cannot have * in a filename in windows; untested as to1691# what p4 would do in such a case.1692if not self.isWindows:1693 path = path.replace("%2A","*")1694 path = path.replace("%23","#") \1695.replace("%40","@") \1696.replace("%25","%")1697return path16981699# Force a checkpoint in fast-import and wait for it to finish1700defcheckpoint(self):1701 self.gitStream.write("checkpoint\n\n")1702 self.gitStream.write("progress checkpoint\n\n")1703 out = self.gitOutput.readline()1704if self.verbose:1705print"checkpoint finished: "+ out17061707defextractFilesFromCommit(self, commit):1708 self.cloneExclude = [re.sub(r"\.\.\.$","", path)1709for path in self.cloneExclude]1710 files = []1711 fnum =01712while commit.has_key("depotFile%s"% fnum):1713 path = commit["depotFile%s"% fnum]17141715if[p for p in self.cloneExclude1716ifp4PathStartsWith(path, p)]:1717 found =False1718else:1719 found = [p for p in self.depotPaths1720ifp4PathStartsWith(path, p)]1721if not found:1722 fnum = fnum +11723continue17241725file= {}1726file["path"] = path1727file["rev"] = commit["rev%s"% fnum]1728file["action"] = commit["action%s"% fnum]1729file["type"] = commit["type%s"% fnum]1730 files.append(file)1731 fnum = fnum +11732return files17331734defstripRepoPath(self, path, prefixes):1735if self.useClientSpec:1736return self.clientSpecDirs.map_in_client(path)17371738if self.keepRepoPath:1739 prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]17401741for p in prefixes:1742ifp4PathStartsWith(path, p):1743 path = path[len(p):]17441745return path17461747defsplitFilesIntoBranches(self, commit):1748 branches = {}1749 fnum =01750while commit.has_key("depotFile%s"% fnum):1751 path = commit["depotFile%s"% fnum]1752 found = [p for p in self.depotPaths1753ifp4PathStartsWith(path, p)]1754if not found:1755 fnum = fnum +11756continue17571758file= {}1759file["path"] = path1760file["rev"] = commit["rev%s"% fnum]1761file["action"] = commit["action%s"% fnum]1762file["type"] = commit["type%s"% fnum]1763 fnum = fnum +117641765 relPath = self.stripRepoPath(path, self.depotPaths)17661767for branch in self.knownBranches.keys():17681769# add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.21770if relPath.startswith(branch +"/"):1771if branch not in branches:1772 branches[branch] = []1773 branches[branch].append(file)1774break17751776return branches17771778# output one file from the P4 stream1779# - helper for streamP4Files17801781defstreamOneP4File(self,file, contents):1782 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)1783 relPath = self.wildcard_decode(relPath)1784if verbose:1785 sys.stderr.write("%s\n"% relPath)17861787(type_base, type_mods) =split_p4_type(file["type"])17881789 git_mode ="100644"1790if"x"in type_mods:1791 git_mode ="100755"1792if type_base =="symlink":1793 git_mode ="120000"1794# p4 print on a symlink contains "target\n"; remove the newline1795 data =''.join(contents)1796 contents = [data[:-1]]17971798if type_base =="utf16":1799# p4 delivers different text in the python output to -G1800# than it does when using "print -o", or normal p4 client1801# operations. utf16 is converted to ascii or utf8, perhaps.1802# But ascii text saved as -t utf16 is completely mangled.1803# Invoke print -o to get the real contents.1804 text =p4_read_pipe(['print','-q','-o','-',file['depotFile']])1805 contents = [ text ]18061807if type_base =="apple":1808# Apple filetype files will be streamed as a concatenation of1809# its appledouble header and the contents. This is useless1810# on both macs and non-macs. If using "print -q -o xx", it1811# will create "xx" with the data, and "%xx" with the header.1812# This is also not very useful.1813#1814# Ideally, someday, this script can learn how to generate1815# appledouble files directly and import those to git, but1816# non-mac machines can never find a use for apple filetype.1817print"\nIgnoring apple filetype file%s"%file['depotFile']1818return18191820# Perhaps windows wants unicode, utf16 newlines translated too;1821# but this is not doing it.1822if self.isWindows and type_base =="text":1823 mangled = []1824for data in contents:1825 data = data.replace("\r\n","\n")1826 mangled.append(data)1827 contents = mangled18281829# Note that we do not try to de-mangle keywords on utf16 files,1830# even though in theory somebody may want that.1831 pattern =p4_keywords_regexp_for_type(type_base, type_mods)1832if pattern:1833 regexp = re.compile(pattern, re.VERBOSE)1834 text =''.join(contents)1835 text = regexp.sub(r'$\1$', text)1836 contents = [ text ]18371838 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))18391840# total length...1841 length =01842for d in contents:1843 length = length +len(d)18441845 self.gitStream.write("data%d\n"% length)1846for d in contents:1847 self.gitStream.write(d)1848 self.gitStream.write("\n")18491850defstreamOneP4Deletion(self,file):1851 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)1852if verbose:1853 sys.stderr.write("delete%s\n"% relPath)1854 self.gitStream.write("D%s\n"% relPath)18551856# handle another chunk of streaming data1857defstreamP4FilesCb(self, marshalled):18581859if marshalled.has_key('depotFile')and self.stream_have_file_info:1860# start of a new file - output the old one first1861 self.streamOneP4File(self.stream_file, self.stream_contents)1862 self.stream_file = {}1863 self.stream_contents = []1864 self.stream_have_file_info =False18651866# pick up the new file information... for the1867# 'data' field we need to append to our array1868for k in marshalled.keys():1869if k =='data':1870 self.stream_contents.append(marshalled['data'])1871else:1872 self.stream_file[k] = marshalled[k]18731874 self.stream_have_file_info =True18751876# Stream directly from "p4 files" into "git fast-import"1877defstreamP4Files(self, files):1878 filesForCommit = []1879 filesToRead = []1880 filesToDelete = []18811882for f in files:1883# if using a client spec, only add the files that have1884# a path in the client1885if self.clientSpecDirs:1886if self.clientSpecDirs.map_in_client(f['path']) =="":1887continue18881889 filesForCommit.append(f)1890if f['action']in self.delete_actions:1891 filesToDelete.append(f)1892else:1893 filesToRead.append(f)18941895# deleted files...1896for f in filesToDelete:1897 self.streamOneP4Deletion(f)18981899iflen(filesToRead) >0:1900 self.stream_file = {}1901 self.stream_contents = []1902 self.stream_have_file_info =False19031904# curry self argument1905defstreamP4FilesCbSelf(entry):1906 self.streamP4FilesCb(entry)19071908 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]19091910p4CmdList(["-x","-","print"],1911 stdin=fileArgs,1912 cb=streamP4FilesCbSelf)19131914# do the last chunk1915if self.stream_file.has_key('depotFile'):1916 self.streamOneP4File(self.stream_file, self.stream_contents)19171918defmake_email(self, userid):1919if userid in self.users:1920return self.users[userid]1921else:1922return"%s<a@b>"% userid19231924# Stream a p4 tag1925defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):1926if verbose:1927print"writing tag%sfor commit%s"% (labelName, commit)1928 gitStream.write("tag%s\n"% labelName)1929 gitStream.write("from%s\n"% commit)19301931if labelDetails.has_key('Owner'):1932 owner = labelDetails["Owner"]1933else:1934 owner =None19351936# Try to use the owner of the p4 label, or failing that,1937# the current p4 user id.1938if owner:1939 email = self.make_email(owner)1940else:1941 email = self.make_email(self.p4UserId())1942 tagger ="%s %s %s"% (email, epoch, self.tz)19431944 gitStream.write("tagger%s\n"% tagger)19451946print"labelDetails=",labelDetails1947if labelDetails.has_key('Description'):1948 description = labelDetails['Description']1949else:1950 description ='Label from git p4'19511952 gitStream.write("data%d\n"%len(description))1953 gitStream.write(description)1954 gitStream.write("\n")19551956defcommit(self, details, files, branch, branchPrefixes, parent =""):1957 epoch = details["time"]1958 author = details["user"]1959 self.branchPrefixes = branchPrefixes19601961if self.verbose:1962print"commit into%s"% branch19631964# start with reading files; if that fails, we should not1965# create a commit.1966 new_files = []1967for f in files:1968if[p for p in branchPrefixes ifp4PathStartsWith(f['path'], p)]:1969 new_files.append(f)1970else:1971 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])19721973 self.gitStream.write("commit%s\n"% branch)1974# gitStream.write("mark :%s\n" % details["change"])1975 self.committedChanges.add(int(details["change"]))1976 committer =""1977if author not in self.users:1978 self.getUserMapFromPerforceServer()1979 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)19801981 self.gitStream.write("committer%s\n"% committer)19821983 self.gitStream.write("data <<EOT\n")1984 self.gitStream.write(details["desc"])1985 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"1986% (','.join(branchPrefixes), details["change"]))1987iflen(details['options']) >0:1988 self.gitStream.write(": options =%s"% details['options'])1989 self.gitStream.write("]\nEOT\n\n")19901991iflen(parent) >0:1992if self.verbose:1993print"parent%s"% parent1994 self.gitStream.write("from%s\n"% parent)19951996 self.streamP4Files(new_files)1997 self.gitStream.write("\n")19981999 change =int(details["change"])20002001if self.labels.has_key(change):2002 label = self.labels[change]2003 labelDetails = label[0]2004 labelRevisions = label[1]2005if self.verbose:2006print"Change%sis labelled%s"% (change, labelDetails)20072008 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2009for p in branchPrefixes])20102011iflen(files) ==len(labelRevisions):20122013 cleanedFiles = {}2014for info in files:2015if info["action"]in self.delete_actions:2016continue2017 cleanedFiles[info["depotFile"]] = info["rev"]20182019if cleanedFiles == labelRevisions:2020 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)20212022else:2023if not self.silent:2024print("Tag%sdoes not match with change%s: files do not match."2025% (labelDetails["label"], change))20262027else:2028if not self.silent:2029print("Tag%sdoes not match with change%s: file count is different."2030% (labelDetails["label"], change))20312032# Build a dictionary of changelists and labels, for "detect-labels" option.2033defgetLabels(self):2034 self.labels = {}20352036 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2037iflen(l) >0and not self.silent:2038print"Finding files belonging to labels in%s"% `self.depotPaths`20392040for output in l:2041 label = output["label"]2042 revisions = {}2043 newestChange =02044if self.verbose:2045print"Querying files for label%s"% label2046forfileinp4CmdList(["files"] +2047["%s...@%s"% (p, label)2048for p in self.depotPaths]):2049 revisions[file["depotFile"]] =file["rev"]2050 change =int(file["change"])2051if change > newestChange:2052 newestChange = change20532054 self.labels[newestChange] = [output, revisions]20552056if self.verbose:2057print"Label changes:%s"% self.labels.keys()20582059# Import p4 labels as git tags. A direct mapping does not2060# exist, so assume that if all the files are at the same revision2061# then we can use that, or it's something more complicated we should2062# just ignore.2063defimportP4Labels(self, stream, p4Labels):2064if verbose:2065print"import p4 labels: "+' '.join(p4Labels)20662067 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2068 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2069iflen(validLabelRegexp) ==0:2070 validLabelRegexp = defaultLabelRegexp2071 m = re.compile(validLabelRegexp)20722073for name in p4Labels:2074 commitFound =False20752076if not m.match(name):2077if verbose:2078print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2079continue20802081if name in ignoredP4Labels:2082continue20832084 labelDetails =p4CmdList(['label',"-o", name])[0]20852086# get the most recent changelist for each file in this label2087 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2088for p in self.depotPaths])20892090if change.has_key('change'):2091# find the corresponding git commit; take the oldest commit2092 changelist =int(change['change'])2093 gitCommit =read_pipe(["git","rev-list","--max-count=1",2094"--reverse",":/\[git-p4:.*change =%d\]"% changelist])2095iflen(gitCommit) ==0:2096print"could not find git commit for changelist%d"% changelist2097else:2098 gitCommit = gitCommit.strip()2099 commitFound =True2100# Convert from p4 time format2101try:2102 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2103exceptValueError:2104print"Could not convert label time%s"% labelDetail['Update']2105 tmwhen =121062107 when =int(time.mktime(tmwhen))2108 self.streamTag(stream, name, labelDetails, gitCommit, when)2109if verbose:2110print"p4 label%smapped to git commit%s"% (name, gitCommit)2111else:2112if verbose:2113print"Label%shas no changelists - possibly deleted?"% name21142115if not commitFound:2116# We can't import this label; don't try again as it will get very2117# expensive repeatedly fetching all the files for labels that will2118# never be imported. If the label is moved in the future, the2119# ignore will need to be removed manually.2120system(["git","config","--add","git-p4.ignoredP4Labels", name])21212122defguessProjectName(self):2123for p in self.depotPaths:2124if p.endswith("/"):2125 p = p[:-1]2126 p = p[p.strip().rfind("/") +1:]2127if not p.endswith("/"):2128 p +="/"2129return p21302131defgetBranchMapping(self):2132 lostAndFoundBranches =set()21332134 user =gitConfig("git-p4.branchUser")2135iflen(user) >0:2136 command ="branches -u%s"% user2137else:2138 command ="branches"21392140for info inp4CmdList(command):2141 details =p4Cmd(["branch","-o", info["branch"]])2142 viewIdx =02143while details.has_key("View%s"% viewIdx):2144 paths = details["View%s"% viewIdx].split(" ")2145 viewIdx = viewIdx +12146# require standard //depot/foo/... //depot/bar/... mapping2147iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2148continue2149 source = paths[0]2150 destination = paths[1]2151## HACK2152ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2153 source = source[len(self.depotPaths[0]):-4]2154 destination = destination[len(self.depotPaths[0]):-4]21552156if destination in self.knownBranches:2157if not self.silent:2158print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2159print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2160continue21612162 self.knownBranches[destination] = source21632164 lostAndFoundBranches.discard(destination)21652166if source not in self.knownBranches:2167 lostAndFoundBranches.add(source)21682169# Perforce does not strictly require branches to be defined, so we also2170# check git config for a branch list.2171#2172# Example of branch definition in git config file:2173# [git-p4]2174# branchList=main:branchA2175# branchList=main:branchB2176# branchList=branchA:branchC2177 configBranches =gitConfigList("git-p4.branchList")2178for branch in configBranches:2179if branch:2180(source, destination) = branch.split(":")2181 self.knownBranches[destination] = source21822183 lostAndFoundBranches.discard(destination)21842185if source not in self.knownBranches:2186 lostAndFoundBranches.add(source)218721882189for branch in lostAndFoundBranches:2190 self.knownBranches[branch] = branch21912192defgetBranchMappingFromGitBranches(self):2193 branches =p4BranchesInGit(self.importIntoRemotes)2194for branch in branches.keys():2195if branch =="master":2196 branch ="main"2197else:2198 branch = branch[len(self.projectName):]2199 self.knownBranches[branch] = branch22002201deflistExistingP4GitBranches(self):2202# branches holds mapping from name to commit2203 branches =p4BranchesInGit(self.importIntoRemotes)2204 self.p4BranchesInGit = branches.keys()2205for branch in branches.keys():2206 self.initialParents[self.refPrefix + branch] = branches[branch]22072208defupdateOptionDict(self, d):2209 option_keys = {}2210if self.keepRepoPath:2211 option_keys['keepRepoPath'] =122122213 d["options"] =' '.join(sorted(option_keys.keys()))22142215defreadOptions(self, d):2216 self.keepRepoPath = (d.has_key('options')2217and('keepRepoPath'in d['options']))22182219defgitRefForBranch(self, branch):2220if branch =="main":2221return self.refPrefix +"master"22222223iflen(branch) <=0:2224return branch22252226return self.refPrefix + self.projectName + branch22272228defgitCommitByP4Change(self, ref, change):2229if self.verbose:2230print"looking in ref "+ ref +" for change%susing bisect..."% change22312232 earliestCommit =""2233 latestCommit =parseRevision(ref)22342235while True:2236if self.verbose:2237print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2238 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2239iflen(next) ==0:2240if self.verbose:2241print"argh"2242return""2243 log =extractLogMessageFromGitCommit(next)2244 settings =extractSettingsGitLog(log)2245 currentChange =int(settings['change'])2246if self.verbose:2247print"current change%s"% currentChange22482249if currentChange == change:2250if self.verbose:2251print"found%s"% next2252return next22532254if currentChange < change:2255 earliestCommit ="^%s"% next2256else:2257 latestCommit ="%s"% next22582259return""22602261defimportNewBranch(self, branch, maxChange):2262# make fast-import flush all changes to disk and update the refs using the checkpoint2263# command so that we can try to find the branch parent in the git history2264 self.gitStream.write("checkpoint\n\n");2265 self.gitStream.flush();2266 branchPrefix = self.depotPaths[0] + branch +"/"2267range="@1,%s"% maxChange2268#print "prefix" + branchPrefix2269 changes =p4ChangesForPaths([branchPrefix],range)2270iflen(changes) <=0:2271return False2272 firstChange = changes[0]2273#print "first change in branch: %s" % firstChange2274 sourceBranch = self.knownBranches[branch]2275 sourceDepotPath = self.depotPaths[0] + sourceBranch2276 sourceRef = self.gitRefForBranch(sourceBranch)2277#print "source " + sourceBranch22782279 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2280#print "branch parent: %s" % branchParentChange2281 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2282iflen(gitParent) >0:2283 self.initialParents[self.gitRefForBranch(branch)] = gitParent2284#print "parent git commit: %s" % gitParent22852286 self.importChanges(changes)2287return True22882289defsearchParent(self, parent, branch, target):2290 parentFound =False2291for blob inread_pipe_lines(["git","rev-list","--reverse","--no-merges", parent]):2292 blob = blob.strip()2293iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2294 parentFound =True2295if self.verbose:2296print"Found parent of%sin commit%s"% (branch, blob)2297break2298if parentFound:2299return blob2300else:2301return None23022303defimportChanges(self, changes):2304 cnt =12305for change in changes:2306 description =p4Cmd(["describe",str(change)])2307 self.updateOptionDict(description)23082309if not self.silent:2310 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2311 sys.stdout.flush()2312 cnt = cnt +123132314try:2315if self.detectBranches:2316 branches = self.splitFilesIntoBranches(description)2317for branch in branches.keys():2318## HACK --hwn2319 branchPrefix = self.depotPaths[0] + branch +"/"23202321 parent =""23222323 filesForCommit = branches[branch]23242325if self.verbose:2326print"branch is%s"% branch23272328 self.updatedBranches.add(branch)23292330if branch not in self.createdBranches:2331 self.createdBranches.add(branch)2332 parent = self.knownBranches[branch]2333if parent == branch:2334 parent =""2335else:2336 fullBranch = self.projectName + branch2337if fullBranch not in self.p4BranchesInGit:2338if not self.silent:2339print("\nImporting new branch%s"% fullBranch);2340if self.importNewBranch(branch, change -1):2341 parent =""2342 self.p4BranchesInGit.append(fullBranch)2343if not self.silent:2344print("\nResuming with change%s"% change);23452346if self.verbose:2347print"parent determined through known branches:%s"% parent23482349 branch = self.gitRefForBranch(branch)2350 parent = self.gitRefForBranch(parent)23512352if self.verbose:2353print"looking for initial parent for%s; current parent is%s"% (branch, parent)23542355iflen(parent) ==0and branch in self.initialParents:2356 parent = self.initialParents[branch]2357del self.initialParents[branch]23582359 blob =None2360iflen(parent) >0:2361 tempBranch = os.path.join(self.tempBranchLocation,"%d"% (change))2362if self.verbose:2363print"Creating temporary branch: "+ tempBranch2364 self.commit(description, filesForCommit, tempBranch, [branchPrefix])2365 self.tempBranches.append(tempBranch)2366 self.checkpoint()2367 blob = self.searchParent(parent, branch, tempBranch)2368if blob:2369 self.commit(description, filesForCommit, branch, [branchPrefix], blob)2370else:2371if self.verbose:2372print"Parent of%snot found. Committing into head of%s"% (branch, parent)2373 self.commit(description, filesForCommit, branch, [branchPrefix], parent)2374else:2375 files = self.extractFilesFromCommit(description)2376 self.commit(description, files, self.branch, self.depotPaths,2377 self.initialParent)2378 self.initialParent =""2379exceptIOError:2380print self.gitError.read()2381 sys.exit(1)23822383defimportHeadRevision(self, revision):2384print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)23852386 details = {}2387 details["user"] ="git perforce import user"2388 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2389% (' '.join(self.depotPaths), revision))2390 details["change"] = revision2391 newestRevision =023922393 fileCnt =02394 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]23952396for info inp4CmdList(["files"] + fileArgs):23972398if'code'in info and info['code'] =='error':2399 sys.stderr.write("p4 returned an error:%s\n"2400% info['data'])2401if info['data'].find("must refer to client") >=0:2402 sys.stderr.write("This particular p4 error is misleading.\n")2403 sys.stderr.write("Perhaps the depot path was misspelled.\n");2404 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2405 sys.exit(1)2406if'p4ExitCode'in info:2407 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2408 sys.exit(1)240924102411 change =int(info["change"])2412if change > newestRevision:2413 newestRevision = change24142415if info["action"]in self.delete_actions:2416# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2417#fileCnt = fileCnt + 12418continue24192420for prop in["depotFile","rev","action","type"]:2421 details["%s%s"% (prop, fileCnt)] = info[prop]24222423 fileCnt = fileCnt +124242425 details["change"] = newestRevision24262427# Use time from top-most change so that all git p4 clones of2428# the same p4 repo have the same commit SHA1s.2429 res =p4CmdList("describe -s%d"% newestRevision)2430 newestTime =None2431for r in res:2432if r.has_key('time'):2433 newestTime =int(r['time'])2434if newestTime is None:2435die("\"describe -s\"on newest change%ddid not give a time")2436 details["time"] = newestTime24372438 self.updateOptionDict(details)2439try:2440 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)2441exceptIOError:2442print"IO error with git fast-import. Is your git version recent enough?"2443print self.gitError.read()244424452446defrun(self, args):2447 self.depotPaths = []2448 self.changeRange =""2449 self.initialParent =""2450 self.previousDepotPaths = []24512452# map from branch depot path to parent branch2453 self.knownBranches = {}2454 self.initialParents = {}2455 self.hasOrigin =originP4BranchesExist()2456if not self.syncWithOrigin:2457 self.hasOrigin =False24582459if self.importIntoRemotes:2460 self.refPrefix ="refs/remotes/p4/"2461else:2462 self.refPrefix ="refs/heads/p4/"24632464if self.syncWithOrigin and self.hasOrigin:2465if not self.silent:2466print"Syncing with origin first by calling git fetch origin"2467system("git fetch origin")24682469iflen(self.branch) ==0:2470 self.branch = self.refPrefix +"master"2471ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2472system("git update-ref%srefs/heads/p4"% self.branch)2473system("git branch -D p4");2474# create it /after/ importing, when master exists2475if notgitBranchExists(self.refPrefix +"HEAD")and self.importIntoRemotes andgitBranchExists(self.branch):2476system("git symbolic-ref%sHEAD%s"% (self.refPrefix, self.branch))24772478# accept either the command-line option, or the configuration variable2479if self.useClientSpec:2480# will use this after clone to set the variable2481 self.useClientSpec_from_options =True2482else:2483ifgitConfig("git-p4.useclientspec","--bool") =="true":2484 self.useClientSpec =True2485if self.useClientSpec:2486 self.clientSpecDirs =getClientSpec()24872488# TODO: should always look at previous commits,2489# merge with previous imports, if possible.2490if args == []:2491if self.hasOrigin:2492createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)2493 self.listExistingP4GitBranches()24942495iflen(self.p4BranchesInGit) >1:2496if not self.silent:2497print"Importing from/into multiple branches"2498 self.detectBranches =True24992500if self.verbose:2501print"branches:%s"% self.p4BranchesInGit25022503 p4Change =02504for branch in self.p4BranchesInGit:2505 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)25062507 settings =extractSettingsGitLog(logMsg)25082509 self.readOptions(settings)2510if(settings.has_key('depot-paths')2511and settings.has_key('change')):2512 change =int(settings['change']) +12513 p4Change =max(p4Change, change)25142515 depotPaths =sorted(settings['depot-paths'])2516if self.previousDepotPaths == []:2517 self.previousDepotPaths = depotPaths2518else:2519 paths = []2520for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2521 prev_list = prev.split("/")2522 cur_list = cur.split("/")2523for i inrange(0,min(len(cur_list),len(prev_list))):2524if cur_list[i] <> prev_list[i]:2525 i = i -12526break25272528 paths.append("/".join(cur_list[:i +1]))25292530 self.previousDepotPaths = paths25312532if p4Change >0:2533 self.depotPaths =sorted(self.previousDepotPaths)2534 self.changeRange ="@%s,#head"% p4Change2535if not self.detectBranches:2536 self.initialParent =parseRevision(self.branch)2537if not self.silent and not self.detectBranches:2538print"Performing incremental import into%sgit branch"% self.branch25392540if not self.branch.startswith("refs/"):2541 self.branch ="refs/heads/"+ self.branch25422543iflen(args) ==0and self.depotPaths:2544if not self.silent:2545print"Depot paths:%s"%' '.join(self.depotPaths)2546else:2547if self.depotPaths and self.depotPaths != args:2548print("previous import used depot path%sand now%swas specified. "2549"This doesn't work!"% (' '.join(self.depotPaths),2550' '.join(args)))2551 sys.exit(1)25522553 self.depotPaths =sorted(args)25542555 revision =""2556 self.users = {}25572558# Make sure no revision specifiers are used when --changesfile2559# is specified.2560 bad_changesfile =False2561iflen(self.changesFile) >0:2562for p in self.depotPaths:2563if p.find("@") >=0or p.find("#") >=0:2564 bad_changesfile =True2565break2566if bad_changesfile:2567die("Option --changesfile is incompatible with revision specifiers")25682569 newPaths = []2570for p in self.depotPaths:2571if p.find("@") != -1:2572 atIdx = p.index("@")2573 self.changeRange = p[atIdx:]2574if self.changeRange =="@all":2575 self.changeRange =""2576elif','not in self.changeRange:2577 revision = self.changeRange2578 self.changeRange =""2579 p = p[:atIdx]2580elif p.find("#") != -1:2581 hashIdx = p.index("#")2582 revision = p[hashIdx:]2583 p = p[:hashIdx]2584elif self.previousDepotPaths == []:2585# pay attention to changesfile, if given, else import2586# the entire p4 tree at the head revision2587iflen(self.changesFile) ==0:2588 revision ="#head"25892590 p = re.sub("\.\.\.$","", p)2591if not p.endswith("/"):2592 p +="/"25932594 newPaths.append(p)25952596 self.depotPaths = newPaths25972598 self.loadUserMapFromCache()2599 self.labels = {}2600if self.detectLabels:2601 self.getLabels();26022603if self.detectBranches:2604## FIXME - what's a P4 projectName ?2605 self.projectName = self.guessProjectName()26062607if self.hasOrigin:2608 self.getBranchMappingFromGitBranches()2609else:2610 self.getBranchMapping()2611if self.verbose:2612print"p4-git branches:%s"% self.p4BranchesInGit2613print"initial parents:%s"% self.initialParents2614for b in self.p4BranchesInGit:2615if b !="master":26162617## FIXME2618 b = b[len(self.projectName):]2619 self.createdBranches.add(b)26202621 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))26222623 importProcess = subprocess.Popen(["git","fast-import"],2624 stdin=subprocess.PIPE, stdout=subprocess.PIPE,2625 stderr=subprocess.PIPE);2626 self.gitOutput = importProcess.stdout2627 self.gitStream = importProcess.stdin2628 self.gitError = importProcess.stderr26292630if revision:2631 self.importHeadRevision(revision)2632else:2633 changes = []26342635iflen(self.changesFile) >0:2636 output =open(self.changesFile).readlines()2637 changeSet =set()2638for line in output:2639 changeSet.add(int(line))26402641for change in changeSet:2642 changes.append(change)26432644 changes.sort()2645else:2646# catch "git p4 sync" with no new branches, in a repo that2647# does not have any existing p4 branches2648iflen(args) ==0and not self.p4BranchesInGit:2649die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.");2650if self.verbose:2651print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),2652 self.changeRange)2653 changes =p4ChangesForPaths(self.depotPaths, self.changeRange)26542655iflen(self.maxChanges) >0:2656 changes = changes[:min(int(self.maxChanges),len(changes))]26572658iflen(changes) ==0:2659if not self.silent:2660print"No changes to import!"2661else:2662if not self.silent and not self.detectBranches:2663print"Import destination:%s"% self.branch26642665 self.updatedBranches =set()26662667 self.importChanges(changes)26682669if not self.silent:2670print""2671iflen(self.updatedBranches) >0:2672 sys.stdout.write("Updated branches: ")2673for b in self.updatedBranches:2674 sys.stdout.write("%s"% b)2675 sys.stdout.write("\n")26762677ifgitConfig("git-p4.importLabels","--bool") =="true":2678 self.importLabels = true26792680if self.importLabels:2681 p4Labels =getP4Labels(self.depotPaths)2682 gitTags =getGitTags()26832684 missingP4Labels = p4Labels - gitTags2685 self.importP4Labels(self.gitStream, missingP4Labels)26862687 self.gitStream.close()2688if importProcess.wait() !=0:2689die("fast-import failed:%s"% self.gitError.read())2690 self.gitOutput.close()2691 self.gitError.close()26922693# Cleanup temporary branches created during import2694if self.tempBranches != []:2695for branch in self.tempBranches:2696read_pipe("git update-ref -d%s"% branch)2697 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))26982699return True27002701classP4Rebase(Command):2702def__init__(self):2703 Command.__init__(self)2704 self.options = [2705 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2706]2707 self.importLabels =False2708 self.description = ("Fetches the latest revision from perforce and "2709+"rebases the current work (branch) against it")27102711defrun(self, args):2712 sync =P4Sync()2713 sync.importLabels = self.importLabels2714 sync.run([])27152716return self.rebase()27172718defrebase(self):2719if os.system("git update-index --refresh") !=0:2720die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");2721iflen(read_pipe("git diff-index HEAD --")) >0:2722die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");27232724[upstream, settings] =findUpstreamBranchPoint()2725iflen(upstream) ==0:2726die("Cannot find upstream branchpoint for rebase")27272728# the branchpoint may be p4/foo~3, so strip off the parent2729 upstream = re.sub("~[0-9]+$","", upstream)27302731print"Rebasing the current branch onto%s"% upstream2732 oldHead =read_pipe("git rev-parse HEAD").strip()2733system("git rebase%s"% upstream)2734system("git diff-tree --stat --summary -M%sHEAD"% oldHead)2735return True27362737classP4Clone(P4Sync):2738def__init__(self):2739 P4Sync.__init__(self)2740 self.description ="Creates a new git repository and imports from Perforce into it"2741 self.usage ="usage: %prog [options] //depot/path[@revRange]"2742 self.options += [2743 optparse.make_option("--destination", dest="cloneDestination",2744 action='store', default=None,2745help="where to leave result of the clone"),2746 optparse.make_option("-/", dest="cloneExclude",2747 action="append",type="string",2748help="exclude depot path"),2749 optparse.make_option("--bare", dest="cloneBare",2750 action="store_true", default=False),2751]2752 self.cloneDestination =None2753 self.needsGit =False2754 self.cloneBare =False27552756# This is required for the "append" cloneExclude action2757defensure_value(self, attr, value):2758if nothasattr(self, attr)orgetattr(self, attr)is None:2759setattr(self, attr, value)2760returngetattr(self, attr)27612762defdefaultDestination(self, args):2763## TODO: use common prefix of args?2764 depotPath = args[0]2765 depotDir = re.sub("(@[^@]*)$","", depotPath)2766 depotDir = re.sub("(#[^#]*)$","", depotDir)2767 depotDir = re.sub(r"\.\.\.$","", depotDir)2768 depotDir = re.sub(r"/$","", depotDir)2769return os.path.split(depotDir)[1]27702771defrun(self, args):2772iflen(args) <1:2773return False27742775if self.keepRepoPath and not self.cloneDestination:2776 sys.stderr.write("Must specify destination for --keep-path\n")2777 sys.exit(1)27782779 depotPaths = args27802781if not self.cloneDestination andlen(depotPaths) >1:2782 self.cloneDestination = depotPaths[-1]2783 depotPaths = depotPaths[:-1]27842785 self.cloneExclude = ["/"+p for p in self.cloneExclude]2786for p in depotPaths:2787if not p.startswith("//"):2788return False27892790if not self.cloneDestination:2791 self.cloneDestination = self.defaultDestination(args)27922793print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)27942795if not os.path.exists(self.cloneDestination):2796 os.makedirs(self.cloneDestination)2797chdir(self.cloneDestination)27982799 init_cmd = ["git","init"]2800if self.cloneBare:2801 init_cmd.append("--bare")2802 subprocess.check_call(init_cmd)28032804if not P4Sync.run(self, depotPaths):2805return False2806if self.branch !="master":2807if self.importIntoRemotes:2808 masterbranch ="refs/remotes/p4/master"2809else:2810 masterbranch ="refs/heads/p4/master"2811ifgitBranchExists(masterbranch):2812system("git branch master%s"% masterbranch)2813if not self.cloneBare:2814system("git checkout -f")2815else:2816print"Could not detect main branch. No checkout/master branch created."28172818# auto-set this variable if invoked with --use-client-spec2819if self.useClientSpec_from_options:2820system("git config --bool git-p4.useclientspec true")28212822return True28232824classP4Branches(Command):2825def__init__(self):2826 Command.__init__(self)2827 self.options = [ ]2828 self.description = ("Shows the git branches that hold imports and their "2829+"corresponding perforce depot paths")2830 self.verbose =False28312832defrun(self, args):2833iforiginP4BranchesExist():2834createOrUpdateBranchesFromOrigin()28352836 cmdline ="git rev-parse --symbolic "2837 cmdline +=" --remotes"28382839for line inread_pipe_lines(cmdline):2840 line = line.strip()28412842if not line.startswith('p4/')or line =="p4/HEAD":2843continue2844 branch = line28452846 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)2847 settings =extractSettingsGitLog(log)28482849print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])2850return True28512852classHelpFormatter(optparse.IndentedHelpFormatter):2853def__init__(self):2854 optparse.IndentedHelpFormatter.__init__(self)28552856defformat_description(self, description):2857if description:2858return description +"\n"2859else:2860return""28612862defprintUsage(commands):2863print"usage:%s<command> [options]"% sys.argv[0]2864print""2865print"valid commands:%s"%", ".join(commands)2866print""2867print"Try%s<command> --help for command specific help."% sys.argv[0]2868print""28692870commands = {2871"debug": P4Debug,2872"submit": P4Submit,2873"commit": P4Submit,2874"sync": P4Sync,2875"rebase": P4Rebase,2876"clone": P4Clone,2877"rollback": P4RollBack,2878"branches": P4Branches2879}288028812882defmain():2883iflen(sys.argv[1:]) ==0:2884printUsage(commands.keys())2885 sys.exit(2)28862887 cmd =""2888 cmdName = sys.argv[1]2889try:2890 klass = commands[cmdName]2891 cmd =klass()2892exceptKeyError:2893print"unknown command%s"% cmdName2894print""2895printUsage(commands.keys())2896 sys.exit(2)28972898 options = cmd.options2899 cmd.gitdir = os.environ.get("GIT_DIR",None)29002901 args = sys.argv[2:]29022903 options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))2904if cmd.needsGit:2905 options.append(optparse.make_option("--git-dir", dest="gitdir"))29062907 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),2908 options,2909 description = cmd.description,2910 formatter =HelpFormatter())29112912(cmd, args) = parser.parse_args(sys.argv[2:], cmd);2913global verbose2914 verbose = cmd.verbose2915if cmd.needsGit:2916if cmd.gitdir ==None:2917 cmd.gitdir = os.path.abspath(".git")2918if notisValidGitDir(cmd.gitdir):2919 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()2920if os.path.exists(cmd.gitdir):2921 cdup =read_pipe("git rev-parse --show-cdup").strip()2922iflen(cdup) >0:2923chdir(cdup);29242925if notisValidGitDir(cmd.gitdir):2926ifisValidGitDir(cmd.gitdir +"/.git"):2927 cmd.gitdir +="/.git"2928else:2929die("fatal: cannot locate git repository at%s"% cmd.gitdir)29302931 os.environ["GIT_DIR"] = cmd.gitdir29322933if not cmd.run(args):2934 parser.print_help()2935 sys.exit(2)293629372938if __name__ =='__main__':2939main()