1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25 26try: 27from subprocess import CalledProcessError 28exceptImportError: 29# from python2.7:subprocess.py 30# Exception classes used by this module. 31classCalledProcessError(Exception): 32"""This exception is raised when a process run by check_call() returns 33 a non-zero exit status. The exit status will be stored in the 34 returncode attribute.""" 35def__init__(self, returncode, cmd): 36 self.returncode = returncode 37 self.cmd = cmd 38def__str__(self): 39return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 40 41verbose =False 42 43# Only labels/tags matching this will be imported/exported 44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 45 46# Grab changes in blocks of this many revisions, unless otherwise requested 47defaultBlockSize =512 48 49defp4_build_cmd(cmd): 50"""Build a suitable p4 command line. 51 52 This consolidates building and returning a p4 command line into one 53 location. It means that hooking into the environment, or other configuration 54 can be done more easily. 55 """ 56 real_cmd = ["p4"] 57 58 user =gitConfig("git-p4.user") 59iflen(user) >0: 60 real_cmd += ["-u",user] 61 62 password =gitConfig("git-p4.password") 63iflen(password) >0: 64 real_cmd += ["-P", password] 65 66 port =gitConfig("git-p4.port") 67iflen(port) >0: 68 real_cmd += ["-p", port] 69 70 host =gitConfig("git-p4.host") 71iflen(host) >0: 72 real_cmd += ["-H", host] 73 74 client =gitConfig("git-p4.client") 75iflen(client) >0: 76 real_cmd += ["-c", client] 77 78 79ifisinstance(cmd,basestring): 80 real_cmd =' '.join(real_cmd) +' '+ cmd 81else: 82 real_cmd += cmd 83return real_cmd 84 85defchdir(path, is_client_path=False): 86"""Do chdir to the given path, and set the PWD environment 87 variable for use by P4. It does not look at getcwd() output. 88 Since we're not using the shell, it is necessary to set the 89 PWD environment variable explicitly. 90 91 Normally, expand the path to force it to be absolute. This 92 addresses the use of relative path names inside P4 settings, 93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 94 as given; it looks for .p4config using PWD. 95 96 If is_client_path, the path was handed to us directly by p4, 97 and may be a symbolic link. Do not call os.getcwd() in this 98 case, because it will cause p4 to think that PWD is not inside 99 the client path. 100 """ 101 102 os.chdir(path) 103if not is_client_path: 104 path = os.getcwd() 105 os.environ['PWD'] = path 106 107defdie(msg): 108if verbose: 109raiseException(msg) 110else: 111 sys.stderr.write(msg +"\n") 112 sys.exit(1) 113 114defwrite_pipe(c, stdin): 115if verbose: 116 sys.stderr.write('Writing pipe:%s\n'%str(c)) 117 118 expand =isinstance(c,basestring) 119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 120 pipe = p.stdin 121 val = pipe.write(stdin) 122 pipe.close() 123if p.wait(): 124die('Command failed:%s'%str(c)) 125 126return val 127 128defp4_write_pipe(c, stdin): 129 real_cmd =p4_build_cmd(c) 130returnwrite_pipe(real_cmd, stdin) 131 132defread_pipe(c, ignore_error=False): 133if verbose: 134 sys.stderr.write('Reading pipe:%s\n'%str(c)) 135 136 expand =isinstance(c,basestring) 137 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 138 pipe = p.stdout 139 val = pipe.read() 140if p.wait()and not ignore_error: 141die('Command failed:%s'%str(c)) 142 143return val 144 145defp4_read_pipe(c, ignore_error=False): 146 real_cmd =p4_build_cmd(c) 147returnread_pipe(real_cmd, ignore_error) 148 149defread_pipe_lines(c): 150if verbose: 151 sys.stderr.write('Reading pipe:%s\n'%str(c)) 152 153 expand =isinstance(c, basestring) 154 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 155 pipe = p.stdout 156 val = pipe.readlines() 157if pipe.close()or p.wait(): 158die('Command failed:%s'%str(c)) 159 160return val 161 162defp4_read_pipe_lines(c): 163"""Specifically invoke p4 on the command supplied. """ 164 real_cmd =p4_build_cmd(c) 165returnread_pipe_lines(real_cmd) 166 167defp4_has_command(cmd): 168"""Ask p4 for help on this command. If it returns an error, the 169 command does not exist in this version of p4.""" 170 real_cmd =p4_build_cmd(["help", cmd]) 171 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 172 stderr=subprocess.PIPE) 173 p.communicate() 174return p.returncode ==0 175 176defp4_has_move_command(): 177"""See if the move command exists, that it supports -k, and that 178 it has not been administratively disabled. The arguments 179 must be correct, but the filenames do not have to exist. Use 180 ones with wildcards so even if they exist, it will fail.""" 181 182if notp4_has_command("move"): 183return False 184 cmd =p4_build_cmd(["move","-k","@from","@to"]) 185 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 186(out, err) = p.communicate() 187# return code will be 1 in either case 188if err.find("Invalid option") >=0: 189return False 190if err.find("disabled") >=0: 191return False 192# assume it failed because @... was invalid changelist 193return True 194 195defsystem(cmd, ignore_error=False): 196 expand =isinstance(cmd,basestring) 197if verbose: 198 sys.stderr.write("executing%s\n"%str(cmd)) 199 retcode = subprocess.call(cmd, shell=expand) 200if retcode and not ignore_error: 201raiseCalledProcessError(retcode, cmd) 202 203return retcode 204 205defp4_system(cmd): 206"""Specifically invoke p4 as the system command. """ 207 real_cmd =p4_build_cmd(cmd) 208 expand =isinstance(real_cmd, basestring) 209 retcode = subprocess.call(real_cmd, shell=expand) 210if retcode: 211raiseCalledProcessError(retcode, real_cmd) 212 213_p4_version_string =None 214defp4_version_string(): 215"""Read the version string, showing just the last line, which 216 hopefully is the interesting version bit. 217 218 $ p4 -V 219 Perforce - The Fast Software Configuration Management System. 220 Copyright 1995-2011 Perforce Software. All rights reserved. 221 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 222 """ 223global _p4_version_string 224if not _p4_version_string: 225 a =p4_read_pipe_lines(["-V"]) 226 _p4_version_string = a[-1].rstrip() 227return _p4_version_string 228 229defp4_integrate(src, dest): 230p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 231 232defp4_sync(f, *options): 233p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 234 235defp4_add(f): 236# forcibly add file names with wildcards 237ifwildcard_present(f): 238p4_system(["add","-f", f]) 239else: 240p4_system(["add", f]) 241 242defp4_delete(f): 243p4_system(["delete",wildcard_encode(f)]) 244 245defp4_edit(f): 246p4_system(["edit",wildcard_encode(f)]) 247 248defp4_revert(f): 249p4_system(["revert",wildcard_encode(f)]) 250 251defp4_reopen(type, f): 252p4_system(["reopen","-t",type,wildcard_encode(f)]) 253 254defp4_move(src, dest): 255p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 256 257defp4_last_change(): 258 results =p4CmdList(["changes","-m","1"]) 259returnint(results[0]['change']) 260 261defp4_describe(change): 262"""Make sure it returns a valid result by checking for 263 the presence of field "time". Return a dict of the 264 results.""" 265 266 ds =p4CmdList(["describe","-s",str(change)]) 267iflen(ds) !=1: 268die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 269 270 d = ds[0] 271 272if"p4ExitCode"in d: 273die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 274str(d))) 275if"code"in d: 276if d["code"] =="error": 277die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 278 279if"time"not in d: 280die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 281 282return d 283 284# 285# Canonicalize the p4 type and return a tuple of the 286# base type, plus any modifiers. See "p4 help filetypes" 287# for a list and explanation. 288# 289defsplit_p4_type(p4type): 290 291 p4_filetypes_historical = { 292"ctempobj":"binary+Sw", 293"ctext":"text+C", 294"cxtext":"text+Cx", 295"ktext":"text+k", 296"kxtext":"text+kx", 297"ltext":"text+F", 298"tempobj":"binary+FSw", 299"ubinary":"binary+F", 300"uresource":"resource+F", 301"uxbinary":"binary+Fx", 302"xbinary":"binary+x", 303"xltext":"text+Fx", 304"xtempobj":"binary+Swx", 305"xtext":"text+x", 306"xunicode":"unicode+x", 307"xutf16":"utf16+x", 308} 309if p4type in p4_filetypes_historical: 310 p4type = p4_filetypes_historical[p4type] 311 mods ="" 312 s = p4type.split("+") 313 base = s[0] 314 mods ="" 315iflen(s) >1: 316 mods = s[1] 317return(base, mods) 318 319# 320# return the raw p4 type of a file (text, text+ko, etc) 321# 322defp4_type(f): 323 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 324return results[0]['headType'] 325 326# 327# Given a type base and modifier, return a regexp matching 328# the keywords that can be expanded in the file 329# 330defp4_keywords_regexp_for_type(base, type_mods): 331if base in("text","unicode","binary"): 332 kwords =None 333if"ko"in type_mods: 334 kwords ='Id|Header' 335elif"k"in type_mods: 336 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 337else: 338return None 339 pattern = r""" 340 \$ # Starts with a dollar, followed by... 341 (%s) # one of the keywords, followed by... 342 (:[^$\n]+)? # possibly an old expansion, followed by... 343 \$ # another dollar 344 """% kwords 345return pattern 346else: 347return None 348 349# 350# Given a file, return a regexp matching the possible 351# RCS keywords that will be expanded, or None for files 352# with kw expansion turned off. 353# 354defp4_keywords_regexp_for_file(file): 355if not os.path.exists(file): 356return None 357else: 358(type_base, type_mods) =split_p4_type(p4_type(file)) 359returnp4_keywords_regexp_for_type(type_base, type_mods) 360 361defsetP4ExecBit(file, mode): 362# Reopens an already open file and changes the execute bit to match 363# the execute bit setting in the passed in mode. 364 365 p4Type ="+x" 366 367if notisModeExec(mode): 368 p4Type =getP4OpenedType(file) 369 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 370 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 371if p4Type[-1] =="+": 372 p4Type = p4Type[0:-1] 373 374p4_reopen(p4Type,file) 375 376defgetP4OpenedType(file): 377# Returns the perforce file type for the given file. 378 379 result =p4_read_pipe(["opened",wildcard_encode(file)]) 380 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 381if match: 382return match.group(1) 383else: 384die("Could not determine file type for%s(result: '%s')"% (file, result)) 385 386# Return the set of all p4 labels 387defgetP4Labels(depotPaths): 388 labels =set() 389ifisinstance(depotPaths,basestring): 390 depotPaths = [depotPaths] 391 392for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 393 label = l['label'] 394 labels.add(label) 395 396return labels 397 398# Return the set of all git tags 399defgetGitTags(): 400 gitTags =set() 401for line inread_pipe_lines(["git","tag"]): 402 tag = line.strip() 403 gitTags.add(tag) 404return gitTags 405 406defdiffTreePattern(): 407# This is a simple generator for the diff tree regex pattern. This could be 408# a class variable if this and parseDiffTreeEntry were a part of a class. 409 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 410while True: 411yield pattern 412 413defparseDiffTreeEntry(entry): 414"""Parses a single diff tree entry into its component elements. 415 416 See git-diff-tree(1) manpage for details about the format of the diff 417 output. This method returns a dictionary with the following elements: 418 419 src_mode - The mode of the source file 420 dst_mode - The mode of the destination file 421 src_sha1 - The sha1 for the source file 422 dst_sha1 - The sha1 fr the destination file 423 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 424 status_score - The score for the status (applicable for 'C' and 'R' 425 statuses). This is None if there is no score. 426 src - The path for the source file. 427 dst - The path for the destination file. This is only present for 428 copy or renames. If it is not present, this is None. 429 430 If the pattern is not matched, None is returned.""" 431 432 match =diffTreePattern().next().match(entry) 433if match: 434return{ 435'src_mode': match.group(1), 436'dst_mode': match.group(2), 437'src_sha1': match.group(3), 438'dst_sha1': match.group(4), 439'status': match.group(5), 440'status_score': match.group(6), 441'src': match.group(7), 442'dst': match.group(10) 443} 444return None 445 446defisModeExec(mode): 447# Returns True if the given git mode represents an executable file, 448# otherwise False. 449return mode[-3:] =="755" 450 451defisModeExecChanged(src_mode, dst_mode): 452returnisModeExec(src_mode) !=isModeExec(dst_mode) 453 454defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 455 456ifisinstance(cmd,basestring): 457 cmd ="-G "+ cmd 458 expand =True 459else: 460 cmd = ["-G"] + cmd 461 expand =False 462 463 cmd =p4_build_cmd(cmd) 464if verbose: 465 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 466 467# Use a temporary file to avoid deadlocks without 468# subprocess.communicate(), which would put another copy 469# of stdout into memory. 470 stdin_file =None 471if stdin is not None: 472 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 473ifisinstance(stdin,basestring): 474 stdin_file.write(stdin) 475else: 476for i in stdin: 477 stdin_file.write(i +'\n') 478 stdin_file.flush() 479 stdin_file.seek(0) 480 481 p4 = subprocess.Popen(cmd, 482 shell=expand, 483 stdin=stdin_file, 484 stdout=subprocess.PIPE) 485 486 result = [] 487try: 488while True: 489 entry = marshal.load(p4.stdout) 490if cb is not None: 491cb(entry) 492else: 493 result.append(entry) 494exceptEOFError: 495pass 496 exitCode = p4.wait() 497if exitCode !=0: 498 entry = {} 499 entry["p4ExitCode"] = exitCode 500 result.append(entry) 501 502return result 503 504defp4Cmd(cmd): 505list=p4CmdList(cmd) 506 result = {} 507for entry inlist: 508 result.update(entry) 509return result; 510 511defp4Where(depotPath): 512if not depotPath.endswith("/"): 513 depotPath +="/" 514 depotPathLong = depotPath +"..." 515 outputList =p4CmdList(["where", depotPathLong]) 516 output =None 517for entry in outputList: 518if"depotFile"in entry: 519# Search for the base client side depot path, as long as it starts with the branch's P4 path. 520# The base path always ends with "/...". 521if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 522 output = entry 523break 524elif"data"in entry: 525 data = entry.get("data") 526 space = data.find(" ") 527if data[:space] == depotPath: 528 output = entry 529break 530if output ==None: 531return"" 532if output["code"] =="error": 533return"" 534 clientPath ="" 535if"path"in output: 536 clientPath = output.get("path") 537elif"data"in output: 538 data = output.get("data") 539 lastSpace = data.rfind(" ") 540 clientPath = data[lastSpace +1:] 541 542if clientPath.endswith("..."): 543 clientPath = clientPath[:-3] 544return clientPath 545 546defcurrentGitBranch(): 547returnread_pipe("git name-rev HEAD").split(" ")[1].strip() 548 549defisValidGitDir(path): 550if(os.path.exists(path +"/HEAD") 551and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 552return True; 553return False 554 555defparseRevision(ref): 556returnread_pipe("git rev-parse%s"% ref).strip() 557 558defbranchExists(ref): 559 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 560 ignore_error=True) 561returnlen(rev) >0 562 563defextractLogMessageFromGitCommit(commit): 564 logMessage ="" 565 566## fixme: title is first line of commit, not 1st paragraph. 567 foundTitle =False 568for log inread_pipe_lines("git cat-file commit%s"% commit): 569if not foundTitle: 570iflen(log) ==1: 571 foundTitle =True 572continue 573 574 logMessage += log 575return logMessage 576 577defextractSettingsGitLog(log): 578 values = {} 579for line in log.split("\n"): 580 line = line.strip() 581 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 582if not m: 583continue 584 585 assignments = m.group(1).split(':') 586for a in assignments: 587 vals = a.split('=') 588 key = vals[0].strip() 589 val = ('='.join(vals[1:])).strip() 590if val.endswith('\"')and val.startswith('"'): 591 val = val[1:-1] 592 593 values[key] = val 594 595 paths = values.get("depot-paths") 596if not paths: 597 paths = values.get("depot-path") 598if paths: 599 values['depot-paths'] = paths.split(',') 600return values 601 602defgitBranchExists(branch): 603 proc = subprocess.Popen(["git","rev-parse", branch], 604 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 605return proc.wait() ==0; 606 607_gitConfig = {} 608 609defgitConfig(key): 610if not _gitConfig.has_key(key): 611 cmd = ["git","config", key ] 612 s =read_pipe(cmd, ignore_error=True) 613 _gitConfig[key] = s.strip() 614return _gitConfig[key] 615 616defgitConfigBool(key): 617"""Return a bool, using git config --bool. It is True only if the 618 variable is set to true, and False if set to false or not present 619 in the config.""" 620 621if not _gitConfig.has_key(key): 622 cmd = ["git","config","--bool", key ] 623 s =read_pipe(cmd, ignore_error=True) 624 v = s.strip() 625 _gitConfig[key] = v =="true" 626return _gitConfig[key] 627 628defgitConfigList(key): 629if not _gitConfig.has_key(key): 630 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 631 _gitConfig[key] = s.strip().split(os.linesep) 632return _gitConfig[key] 633 634defp4BranchesInGit(branchesAreInRemotes=True): 635"""Find all the branches whose names start with "p4/", looking 636 in remotes or heads as specified by the argument. Return 637 a dictionary of{ branch: revision }for each one found. 638 The branch names are the short names, without any 639 "p4/" prefix.""" 640 641 branches = {} 642 643 cmdline ="git rev-parse --symbolic " 644if branchesAreInRemotes: 645 cmdline +="--remotes" 646else: 647 cmdline +="--branches" 648 649for line inread_pipe_lines(cmdline): 650 line = line.strip() 651 652# only import to p4/ 653if not line.startswith('p4/'): 654continue 655# special symbolic ref to p4/master 656if line =="p4/HEAD": 657continue 658 659# strip off p4/ prefix 660 branch = line[len("p4/"):] 661 662 branches[branch] =parseRevision(line) 663 664return branches 665 666defbranch_exists(branch): 667"""Make sure that the given ref name really exists.""" 668 669 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 670 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 671 out, _ = p.communicate() 672if p.returncode: 673return False 674# expect exactly one line of output: the branch name 675return out.rstrip() == branch 676 677deffindUpstreamBranchPoint(head ="HEAD"): 678 branches =p4BranchesInGit() 679# map from depot-path to branch name 680 branchByDepotPath = {} 681for branch in branches.keys(): 682 tip = branches[branch] 683 log =extractLogMessageFromGitCommit(tip) 684 settings =extractSettingsGitLog(log) 685if settings.has_key("depot-paths"): 686 paths =",".join(settings["depot-paths"]) 687 branchByDepotPath[paths] ="remotes/p4/"+ branch 688 689 settings =None 690 parent =0 691while parent <65535: 692 commit = head +"~%s"% parent 693 log =extractLogMessageFromGitCommit(commit) 694 settings =extractSettingsGitLog(log) 695if settings.has_key("depot-paths"): 696 paths =",".join(settings["depot-paths"]) 697if branchByDepotPath.has_key(paths): 698return[branchByDepotPath[paths], settings] 699 700 parent = parent +1 701 702return["", settings] 703 704defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 705if not silent: 706print("Creating/updating branch(es) in%sbased on origin branch(es)" 707% localRefPrefix) 708 709 originPrefix ="origin/p4/" 710 711for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 712 line = line.strip() 713if(not line.startswith(originPrefix))or line.endswith("HEAD"): 714continue 715 716 headName = line[len(originPrefix):] 717 remoteHead = localRefPrefix + headName 718 originHead = line 719 720 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 721if(not original.has_key('depot-paths') 722or not original.has_key('change')): 723continue 724 725 update =False 726if notgitBranchExists(remoteHead): 727if verbose: 728print"creating%s"% remoteHead 729 update =True 730else: 731 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 732if settings.has_key('change') >0: 733if settings['depot-paths'] == original['depot-paths']: 734 originP4Change =int(original['change']) 735 p4Change =int(settings['change']) 736if originP4Change > p4Change: 737print("%s(%s) is newer than%s(%s). " 738"Updating p4 branch from origin." 739% (originHead, originP4Change, 740 remoteHead, p4Change)) 741 update =True 742else: 743print("Ignoring:%swas imported from%swhile " 744"%swas imported from%s" 745% (originHead,','.join(original['depot-paths']), 746 remoteHead,','.join(settings['depot-paths']))) 747 748if update: 749system("git update-ref%s %s"% (remoteHead, originHead)) 750 751deforiginP4BranchesExist(): 752returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 753 754 755defp4ParseNumericChangeRange(parts): 756 changeStart =int(parts[0][1:]) 757if parts[1] =='#head': 758 changeEnd =p4_last_change() 759else: 760 changeEnd =int(parts[1]) 761 762return(changeStart, changeEnd) 763 764defchooseBlockSize(blockSize): 765if blockSize: 766return blockSize 767else: 768return defaultBlockSize 769 770defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 771assert depotPaths 772 773# Parse the change range into start and end. Try to find integer 774# revision ranges as these can be broken up into blocks to avoid 775# hitting server-side limits (maxrows, maxscanresults). But if 776# that doesn't work, fall back to using the raw revision specifier 777# strings, without using block mode. 778 779if changeRange is None or changeRange =='': 780 changeStart =1 781 changeEnd =p4_last_change() 782 block_size =chooseBlockSize(requestedBlockSize) 783else: 784 parts = changeRange.split(',') 785assertlen(parts) ==2 786try: 787(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 788 block_size =chooseBlockSize(requestedBlockSize) 789except: 790 changeStart = parts[0][1:] 791 changeEnd = parts[1] 792if requestedBlockSize: 793die("cannot use --changes-block-size with non-numeric revisions") 794 block_size =None 795 796# Accumulate change numbers in a dictionary to avoid duplicates 797 changes = {} 798 799for p in depotPaths: 800# Retrieve changes a block at a time, to prevent running 801# into a MaxResults/MaxScanRows error from the server. 802 803while True: 804 cmd = ['changes'] 805 806if block_size: 807 end =min(changeEnd, changeStart + block_size) 808 revisionRange ="%d,%d"% (changeStart, end) 809else: 810 revisionRange ="%s,%s"% (changeStart, changeEnd) 811 812 cmd += ["%s...@%s"% (p, revisionRange)] 813 814for line inp4_read_pipe_lines(cmd): 815 changeNum =int(line.split(" ")[1]) 816 changes[changeNum] =True 817 818if not block_size: 819break 820 821if end >= changeEnd: 822break 823 824 changeStart = end +1 825 826 changelist = changes.keys() 827 changelist.sort() 828return changelist 829 830defp4PathStartsWith(path, prefix): 831# This method tries to remedy a potential mixed-case issue: 832# 833# If UserA adds //depot/DirA/file1 834# and UserB adds //depot/dira/file2 835# 836# we may or may not have a problem. If you have core.ignorecase=true, 837# we treat DirA and dira as the same directory 838ifgitConfigBool("core.ignorecase"): 839return path.lower().startswith(prefix.lower()) 840return path.startswith(prefix) 841 842defgetClientSpec(): 843"""Look at the p4 client spec, create a View() object that contains 844 all the mappings, and return it.""" 845 846 specList =p4CmdList("client -o") 847iflen(specList) !=1: 848die('Output from "client -o" is%dlines, expecting 1'% 849len(specList)) 850 851# dictionary of all client parameters 852 entry = specList[0] 853 854# the //client/ name 855 client_name = entry["Client"] 856 857# just the keys that start with "View" 858 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 859 860# hold this new View 861 view =View(client_name) 862 863# append the lines, in order, to the view 864for view_num inrange(len(view_keys)): 865 k ="View%d"% view_num 866if k not in view_keys: 867die("Expected view key%smissing"% k) 868 view.append(entry[k]) 869 870return view 871 872defgetClientRoot(): 873"""Grab the client directory.""" 874 875 output =p4CmdList("client -o") 876iflen(output) !=1: 877die('Output from "client -o" is%dlines, expecting 1'%len(output)) 878 879 entry = output[0] 880if"Root"not in entry: 881die('Client has no "Root"') 882 883return entry["Root"] 884 885# 886# P4 wildcards are not allowed in filenames. P4 complains 887# if you simply add them, but you can force it with "-f", in 888# which case it translates them into %xx encoding internally. 889# 890defwildcard_decode(path): 891# Search for and fix just these four characters. Do % last so 892# that fixing it does not inadvertently create new %-escapes. 893# Cannot have * in a filename in windows; untested as to 894# what p4 would do in such a case. 895if not platform.system() =="Windows": 896 path = path.replace("%2A","*") 897 path = path.replace("%23","#") \ 898.replace("%40","@") \ 899.replace("%25","%") 900return path 901 902defwildcard_encode(path): 903# do % first to avoid double-encoding the %s introduced here 904 path = path.replace("%","%25") \ 905.replace("*","%2A") \ 906.replace("#","%23") \ 907.replace("@","%40") 908return path 909 910defwildcard_present(path): 911 m = re.search("[*#@%]", path) 912return m is not None 913 914class Command: 915def__init__(self): 916 self.usage ="usage: %prog [options]" 917 self.needsGit =True 918 self.verbose =False 919 920class P4UserMap: 921def__init__(self): 922 self.userMapFromPerforceServer =False 923 self.myP4UserId =None 924 925defp4UserId(self): 926if self.myP4UserId: 927return self.myP4UserId 928 929 results =p4CmdList("user -o") 930for r in results: 931if r.has_key('User'): 932 self.myP4UserId = r['User'] 933return r['User'] 934die("Could not find your p4 user id") 935 936defp4UserIsMe(self, p4User): 937# return True if the given p4 user is actually me 938 me = self.p4UserId() 939if not p4User or p4User != me: 940return False 941else: 942return True 943 944defgetUserCacheFilename(self): 945 home = os.environ.get("HOME", os.environ.get("USERPROFILE")) 946return home +"/.gitp4-usercache.txt" 947 948defgetUserMapFromPerforceServer(self): 949if self.userMapFromPerforceServer: 950return 951 self.users = {} 952 self.emails = {} 953 954for output inp4CmdList("users"): 955if not output.has_key("User"): 956continue 957 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">" 958 self.emails[output["Email"]] = output["User"] 959 960 961 s ='' 962for(key, val)in self.users.items(): 963 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1)) 964 965open(self.getUserCacheFilename(),"wb").write(s) 966 self.userMapFromPerforceServer =True 967 968defloadUserMapFromCache(self): 969 self.users = {} 970 self.userMapFromPerforceServer =False 971try: 972 cache =open(self.getUserCacheFilename(),"rb") 973 lines = cache.readlines() 974 cache.close() 975for line in lines: 976 entry = line.strip().split("\t") 977 self.users[entry[0]] = entry[1] 978exceptIOError: 979 self.getUserMapFromPerforceServer() 980 981classP4Debug(Command): 982def__init__(self): 983 Command.__init__(self) 984 self.options = [] 985 self.description ="A tool to debug the output of p4 -G." 986 self.needsGit =False 987 988defrun(self, args): 989 j =0 990for output inp4CmdList(args): 991print'Element:%d'% j 992 j +=1 993print output 994return True 995 996classP4RollBack(Command): 997def__init__(self): 998 Command.__init__(self) 999 self.options = [1000 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1001]1002 self.description ="A tool to debug the multi-branch import. Don't use :)"1003 self.rollbackLocalBranches =False10041005defrun(self, args):1006iflen(args) !=1:1007return False1008 maxChange =int(args[0])10091010if"p4ExitCode"inp4Cmd("changes -m 1"):1011die("Problems executing p4");10121013if self.rollbackLocalBranches:1014 refPrefix ="refs/heads/"1015 lines =read_pipe_lines("git rev-parse --symbolic --branches")1016else:1017 refPrefix ="refs/remotes/"1018 lines =read_pipe_lines("git rev-parse --symbolic --remotes")10191020for line in lines:1021if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1022 line = line.strip()1023 ref = refPrefix + line1024 log =extractLogMessageFromGitCommit(ref)1025 settings =extractSettingsGitLog(log)10261027 depotPaths = settings['depot-paths']1028 change = settings['change']10291030 changed =False10311032iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1033for p in depotPaths]))) ==0:1034print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange)1035system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1036continue10371038while change andint(change) > maxChange:1039 changed =True1040if self.verbose:1041print"%sis at%s; rewinding towards%s"% (ref, change, maxChange)1042system("git update-ref%s\"%s^\""% (ref, ref))1043 log =extractLogMessageFromGitCommit(ref)1044 settings =extractSettingsGitLog(log)104510461047 depotPaths = settings['depot-paths']1048 change = settings['change']10491050if changed:1051print"%srewound to%s"% (ref, change)10521053return True10541055classP4Submit(Command, P4UserMap):10561057 conflict_behavior_choices = ("ask","skip","quit")10581059def__init__(self):1060 Command.__init__(self)1061 P4UserMap.__init__(self)1062 self.options = [1063 optparse.make_option("--origin", dest="origin"),1064 optparse.make_option("-M", dest="detectRenames", action="store_true"),1065# preserve the user, requires relevant p4 permissions1066 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1067 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1068 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1069 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1070 optparse.make_option("--conflict", dest="conflict_behavior",1071 choices=self.conflict_behavior_choices),1072 optparse.make_option("--branch", dest="branch"),1073]1074 self.description ="Submit changes from git to the perforce depot."1075 self.usage +=" [name of git branch to submit into perforce depot]"1076 self.origin =""1077 self.detectRenames =False1078 self.preserveUser =gitConfigBool("git-p4.preserveUser")1079 self.dry_run =False1080 self.prepare_p4_only =False1081 self.conflict_behavior =None1082 self.isWindows = (platform.system() =="Windows")1083 self.exportLabels =False1084 self.p4HasMoveCommand =p4_has_move_command()1085 self.branch =None10861087defcheck(self):1088iflen(p4CmdList("opened ...")) >0:1089die("You have files opened with perforce! Close them before starting the sync.")10901091defseparate_jobs_from_description(self, message):1092"""Extract and return a possible Jobs field in the commit1093 message. It goes into a separate section in the p4 change1094 specification.10951096 A jobs line starts with "Jobs:" and looks like a new field1097 in a form. Values are white-space separated on the same1098 line or on following lines that start with a tab.10991100 This does not parse and extract the full git commit message1101 like a p4 form. It just sees the Jobs: line as a marker1102 to pass everything from then on directly into the p4 form,1103 but outside the description section.11041105 Return a tuple (stripped log message, jobs string)."""11061107 m = re.search(r'^Jobs:', message, re.MULTILINE)1108if m is None:1109return(message,None)11101111 jobtext = message[m.start():]1112 stripped_message = message[:m.start()].rstrip()1113return(stripped_message, jobtext)11141115defprepareLogMessage(self, template, message, jobs):1116"""Edits the template returned from "p4 change -o" to insert1117 the message in the Description field, and the jobs text in1118 the Jobs field."""1119 result =""11201121 inDescriptionSection =False11221123for line in template.split("\n"):1124if line.startswith("#"):1125 result += line +"\n"1126continue11271128if inDescriptionSection:1129if line.startswith("Files:")or line.startswith("Jobs:"):1130 inDescriptionSection =False1131# insert Jobs section1132if jobs:1133 result += jobs +"\n"1134else:1135continue1136else:1137if line.startswith("Description:"):1138 inDescriptionSection =True1139 line +="\n"1140for messageLine in message.split("\n"):1141 line +="\t"+ messageLine +"\n"11421143 result += line +"\n"11441145return result11461147defpatchRCSKeywords(self,file, pattern):1148# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1149(handle, outFileName) = tempfile.mkstemp(dir='.')1150try:1151 outFile = os.fdopen(handle,"w+")1152 inFile =open(file,"r")1153 regexp = re.compile(pattern, re.VERBOSE)1154for line in inFile.readlines():1155 line = regexp.sub(r'$\1$', line)1156 outFile.write(line)1157 inFile.close()1158 outFile.close()1159# Forcibly overwrite the original file1160 os.unlink(file)1161 shutil.move(outFileName,file)1162except:1163# cleanup our temporary file1164 os.unlink(outFileName)1165print"Failed to strip RCS keywords in%s"%file1166raise11671168print"Patched up RCS keywords in%s"%file11691170defp4UserForCommit(self,id):1171# Return the tuple (perforce user,git email) for a given git commit id1172 self.getUserMapFromPerforceServer()1173 gitEmail =read_pipe(["git","log","--max-count=1",1174"--format=%ae",id])1175 gitEmail = gitEmail.strip()1176if not self.emails.has_key(gitEmail):1177return(None,gitEmail)1178else:1179return(self.emails[gitEmail],gitEmail)11801181defcheckValidP4Users(self,commits):1182# check if any git authors cannot be mapped to p4 users1183foridin commits:1184(user,email) = self.p4UserForCommit(id)1185if not user:1186 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1187ifgitConfigBool("git-p4.allowMissingP4Users"):1188print"%s"% msg1189else:1190die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)11911192deflastP4Changelist(self):1193# Get back the last changelist number submitted in this client spec. This1194# then gets used to patch up the username in the change. If the same1195# client spec is being used by multiple processes then this might go1196# wrong.1197 results =p4CmdList("client -o")# find the current client1198 client =None1199for r in results:1200if r.has_key('Client'):1201 client = r['Client']1202break1203if not client:1204die("could not get client spec")1205 results =p4CmdList(["changes","-c", client,"-m","1"])1206for r in results:1207if r.has_key('change'):1208return r['change']1209die("Could not get changelist number for last submit - cannot patch up user details")12101211defmodifyChangelistUser(self, changelist, newUser):1212# fixup the user field of a changelist after it has been submitted.1213 changes =p4CmdList("change -o%s"% changelist)1214iflen(changes) !=1:1215die("Bad output from p4 change modifying%sto user%s"%1216(changelist, newUser))12171218 c = changes[0]1219if c['User'] == newUser:return# nothing to do1220 c['User'] = newUser1221input= marshal.dumps(c)12221223 result =p4CmdList("change -f -i", stdin=input)1224for r in result:1225if r.has_key('code'):1226if r['code'] =='error':1227die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1228if r.has_key('data'):1229print("Updated user field for changelist%sto%s"% (changelist, newUser))1230return1231die("Could not modify user field of changelist%sto%s"% (changelist, newUser))12321233defcanChangeChangelists(self):1234# check to see if we have p4 admin or super-user permissions, either of1235# which are required to modify changelists.1236 results =p4CmdList(["protects", self.depotPath])1237for r in results:1238if r.has_key('perm'):1239if r['perm'] =='admin':1240return11241if r['perm'] =='super':1242return11243return012441245defprepareSubmitTemplate(self):1246"""Run "p4 change -o" to grab a change specification template.1247 This does not use "p4 -G", as it is nice to keep the submission1248 template in original order, since a human might edit it.12491250 Remove lines in the Files section that show changes to files1251 outside the depot path we're committing into."""12521253 template =""1254 inFilesSection =False1255for line inp4_read_pipe_lines(['change','-o']):1256if line.endswith("\r\n"):1257 line = line[:-2] +"\n"1258if inFilesSection:1259if line.startswith("\t"):1260# path starts and ends with a tab1261 path = line[1:]1262 lastTab = path.rfind("\t")1263if lastTab != -1:1264 path = path[:lastTab]1265if notp4PathStartsWith(path, self.depotPath):1266continue1267else:1268 inFilesSection =False1269else:1270if line.startswith("Files:"):1271 inFilesSection =True12721273 template += line12741275return template12761277defedit_template(self, template_file):1278"""Invoke the editor to let the user change the submission1279 message. Return true if okay to continue with the submit."""12801281# if configured to skip the editing part, just submit1282ifgitConfigBool("git-p4.skipSubmitEdit"):1283return True12841285# look at the modification time, to check later if the user saved1286# the file1287 mtime = os.stat(template_file).st_mtime12881289# invoke the editor1290if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1291 editor = os.environ.get("P4EDITOR")1292else:1293 editor =read_pipe("git var GIT_EDITOR").strip()1294system(["sh","-c", ('%s"$@"'% editor), editor, template_file])12951296# If the file was not saved, prompt to see if this patch should1297# be skipped. But skip this verification step if configured so.1298ifgitConfigBool("git-p4.skipSubmitEditCheck"):1299return True13001301# modification time updated means user saved the file1302if os.stat(template_file).st_mtime > mtime:1303return True13041305while True:1306 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1307if response =='y':1308return True1309if response =='n':1310return False13111312defget_diff_description(self, editedFiles, filesToAdd):1313# diff1314if os.environ.has_key("P4DIFF"):1315del(os.environ["P4DIFF"])1316 diff =""1317for editedFile in editedFiles:1318 diff +=p4_read_pipe(['diff','-du',1319wildcard_encode(editedFile)])13201321# new file diff1322 newdiff =""1323for newFile in filesToAdd:1324 newdiff +="==== new file ====\n"1325 newdiff +="--- /dev/null\n"1326 newdiff +="+++%s\n"% newFile1327 f =open(newFile,"r")1328for line in f.readlines():1329 newdiff +="+"+ line1330 f.close()13311332return(diff + newdiff).replace('\r\n','\n')13331334defapplyCommit(self,id):1335"""Apply one commit, return True if it succeeded."""13361337print"Applying",read_pipe(["git","show","-s",1338"--format=format:%h%s",id])13391340(p4User, gitEmail) = self.p4UserForCommit(id)13411342 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1343 filesToAdd =set()1344 filesToDelete =set()1345 editedFiles =set()1346 pureRenameCopy =set()1347 filesToChangeExecBit = {}13481349for line in diff:1350 diff =parseDiffTreeEntry(line)1351 modifier = diff['status']1352 path = diff['src']1353if modifier =="M":1354p4_edit(path)1355ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1356 filesToChangeExecBit[path] = diff['dst_mode']1357 editedFiles.add(path)1358elif modifier =="A":1359 filesToAdd.add(path)1360 filesToChangeExecBit[path] = diff['dst_mode']1361if path in filesToDelete:1362 filesToDelete.remove(path)1363elif modifier =="D":1364 filesToDelete.add(path)1365if path in filesToAdd:1366 filesToAdd.remove(path)1367elif modifier =="C":1368 src, dest = diff['src'], diff['dst']1369p4_integrate(src, dest)1370 pureRenameCopy.add(dest)1371if diff['src_sha1'] != diff['dst_sha1']:1372p4_edit(dest)1373 pureRenameCopy.discard(dest)1374ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1375p4_edit(dest)1376 pureRenameCopy.discard(dest)1377 filesToChangeExecBit[dest] = diff['dst_mode']1378if self.isWindows:1379# turn off read-only attribute1380 os.chmod(dest, stat.S_IWRITE)1381 os.unlink(dest)1382 editedFiles.add(dest)1383elif modifier =="R":1384 src, dest = diff['src'], diff['dst']1385if self.p4HasMoveCommand:1386p4_edit(src)# src must be open before move1387p4_move(src, dest)# opens for (move/delete, move/add)1388else:1389p4_integrate(src, dest)1390if diff['src_sha1'] != diff['dst_sha1']:1391p4_edit(dest)1392else:1393 pureRenameCopy.add(dest)1394ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1395if not self.p4HasMoveCommand:1396p4_edit(dest)# with move: already open, writable1397 filesToChangeExecBit[dest] = diff['dst_mode']1398if not self.p4HasMoveCommand:1399if self.isWindows:1400 os.chmod(dest, stat.S_IWRITE)1401 os.unlink(dest)1402 filesToDelete.add(src)1403 editedFiles.add(dest)1404else:1405die("unknown modifier%sfor%s"% (modifier, path))14061407 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1408 patchcmd = diffcmd +" | git apply "1409 tryPatchCmd = patchcmd +"--check -"1410 applyPatchCmd = patchcmd +"--check --apply -"1411 patch_succeeded =True14121413if os.system(tryPatchCmd) !=0:1414 fixed_rcs_keywords =False1415 patch_succeeded =False1416print"Unfortunately applying the change failed!"14171418# Patch failed, maybe it's just RCS keyword woes. Look through1419# the patch to see if that's possible.1420ifgitConfigBool("git-p4.attemptRCSCleanup"):1421file=None1422 pattern =None1423 kwfiles = {}1424forfilein editedFiles | filesToDelete:1425# did this file's delta contain RCS keywords?1426 pattern =p4_keywords_regexp_for_file(file)14271428if pattern:1429# this file is a possibility...look for RCS keywords.1430 regexp = re.compile(pattern, re.VERBOSE)1431for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1432if regexp.search(line):1433if verbose:1434print"got keyword match on%sin%sin%s"% (pattern, line,file)1435 kwfiles[file] = pattern1436break14371438forfilein kwfiles:1439if verbose:1440print"zapping%swith%s"% (line,pattern)1441# File is being deleted, so not open in p4. Must1442# disable the read-only bit on windows.1443if self.isWindows andfilenot in editedFiles:1444 os.chmod(file, stat.S_IWRITE)1445 self.patchRCSKeywords(file, kwfiles[file])1446 fixed_rcs_keywords =True14471448if fixed_rcs_keywords:1449print"Retrying the patch with RCS keywords cleaned up"1450if os.system(tryPatchCmd) ==0:1451 patch_succeeded =True14521453if not patch_succeeded:1454for f in editedFiles:1455p4_revert(f)1456return False14571458#1459# Apply the patch for real, and do add/delete/+x handling.1460#1461system(applyPatchCmd)14621463for f in filesToAdd:1464p4_add(f)1465for f in filesToDelete:1466p4_revert(f)1467p4_delete(f)14681469# Set/clear executable bits1470for f in filesToChangeExecBit.keys():1471 mode = filesToChangeExecBit[f]1472setP4ExecBit(f, mode)14731474#1475# Build p4 change description, starting with the contents1476# of the git commit message.1477#1478 logMessage =extractLogMessageFromGitCommit(id)1479 logMessage = logMessage.strip()1480(logMessage, jobs) = self.separate_jobs_from_description(logMessage)14811482 template = self.prepareSubmitTemplate()1483 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)14841485if self.preserveUser:1486 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User14871488if self.checkAuthorship and not self.p4UserIsMe(p4User):1489 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1490 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1491 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"14921493 separatorLine ="######## everything below this line is just the diff #######\n"1494if not self.prepare_p4_only:1495 submitTemplate += separatorLine1496 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)14971498(handle, fileName) = tempfile.mkstemp()1499 tmpFile = os.fdopen(handle,"w+b")1500if self.isWindows:1501 submitTemplate = submitTemplate.replace("\n","\r\n")1502 tmpFile.write(submitTemplate)1503 tmpFile.close()15041505if self.prepare_p4_only:1506#1507# Leave the p4 tree prepared, and the submit template around1508# and let the user decide what to do next1509#1510print1511print"P4 workspace prepared for submission."1512print"To submit or revert, go to client workspace"1513print" "+ self.clientPath1514print1515print"To submit, use\"p4 submit\"to write a new description,"1516print"or\"p4 submit -i <%s\"to use the one prepared by" \1517"\"git p4\"."% fileName1518print"You can delete the file\"%s\"when finished."% fileName15191520if self.preserveUser and p4User and not self.p4UserIsMe(p4User):1521print"To preserve change ownership by user%s, you must\n" \1522"do\"p4 change -f <change>\"after submitting and\n" \1523"edit the User field."1524if pureRenameCopy:1525print"After submitting, renamed files must be re-synced."1526print"Invoke\"p4 sync -f\"on each of these files:"1527for f in pureRenameCopy:1528print" "+ f15291530print1531print"To revert the changes, use\"p4 revert ...\", and delete"1532print"the submit template file\"%s\""% fileName1533if filesToAdd:1534print"Since the commit adds new files, they must be deleted:"1535for f in filesToAdd:1536print" "+ f1537print1538return True15391540#1541# Let the user edit the change description, then submit it.1542#1543if self.edit_template(fileName):1544# read the edited message and submit1545 ret =True1546 tmpFile =open(fileName,"rb")1547 message = tmpFile.read()1548 tmpFile.close()1549if self.isWindows:1550 message = message.replace("\r\n","\n")1551 submitTemplate = message[:message.index(separatorLine)]1552p4_write_pipe(['submit','-i'], submitTemplate)15531554if self.preserveUser:1555if p4User:1556# Get last changelist number. Cannot easily get it from1557# the submit command output as the output is1558# unmarshalled.1559 changelist = self.lastP4Changelist()1560 self.modifyChangelistUser(changelist, p4User)15611562# The rename/copy happened by applying a patch that created a1563# new file. This leaves it writable, which confuses p4.1564for f in pureRenameCopy:1565p4_sync(f,"-f")15661567else:1568# skip this patch1569 ret =False1570print"Submission cancelled, undoing p4 changes."1571for f in editedFiles:1572p4_revert(f)1573for f in filesToAdd:1574p4_revert(f)1575 os.remove(f)1576for f in filesToDelete:1577p4_revert(f)15781579 os.remove(fileName)1580return ret15811582# Export git tags as p4 labels. Create a p4 label and then tag1583# with that.1584defexportGitTags(self, gitTags):1585 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1586iflen(validLabelRegexp) ==0:1587 validLabelRegexp = defaultLabelRegexp1588 m = re.compile(validLabelRegexp)15891590for name in gitTags:15911592if not m.match(name):1593if verbose:1594print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1595continue15961597# Get the p4 commit this corresponds to1598 logMessage =extractLogMessageFromGitCommit(name)1599 values =extractSettingsGitLog(logMessage)16001601if not values.has_key('change'):1602# a tag pointing to something not sent to p4; ignore1603if verbose:1604print"git tag%sdoes not give a p4 commit"% name1605continue1606else:1607 changelist = values['change']16081609# Get the tag details.1610 inHeader =True1611 isAnnotated =False1612 body = []1613for l inread_pipe_lines(["git","cat-file","-p", name]):1614 l = l.strip()1615if inHeader:1616if re.match(r'tag\s+', l):1617 isAnnotated =True1618elif re.match(r'\s*$', l):1619 inHeader =False1620continue1621else:1622 body.append(l)16231624if not isAnnotated:1625 body = ["lightweight tag imported by git p4\n"]16261627# Create the label - use the same view as the client spec we are using1628 clientSpec =getClientSpec()16291630 labelTemplate ="Label:%s\n"% name1631 labelTemplate +="Description:\n"1632for b in body:1633 labelTemplate +="\t"+ b +"\n"1634 labelTemplate +="View:\n"1635for depot_side in clientSpec.mappings:1636 labelTemplate +="\t%s\n"% depot_side16371638if self.dry_run:1639print"Would create p4 label%sfor tag"% name1640elif self.prepare_p4_only:1641print"Not creating p4 label%sfor tag due to option" \1642" --prepare-p4-only"% name1643else:1644p4_write_pipe(["label","-i"], labelTemplate)16451646# Use the label1647p4_system(["tag","-l", name] +1648["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])16491650if verbose:1651print"created p4 label for tag%s"% name16521653defrun(self, args):1654iflen(args) ==0:1655 self.master =currentGitBranch()1656iflen(self.master) ==0or notgitBranchExists("refs/heads/%s"% self.master):1657die("Detecting current git branch failed!")1658eliflen(args) ==1:1659 self.master = args[0]1660if notbranchExists(self.master):1661die("Branch%sdoes not exist"% self.master)1662else:1663return False16641665 allowSubmit =gitConfig("git-p4.allowSubmit")1666iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1667die("%sis not in git-p4.allowSubmit"% self.master)16681669[upstream, settings] =findUpstreamBranchPoint()1670 self.depotPath = settings['depot-paths'][0]1671iflen(self.origin) ==0:1672 self.origin = upstream16731674if self.preserveUser:1675if not self.canChangeChangelists():1676die("Cannot preserve user names without p4 super-user or admin permissions")16771678# if not set from the command line, try the config file1679if self.conflict_behavior is None:1680 val =gitConfig("git-p4.conflict")1681if val:1682if val not in self.conflict_behavior_choices:1683die("Invalid value '%s' for config git-p4.conflict"% val)1684else:1685 val ="ask"1686 self.conflict_behavior = val16871688if self.verbose:1689print"Origin branch is "+ self.origin16901691iflen(self.depotPath) ==0:1692print"Internal error: cannot locate perforce depot path from existing branches"1693 sys.exit(128)16941695 self.useClientSpec =False1696ifgitConfigBool("git-p4.useclientspec"):1697 self.useClientSpec =True1698if self.useClientSpec:1699 self.clientSpecDirs =getClientSpec()17001701# Check for the existance of P4 branches1702 branchesDetected = (len(p4BranchesInGit().keys()) >1)17031704if self.useClientSpec and not branchesDetected:1705# all files are relative to the client spec1706 self.clientPath =getClientRoot()1707else:1708 self.clientPath =p4Where(self.depotPath)17091710if self.clientPath =="":1711die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)17121713print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1714 self.oldWorkingDirectory = os.getcwd()17151716# ensure the clientPath exists1717 new_client_dir =False1718if not os.path.exists(self.clientPath):1719 new_client_dir =True1720 os.makedirs(self.clientPath)17211722chdir(self.clientPath, is_client_path=True)1723if self.dry_run:1724print"Would synchronize p4 checkout in%s"% self.clientPath1725else:1726print"Synchronizing p4 checkout..."1727if new_client_dir:1728# old one was destroyed, and maybe nobody told p41729p4_sync("...","-f")1730else:1731p4_sync("...")1732 self.check()17331734 commits = []1735for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, self.master)]):1736 commits.append(line.strip())1737 commits.reverse()17381739if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):1740 self.checkAuthorship =False1741else:1742 self.checkAuthorship =True17431744if self.preserveUser:1745 self.checkValidP4Users(commits)17461747#1748# Build up a set of options to be passed to diff when1749# submitting each commit to p4.1750#1751if self.detectRenames:1752# command-line -M arg1753 self.diffOpts ="-M"1754else:1755# If not explicitly set check the config variable1756 detectRenames =gitConfig("git-p4.detectRenames")17571758if detectRenames.lower() =="false"or detectRenames =="":1759 self.diffOpts =""1760elif detectRenames.lower() =="true":1761 self.diffOpts ="-M"1762else:1763 self.diffOpts ="-M%s"% detectRenames17641765# no command-line arg for -C or --find-copies-harder, just1766# config variables1767 detectCopies =gitConfig("git-p4.detectCopies")1768if detectCopies.lower() =="false"or detectCopies =="":1769pass1770elif detectCopies.lower() =="true":1771 self.diffOpts +=" -C"1772else:1773 self.diffOpts +=" -C%s"% detectCopies17741775ifgitConfigBool("git-p4.detectCopiesHarder"):1776 self.diffOpts +=" --find-copies-harder"17771778#1779# Apply the commits, one at a time. On failure, ask if should1780# continue to try the rest of the patches, or quit.1781#1782if self.dry_run:1783print"Would apply"1784 applied = []1785 last =len(commits) -11786for i, commit inenumerate(commits):1787if self.dry_run:1788print" ",read_pipe(["git","show","-s",1789"--format=format:%h%s", commit])1790 ok =True1791else:1792 ok = self.applyCommit(commit)1793if ok:1794 applied.append(commit)1795else:1796if self.prepare_p4_only and i < last:1797print"Processing only the first commit due to option" \1798" --prepare-p4-only"1799break1800if i < last:1801 quit =False1802while True:1803# prompt for what to do, or use the option/variable1804if self.conflict_behavior =="ask":1805print"What do you want to do?"1806 response =raw_input("[s]kip this commit but apply"1807" the rest, or [q]uit? ")1808if not response:1809continue1810elif self.conflict_behavior =="skip":1811 response ="s"1812elif self.conflict_behavior =="quit":1813 response ="q"1814else:1815die("Unknown conflict_behavior '%s'"%1816 self.conflict_behavior)18171818if response[0] =="s":1819print"Skipping this commit, but applying the rest"1820break1821if response[0] =="q":1822print"Quitting"1823 quit =True1824break1825if quit:1826break18271828chdir(self.oldWorkingDirectory)18291830if self.dry_run:1831pass1832elif self.prepare_p4_only:1833pass1834eliflen(commits) ==len(applied):1835print"All commits applied!"18361837 sync =P4Sync()1838if self.branch:1839 sync.branch = self.branch1840 sync.run([])18411842 rebase =P4Rebase()1843 rebase.rebase()18441845else:1846iflen(applied) ==0:1847print"No commits applied."1848else:1849print"Applied only the commits marked with '*':"1850for c in commits:1851if c in applied:1852 star ="*"1853else:1854 star =" "1855print star,read_pipe(["git","show","-s",1856"--format=format:%h%s", c])1857print"You will have to do 'git p4 sync' and rebase."18581859ifgitConfigBool("git-p4.exportLabels"):1860 self.exportLabels =True18611862if self.exportLabels:1863 p4Labels =getP4Labels(self.depotPath)1864 gitTags =getGitTags()18651866 missingGitTags = gitTags - p4Labels1867 self.exportGitTags(missingGitTags)18681869# exit with error unless everything applied perfectly1870iflen(commits) !=len(applied):1871 sys.exit(1)18721873return True18741875classView(object):1876"""Represent a p4 view ("p4 help views"), and map files in a1877 repo according to the view."""18781879def__init__(self, client_name):1880 self.mappings = []1881 self.client_prefix ="//%s/"% client_name1882# cache results of "p4 where" to lookup client file locations1883 self.client_spec_path_cache = {}18841885defappend(self, view_line):1886"""Parse a view line, splitting it into depot and client1887 sides. Append to self.mappings, preserving order. This1888 is only needed for tag creation."""18891890# Split the view line into exactly two words. P4 enforces1891# structure on these lines that simplifies this quite a bit.1892#1893# Either or both words may be double-quoted.1894# Single quotes do not matter.1895# Double-quote marks cannot occur inside the words.1896# A + or - prefix is also inside the quotes.1897# There are no quotes unless they contain a space.1898# The line is already white-space stripped.1899# The two words are separated by a single space.1900#1901if view_line[0] =='"':1902# First word is double quoted. Find its end.1903 close_quote_index = view_line.find('"',1)1904if close_quote_index <=0:1905die("No first-word closing quote found:%s"% view_line)1906 depot_side = view_line[1:close_quote_index]1907# skip closing quote and space1908 rhs_index = close_quote_index +1+11909else:1910 space_index = view_line.find(" ")1911if space_index <=0:1912die("No word-splitting space found:%s"% view_line)1913 depot_side = view_line[0:space_index]1914 rhs_index = space_index +119151916# prefix + means overlay on previous mapping1917if depot_side.startswith("+"):1918 depot_side = depot_side[1:]19191920# prefix - means exclude this path, leave out of mappings1921 exclude =False1922if depot_side.startswith("-"):1923 exclude =True1924 depot_side = depot_side[1:]19251926if not exclude:1927 self.mappings.append(depot_side)19281929defconvert_client_path(self, clientFile):1930# chop off //client/ part to make it relative1931if not clientFile.startswith(self.client_prefix):1932die("No prefix '%s' on clientFile '%s'"%1933(self.client_prefix, clientFile))1934return clientFile[len(self.client_prefix):]19351936defupdate_client_spec_path_cache(self, files):1937""" Caching file paths by "p4 where" batch query """19381939# List depot file paths exclude that already cached1940 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]19411942iflen(fileArgs) ==0:1943return# All files in cache19441945 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)1946for res in where_result:1947if"code"in res and res["code"] =="error":1948# assume error is "... file(s) not in client view"1949continue1950if"clientFile"not in res:1951die("No clientFile in 'p4 where' output")1952if"unmap"in res:1953# it will list all of them, but only one not unmap-ped1954continue1955ifgitConfigBool("core.ignorecase"):1956 res['depotFile'] = res['depotFile'].lower()1957 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])19581959# not found files or unmap files set to ""1960for depotFile in fileArgs:1961ifgitConfigBool("core.ignorecase"):1962 depotFile = depotFile.lower()1963if depotFile not in self.client_spec_path_cache:1964 self.client_spec_path_cache[depotFile] =""19651966defmap_in_client(self, depot_path):1967"""Return the relative location in the client where this1968 depot file should live. Returns "" if the file should1969 not be mapped in the client."""19701971ifgitConfigBool("core.ignorecase"):1972 depot_path = depot_path.lower()19731974if depot_path in self.client_spec_path_cache:1975return self.client_spec_path_cache[depot_path]19761977die("Error:%sis not found in client spec path"% depot_path )1978return""19791980classP4Sync(Command, P4UserMap):1981 delete_actions = ("delete","move/delete","purge")19821983def__init__(self):1984 Command.__init__(self)1985 P4UserMap.__init__(self)1986 self.options = [1987 optparse.make_option("--branch", dest="branch"),1988 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),1989 optparse.make_option("--changesfile", dest="changesFile"),1990 optparse.make_option("--silent", dest="silent", action="store_true"),1991 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),1992 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),1993 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",1994help="Import into refs/heads/ , not refs/remotes"),1995 optparse.make_option("--max-changes", dest="maxChanges",1996help="Maximum number of changes to import"),1997 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",1998help="Internal block size to use when iteratively calling p4 changes"),1999 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2000help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2001 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2002help="Only sync files that are included in the Perforce Client Spec"),2003 optparse.make_option("-/", dest="cloneExclude",2004 action="append",type="string",2005help="exclude depot path"),2006]2007 self.description ="""Imports from Perforce into a git repository.\n2008 example:2009 //depot/my/project/ -- to import the current head2010 //depot/my/project/@all -- to import everything2011 //depot/my/project/@1,6 -- to import only from revision 1 to 620122013 (a ... is not needed in the path p4 specification, it's added implicitly)"""20142015 self.usage +=" //depot/path[@revRange]"2016 self.silent =False2017 self.createdBranches =set()2018 self.committedChanges =set()2019 self.branch =""2020 self.detectBranches =False2021 self.detectLabels =False2022 self.importLabels =False2023 self.changesFile =""2024 self.syncWithOrigin =True2025 self.importIntoRemotes =True2026 self.maxChanges =""2027 self.changes_block_size =None2028 self.keepRepoPath =False2029 self.depotPaths =None2030 self.p4BranchesInGit = []2031 self.cloneExclude = []2032 self.useClientSpec =False2033 self.useClientSpec_from_options =False2034 self.clientSpecDirs =None2035 self.tempBranches = []2036 self.tempBranchLocation ="git-p4-tmp"20372038ifgitConfig("git-p4.syncFromOrigin") =="false":2039 self.syncWithOrigin =False20402041# This is required for the "append" cloneExclude action2042defensure_value(self, attr, value):2043if nothasattr(self, attr)orgetattr(self, attr)is None:2044setattr(self, attr, value)2045returngetattr(self, attr)20462047# Force a checkpoint in fast-import and wait for it to finish2048defcheckpoint(self):2049 self.gitStream.write("checkpoint\n\n")2050 self.gitStream.write("progress checkpoint\n\n")2051 out = self.gitOutput.readline()2052if self.verbose:2053print"checkpoint finished: "+ out20542055defextractFilesFromCommit(self, commit):2056 self.cloneExclude = [re.sub(r"\.\.\.$","", path)2057for path in self.cloneExclude]2058 files = []2059 fnum =02060while commit.has_key("depotFile%s"% fnum):2061 path = commit["depotFile%s"% fnum]20622063if[p for p in self.cloneExclude2064ifp4PathStartsWith(path, p)]:2065 found =False2066else:2067 found = [p for p in self.depotPaths2068ifp4PathStartsWith(path, p)]2069if not found:2070 fnum = fnum +12071continue20722073file= {}2074file["path"] = path2075file["rev"] = commit["rev%s"% fnum]2076file["action"] = commit["action%s"% fnum]2077file["type"] = commit["type%s"% fnum]2078 files.append(file)2079 fnum = fnum +12080return files20812082defstripRepoPath(self, path, prefixes):2083"""When streaming files, this is called to map a p4 depot path2084 to where it should go in git. The prefixes are either2085 self.depotPaths, or self.branchPrefixes in the case of2086 branch detection."""20872088if self.useClientSpec:2089# branch detection moves files up a level (the branch name)2090# from what client spec interpretation gives2091 path = self.clientSpecDirs.map_in_client(path)2092if self.detectBranches:2093for b in self.knownBranches:2094if path.startswith(b +"/"):2095 path = path[len(b)+1:]20962097elif self.keepRepoPath:2098# Preserve everything in relative path name except leading2099# //depot/; just look at first prefix as they all should2100# be in the same depot.2101 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2102ifp4PathStartsWith(path, depot):2103 path = path[len(depot):]21042105else:2106for p in prefixes:2107ifp4PathStartsWith(path, p):2108 path = path[len(p):]2109break21102111 path =wildcard_decode(path)2112return path21132114defsplitFilesIntoBranches(self, commit):2115"""Look at each depotFile in the commit to figure out to what2116 branch it belongs."""21172118if self.clientSpecDirs:2119 files = self.extractFilesFromCommit(commit)2120 self.clientSpecDirs.update_client_spec_path_cache(files)21212122 branches = {}2123 fnum =02124while commit.has_key("depotFile%s"% fnum):2125 path = commit["depotFile%s"% fnum]2126 found = [p for p in self.depotPaths2127ifp4PathStartsWith(path, p)]2128if not found:2129 fnum = fnum +12130continue21312132file= {}2133file["path"] = path2134file["rev"] = commit["rev%s"% fnum]2135file["action"] = commit["action%s"% fnum]2136file["type"] = commit["type%s"% fnum]2137 fnum = fnum +121382139# start with the full relative path where this file would2140# go in a p4 client2141if self.useClientSpec:2142 relPath = self.clientSpecDirs.map_in_client(path)2143else:2144 relPath = self.stripRepoPath(path, self.depotPaths)21452146for branch in self.knownBranches.keys():2147# add a trailing slash so that a commit into qt/4.2foo2148# doesn't end up in qt/4.2, e.g.2149if relPath.startswith(branch +"/"):2150if branch not in branches:2151 branches[branch] = []2152 branches[branch].append(file)2153break21542155return branches21562157# output one file from the P4 stream2158# - helper for streamP4Files21592160defstreamOneP4File(self,file, contents):2161 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2162if verbose:2163 sys.stderr.write("%s\n"% relPath)21642165(type_base, type_mods) =split_p4_type(file["type"])21662167 git_mode ="100644"2168if"x"in type_mods:2169 git_mode ="100755"2170if type_base =="symlink":2171 git_mode ="120000"2172# p4 print on a symlink sometimes contains "target\n";2173# if it does, remove the newline2174 data =''.join(contents)2175if not data:2176# Some version of p4 allowed creating a symlink that pointed2177# to nothing. This causes p4 errors when checking out such2178# a change, and errors here too. Work around it by ignoring2179# the bad symlink; hopefully a future change fixes it.2180print"\nIgnoring empty symlink in%s"%file['depotFile']2181return2182elif data[-1] =='\n':2183 contents = [data[:-1]]2184else:2185 contents = [data]21862187if type_base =="utf16":2188# p4 delivers different text in the python output to -G2189# than it does when using "print -o", or normal p4 client2190# operations. utf16 is converted to ascii or utf8, perhaps.2191# But ascii text saved as -t utf16 is completely mangled.2192# Invoke print -o to get the real contents.2193#2194# On windows, the newlines will always be mangled by print, so put2195# them back too. This is not needed to the cygwin windows version,2196# just the native "NT" type.2197#2198 text =p4_read_pipe(['print','-q','-o','-',"%s@%s"% (file['depotFile'],file['change']) ])2199ifp4_version_string().find("/NT") >=0:2200 text = text.replace("\r\n","\n")2201 contents = [ text ]22022203if type_base =="apple":2204# Apple filetype files will be streamed as a concatenation of2205# its appledouble header and the contents. This is useless2206# on both macs and non-macs. If using "print -q -o xx", it2207# will create "xx" with the data, and "%xx" with the header.2208# This is also not very useful.2209#2210# Ideally, someday, this script can learn how to generate2211# appledouble files directly and import those to git, but2212# non-mac machines can never find a use for apple filetype.2213print"\nIgnoring apple filetype file%s"%file['depotFile']2214return22152216# Note that we do not try to de-mangle keywords on utf16 files,2217# even though in theory somebody may want that.2218 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2219if pattern:2220 regexp = re.compile(pattern, re.VERBOSE)2221 text =''.join(contents)2222 text = regexp.sub(r'$\1$', text)2223 contents = [ text ]22242225 self.gitStream.write("M%sinline%s\n"% (git_mode, relPath))22262227# total length...2228 length =02229for d in contents:2230 length = length +len(d)22312232 self.gitStream.write("data%d\n"% length)2233for d in contents:2234 self.gitStream.write(d)2235 self.gitStream.write("\n")22362237defstreamOneP4Deletion(self,file):2238 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2239if verbose:2240 sys.stderr.write("delete%s\n"% relPath)2241 self.gitStream.write("D%s\n"% relPath)22422243# handle another chunk of streaming data2244defstreamP4FilesCb(self, marshalled):22452246# catch p4 errors and complain2247 err =None2248if"code"in marshalled:2249if marshalled["code"] =="error":2250if"data"in marshalled:2251 err = marshalled["data"].rstrip()2252if err:2253 f =None2254if self.stream_have_file_info:2255if"depotFile"in self.stream_file:2256 f = self.stream_file["depotFile"]2257# force a failure in fast-import, else an empty2258# commit will be made2259 self.gitStream.write("\n")2260 self.gitStream.write("die-now\n")2261 self.gitStream.close()2262# ignore errors, but make sure it exits first2263 self.importProcess.wait()2264if f:2265die("Error from p4 print for%s:%s"% (f, err))2266else:2267die("Error from p4 print:%s"% err)22682269if marshalled.has_key('depotFile')and self.stream_have_file_info:2270# start of a new file - output the old one first2271 self.streamOneP4File(self.stream_file, self.stream_contents)2272 self.stream_file = {}2273 self.stream_contents = []2274 self.stream_have_file_info =False22752276# pick up the new file information... for the2277# 'data' field we need to append to our array2278for k in marshalled.keys():2279if k =='data':2280 self.stream_contents.append(marshalled['data'])2281else:2282 self.stream_file[k] = marshalled[k]22832284 self.stream_have_file_info =True22852286# Stream directly from "p4 files" into "git fast-import"2287defstreamP4Files(self, files):2288 filesForCommit = []2289 filesToRead = []2290 filesToDelete = []22912292for f in files:2293# if using a client spec, only add the files that have2294# a path in the client2295if self.clientSpecDirs:2296if self.clientSpecDirs.map_in_client(f['path']) =="":2297continue22982299 filesForCommit.append(f)2300if f['action']in self.delete_actions:2301 filesToDelete.append(f)2302else:2303 filesToRead.append(f)23042305# deleted files...2306for f in filesToDelete:2307 self.streamOneP4Deletion(f)23082309iflen(filesToRead) >0:2310 self.stream_file = {}2311 self.stream_contents = []2312 self.stream_have_file_info =False23132314# curry self argument2315defstreamP4FilesCbSelf(entry):2316 self.streamP4FilesCb(entry)23172318 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]23192320p4CmdList(["-x","-","print"],2321 stdin=fileArgs,2322 cb=streamP4FilesCbSelf)23232324# do the last chunk2325if self.stream_file.has_key('depotFile'):2326 self.streamOneP4File(self.stream_file, self.stream_contents)23272328defmake_email(self, userid):2329if userid in self.users:2330return self.users[userid]2331else:2332return"%s<a@b>"% userid23332334# Stream a p4 tag2335defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2336if verbose:2337print"writing tag%sfor commit%s"% (labelName, commit)2338 gitStream.write("tag%s\n"% labelName)2339 gitStream.write("from%s\n"% commit)23402341if labelDetails.has_key('Owner'):2342 owner = labelDetails["Owner"]2343else:2344 owner =None23452346# Try to use the owner of the p4 label, or failing that,2347# the current p4 user id.2348if owner:2349 email = self.make_email(owner)2350else:2351 email = self.make_email(self.p4UserId())2352 tagger ="%s %s %s"% (email, epoch, self.tz)23532354 gitStream.write("tagger%s\n"% tagger)23552356print"labelDetails=",labelDetails2357if labelDetails.has_key('Description'):2358 description = labelDetails['Description']2359else:2360 description ='Label from git p4'23612362 gitStream.write("data%d\n"%len(description))2363 gitStream.write(description)2364 gitStream.write("\n")23652366defcommit(self, details, files, branch, parent =""):2367 epoch = details["time"]2368 author = details["user"]23692370if self.verbose:2371print"commit into%s"% branch23722373# start with reading files; if that fails, we should not2374# create a commit.2375 new_files = []2376for f in files:2377if[p for p in self.branchPrefixes ifp4PathStartsWith(f['path'], p)]:2378 new_files.append(f)2379else:2380 sys.stderr.write("Ignoring file outside of prefix:%s\n"% f['path'])23812382if self.clientSpecDirs:2383 self.clientSpecDirs.update_client_spec_path_cache(files)23842385 self.gitStream.write("commit%s\n"% branch)2386# gitStream.write("mark :%s\n" % details["change"])2387 self.committedChanges.add(int(details["change"]))2388 committer =""2389if author not in self.users:2390 self.getUserMapFromPerforceServer()2391 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)23922393 self.gitStream.write("committer%s\n"% committer)23942395 self.gitStream.write("data <<EOT\n")2396 self.gitStream.write(details["desc"])2397 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2398(','.join(self.branchPrefixes), details["change"]))2399iflen(details['options']) >0:2400 self.gitStream.write(": options =%s"% details['options'])2401 self.gitStream.write("]\nEOT\n\n")24022403iflen(parent) >0:2404if self.verbose:2405print"parent%s"% parent2406 self.gitStream.write("from%s\n"% parent)24072408 self.streamP4Files(new_files)2409 self.gitStream.write("\n")24102411 change =int(details["change"])24122413if self.labels.has_key(change):2414 label = self.labels[change]2415 labelDetails = label[0]2416 labelRevisions = label[1]2417if self.verbose:2418print"Change%sis labelled%s"% (change, labelDetails)24192420 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2421for p in self.branchPrefixes])24222423iflen(files) ==len(labelRevisions):24242425 cleanedFiles = {}2426for info in files:2427if info["action"]in self.delete_actions:2428continue2429 cleanedFiles[info["depotFile"]] = info["rev"]24302431if cleanedFiles == labelRevisions:2432 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)24332434else:2435if not self.silent:2436print("Tag%sdoes not match with change%s: files do not match."2437% (labelDetails["label"], change))24382439else:2440if not self.silent:2441print("Tag%sdoes not match with change%s: file count is different."2442% (labelDetails["label"], change))24432444# Build a dictionary of changelists and labels, for "detect-labels" option.2445defgetLabels(self):2446 self.labels = {}24472448 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2449iflen(l) >0and not self.silent:2450print"Finding files belonging to labels in%s"% `self.depotPaths`24512452for output in l:2453 label = output["label"]2454 revisions = {}2455 newestChange =02456if self.verbose:2457print"Querying files for label%s"% label2458forfileinp4CmdList(["files"] +2459["%s...@%s"% (p, label)2460for p in self.depotPaths]):2461 revisions[file["depotFile"]] =file["rev"]2462 change =int(file["change"])2463if change > newestChange:2464 newestChange = change24652466 self.labels[newestChange] = [output, revisions]24672468if self.verbose:2469print"Label changes:%s"% self.labels.keys()24702471# Import p4 labels as git tags. A direct mapping does not2472# exist, so assume that if all the files are at the same revision2473# then we can use that, or it's something more complicated we should2474# just ignore.2475defimportP4Labels(self, stream, p4Labels):2476if verbose:2477print"import p4 labels: "+' '.join(p4Labels)24782479 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2480 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2481iflen(validLabelRegexp) ==0:2482 validLabelRegexp = defaultLabelRegexp2483 m = re.compile(validLabelRegexp)24842485for name in p4Labels:2486 commitFound =False24872488if not m.match(name):2489if verbose:2490print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2491continue24922493if name in ignoredP4Labels:2494continue24952496 labelDetails =p4CmdList(['label',"-o", name])[0]24972498# get the most recent changelist for each file in this label2499 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2500for p in self.depotPaths])25012502if change.has_key('change'):2503# find the corresponding git commit; take the oldest commit2504 changelist =int(change['change'])2505 gitCommit =read_pipe(["git","rev-list","--max-count=1",2506"--reverse",":/\[git-p4:.*change =%d\]"% changelist])2507iflen(gitCommit) ==0:2508print"could not find git commit for changelist%d"% changelist2509else:2510 gitCommit = gitCommit.strip()2511 commitFound =True2512# Convert from p4 time format2513try:2514 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2515exceptValueError:2516print"Could not convert label time%s"% labelDetails['Update']2517 tmwhen =125182519 when =int(time.mktime(tmwhen))2520 self.streamTag(stream, name, labelDetails, gitCommit, when)2521if verbose:2522print"p4 label%smapped to git commit%s"% (name, gitCommit)2523else:2524if verbose:2525print"Label%shas no changelists - possibly deleted?"% name25262527if not commitFound:2528# We can't import this label; don't try again as it will get very2529# expensive repeatedly fetching all the files for labels that will2530# never be imported. If the label is moved in the future, the2531# ignore will need to be removed manually.2532system(["git","config","--add","git-p4.ignoredP4Labels", name])25332534defguessProjectName(self):2535for p in self.depotPaths:2536if p.endswith("/"):2537 p = p[:-1]2538 p = p[p.strip().rfind("/") +1:]2539if not p.endswith("/"):2540 p +="/"2541return p25422543defgetBranchMapping(self):2544 lostAndFoundBranches =set()25452546 user =gitConfig("git-p4.branchUser")2547iflen(user) >0:2548 command ="branches -u%s"% user2549else:2550 command ="branches"25512552for info inp4CmdList(command):2553 details =p4Cmd(["branch","-o", info["branch"]])2554 viewIdx =02555while details.has_key("View%s"% viewIdx):2556 paths = details["View%s"% viewIdx].split(" ")2557 viewIdx = viewIdx +12558# require standard //depot/foo/... //depot/bar/... mapping2559iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2560continue2561 source = paths[0]2562 destination = paths[1]2563## HACK2564ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2565 source = source[len(self.depotPaths[0]):-4]2566 destination = destination[len(self.depotPaths[0]):-4]25672568if destination in self.knownBranches:2569if not self.silent:2570print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2571print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2572continue25732574 self.knownBranches[destination] = source25752576 lostAndFoundBranches.discard(destination)25772578if source not in self.knownBranches:2579 lostAndFoundBranches.add(source)25802581# Perforce does not strictly require branches to be defined, so we also2582# check git config for a branch list.2583#2584# Example of branch definition in git config file:2585# [git-p4]2586# branchList=main:branchA2587# branchList=main:branchB2588# branchList=branchA:branchC2589 configBranches =gitConfigList("git-p4.branchList")2590for branch in configBranches:2591if branch:2592(source, destination) = branch.split(":")2593 self.knownBranches[destination] = source25942595 lostAndFoundBranches.discard(destination)25962597if source not in self.knownBranches:2598 lostAndFoundBranches.add(source)259926002601for branch in lostAndFoundBranches:2602 self.knownBranches[branch] = branch26032604defgetBranchMappingFromGitBranches(self):2605 branches =p4BranchesInGit(self.importIntoRemotes)2606for branch in branches.keys():2607if branch =="master":2608 branch ="main"2609else:2610 branch = branch[len(self.projectName):]2611 self.knownBranches[branch] = branch26122613defupdateOptionDict(self, d):2614 option_keys = {}2615if self.keepRepoPath:2616 option_keys['keepRepoPath'] =126172618 d["options"] =' '.join(sorted(option_keys.keys()))26192620defreadOptions(self, d):2621 self.keepRepoPath = (d.has_key('options')2622and('keepRepoPath'in d['options']))26232624defgitRefForBranch(self, branch):2625if branch =="main":2626return self.refPrefix +"master"26272628iflen(branch) <=0:2629return branch26302631return self.refPrefix + self.projectName + branch26322633defgitCommitByP4Change(self, ref, change):2634if self.verbose:2635print"looking in ref "+ ref +" for change%susing bisect..."% change26362637 earliestCommit =""2638 latestCommit =parseRevision(ref)26392640while True:2641if self.verbose:2642print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2643 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2644iflen(next) ==0:2645if self.verbose:2646print"argh"2647return""2648 log =extractLogMessageFromGitCommit(next)2649 settings =extractSettingsGitLog(log)2650 currentChange =int(settings['change'])2651if self.verbose:2652print"current change%s"% currentChange26532654if currentChange == change:2655if self.verbose:2656print"found%s"% next2657return next26582659if currentChange < change:2660 earliestCommit ="^%s"% next2661else:2662 latestCommit ="%s"% next26632664return""26652666defimportNewBranch(self, branch, maxChange):2667# make fast-import flush all changes to disk and update the refs using the checkpoint2668# command so that we can try to find the branch parent in the git history2669 self.gitStream.write("checkpoint\n\n");2670 self.gitStream.flush();2671 branchPrefix = self.depotPaths[0] + branch +"/"2672range="@1,%s"% maxChange2673#print "prefix" + branchPrefix2674 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)2675iflen(changes) <=0:2676return False2677 firstChange = changes[0]2678#print "first change in branch: %s" % firstChange2679 sourceBranch = self.knownBranches[branch]2680 sourceDepotPath = self.depotPaths[0] + sourceBranch2681 sourceRef = self.gitRefForBranch(sourceBranch)2682#print "source " + sourceBranch26832684 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])2685#print "branch parent: %s" % branchParentChange2686 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)2687iflen(gitParent) >0:2688 self.initialParents[self.gitRefForBranch(branch)] = gitParent2689#print "parent git commit: %s" % gitParent26902691 self.importChanges(changes)2692return True26932694defsearchParent(self, parent, branch, target):2695 parentFound =False2696for blob inread_pipe_lines(["git","rev-list","--reverse",2697"--no-merges", parent]):2698 blob = blob.strip()2699iflen(read_pipe(["git","diff-tree", blob, target])) ==0:2700 parentFound =True2701if self.verbose:2702print"Found parent of%sin commit%s"% (branch, blob)2703break2704if parentFound:2705return blob2706else:2707return None27082709defimportChanges(self, changes):2710 cnt =12711for change in changes:2712 description =p4_describe(change)2713 self.updateOptionDict(description)27142715if not self.silent:2716 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))2717 sys.stdout.flush()2718 cnt = cnt +127192720try:2721if self.detectBranches:2722 branches = self.splitFilesIntoBranches(description)2723for branch in branches.keys():2724## HACK --hwn2725 branchPrefix = self.depotPaths[0] + branch +"/"2726 self.branchPrefixes = [ branchPrefix ]27272728 parent =""27292730 filesForCommit = branches[branch]27312732if self.verbose:2733print"branch is%s"% branch27342735 self.updatedBranches.add(branch)27362737if branch not in self.createdBranches:2738 self.createdBranches.add(branch)2739 parent = self.knownBranches[branch]2740if parent == branch:2741 parent =""2742else:2743 fullBranch = self.projectName + branch2744if fullBranch not in self.p4BranchesInGit:2745if not self.silent:2746print("\nImporting new branch%s"% fullBranch);2747if self.importNewBranch(branch, change -1):2748 parent =""2749 self.p4BranchesInGit.append(fullBranch)2750if not self.silent:2751print("\nResuming with change%s"% change);27522753if self.verbose:2754print"parent determined through known branches:%s"% parent27552756 branch = self.gitRefForBranch(branch)2757 parent = self.gitRefForBranch(parent)27582759if self.verbose:2760print"looking for initial parent for%s; current parent is%s"% (branch, parent)27612762iflen(parent) ==0and branch in self.initialParents:2763 parent = self.initialParents[branch]2764del self.initialParents[branch]27652766 blob =None2767iflen(parent) >0:2768 tempBranch ="%s/%d"% (self.tempBranchLocation, change)2769if self.verbose:2770print"Creating temporary branch: "+ tempBranch2771 self.commit(description, filesForCommit, tempBranch)2772 self.tempBranches.append(tempBranch)2773 self.checkpoint()2774 blob = self.searchParent(parent, branch, tempBranch)2775if blob:2776 self.commit(description, filesForCommit, branch, blob)2777else:2778if self.verbose:2779print"Parent of%snot found. Committing into head of%s"% (branch, parent)2780 self.commit(description, filesForCommit, branch, parent)2781else:2782 files = self.extractFilesFromCommit(description)2783 self.commit(description, files, self.branch,2784 self.initialParent)2785# only needed once, to connect to the previous commit2786 self.initialParent =""2787exceptIOError:2788print self.gitError.read()2789 sys.exit(1)27902791defimportHeadRevision(self, revision):2792print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)27932794 details = {}2795 details["user"] ="git perforce import user"2796 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"2797% (' '.join(self.depotPaths), revision))2798 details["change"] = revision2799 newestRevision =028002801 fileCnt =02802 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]28032804for info inp4CmdList(["files"] + fileArgs):28052806if'code'in info and info['code'] =='error':2807 sys.stderr.write("p4 returned an error:%s\n"2808% info['data'])2809if info['data'].find("must refer to client") >=0:2810 sys.stderr.write("This particular p4 error is misleading.\n")2811 sys.stderr.write("Perhaps the depot path was misspelled.\n");2812 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))2813 sys.exit(1)2814if'p4ExitCode'in info:2815 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])2816 sys.exit(1)281728182819 change =int(info["change"])2820if change > newestRevision:2821 newestRevision = change28222823if info["action"]in self.delete_actions:2824# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!2825#fileCnt = fileCnt + 12826continue28272828for prop in["depotFile","rev","action","type"]:2829 details["%s%s"% (prop, fileCnt)] = info[prop]28302831 fileCnt = fileCnt +128322833 details["change"] = newestRevision28342835# Use time from top-most change so that all git p4 clones of2836# the same p4 repo have the same commit SHA1s.2837 res =p4_describe(newestRevision)2838 details["time"] = res["time"]28392840 self.updateOptionDict(details)2841try:2842 self.commit(details, self.extractFilesFromCommit(details), self.branch)2843exceptIOError:2844print"IO error with git fast-import. Is your git version recent enough?"2845print self.gitError.read()284628472848defrun(self, args):2849 self.depotPaths = []2850 self.changeRange =""2851 self.previousDepotPaths = []2852 self.hasOrigin =False28532854# map from branch depot path to parent branch2855 self.knownBranches = {}2856 self.initialParents = {}28572858if self.importIntoRemotes:2859 self.refPrefix ="refs/remotes/p4/"2860else:2861 self.refPrefix ="refs/heads/p4/"28622863if self.syncWithOrigin:2864 self.hasOrigin =originP4BranchesExist()2865if self.hasOrigin:2866if not self.silent:2867print'Syncing with origin first, using "git fetch origin"'2868system("git fetch origin")28692870 branch_arg_given =bool(self.branch)2871iflen(self.branch) ==0:2872 self.branch = self.refPrefix +"master"2873ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:2874system("git update-ref%srefs/heads/p4"% self.branch)2875system("git branch -D p4")28762877# accept either the command-line option, or the configuration variable2878if self.useClientSpec:2879# will use this after clone to set the variable2880 self.useClientSpec_from_options =True2881else:2882ifgitConfigBool("git-p4.useclientspec"):2883 self.useClientSpec =True2884if self.useClientSpec:2885 self.clientSpecDirs =getClientSpec()28862887# TODO: should always look at previous commits,2888# merge with previous imports, if possible.2889if args == []:2890if self.hasOrigin:2891createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)28922893# branches holds mapping from branch name to sha12894 branches =p4BranchesInGit(self.importIntoRemotes)28952896# restrict to just this one, disabling detect-branches2897if branch_arg_given:2898 short = self.branch.split("/")[-1]2899if short in branches:2900 self.p4BranchesInGit = [ short ]2901else:2902 self.p4BranchesInGit = branches.keys()29032904iflen(self.p4BranchesInGit) >1:2905if not self.silent:2906print"Importing from/into multiple branches"2907 self.detectBranches =True2908for branch in branches.keys():2909 self.initialParents[self.refPrefix + branch] = \2910 branches[branch]29112912if self.verbose:2913print"branches:%s"% self.p4BranchesInGit29142915 p4Change =02916for branch in self.p4BranchesInGit:2917 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)29182919 settings =extractSettingsGitLog(logMsg)29202921 self.readOptions(settings)2922if(settings.has_key('depot-paths')2923and settings.has_key('change')):2924 change =int(settings['change']) +12925 p4Change =max(p4Change, change)29262927 depotPaths =sorted(settings['depot-paths'])2928if self.previousDepotPaths == []:2929 self.previousDepotPaths = depotPaths2930else:2931 paths = []2932for(prev, cur)inzip(self.previousDepotPaths, depotPaths):2933 prev_list = prev.split("/")2934 cur_list = cur.split("/")2935for i inrange(0,min(len(cur_list),len(prev_list))):2936if cur_list[i] <> prev_list[i]:2937 i = i -12938break29392940 paths.append("/".join(cur_list[:i +1]))29412942 self.previousDepotPaths = paths29432944if p4Change >0:2945 self.depotPaths =sorted(self.previousDepotPaths)2946 self.changeRange ="@%s,#head"% p4Change2947if not self.silent and not self.detectBranches:2948print"Performing incremental import into%sgit branch"% self.branch29492950# accept multiple ref name abbreviations:2951# refs/foo/bar/branch -> use it exactly2952# p4/branch -> prepend refs/remotes/ or refs/heads/2953# branch -> prepend refs/remotes/p4/ or refs/heads/p4/2954if not self.branch.startswith("refs/"):2955if self.importIntoRemotes:2956 prepend ="refs/remotes/"2957else:2958 prepend ="refs/heads/"2959if not self.branch.startswith("p4/"):2960 prepend +="p4/"2961 self.branch = prepend + self.branch29622963iflen(args) ==0and self.depotPaths:2964if not self.silent:2965print"Depot paths:%s"%' '.join(self.depotPaths)2966else:2967if self.depotPaths and self.depotPaths != args:2968print("previous import used depot path%sand now%swas specified. "2969"This doesn't work!"% (' '.join(self.depotPaths),2970' '.join(args)))2971 sys.exit(1)29722973 self.depotPaths =sorted(args)29742975 revision =""2976 self.users = {}29772978# Make sure no revision specifiers are used when --changesfile2979# is specified.2980 bad_changesfile =False2981iflen(self.changesFile) >0:2982for p in self.depotPaths:2983if p.find("@") >=0or p.find("#") >=0:2984 bad_changesfile =True2985break2986if bad_changesfile:2987die("Option --changesfile is incompatible with revision specifiers")29882989 newPaths = []2990for p in self.depotPaths:2991if p.find("@") != -1:2992 atIdx = p.index("@")2993 self.changeRange = p[atIdx:]2994if self.changeRange =="@all":2995 self.changeRange =""2996elif','not in self.changeRange:2997 revision = self.changeRange2998 self.changeRange =""2999 p = p[:atIdx]3000elif p.find("#") != -1:3001 hashIdx = p.index("#")3002 revision = p[hashIdx:]3003 p = p[:hashIdx]3004elif self.previousDepotPaths == []:3005# pay attention to changesfile, if given, else import3006# the entire p4 tree at the head revision3007iflen(self.changesFile) ==0:3008 revision ="#head"30093010 p = re.sub("\.\.\.$","", p)3011if not p.endswith("/"):3012 p +="/"30133014 newPaths.append(p)30153016 self.depotPaths = newPaths30173018# --detect-branches may change this for each branch3019 self.branchPrefixes = self.depotPaths30203021 self.loadUserMapFromCache()3022 self.labels = {}3023if self.detectLabels:3024 self.getLabels();30253026if self.detectBranches:3027## FIXME - what's a P4 projectName ?3028 self.projectName = self.guessProjectName()30293030if self.hasOrigin:3031 self.getBranchMappingFromGitBranches()3032else:3033 self.getBranchMapping()3034if self.verbose:3035print"p4-git branches:%s"% self.p4BranchesInGit3036print"initial parents:%s"% self.initialParents3037for b in self.p4BranchesInGit:3038if b !="master":30393040## FIXME3041 b = b[len(self.projectName):]3042 self.createdBranches.add(b)30433044 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))30453046 self.importProcess = subprocess.Popen(["git","fast-import"],3047 stdin=subprocess.PIPE,3048 stdout=subprocess.PIPE,3049 stderr=subprocess.PIPE);3050 self.gitOutput = self.importProcess.stdout3051 self.gitStream = self.importProcess.stdin3052 self.gitError = self.importProcess.stderr30533054if revision:3055 self.importHeadRevision(revision)3056else:3057 changes = []30583059iflen(self.changesFile) >0:3060 output =open(self.changesFile).readlines()3061 changeSet =set()3062for line in output:3063 changeSet.add(int(line))30643065for change in changeSet:3066 changes.append(change)30673068 changes.sort()3069else:3070# catch "git p4 sync" with no new branches, in a repo that3071# does not have any existing p4 branches3072iflen(args) ==0:3073if not self.p4BranchesInGit:3074die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")30753076# The default branch is master, unless --branch is used to3077# specify something else. Make sure it exists, or complain3078# nicely about how to use --branch.3079if not self.detectBranches:3080if notbranch_exists(self.branch):3081if branch_arg_given:3082die("Error: branch%sdoes not exist."% self.branch)3083else:3084die("Error: no branch%s; perhaps specify one with --branch."%3085 self.branch)30863087if self.verbose:3088print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3089 self.changeRange)3090 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)30913092iflen(self.maxChanges) >0:3093 changes = changes[:min(int(self.maxChanges),len(changes))]30943095iflen(changes) ==0:3096if not self.silent:3097print"No changes to import!"3098else:3099if not self.silent and not self.detectBranches:3100print"Import destination:%s"% self.branch31013102 self.updatedBranches =set()31033104if not self.detectBranches:3105if args:3106# start a new branch3107 self.initialParent =""3108else:3109# build on a previous revision3110 self.initialParent =parseRevision(self.branch)31113112 self.importChanges(changes)31133114if not self.silent:3115print""3116iflen(self.updatedBranches) >0:3117 sys.stdout.write("Updated branches: ")3118for b in self.updatedBranches:3119 sys.stdout.write("%s"% b)3120 sys.stdout.write("\n")31213122ifgitConfigBool("git-p4.importLabels"):3123 self.importLabels =True31243125if self.importLabels:3126 p4Labels =getP4Labels(self.depotPaths)3127 gitTags =getGitTags()31283129 missingP4Labels = p4Labels - gitTags3130 self.importP4Labels(self.gitStream, missingP4Labels)31313132 self.gitStream.close()3133if self.importProcess.wait() !=0:3134die("fast-import failed:%s"% self.gitError.read())3135 self.gitOutput.close()3136 self.gitError.close()31373138# Cleanup temporary branches created during import3139if self.tempBranches != []:3140for branch in self.tempBranches:3141read_pipe("git update-ref -d%s"% branch)3142 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))31433144# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3145# a convenient shortcut refname "p4".3146if self.importIntoRemotes:3147 head_ref = self.refPrefix +"HEAD"3148if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3149system(["git","symbolic-ref", head_ref, self.branch])31503151return True31523153classP4Rebase(Command):3154def__init__(self):3155 Command.__init__(self)3156 self.options = [3157 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3158]3159 self.importLabels =False3160 self.description = ("Fetches the latest revision from perforce and "3161+"rebases the current work (branch) against it")31623163defrun(self, args):3164 sync =P4Sync()3165 sync.importLabels = self.importLabels3166 sync.run([])31673168return self.rebase()31693170defrebase(self):3171if os.system("git update-index --refresh") !=0:3172die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");3173iflen(read_pipe("git diff-index HEAD --")) >0:3174die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");31753176[upstream, settings] =findUpstreamBranchPoint()3177iflen(upstream) ==0:3178die("Cannot find upstream branchpoint for rebase")31793180# the branchpoint may be p4/foo~3, so strip off the parent3181 upstream = re.sub("~[0-9]+$","", upstream)31823183print"Rebasing the current branch onto%s"% upstream3184 oldHead =read_pipe("git rev-parse HEAD").strip()3185system("git rebase%s"% upstream)3186system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3187return True31883189classP4Clone(P4Sync):3190def__init__(self):3191 P4Sync.__init__(self)3192 self.description ="Creates a new git repository and imports from Perforce into it"3193 self.usage ="usage: %prog [options] //depot/path[@revRange]"3194 self.options += [3195 optparse.make_option("--destination", dest="cloneDestination",3196 action='store', default=None,3197help="where to leave result of the clone"),3198 optparse.make_option("--bare", dest="cloneBare",3199 action="store_true", default=False),3200]3201 self.cloneDestination =None3202 self.needsGit =False3203 self.cloneBare =False32043205defdefaultDestination(self, args):3206## TODO: use common prefix of args?3207 depotPath = args[0]3208 depotDir = re.sub("(@[^@]*)$","", depotPath)3209 depotDir = re.sub("(#[^#]*)$","", depotDir)3210 depotDir = re.sub(r"\.\.\.$","", depotDir)3211 depotDir = re.sub(r"/$","", depotDir)3212return os.path.split(depotDir)[1]32133214defrun(self, args):3215iflen(args) <1:3216return False32173218if self.keepRepoPath and not self.cloneDestination:3219 sys.stderr.write("Must specify destination for --keep-path\n")3220 sys.exit(1)32213222 depotPaths = args32233224if not self.cloneDestination andlen(depotPaths) >1:3225 self.cloneDestination = depotPaths[-1]3226 depotPaths = depotPaths[:-1]32273228 self.cloneExclude = ["/"+p for p in self.cloneExclude]3229for p in depotPaths:3230if not p.startswith("//"):3231 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3232return False32333234if not self.cloneDestination:3235 self.cloneDestination = self.defaultDestination(args)32363237print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)32383239if not os.path.exists(self.cloneDestination):3240 os.makedirs(self.cloneDestination)3241chdir(self.cloneDestination)32423243 init_cmd = ["git","init"]3244if self.cloneBare:3245 init_cmd.append("--bare")3246 retcode = subprocess.call(init_cmd)3247if retcode:3248raiseCalledProcessError(retcode, init_cmd)32493250if not P4Sync.run(self, depotPaths):3251return False32523253# create a master branch and check out a work tree3254ifgitBranchExists(self.branch):3255system(["git","branch","master", self.branch ])3256if not self.cloneBare:3257system(["git","checkout","-f"])3258else:3259print'Not checking out any branch, use ' \3260'"git checkout -q -b master <branch>"'32613262# auto-set this variable if invoked with --use-client-spec3263if self.useClientSpec_from_options:3264system("git config --bool git-p4.useclientspec true")32653266return True32673268classP4Branches(Command):3269def__init__(self):3270 Command.__init__(self)3271 self.options = [ ]3272 self.description = ("Shows the git branches that hold imports and their "3273+"corresponding perforce depot paths")3274 self.verbose =False32753276defrun(self, args):3277iforiginP4BranchesExist():3278createOrUpdateBranchesFromOrigin()32793280 cmdline ="git rev-parse --symbolic "3281 cmdline +=" --remotes"32823283for line inread_pipe_lines(cmdline):3284 line = line.strip()32853286if not line.startswith('p4/')or line =="p4/HEAD":3287continue3288 branch = line32893290 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)3291 settings =extractSettingsGitLog(log)32923293print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])3294return True32953296classHelpFormatter(optparse.IndentedHelpFormatter):3297def__init__(self):3298 optparse.IndentedHelpFormatter.__init__(self)32993300defformat_description(self, description):3301if description:3302return description +"\n"3303else:3304return""33053306defprintUsage(commands):3307print"usage:%s<command> [options]"% sys.argv[0]3308print""3309print"valid commands:%s"%", ".join(commands)3310print""3311print"Try%s<command> --help for command specific help."% sys.argv[0]3312print""33133314commands = {3315"debug": P4Debug,3316"submit": P4Submit,3317"commit": P4Submit,3318"sync": P4Sync,3319"rebase": P4Rebase,3320"clone": P4Clone,3321"rollback": P4RollBack,3322"branches": P4Branches3323}332433253326defmain():3327iflen(sys.argv[1:]) ==0:3328printUsage(commands.keys())3329 sys.exit(2)33303331 cmdName = sys.argv[1]3332try:3333 klass = commands[cmdName]3334 cmd =klass()3335exceptKeyError:3336print"unknown command%s"% cmdName3337print""3338printUsage(commands.keys())3339 sys.exit(2)33403341 options = cmd.options3342 cmd.gitdir = os.environ.get("GIT_DIR",None)33433344 args = sys.argv[2:]33453346 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))3347if cmd.needsGit:3348 options.append(optparse.make_option("--git-dir", dest="gitdir"))33493350 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3351 options,3352 description = cmd.description,3353 formatter =HelpFormatter())33543355(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3356global verbose3357 verbose = cmd.verbose3358if cmd.needsGit:3359if cmd.gitdir ==None:3360 cmd.gitdir = os.path.abspath(".git")3361if notisValidGitDir(cmd.gitdir):3362 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3363if os.path.exists(cmd.gitdir):3364 cdup =read_pipe("git rev-parse --show-cdup").strip()3365iflen(cdup) >0:3366chdir(cdup);33673368if notisValidGitDir(cmd.gitdir):3369ifisValidGitDir(cmd.gitdir +"/.git"):3370 cmd.gitdir +="/.git"3371else:3372die("fatal: cannot locate git repository at%s"% cmd.gitdir)33733374 os.environ["GIT_DIR"] = cmd.gitdir33753376if not cmd.run(args):3377 parser.print_help()3378 sys.exit(2)337933803381if __name__ =='__main__':3382main()