1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25import zipfile 26import zlib 27import ctypes 28import errno 29 30# support basestring in python3 31try: 32unicode=unicode 33exceptNameError: 34# 'unicode' is undefined, must be Python 3 35str=str 36unicode=str 37bytes=bytes 38 basestring = (str,bytes) 39else: 40# 'unicode' exists, must be Python 2 41str=str 42unicode=unicode 43bytes=str 44 basestring = basestring 45 46try: 47from subprocess import CalledProcessError 48exceptImportError: 49# from python2.7:subprocess.py 50# Exception classes used by this module. 51classCalledProcessError(Exception): 52"""This exception is raised when a process run by check_call() returns 53 a non-zero exit status. The exit status will be stored in the 54 returncode attribute.""" 55def__init__(self, returncode, cmd): 56 self.returncode = returncode 57 self.cmd = cmd 58def__str__(self): 59return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 60 61verbose =False 62 63# Only labels/tags matching this will be imported/exported 64defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 65 66# The block size is reduced automatically if required 67defaultBlockSize =1<<20 68 69p4_access_checked =False 70 71defp4_build_cmd(cmd): 72"""Build a suitable p4 command line. 73 74 This consolidates building and returning a p4 command line into one 75 location. It means that hooking into the environment, or other configuration 76 can be done more easily. 77 """ 78 real_cmd = ["p4"] 79 80 user =gitConfig("git-p4.user") 81iflen(user) >0: 82 real_cmd += ["-u",user] 83 84 password =gitConfig("git-p4.password") 85iflen(password) >0: 86 real_cmd += ["-P", password] 87 88 port =gitConfig("git-p4.port") 89iflen(port) >0: 90 real_cmd += ["-p", port] 91 92 host =gitConfig("git-p4.host") 93iflen(host) >0: 94 real_cmd += ["-H", host] 95 96 client =gitConfig("git-p4.client") 97iflen(client) >0: 98 real_cmd += ["-c", client] 99 100 retries =gitConfigInt("git-p4.retries") 101if retries is None: 102# Perform 3 retries by default 103 retries =3 104if retries >0: 105# Provide a way to not pass this option by setting git-p4.retries to 0 106 real_cmd += ["-r",str(retries)] 107 108ifisinstance(cmd,basestring): 109 real_cmd =' '.join(real_cmd) +' '+ cmd 110else: 111 real_cmd += cmd 112 113# now check that we can actually talk to the server 114global p4_access_checked 115if not p4_access_checked: 116 p4_access_checked =True# suppress access checks in p4_check_access itself 117p4_check_access() 118 119return real_cmd 120 121defgit_dir(path): 122""" Return TRUE if the given path is a git directory (/path/to/dir/.git). 123 This won't automatically add ".git" to a directory. 124 """ 125 d =read_pipe(["git","--git-dir", path,"rev-parse","--git-dir"],True).strip() 126if not d orlen(d) ==0: 127return None 128else: 129return d 130 131defchdir(path, is_client_path=False): 132"""Do chdir to the given path, and set the PWD environment 133 variable for use by P4. It does not look at getcwd() output. 134 Since we're not using the shell, it is necessary to set the 135 PWD environment variable explicitly. 136 137 Normally, expand the path to force it to be absolute. This 138 addresses the use of relative path names inside P4 settings, 139 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 140 as given; it looks for .p4config using PWD. 141 142 If is_client_path, the path was handed to us directly by p4, 143 and may be a symbolic link. Do not call os.getcwd() in this 144 case, because it will cause p4 to think that PWD is not inside 145 the client path. 146 """ 147 148 os.chdir(path) 149if not is_client_path: 150 path = os.getcwd() 151 os.environ['PWD'] = path 152 153defcalcDiskFree(): 154"""Return free space in bytes on the disk of the given dirname.""" 155if platform.system() =='Windows': 156 free_bytes = ctypes.c_ulonglong(0) 157 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()),None,None, ctypes.pointer(free_bytes)) 158return free_bytes.value 159else: 160 st = os.statvfs(os.getcwd()) 161return st.f_bavail * st.f_frsize 162 163defdie(msg): 164if verbose: 165raiseException(msg) 166else: 167 sys.stderr.write(msg +"\n") 168 sys.exit(1) 169 170defwrite_pipe(c, stdin): 171if verbose: 172 sys.stderr.write('Writing pipe:%s\n'%str(c)) 173 174 expand =isinstance(c,basestring) 175 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 176 pipe = p.stdin 177 val = pipe.write(stdin) 178 pipe.close() 179if p.wait(): 180die('Command failed:%s'%str(c)) 181 182return val 183 184defp4_write_pipe(c, stdin): 185 real_cmd =p4_build_cmd(c) 186returnwrite_pipe(real_cmd, stdin) 187 188defread_pipe_full(c): 189""" Read output from command. Returns a tuple 190 of the return status, stdout text and stderr 191 text. 192 """ 193if verbose: 194 sys.stderr.write('Reading pipe:%s\n'%str(c)) 195 196 expand =isinstance(c,basestring) 197 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) 198(out, err) = p.communicate() 199return(p.returncode, out, err) 200 201defread_pipe(c, ignore_error=False): 202""" Read output from command. Returns the output text on 203 success. On failure, terminates execution, unless 204 ignore_error is True, when it returns an empty string. 205 """ 206(retcode, out, err) =read_pipe_full(c) 207if retcode !=0: 208if ignore_error: 209 out ="" 210else: 211die('Command failed:%s\nError:%s'% (str(c), err)) 212return out 213 214defread_pipe_text(c): 215""" Read output from a command with trailing whitespace stripped. 216 On error, returns None. 217 """ 218(retcode, out, err) =read_pipe_full(c) 219if retcode !=0: 220return None 221else: 222return out.rstrip() 223 224defp4_read_pipe(c, ignore_error=False): 225 real_cmd =p4_build_cmd(c) 226returnread_pipe(real_cmd, ignore_error) 227 228defread_pipe_lines(c): 229if verbose: 230 sys.stderr.write('Reading pipe:%s\n'%str(c)) 231 232 expand =isinstance(c, basestring) 233 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 234 pipe = p.stdout 235 val = pipe.readlines() 236if pipe.close()or p.wait(): 237die('Command failed:%s'%str(c)) 238 239return val 240 241defp4_read_pipe_lines(c): 242"""Specifically invoke p4 on the command supplied. """ 243 real_cmd =p4_build_cmd(c) 244returnread_pipe_lines(real_cmd) 245 246defp4_has_command(cmd): 247"""Ask p4 for help on this command. If it returns an error, the 248 command does not exist in this version of p4.""" 249 real_cmd =p4_build_cmd(["help", cmd]) 250 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 251 stderr=subprocess.PIPE) 252 p.communicate() 253return p.returncode ==0 254 255defp4_has_move_command(): 256"""See if the move command exists, that it supports -k, and that 257 it has not been administratively disabled. The arguments 258 must be correct, but the filenames do not have to exist. Use 259 ones with wildcards so even if they exist, it will fail.""" 260 261if notp4_has_command("move"): 262return False 263 cmd =p4_build_cmd(["move","-k","@from","@to"]) 264 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 265(out, err) = p.communicate() 266# return code will be 1 in either case 267if err.find("Invalid option") >=0: 268return False 269if err.find("disabled") >=0: 270return False 271# assume it failed because @... was invalid changelist 272return True 273 274defsystem(cmd, ignore_error=False): 275 expand =isinstance(cmd,basestring) 276if verbose: 277 sys.stderr.write("executing%s\n"%str(cmd)) 278 retcode = subprocess.call(cmd, shell=expand) 279if retcode and not ignore_error: 280raiseCalledProcessError(retcode, cmd) 281 282return retcode 283 284defp4_system(cmd): 285"""Specifically invoke p4 as the system command. """ 286 real_cmd =p4_build_cmd(cmd) 287 expand =isinstance(real_cmd, basestring) 288 retcode = subprocess.call(real_cmd, shell=expand) 289if retcode: 290raiseCalledProcessError(retcode, real_cmd) 291 292defdie_bad_access(s): 293die("failure accessing depot:{0}".format(s.rstrip())) 294 295defp4_check_access(min_expiration=1): 296""" Check if we can access Perforce - account still logged in 297 """ 298 results =p4CmdList(["login","-s"]) 299 300iflen(results) ==0: 301# should never get here: always get either some results, or a p4ExitCode 302assert("could not parse response from perforce") 303 304 result = results[0] 305 306if'p4ExitCode'in result: 307# p4 returned non-zero status, e.g. P4PORT invalid, or p4 not in path 308die_bad_access("could not run p4") 309 310 code = result.get("code") 311if not code: 312# we get here if we couldn't connect and there was nothing to unmarshal 313die_bad_access("could not connect") 314 315elif code =="stat": 316 expiry = result.get("TicketExpiration") 317if expiry: 318 expiry =int(expiry) 319if expiry > min_expiration: 320# ok to carry on 321return 322else: 323die_bad_access("perforce ticket expires in{0}seconds".format(expiry)) 324 325else: 326# account without a timeout - all ok 327return 328 329elif code =="error": 330 data = result.get("data") 331if data: 332die_bad_access("p4 error:{0}".format(data)) 333else: 334die_bad_access("unknown error") 335elif code =="info": 336return 337else: 338die_bad_access("unknown error code{0}".format(code)) 339 340_p4_version_string =None 341defp4_version_string(): 342"""Read the version string, showing just the last line, which 343 hopefully is the interesting version bit. 344 345 $ p4 -V 346 Perforce - The Fast Software Configuration Management System. 347 Copyright 1995-2011 Perforce Software. All rights reserved. 348 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 349 """ 350global _p4_version_string 351if not _p4_version_string: 352 a =p4_read_pipe_lines(["-V"]) 353 _p4_version_string = a[-1].rstrip() 354return _p4_version_string 355 356defp4_integrate(src, dest): 357p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 358 359defp4_sync(f, *options): 360p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 361 362defp4_add(f): 363# forcibly add file names with wildcards 364ifwildcard_present(f): 365p4_system(["add","-f", f]) 366else: 367p4_system(["add", f]) 368 369defp4_delete(f): 370p4_system(["delete",wildcard_encode(f)]) 371 372defp4_edit(f, *options): 373p4_system(["edit"] +list(options) + [wildcard_encode(f)]) 374 375defp4_revert(f): 376p4_system(["revert",wildcard_encode(f)]) 377 378defp4_reopen(type, f): 379p4_system(["reopen","-t",type,wildcard_encode(f)]) 380 381defp4_reopen_in_change(changelist, files): 382 cmd = ["reopen","-c",str(changelist)] + files 383p4_system(cmd) 384 385defp4_move(src, dest): 386p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 387 388defp4_last_change(): 389 results =p4CmdList(["changes","-m","1"], skip_info=True) 390returnint(results[0]['change']) 391 392defp4_describe(change, shelved=False): 393"""Make sure it returns a valid result by checking for 394 the presence of field "time". Return a dict of the 395 results.""" 396 397 cmd = ["describe","-s"] 398if shelved: 399 cmd += ["-S"] 400 cmd += [str(change)] 401 402 ds =p4CmdList(cmd, skip_info=True) 403iflen(ds) !=1: 404die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 405 406 d = ds[0] 407 408if"p4ExitCode"in d: 409die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 410str(d))) 411if"code"in d: 412if d["code"] =="error": 413die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 414 415if"time"not in d: 416die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 417 418return d 419 420# 421# Canonicalize the p4 type and return a tuple of the 422# base type, plus any modifiers. See "p4 help filetypes" 423# for a list and explanation. 424# 425defsplit_p4_type(p4type): 426 427 p4_filetypes_historical = { 428"ctempobj":"binary+Sw", 429"ctext":"text+C", 430"cxtext":"text+Cx", 431"ktext":"text+k", 432"kxtext":"text+kx", 433"ltext":"text+F", 434"tempobj":"binary+FSw", 435"ubinary":"binary+F", 436"uresource":"resource+F", 437"uxbinary":"binary+Fx", 438"xbinary":"binary+x", 439"xltext":"text+Fx", 440"xtempobj":"binary+Swx", 441"xtext":"text+x", 442"xunicode":"unicode+x", 443"xutf16":"utf16+x", 444} 445if p4type in p4_filetypes_historical: 446 p4type = p4_filetypes_historical[p4type] 447 mods ="" 448 s = p4type.split("+") 449 base = s[0] 450 mods ="" 451iflen(s) >1: 452 mods = s[1] 453return(base, mods) 454 455# 456# return the raw p4 type of a file (text, text+ko, etc) 457# 458defp4_type(f): 459 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 460return results[0]['headType'] 461 462# 463# Given a type base and modifier, return a regexp matching 464# the keywords that can be expanded in the file 465# 466defp4_keywords_regexp_for_type(base, type_mods): 467if base in("text","unicode","binary"): 468 kwords =None 469if"ko"in type_mods: 470 kwords ='Id|Header' 471elif"k"in type_mods: 472 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 473else: 474return None 475 pattern = r""" 476 \$ # Starts with a dollar, followed by... 477 (%s) # one of the keywords, followed by... 478 (:[^$\n]+)? # possibly an old expansion, followed by... 479 \$ # another dollar 480 """% kwords 481return pattern 482else: 483return None 484 485# 486# Given a file, return a regexp matching the possible 487# RCS keywords that will be expanded, or None for files 488# with kw expansion turned off. 489# 490defp4_keywords_regexp_for_file(file): 491if not os.path.exists(file): 492return None 493else: 494(type_base, type_mods) =split_p4_type(p4_type(file)) 495returnp4_keywords_regexp_for_type(type_base, type_mods) 496 497defsetP4ExecBit(file, mode): 498# Reopens an already open file and changes the execute bit to match 499# the execute bit setting in the passed in mode. 500 501 p4Type ="+x" 502 503if notisModeExec(mode): 504 p4Type =getP4OpenedType(file) 505 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 506 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 507if p4Type[-1] =="+": 508 p4Type = p4Type[0:-1] 509 510p4_reopen(p4Type,file) 511 512defgetP4OpenedType(file): 513# Returns the perforce file type for the given file. 514 515 result =p4_read_pipe(["opened",wildcard_encode(file)]) 516 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 517if match: 518return match.group(1) 519else: 520die("Could not determine file type for%s(result: '%s')"% (file, result)) 521 522# Return the set of all p4 labels 523defgetP4Labels(depotPaths): 524 labels =set() 525ifisinstance(depotPaths,basestring): 526 depotPaths = [depotPaths] 527 528for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 529 label = l['label'] 530 labels.add(label) 531 532return labels 533 534# Return the set of all git tags 535defgetGitTags(): 536 gitTags =set() 537for line inread_pipe_lines(["git","tag"]): 538 tag = line.strip() 539 gitTags.add(tag) 540return gitTags 541 542defdiffTreePattern(): 543# This is a simple generator for the diff tree regex pattern. This could be 544# a class variable if this and parseDiffTreeEntry were a part of a class. 545 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 546while True: 547yield pattern 548 549defparseDiffTreeEntry(entry): 550"""Parses a single diff tree entry into its component elements. 551 552 See git-diff-tree(1) manpage for details about the format of the diff 553 output. This method returns a dictionary with the following elements: 554 555 src_mode - The mode of the source file 556 dst_mode - The mode of the destination file 557 src_sha1 - The sha1 for the source file 558 dst_sha1 - The sha1 fr the destination file 559 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 560 status_score - The score for the status (applicable for 'C' and 'R' 561 statuses). This is None if there is no score. 562 src - The path for the source file. 563 dst - The path for the destination file. This is only present for 564 copy or renames. If it is not present, this is None. 565 566 If the pattern is not matched, None is returned.""" 567 568 match =diffTreePattern().next().match(entry) 569if match: 570return{ 571'src_mode': match.group(1), 572'dst_mode': match.group(2), 573'src_sha1': match.group(3), 574'dst_sha1': match.group(4), 575'status': match.group(5), 576'status_score': match.group(6), 577'src': match.group(7), 578'dst': match.group(10) 579} 580return None 581 582defisModeExec(mode): 583# Returns True if the given git mode represents an executable file, 584# otherwise False. 585return mode[-3:] =="755" 586 587classP4Exception(Exception): 588""" Base class for exceptions from the p4 client """ 589def__init__(self, exit_code): 590 self.p4ExitCode = exit_code 591 592classP4ServerException(P4Exception): 593""" Base class for exceptions where we get some kind of marshalled up result from the server """ 594def__init__(self, exit_code, p4_result): 595super(P4ServerException, self).__init__(exit_code) 596 self.p4_result = p4_result 597 self.code = p4_result[0]['code'] 598 self.data = p4_result[0]['data'] 599 600classP4RequestSizeException(P4ServerException): 601""" One of the maxresults or maxscanrows errors """ 602def__init__(self, exit_code, p4_result, limit): 603super(P4RequestSizeException, self).__init__(exit_code, p4_result) 604 self.limit = limit 605 606defisModeExecChanged(src_mode, dst_mode): 607returnisModeExec(src_mode) !=isModeExec(dst_mode) 608 609defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, 610 errors_as_exceptions=False): 611 612ifisinstance(cmd,basestring): 613 cmd ="-G "+ cmd 614 expand =True 615else: 616 cmd = ["-G"] + cmd 617 expand =False 618 619 cmd =p4_build_cmd(cmd) 620if verbose: 621 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 622 623# Use a temporary file to avoid deadlocks without 624# subprocess.communicate(), which would put another copy 625# of stdout into memory. 626 stdin_file =None 627if stdin is not None: 628 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 629ifisinstance(stdin,basestring): 630 stdin_file.write(stdin) 631else: 632for i in stdin: 633 stdin_file.write(i +'\n') 634 stdin_file.flush() 635 stdin_file.seek(0) 636 637 p4 = subprocess.Popen(cmd, 638 shell=expand, 639 stdin=stdin_file, 640 stdout=subprocess.PIPE) 641 642 result = [] 643try: 644while True: 645 entry = marshal.load(p4.stdout) 646if skip_info: 647if'code'in entry and entry['code'] =='info': 648continue 649if cb is not None: 650cb(entry) 651else: 652 result.append(entry) 653exceptEOFError: 654pass 655 exitCode = p4.wait() 656if exitCode !=0: 657if errors_as_exceptions: 658iflen(result) >0: 659 data = result[0].get('data') 660if data: 661 m = re.search('Too many rows scanned \(over (\d+)\)', data) 662if not m: 663 m = re.search('Request too large \(over (\d+)\)', data) 664 665if m: 666 limit =int(m.group(1)) 667raiseP4RequestSizeException(exitCode, result, limit) 668 669raiseP4ServerException(exitCode, result) 670else: 671raiseP4Exception(exitCode) 672else: 673 entry = {} 674 entry["p4ExitCode"] = exitCode 675 result.append(entry) 676 677return result 678 679defp4Cmd(cmd): 680list=p4CmdList(cmd) 681 result = {} 682for entry inlist: 683 result.update(entry) 684return result; 685 686defp4Where(depotPath): 687if not depotPath.endswith("/"): 688 depotPath +="/" 689 depotPathLong = depotPath +"..." 690 outputList =p4CmdList(["where", depotPathLong]) 691 output =None 692for entry in outputList: 693if"depotFile"in entry: 694# Search for the base client side depot path, as long as it starts with the branch's P4 path. 695# The base path always ends with "/...". 696if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 697 output = entry 698break 699elif"data"in entry: 700 data = entry.get("data") 701 space = data.find(" ") 702if data[:space] == depotPath: 703 output = entry 704break 705if output ==None: 706return"" 707if output["code"] =="error": 708return"" 709 clientPath ="" 710if"path"in output: 711 clientPath = output.get("path") 712elif"data"in output: 713 data = output.get("data") 714 lastSpace = data.rfind(" ") 715 clientPath = data[lastSpace +1:] 716 717if clientPath.endswith("..."): 718 clientPath = clientPath[:-3] 719return clientPath 720 721defcurrentGitBranch(): 722returnread_pipe_text(["git","symbolic-ref","--short","-q","HEAD"]) 723 724defisValidGitDir(path): 725returngit_dir(path) !=None 726 727defparseRevision(ref): 728returnread_pipe("git rev-parse%s"% ref).strip() 729 730defbranchExists(ref): 731 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 732 ignore_error=True) 733returnlen(rev) >0 734 735defextractLogMessageFromGitCommit(commit): 736 logMessage ="" 737 738## fixme: title is first line of commit, not 1st paragraph. 739 foundTitle =False 740for log inread_pipe_lines(["git","cat-file","commit", commit]): 741if not foundTitle: 742iflen(log) ==1: 743 foundTitle =True 744continue 745 746 logMessage += log 747return logMessage 748 749defextractSettingsGitLog(log): 750 values = {} 751for line in log.split("\n"): 752 line = line.strip() 753 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 754if not m: 755continue 756 757 assignments = m.group(1).split(':') 758for a in assignments: 759 vals = a.split('=') 760 key = vals[0].strip() 761 val = ('='.join(vals[1:])).strip() 762if val.endswith('\"')and val.startswith('"'): 763 val = val[1:-1] 764 765 values[key] = val 766 767 paths = values.get("depot-paths") 768if not paths: 769 paths = values.get("depot-path") 770if paths: 771 values['depot-paths'] = paths.split(',') 772return values 773 774defgitBranchExists(branch): 775 proc = subprocess.Popen(["git","rev-parse", branch], 776 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 777return proc.wait() ==0; 778 779defgitUpdateRef(ref, newvalue): 780 subprocess.check_call(["git","update-ref", ref, newvalue]) 781 782defgitDeleteRef(ref): 783 subprocess.check_call(["git","update-ref","-d", ref]) 784 785_gitConfig = {} 786 787defgitConfig(key, typeSpecifier=None): 788if key not in _gitConfig: 789 cmd = ["git","config"] 790if typeSpecifier: 791 cmd += [ typeSpecifier ] 792 cmd += [ key ] 793 s =read_pipe(cmd, ignore_error=True) 794 _gitConfig[key] = s.strip() 795return _gitConfig[key] 796 797defgitConfigBool(key): 798"""Return a bool, using git config --bool. It is True only if the 799 variable is set to true, and False if set to false or not present 800 in the config.""" 801 802if key not in _gitConfig: 803 _gitConfig[key] =gitConfig(key,'--bool') =="true" 804return _gitConfig[key] 805 806defgitConfigInt(key): 807if key not in _gitConfig: 808 cmd = ["git","config","--int", key ] 809 s =read_pipe(cmd, ignore_error=True) 810 v = s.strip() 811try: 812 _gitConfig[key] =int(gitConfig(key,'--int')) 813exceptValueError: 814 _gitConfig[key] =None 815return _gitConfig[key] 816 817defgitConfigList(key): 818if key not in _gitConfig: 819 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 820 _gitConfig[key] = s.strip().splitlines() 821if _gitConfig[key] == ['']: 822 _gitConfig[key] = [] 823return _gitConfig[key] 824 825defp4BranchesInGit(branchesAreInRemotes=True): 826"""Find all the branches whose names start with "p4/", looking 827 in remotes or heads as specified by the argument. Return 828 a dictionary of{ branch: revision }for each one found. 829 The branch names are the short names, without any 830 "p4/" prefix.""" 831 832 branches = {} 833 834 cmdline ="git rev-parse --symbolic " 835if branchesAreInRemotes: 836 cmdline +="--remotes" 837else: 838 cmdline +="--branches" 839 840for line inread_pipe_lines(cmdline): 841 line = line.strip() 842 843# only import to p4/ 844if not line.startswith('p4/'): 845continue 846# special symbolic ref to p4/master 847if line =="p4/HEAD": 848continue 849 850# strip off p4/ prefix 851 branch = line[len("p4/"):] 852 853 branches[branch] =parseRevision(line) 854 855return branches 856 857defbranch_exists(branch): 858"""Make sure that the given ref name really exists.""" 859 860 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 861 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 862 out, _ = p.communicate() 863if p.returncode: 864return False 865# expect exactly one line of output: the branch name 866return out.rstrip() == branch 867 868deffindUpstreamBranchPoint(head ="HEAD"): 869 branches =p4BranchesInGit() 870# map from depot-path to branch name 871 branchByDepotPath = {} 872for branch in branches.keys(): 873 tip = branches[branch] 874 log =extractLogMessageFromGitCommit(tip) 875 settings =extractSettingsGitLog(log) 876if"depot-paths"in settings: 877 paths =",".join(settings["depot-paths"]) 878 branchByDepotPath[paths] ="remotes/p4/"+ branch 879 880 settings =None 881 parent =0 882while parent <65535: 883 commit = head +"~%s"% parent 884 log =extractLogMessageFromGitCommit(commit) 885 settings =extractSettingsGitLog(log) 886if"depot-paths"in settings: 887 paths =",".join(settings["depot-paths"]) 888if paths in branchByDepotPath: 889return[branchByDepotPath[paths], settings] 890 891 parent = parent +1 892 893return["", settings] 894 895defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 896if not silent: 897print("Creating/updating branch(es) in%sbased on origin branch(es)" 898% localRefPrefix) 899 900 originPrefix ="origin/p4/" 901 902for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 903 line = line.strip() 904if(not line.startswith(originPrefix))or line.endswith("HEAD"): 905continue 906 907 headName = line[len(originPrefix):] 908 remoteHead = localRefPrefix + headName 909 originHead = line 910 911 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 912if('depot-paths'not in original 913or'change'not in original): 914continue 915 916 update =False 917if notgitBranchExists(remoteHead): 918if verbose: 919print("creating%s"% remoteHead) 920 update =True 921else: 922 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 923if'change'in settings: 924if settings['depot-paths'] == original['depot-paths']: 925 originP4Change =int(original['change']) 926 p4Change =int(settings['change']) 927if originP4Change > p4Change: 928print("%s(%s) is newer than%s(%s). " 929"Updating p4 branch from origin." 930% (originHead, originP4Change, 931 remoteHead, p4Change)) 932 update =True 933else: 934print("Ignoring:%swas imported from%swhile " 935"%swas imported from%s" 936% (originHead,','.join(original['depot-paths']), 937 remoteHead,','.join(settings['depot-paths']))) 938 939if update: 940system("git update-ref%s %s"% (remoteHead, originHead)) 941 942deforiginP4BranchesExist(): 943returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 944 945 946defp4ParseNumericChangeRange(parts): 947 changeStart =int(parts[0][1:]) 948if parts[1] =='#head': 949 changeEnd =p4_last_change() 950else: 951 changeEnd =int(parts[1]) 952 953return(changeStart, changeEnd) 954 955defchooseBlockSize(blockSize): 956if blockSize: 957return blockSize 958else: 959return defaultBlockSize 960 961defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 962assert depotPaths 963 964# Parse the change range into start and end. Try to find integer 965# revision ranges as these can be broken up into blocks to avoid 966# hitting server-side limits (maxrows, maxscanresults). But if 967# that doesn't work, fall back to using the raw revision specifier 968# strings, without using block mode. 969 970if changeRange is None or changeRange =='': 971 changeStart =1 972 changeEnd =p4_last_change() 973 block_size =chooseBlockSize(requestedBlockSize) 974else: 975 parts = changeRange.split(',') 976assertlen(parts) ==2 977try: 978(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 979 block_size =chooseBlockSize(requestedBlockSize) 980exceptValueError: 981 changeStart = parts[0][1:] 982 changeEnd = parts[1] 983if requestedBlockSize: 984die("cannot use --changes-block-size with non-numeric revisions") 985 block_size =None 986 987 changes =set() 988 989# Retrieve changes a block at a time, to prevent running 990# into a MaxResults/MaxScanRows error from the server. If 991# we _do_ hit one of those errors, turn down the block size 992 993while True: 994 cmd = ['changes'] 995 996if block_size: 997 end =min(changeEnd, changeStart + block_size) 998 revisionRange ="%d,%d"% (changeStart, end) 999else:1000 revisionRange ="%s,%s"% (changeStart, changeEnd)10011002for p in depotPaths:1003 cmd += ["%s...@%s"% (p, revisionRange)]10041005# fetch the changes1006try:1007 result =p4CmdList(cmd, errors_as_exceptions=True)1008except P4RequestSizeException as e:1009if not block_size:1010 block_size = e.limit1011elif block_size > e.limit:1012 block_size = e.limit1013else:1014 block_size =max(2, block_size //2)10151016if verbose:print("block size error, retrying with block size{0}".format(block_size))1017continue1018except P4Exception as e:1019die('Error retrieving changes description ({0})'.format(e.p4ExitCode))10201021# Insert changes in chronological order1022for entry inreversed(result):1023if'change'not in entry:1024continue1025 changes.add(int(entry['change']))10261027if not block_size:1028break10291030if end >= changeEnd:1031break10321033 changeStart = end +110341035 changes =sorted(changes)1036return changes10371038defp4PathStartsWith(path, prefix):1039# This method tries to remedy a potential mixed-case issue:1040#1041# If UserA adds //depot/DirA/file11042# and UserB adds //depot/dira/file21043#1044# we may or may not have a problem. If you have core.ignorecase=true,1045# we treat DirA and dira as the same directory1046ifgitConfigBool("core.ignorecase"):1047return path.lower().startswith(prefix.lower())1048return path.startswith(prefix)10491050defgetClientSpec():1051"""Look at the p4 client spec, create a View() object that contains1052 all the mappings, and return it."""10531054 specList =p4CmdList("client -o")1055iflen(specList) !=1:1056die('Output from "client -o" is%dlines, expecting 1'%1057len(specList))10581059# dictionary of all client parameters1060 entry = specList[0]10611062# the //client/ name1063 client_name = entry["Client"]10641065# just the keys that start with "View"1066 view_keys = [ k for k in entry.keys()if k.startswith("View") ]10671068# hold this new View1069 view =View(client_name)10701071# append the lines, in order, to the view1072for view_num inrange(len(view_keys)):1073 k ="View%d"% view_num1074if k not in view_keys:1075die("Expected view key%smissing"% k)1076 view.append(entry[k])10771078return view10791080defgetClientRoot():1081"""Grab the client directory."""10821083 output =p4CmdList("client -o")1084iflen(output) !=1:1085die('Output from "client -o" is%dlines, expecting 1'%len(output))10861087 entry = output[0]1088if"Root"not in entry:1089die('Client has no "Root"')10901091return entry["Root"]10921093#1094# P4 wildcards are not allowed in filenames. P4 complains1095# if you simply add them, but you can force it with "-f", in1096# which case it translates them into %xx encoding internally.1097#1098defwildcard_decode(path):1099# Search for and fix just these four characters. Do % last so1100# that fixing it does not inadvertently create new %-escapes.1101# Cannot have * in a filename in windows; untested as to1102# what p4 would do in such a case.1103if not platform.system() =="Windows":1104 path = path.replace("%2A","*")1105 path = path.replace("%23","#") \1106.replace("%40","@") \1107.replace("%25","%")1108return path11091110defwildcard_encode(path):1111# do % first to avoid double-encoding the %s introduced here1112 path = path.replace("%","%25") \1113.replace("*","%2A") \1114.replace("#","%23") \1115.replace("@","%40")1116return path11171118defwildcard_present(path):1119 m = re.search("[*#@%]", path)1120return m is not None11211122classLargeFileSystem(object):1123"""Base class for large file system support."""11241125def__init__(self, writeToGitStream):1126 self.largeFiles =set()1127 self.writeToGitStream = writeToGitStream11281129defgeneratePointer(self, cloneDestination, contentFile):1130"""Return the content of a pointer file that is stored in Git instead of1131 the actual content."""1132assert False,"Method 'generatePointer' required in "+ self.__class__.__name__11331134defpushFile(self, localLargeFile):1135"""Push the actual content which is not stored in the Git repository to1136 a server."""1137assert False,"Method 'pushFile' required in "+ self.__class__.__name__11381139defhasLargeFileExtension(self, relPath):1140returnreduce(1141lambda a, b: a or b,1142[relPath.endswith('.'+ e)for e ingitConfigList('git-p4.largeFileExtensions')],1143False1144)11451146defgenerateTempFile(self, contents):1147 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)1148for d in contents:1149 contentFile.write(d)1150 contentFile.close()1151return contentFile.name11521153defexceedsLargeFileThreshold(self, relPath, contents):1154ifgitConfigInt('git-p4.largeFileThreshold'):1155 contentsSize =sum(len(d)for d in contents)1156if contentsSize >gitConfigInt('git-p4.largeFileThreshold'):1157return True1158ifgitConfigInt('git-p4.largeFileCompressedThreshold'):1159 contentsSize =sum(len(d)for d in contents)1160if contentsSize <=gitConfigInt('git-p4.largeFileCompressedThreshold'):1161return False1162 contentTempFile = self.generateTempFile(contents)1163 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)1164 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')1165 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)1166 zf.close()1167 compressedContentsSize = zf.infolist()[0].compress_size1168 os.remove(contentTempFile)1169 os.remove(compressedContentFile.name)1170if compressedContentsSize >gitConfigInt('git-p4.largeFileCompressedThreshold'):1171return True1172return False11731174defaddLargeFile(self, relPath):1175 self.largeFiles.add(relPath)11761177defremoveLargeFile(self, relPath):1178 self.largeFiles.remove(relPath)11791180defisLargeFile(self, relPath):1181return relPath in self.largeFiles11821183defprocessContent(self, git_mode, relPath, contents):1184"""Processes the content of git fast import. This method decides if a1185 file is stored in the large file system and handles all necessary1186 steps."""1187if self.exceedsLargeFileThreshold(relPath, contents)or self.hasLargeFileExtension(relPath):1188 contentTempFile = self.generateTempFile(contents)1189(pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)1190if pointer_git_mode:1191 git_mode = pointer_git_mode1192if localLargeFile:1193# Move temp file to final location in large file system1194 largeFileDir = os.path.dirname(localLargeFile)1195if not os.path.isdir(largeFileDir):1196 os.makedirs(largeFileDir)1197 shutil.move(contentTempFile, localLargeFile)1198 self.addLargeFile(relPath)1199ifgitConfigBool('git-p4.largeFilePush'):1200 self.pushFile(localLargeFile)1201if verbose:1202 sys.stderr.write("%smoved to large file system (%s)\n"% (relPath, localLargeFile))1203return(git_mode, contents)12041205classMockLFS(LargeFileSystem):1206"""Mock large file system for testing."""12071208defgeneratePointer(self, contentFile):1209"""The pointer content is the original content prefixed with "pointer-".1210 The local filename of the large file storage is derived from the file content.1211 """1212withopen(contentFile,'r')as f:1213 content =next(f)1214 gitMode ='100644'1215 pointerContents ='pointer-'+ content1216 localLargeFile = os.path.join(os.getcwd(),'.git','mock-storage','local', content[:-1])1217return(gitMode, pointerContents, localLargeFile)12181219defpushFile(self, localLargeFile):1220"""The remote filename of the large file storage is the same as the local1221 one but in a different directory.1222 """1223 remotePath = os.path.join(os.path.dirname(localLargeFile),'..','remote')1224if not os.path.exists(remotePath):1225 os.makedirs(remotePath)1226 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))12271228classGitLFS(LargeFileSystem):1229"""Git LFS as backend for the git-p4 large file system.1230 See https://git-lfs.github.com/ for details."""12311232def__init__(self, *args):1233 LargeFileSystem.__init__(self, *args)1234 self.baseGitAttributes = []12351236defgeneratePointer(self, contentFile):1237"""Generate a Git LFS pointer for the content. Return LFS Pointer file1238 mode and content which is stored in the Git repository instead of1239 the actual content. Return also the new location of the actual1240 content.1241 """1242if os.path.getsize(contentFile) ==0:1243return(None,'',None)12441245 pointerProcess = subprocess.Popen(1246['git','lfs','pointer','--file='+ contentFile],1247 stdout=subprocess.PIPE1248)1249 pointerFile = pointerProcess.stdout.read()1250if pointerProcess.wait():1251 os.remove(contentFile)1252die('git-lfs pointer command failed. Did you install the extension?')12531254# Git LFS removed the preamble in the output of the 'pointer' command1255# starting from version 1.2.0. Check for the preamble here to support1256# earlier versions.1257# c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b431258if pointerFile.startswith('Git LFS pointer for'):1259 pointerFile = re.sub(r'Git LFS pointer for.*\n\n','', pointerFile)12601261 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)1262 localLargeFile = os.path.join(1263 os.getcwd(),1264'.git','lfs','objects', oid[:2], oid[2:4],1265 oid,1266)1267# LFS Spec states that pointer files should not have the executable bit set.1268 gitMode ='100644'1269return(gitMode, pointerFile, localLargeFile)12701271defpushFile(self, localLargeFile):1272 uploadProcess = subprocess.Popen(1273['git','lfs','push','--object-id','origin', os.path.basename(localLargeFile)]1274)1275if uploadProcess.wait():1276die('git-lfs push command failed. Did you define a remote?')12771278defgenerateGitAttributes(self):1279return(1280 self.baseGitAttributes +1281[1282'\n',1283'#\n',1284'# Git LFS (see https://git-lfs.github.com/)\n',1285'#\n',1286] +1287['*.'+ f.replace(' ','[[:space:]]') +' filter=lfs diff=lfs merge=lfs -text\n'1288for f insorted(gitConfigList('git-p4.largeFileExtensions'))1289] +1290['/'+ f.replace(' ','[[:space:]]') +' filter=lfs diff=lfs merge=lfs -text\n'1291for f insorted(self.largeFiles)if not self.hasLargeFileExtension(f)1292]1293)12941295defaddLargeFile(self, relPath):1296 LargeFileSystem.addLargeFile(self, relPath)1297 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())12981299defremoveLargeFile(self, relPath):1300 LargeFileSystem.removeLargeFile(self, relPath)1301 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())13021303defprocessContent(self, git_mode, relPath, contents):1304if relPath =='.gitattributes':1305 self.baseGitAttributes = contents1306return(git_mode, self.generateGitAttributes())1307else:1308return LargeFileSystem.processContent(self, git_mode, relPath, contents)13091310class Command:1311 delete_actions = ("delete","move/delete","purge")1312 add_actions = ("add","branch","move/add")13131314def__init__(self):1315 self.usage ="usage: %prog [options]"1316 self.needsGit =True1317 self.verbose =False13181319# This is required for the "append" update_shelve action1320defensure_value(self, attr, value):1321if nothasattr(self, attr)orgetattr(self, attr)is None:1322setattr(self, attr, value)1323returngetattr(self, attr)13241325class P4UserMap:1326def__init__(self):1327 self.userMapFromPerforceServer =False1328 self.myP4UserId =None13291330defp4UserId(self):1331if self.myP4UserId:1332return self.myP4UserId13331334 results =p4CmdList("user -o")1335for r in results:1336if'User'in r:1337 self.myP4UserId = r['User']1338return r['User']1339die("Could not find your p4 user id")13401341defp4UserIsMe(self, p4User):1342# return True if the given p4 user is actually me1343 me = self.p4UserId()1344if not p4User or p4User != me:1345return False1346else:1347return True13481349defgetUserCacheFilename(self):1350 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))1351return home +"/.gitp4-usercache.txt"13521353defgetUserMapFromPerforceServer(self):1354if self.userMapFromPerforceServer:1355return1356 self.users = {}1357 self.emails = {}13581359for output inp4CmdList("users"):1360if"User"not in output:1361continue1362 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">"1363 self.emails[output["Email"]] = output["User"]13641365 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)1366for mapUserConfig ingitConfigList("git-p4.mapUser"):1367 mapUser = mapUserConfigRegex.findall(mapUserConfig)1368if mapUser andlen(mapUser[0]) ==3:1369 user = mapUser[0][0]1370 fullname = mapUser[0][1]1371 email = mapUser[0][2]1372 self.users[user] = fullname +" <"+ email +">"1373 self.emails[email] = user13741375 s =''1376for(key, val)in self.users.items():1377 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1))13781379open(self.getUserCacheFilename(),"wb").write(s)1380 self.userMapFromPerforceServer =True13811382defloadUserMapFromCache(self):1383 self.users = {}1384 self.userMapFromPerforceServer =False1385try:1386 cache =open(self.getUserCacheFilename(),"rb")1387 lines = cache.readlines()1388 cache.close()1389for line in lines:1390 entry = line.strip().split("\t")1391 self.users[entry[0]] = entry[1]1392exceptIOError:1393 self.getUserMapFromPerforceServer()13941395classP4Debug(Command):1396def__init__(self):1397 Command.__init__(self)1398 self.options = []1399 self.description ="A tool to debug the output of p4 -G."1400 self.needsGit =False14011402defrun(self, args):1403 j =01404for output inp4CmdList(args):1405print('Element:%d'% j)1406 j +=11407print(output)1408return True14091410classP4RollBack(Command):1411def__init__(self):1412 Command.__init__(self)1413 self.options = [1414 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1415]1416 self.description ="A tool to debug the multi-branch import. Don't use :)"1417 self.rollbackLocalBranches =False14181419defrun(self, args):1420iflen(args) !=1:1421return False1422 maxChange =int(args[0])14231424if"p4ExitCode"inp4Cmd("changes -m 1"):1425die("Problems executing p4");14261427if self.rollbackLocalBranches:1428 refPrefix ="refs/heads/"1429 lines =read_pipe_lines("git rev-parse --symbolic --branches")1430else:1431 refPrefix ="refs/remotes/"1432 lines =read_pipe_lines("git rev-parse --symbolic --remotes")14331434for line in lines:1435if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1436 line = line.strip()1437 ref = refPrefix + line1438 log =extractLogMessageFromGitCommit(ref)1439 settings =extractSettingsGitLog(log)14401441 depotPaths = settings['depot-paths']1442 change = settings['change']14431444 changed =False14451446iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1447for p in depotPaths]))) ==0:1448print("Branch%sdid not exist at change%s, deleting."% (ref, maxChange))1449system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1450continue14511452while change andint(change) > maxChange:1453 changed =True1454if self.verbose:1455print("%sis at%s; rewinding towards%s"% (ref, change, maxChange))1456system("git update-ref%s\"%s^\""% (ref, ref))1457 log =extractLogMessageFromGitCommit(ref)1458 settings =extractSettingsGitLog(log)145914601461 depotPaths = settings['depot-paths']1462 change = settings['change']14631464if changed:1465print("%srewound to%s"% (ref, change))14661467return True14681469classP4Submit(Command, P4UserMap):14701471 conflict_behavior_choices = ("ask","skip","quit")14721473def__init__(self):1474 Command.__init__(self)1475 P4UserMap.__init__(self)1476 self.options = [1477 optparse.make_option("--origin", dest="origin"),1478 optparse.make_option("-M", dest="detectRenames", action="store_true"),1479# preserve the user, requires relevant p4 permissions1480 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1481 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1482 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1483 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1484 optparse.make_option("--conflict", dest="conflict_behavior",1485 choices=self.conflict_behavior_choices),1486 optparse.make_option("--branch", dest="branch"),1487 optparse.make_option("--shelve", dest="shelve", action="store_true",1488help="Shelve instead of submit. Shelved files are reverted, "1489"restoring the workspace to the state before the shelve"),1490 optparse.make_option("--update-shelve", dest="update_shelve", action="append",type="int",1491 metavar="CHANGELIST",1492help="update an existing shelved changelist, implies --shelve, "1493"repeat in-order for multiple shelved changelists"),1494 optparse.make_option("--commit", dest="commit", metavar="COMMIT",1495help="submit only the specified commit(s), one commit or xxx..xxx"),1496 optparse.make_option("--disable-rebase", dest="disable_rebase", action="store_true",1497help="Disable rebase after submit is completed. Can be useful if you "1498"work from a local git branch that is not master"),1499 optparse.make_option("--disable-p4sync", dest="disable_p4sync", action="store_true",1500help="Skip Perforce sync of p4/master after submit or shelve"),1501]1502 self.description ="""Submit changes from git to the perforce depot.\n1503 The `p4-pre-submit` hook is executed if it exists and is executable.1504 The hook takes no parameters and nothing from standard input. Exiting with1505 non-zero status from this script prevents `git-p4 submit` from launching.15061507 One usage scenario is to run unit tests in the hook."""15081509 self.usage +=" [name of git branch to submit into perforce depot]"1510 self.origin =""1511 self.detectRenames =False1512 self.preserveUser =gitConfigBool("git-p4.preserveUser")1513 self.dry_run =False1514 self.shelve =False1515 self.update_shelve =list()1516 self.commit =""1517 self.disable_rebase =gitConfigBool("git-p4.disableRebase")1518 self.disable_p4sync =gitConfigBool("git-p4.disableP4Sync")1519 self.prepare_p4_only =False1520 self.conflict_behavior =None1521 self.isWindows = (platform.system() =="Windows")1522 self.exportLabels =False1523 self.p4HasMoveCommand =p4_has_move_command()1524 self.branch =None15251526ifgitConfig('git-p4.largeFileSystem'):1527die("Large file system not supported for git-p4 submit command. Please remove it from config.")15281529defcheck(self):1530iflen(p4CmdList("opened ...")) >0:1531die("You have files opened with perforce! Close them before starting the sync.")15321533defseparate_jobs_from_description(self, message):1534"""Extract and return a possible Jobs field in the commit1535 message. It goes into a separate section in the p4 change1536 specification.15371538 A jobs line starts with "Jobs:" and looks like a new field1539 in a form. Values are white-space separated on the same1540 line or on following lines that start with a tab.15411542 This does not parse and extract the full git commit message1543 like a p4 form. It just sees the Jobs: line as a marker1544 to pass everything from then on directly into the p4 form,1545 but outside the description section.15461547 Return a tuple (stripped log message, jobs string)."""15481549 m = re.search(r'^Jobs:', message, re.MULTILINE)1550if m is None:1551return(message,None)15521553 jobtext = message[m.start():]1554 stripped_message = message[:m.start()].rstrip()1555return(stripped_message, jobtext)15561557defprepareLogMessage(self, template, message, jobs):1558"""Edits the template returned from "p4 change -o" to insert1559 the message in the Description field, and the jobs text in1560 the Jobs field."""1561 result =""15621563 inDescriptionSection =False15641565for line in template.split("\n"):1566if line.startswith("#"):1567 result += line +"\n"1568continue15691570if inDescriptionSection:1571if line.startswith("Files:")or line.startswith("Jobs:"):1572 inDescriptionSection =False1573# insert Jobs section1574if jobs:1575 result += jobs +"\n"1576else:1577continue1578else:1579if line.startswith("Description:"):1580 inDescriptionSection =True1581 line +="\n"1582for messageLine in message.split("\n"):1583 line +="\t"+ messageLine +"\n"15841585 result += line +"\n"15861587return result15881589defpatchRCSKeywords(self,file, pattern):1590# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1591(handle, outFileName) = tempfile.mkstemp(dir='.')1592try:1593 outFile = os.fdopen(handle,"w+")1594 inFile =open(file,"r")1595 regexp = re.compile(pattern, re.VERBOSE)1596for line in inFile.readlines():1597 line = regexp.sub(r'$\1$', line)1598 outFile.write(line)1599 inFile.close()1600 outFile.close()1601# Forcibly overwrite the original file1602 os.unlink(file)1603 shutil.move(outFileName,file)1604except:1605# cleanup our temporary file1606 os.unlink(outFileName)1607print("Failed to strip RCS keywords in%s"%file)1608raise16091610print("Patched up RCS keywords in%s"%file)16111612defp4UserForCommit(self,id):1613# Return the tuple (perforce user,git email) for a given git commit id1614 self.getUserMapFromPerforceServer()1615 gitEmail =read_pipe(["git","log","--max-count=1",1616"--format=%ae",id])1617 gitEmail = gitEmail.strip()1618if gitEmail not in self.emails:1619return(None,gitEmail)1620else:1621return(self.emails[gitEmail],gitEmail)16221623defcheckValidP4Users(self,commits):1624# check if any git authors cannot be mapped to p4 users1625foridin commits:1626(user,email) = self.p4UserForCommit(id)1627if not user:1628 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1629ifgitConfigBool("git-p4.allowMissingP4Users"):1630print("%s"% msg)1631else:1632die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)16331634deflastP4Changelist(self):1635# Get back the last changelist number submitted in this client spec. This1636# then gets used to patch up the username in the change. If the same1637# client spec is being used by multiple processes then this might go1638# wrong.1639 results =p4CmdList("client -o")# find the current client1640 client =None1641for r in results:1642if'Client'in r:1643 client = r['Client']1644break1645if not client:1646die("could not get client spec")1647 results =p4CmdList(["changes","-c", client,"-m","1"])1648for r in results:1649if'change'in r:1650return r['change']1651die("Could not get changelist number for last submit - cannot patch up user details")16521653defmodifyChangelistUser(self, changelist, newUser):1654# fixup the user field of a changelist after it has been submitted.1655 changes =p4CmdList("change -o%s"% changelist)1656iflen(changes) !=1:1657die("Bad output from p4 change modifying%sto user%s"%1658(changelist, newUser))16591660 c = changes[0]1661if c['User'] == newUser:return# nothing to do1662 c['User'] = newUser1663input= marshal.dumps(c)16641665 result =p4CmdList("change -f -i", stdin=input)1666for r in result:1667if'code'in r:1668if r['code'] =='error':1669die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1670if'data'in r:1671print("Updated user field for changelist%sto%s"% (changelist, newUser))1672return1673die("Could not modify user field of changelist%sto%s"% (changelist, newUser))16741675defcanChangeChangelists(self):1676# check to see if we have p4 admin or super-user permissions, either of1677# which are required to modify changelists.1678 results =p4CmdList(["protects", self.depotPath])1679for r in results:1680if'perm'in r:1681if r['perm'] =='admin':1682return11683if r['perm'] =='super':1684return11685return016861687defprepareSubmitTemplate(self, changelist=None):1688"""Run "p4 change -o" to grab a change specification template.1689 This does not use "p4 -G", as it is nice to keep the submission1690 template in original order, since a human might edit it.16911692 Remove lines in the Files section that show changes to files1693 outside the depot path we're committing into."""16941695[upstream, settings] =findUpstreamBranchPoint()16961697 template ="""\1698# A Perforce Change Specification.1699#1700# Change: The change number. 'new' on a new changelist.1701# Date: The date this specification was last modified.1702# Client: The client on which the changelist was created. Read-only.1703# User: The user who created the changelist.1704# Status: Either 'pending' or 'submitted'. Read-only.1705# Type: Either 'public' or 'restricted'. Default is 'public'.1706# Description: Comments about the changelist. Required.1707# Jobs: What opened jobs are to be closed by this changelist.1708# You may delete jobs from this list. (New changelists only.)1709# Files: What opened files from the default changelist are to be added1710# to this changelist. You may delete files from this list.1711# (New changelists only.)1712"""1713 files_list = []1714 inFilesSection =False1715 change_entry =None1716 args = ['change','-o']1717if changelist:1718 args.append(str(changelist))1719for entry inp4CmdList(args):1720if'code'not in entry:1721continue1722if entry['code'] =='stat':1723 change_entry = entry1724break1725if not change_entry:1726die('Failed to decode output of p4 change -o')1727for key, value in change_entry.iteritems():1728if key.startswith('File'):1729if'depot-paths'in settings:1730if not[p for p in settings['depot-paths']1731ifp4PathStartsWith(value, p)]:1732continue1733else:1734if notp4PathStartsWith(value, self.depotPath):1735continue1736 files_list.append(value)1737continue1738# Output in the order expected by prepareLogMessage1739for key in['Change','Client','User','Status','Description','Jobs']:1740if key not in change_entry:1741continue1742 template +='\n'1743 template += key +':'1744if key =='Description':1745 template +='\n'1746for field_line in change_entry[key].splitlines():1747 template +='\t'+field_line+'\n'1748iflen(files_list) >0:1749 template +='\n'1750 template +='Files:\n'1751for path in files_list:1752 template +='\t'+path+'\n'1753return template17541755defedit_template(self, template_file):1756"""Invoke the editor to let the user change the submission1757 message. Return true if okay to continue with the submit."""17581759# if configured to skip the editing part, just submit1760ifgitConfigBool("git-p4.skipSubmitEdit"):1761return True17621763# look at the modification time, to check later if the user saved1764# the file1765 mtime = os.stat(template_file).st_mtime17661767# invoke the editor1768if"P4EDITOR"in os.environ and(os.environ.get("P4EDITOR") !=""):1769 editor = os.environ.get("P4EDITOR")1770else:1771 editor =read_pipe("git var GIT_EDITOR").strip()1772system(["sh","-c", ('%s"$@"'% editor), editor, template_file])17731774# If the file was not saved, prompt to see if this patch should1775# be skipped. But skip this verification step if configured so.1776ifgitConfigBool("git-p4.skipSubmitEditCheck"):1777return True17781779# modification time updated means user saved the file1780if os.stat(template_file).st_mtime > mtime:1781return True17821783while True:1784 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1785if response =='y':1786return True1787if response =='n':1788return False17891790defget_diff_description(self, editedFiles, filesToAdd, symlinks):1791# diff1792if"P4DIFF"in os.environ:1793del(os.environ["P4DIFF"])1794 diff =""1795for editedFile in editedFiles:1796 diff +=p4_read_pipe(['diff','-du',1797wildcard_encode(editedFile)])17981799# new file diff1800 newdiff =""1801for newFile in filesToAdd:1802 newdiff +="==== new file ====\n"1803 newdiff +="--- /dev/null\n"1804 newdiff +="+++%s\n"% newFile18051806 is_link = os.path.islink(newFile)1807 expect_link = newFile in symlinks18081809if is_link and expect_link:1810 newdiff +="+%s\n"% os.readlink(newFile)1811else:1812 f =open(newFile,"r")1813for line in f.readlines():1814 newdiff +="+"+ line1815 f.close()18161817return(diff + newdiff).replace('\r\n','\n')18181819defapplyCommit(self,id):1820"""Apply one commit, return True if it succeeded."""18211822print("Applying",read_pipe(["git","show","-s",1823"--format=format:%h%s",id]))18241825(p4User, gitEmail) = self.p4UserForCommit(id)18261827 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1828 filesToAdd =set()1829 filesToChangeType =set()1830 filesToDelete =set()1831 editedFiles =set()1832 pureRenameCopy =set()1833 symlinks =set()1834 filesToChangeExecBit = {}1835 all_files =list()18361837for line in diff:1838 diff =parseDiffTreeEntry(line)1839 modifier = diff['status']1840 path = diff['src']1841 all_files.append(path)18421843if modifier =="M":1844p4_edit(path)1845ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1846 filesToChangeExecBit[path] = diff['dst_mode']1847 editedFiles.add(path)1848elif modifier =="A":1849 filesToAdd.add(path)1850 filesToChangeExecBit[path] = diff['dst_mode']1851if path in filesToDelete:1852 filesToDelete.remove(path)18531854 dst_mode =int(diff['dst_mode'],8)1855if dst_mode ==0o120000:1856 symlinks.add(path)18571858elif modifier =="D":1859 filesToDelete.add(path)1860if path in filesToAdd:1861 filesToAdd.remove(path)1862elif modifier =="C":1863 src, dest = diff['src'], diff['dst']1864 all_files.append(dest)1865p4_integrate(src, dest)1866 pureRenameCopy.add(dest)1867if diff['src_sha1'] != diff['dst_sha1']:1868p4_edit(dest)1869 pureRenameCopy.discard(dest)1870ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1871p4_edit(dest)1872 pureRenameCopy.discard(dest)1873 filesToChangeExecBit[dest] = diff['dst_mode']1874if self.isWindows:1875# turn off read-only attribute1876 os.chmod(dest, stat.S_IWRITE)1877 os.unlink(dest)1878 editedFiles.add(dest)1879elif modifier =="R":1880 src, dest = diff['src'], diff['dst']1881 all_files.append(dest)1882if self.p4HasMoveCommand:1883p4_edit(src)# src must be open before move1884p4_move(src, dest)# opens for (move/delete, move/add)1885else:1886p4_integrate(src, dest)1887if diff['src_sha1'] != diff['dst_sha1']:1888p4_edit(dest)1889else:1890 pureRenameCopy.add(dest)1891ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1892if not self.p4HasMoveCommand:1893p4_edit(dest)# with move: already open, writable1894 filesToChangeExecBit[dest] = diff['dst_mode']1895if not self.p4HasMoveCommand:1896if self.isWindows:1897 os.chmod(dest, stat.S_IWRITE)1898 os.unlink(dest)1899 filesToDelete.add(src)1900 editedFiles.add(dest)1901elif modifier =="T":1902 filesToChangeType.add(path)1903else:1904die("unknown modifier%sfor%s"% (modifier, path))19051906 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1907 patchcmd = diffcmd +" | git apply "1908 tryPatchCmd = patchcmd +"--check -"1909 applyPatchCmd = patchcmd +"--check --apply -"1910 patch_succeeded =True19111912if os.system(tryPatchCmd) !=0:1913 fixed_rcs_keywords =False1914 patch_succeeded =False1915print("Unfortunately applying the change failed!")19161917# Patch failed, maybe it's just RCS keyword woes. Look through1918# the patch to see if that's possible.1919ifgitConfigBool("git-p4.attemptRCSCleanup"):1920file=None1921 pattern =None1922 kwfiles = {}1923forfilein editedFiles | filesToDelete:1924# did this file's delta contain RCS keywords?1925 pattern =p4_keywords_regexp_for_file(file)19261927if pattern:1928# this file is a possibility...look for RCS keywords.1929 regexp = re.compile(pattern, re.VERBOSE)1930for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1931if regexp.search(line):1932if verbose:1933print("got keyword match on%sin%sin%s"% (pattern, line,file))1934 kwfiles[file] = pattern1935break19361937forfilein kwfiles:1938if verbose:1939print("zapping%swith%s"% (line,pattern))1940# File is being deleted, so not open in p4. Must1941# disable the read-only bit on windows.1942if self.isWindows andfilenot in editedFiles:1943 os.chmod(file, stat.S_IWRITE)1944 self.patchRCSKeywords(file, kwfiles[file])1945 fixed_rcs_keywords =True19461947if fixed_rcs_keywords:1948print("Retrying the patch with RCS keywords cleaned up")1949if os.system(tryPatchCmd) ==0:1950 patch_succeeded =True19511952if not patch_succeeded:1953for f in editedFiles:1954p4_revert(f)1955return False19561957#1958# Apply the patch for real, and do add/delete/+x handling.1959#1960system(applyPatchCmd)19611962for f in filesToChangeType:1963p4_edit(f,"-t","auto")1964for f in filesToAdd:1965p4_add(f)1966for f in filesToDelete:1967p4_revert(f)1968p4_delete(f)19691970# Set/clear executable bits1971for f in filesToChangeExecBit.keys():1972 mode = filesToChangeExecBit[f]1973setP4ExecBit(f, mode)19741975 update_shelve =01976iflen(self.update_shelve) >0:1977 update_shelve = self.update_shelve.pop(0)1978p4_reopen_in_change(update_shelve, all_files)19791980#1981# Build p4 change description, starting with the contents1982# of the git commit message.1983#1984 logMessage =extractLogMessageFromGitCommit(id)1985 logMessage = logMessage.strip()1986(logMessage, jobs) = self.separate_jobs_from_description(logMessage)19871988 template = self.prepareSubmitTemplate(update_shelve)1989 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)19901991if self.preserveUser:1992 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User19931994if self.checkAuthorship and not self.p4UserIsMe(p4User):1995 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1996 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1997 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"19981999 separatorLine ="######## everything below this line is just the diff #######\n"2000if not self.prepare_p4_only:2001 submitTemplate += separatorLine2002 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)20032004(handle, fileName) = tempfile.mkstemp()2005 tmpFile = os.fdopen(handle,"w+b")2006if self.isWindows:2007 submitTemplate = submitTemplate.replace("\n","\r\n")2008 tmpFile.write(submitTemplate)2009 tmpFile.close()20102011if self.prepare_p4_only:2012#2013# Leave the p4 tree prepared, and the submit template around2014# and let the user decide what to do next2015#2016print()2017print("P4 workspace prepared for submission.")2018print("To submit or revert, go to client workspace")2019print(" "+ self.clientPath)2020print()2021print("To submit, use\"p4 submit\"to write a new description,")2022print("or\"p4 submit -i <%s\"to use the one prepared by" \2023"\"git p4\"."% fileName)2024print("You can delete the file\"%s\"when finished."% fileName)20252026if self.preserveUser and p4User and not self.p4UserIsMe(p4User):2027print("To preserve change ownership by user%s, you must\n" \2028"do\"p4 change -f <change>\"after submitting and\n" \2029"edit the User field.")2030if pureRenameCopy:2031print("After submitting, renamed files must be re-synced.")2032print("Invoke\"p4 sync -f\"on each of these files:")2033for f in pureRenameCopy:2034print(" "+ f)20352036print()2037print("To revert the changes, use\"p4 revert ...\", and delete")2038print("the submit template file\"%s\""% fileName)2039if filesToAdd:2040print("Since the commit adds new files, they must be deleted:")2041for f in filesToAdd:2042print(" "+ f)2043print()2044return True20452046#2047# Let the user edit the change description, then submit it.2048#2049 submitted =False20502051try:2052if self.edit_template(fileName):2053# read the edited message and submit2054 tmpFile =open(fileName,"rb")2055 message = tmpFile.read()2056 tmpFile.close()2057if self.isWindows:2058 message = message.replace("\r\n","\n")2059 submitTemplate = message[:message.index(separatorLine)]20602061if update_shelve:2062p4_write_pipe(['shelve','-r','-i'], submitTemplate)2063elif self.shelve:2064p4_write_pipe(['shelve','-i'], submitTemplate)2065else:2066p4_write_pipe(['submit','-i'], submitTemplate)2067# The rename/copy happened by applying a patch that created a2068# new file. This leaves it writable, which confuses p4.2069for f in pureRenameCopy:2070p4_sync(f,"-f")20712072if self.preserveUser:2073if p4User:2074# Get last changelist number. Cannot easily get it from2075# the submit command output as the output is2076# unmarshalled.2077 changelist = self.lastP4Changelist()2078 self.modifyChangelistUser(changelist, p4User)20792080 submitted =True20812082finally:2083# skip this patch2084if not submitted or self.shelve:2085if self.shelve:2086print("Reverting shelved files.")2087else:2088print("Submission cancelled, undoing p4 changes.")2089for f in editedFiles | filesToDelete:2090p4_revert(f)2091for f in filesToAdd:2092p4_revert(f)2093 os.remove(f)20942095 os.remove(fileName)2096return submitted20972098# Export git tags as p4 labels. Create a p4 label and then tag2099# with that.2100defexportGitTags(self, gitTags):2101 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")2102iflen(validLabelRegexp) ==0:2103 validLabelRegexp = defaultLabelRegexp2104 m = re.compile(validLabelRegexp)21052106for name in gitTags:21072108if not m.match(name):2109if verbose:2110print("tag%sdoes not match regexp%s"% (name, validLabelRegexp))2111continue21122113# Get the p4 commit this corresponds to2114 logMessage =extractLogMessageFromGitCommit(name)2115 values =extractSettingsGitLog(logMessage)21162117if'change'not in values:2118# a tag pointing to something not sent to p4; ignore2119if verbose:2120print("git tag%sdoes not give a p4 commit"% name)2121continue2122else:2123 changelist = values['change']21242125# Get the tag details.2126 inHeader =True2127 isAnnotated =False2128 body = []2129for l inread_pipe_lines(["git","cat-file","-p", name]):2130 l = l.strip()2131if inHeader:2132if re.match(r'tag\s+', l):2133 isAnnotated =True2134elif re.match(r'\s*$', l):2135 inHeader =False2136continue2137else:2138 body.append(l)21392140if not isAnnotated:2141 body = ["lightweight tag imported by git p4\n"]21422143# Create the label - use the same view as the client spec we are using2144 clientSpec =getClientSpec()21452146 labelTemplate ="Label:%s\n"% name2147 labelTemplate +="Description:\n"2148for b in body:2149 labelTemplate +="\t"+ b +"\n"2150 labelTemplate +="View:\n"2151for depot_side in clientSpec.mappings:2152 labelTemplate +="\t%s\n"% depot_side21532154if self.dry_run:2155print("Would create p4 label%sfor tag"% name)2156elif self.prepare_p4_only:2157print("Not creating p4 label%sfor tag due to option" \2158" --prepare-p4-only"% name)2159else:2160p4_write_pipe(["label","-i"], labelTemplate)21612162# Use the label2163p4_system(["tag","-l", name] +2164["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])21652166if verbose:2167print("created p4 label for tag%s"% name)21682169defrun(self, args):2170iflen(args) ==0:2171 self.master =currentGitBranch()2172eliflen(args) ==1:2173 self.master = args[0]2174if notbranchExists(self.master):2175die("Branch%sdoes not exist"% self.master)2176else:2177return False21782179for i in self.update_shelve:2180if i <=0:2181 sys.exit("invalid changelist%d"% i)21822183if self.master:2184 allowSubmit =gitConfig("git-p4.allowSubmit")2185iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):2186die("%sis not in git-p4.allowSubmit"% self.master)21872188[upstream, settings] =findUpstreamBranchPoint()2189 self.depotPath = settings['depot-paths'][0]2190iflen(self.origin) ==0:2191 self.origin = upstream21922193iflen(self.update_shelve) >0:2194 self.shelve =True21952196if self.preserveUser:2197if not self.canChangeChangelists():2198die("Cannot preserve user names without p4 super-user or admin permissions")21992200# if not set from the command line, try the config file2201if self.conflict_behavior is None:2202 val =gitConfig("git-p4.conflict")2203if val:2204if val not in self.conflict_behavior_choices:2205die("Invalid value '%s' for config git-p4.conflict"% val)2206else:2207 val ="ask"2208 self.conflict_behavior = val22092210if self.verbose:2211print("Origin branch is "+ self.origin)22122213iflen(self.depotPath) ==0:2214print("Internal error: cannot locate perforce depot path from existing branches")2215 sys.exit(128)22162217 self.useClientSpec =False2218ifgitConfigBool("git-p4.useclientspec"):2219 self.useClientSpec =True2220if self.useClientSpec:2221 self.clientSpecDirs =getClientSpec()22222223# Check for the existence of P4 branches2224 branchesDetected = (len(p4BranchesInGit().keys()) >1)22252226if self.useClientSpec and not branchesDetected:2227# all files are relative to the client spec2228 self.clientPath =getClientRoot()2229else:2230 self.clientPath =p4Where(self.depotPath)22312232if self.clientPath =="":2233die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)22342235print("Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath))2236 self.oldWorkingDirectory = os.getcwd()22372238# ensure the clientPath exists2239 new_client_dir =False2240if not os.path.exists(self.clientPath):2241 new_client_dir =True2242 os.makedirs(self.clientPath)22432244chdir(self.clientPath, is_client_path=True)2245if self.dry_run:2246print("Would synchronize p4 checkout in%s"% self.clientPath)2247else:2248print("Synchronizing p4 checkout...")2249if new_client_dir:2250# old one was destroyed, and maybe nobody told p42251p4_sync("...","-f")2252else:2253p4_sync("...")2254 self.check()22552256 commits = []2257if self.master:2258 committish = self.master2259else:2260 committish ='HEAD'22612262if self.commit !="":2263if self.commit.find("..") != -1:2264 limits_ish = self.commit.split("..")2265for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (limits_ish[0], limits_ish[1])]):2266 commits.append(line.strip())2267 commits.reverse()2268else:2269 commits.append(self.commit)2270else:2271for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, committish)]):2272 commits.append(line.strip())2273 commits.reverse()22742275if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):2276 self.checkAuthorship =False2277else:2278 self.checkAuthorship =True22792280if self.preserveUser:2281 self.checkValidP4Users(commits)22822283#2284# Build up a set of options to be passed to diff when2285# submitting each commit to p4.2286#2287if self.detectRenames:2288# command-line -M arg2289 self.diffOpts ="-M"2290else:2291# If not explicitly set check the config variable2292 detectRenames =gitConfig("git-p4.detectRenames")22932294if detectRenames.lower() =="false"or detectRenames =="":2295 self.diffOpts =""2296elif detectRenames.lower() =="true":2297 self.diffOpts ="-M"2298else:2299 self.diffOpts ="-M%s"% detectRenames23002301# no command-line arg for -C or --find-copies-harder, just2302# config variables2303 detectCopies =gitConfig("git-p4.detectCopies")2304if detectCopies.lower() =="false"or detectCopies =="":2305pass2306elif detectCopies.lower() =="true":2307 self.diffOpts +=" -C"2308else:2309 self.diffOpts +=" -C%s"% detectCopies23102311ifgitConfigBool("git-p4.detectCopiesHarder"):2312 self.diffOpts +=" --find-copies-harder"23132314 num_shelves =len(self.update_shelve)2315if num_shelves >0and num_shelves !=len(commits):2316 sys.exit("number of commits (%d) must match number of shelved changelist (%d)"%2317(len(commits), num_shelves))23182319 hooks_path =gitConfig("core.hooksPath")2320iflen(hooks_path) <=0:2321 hooks_path = os.path.join(os.environ.get("GIT_DIR",".git"),"hooks")23222323 hook_file = os.path.join(hooks_path,"p4-pre-submit")2324if os.path.isfile(hook_file)and os.access(hook_file, os.X_OK)and subprocess.call([hook_file]) !=0:2325 sys.exit(1)23262327#2328# Apply the commits, one at a time. On failure, ask if should2329# continue to try the rest of the patches, or quit.2330#2331if self.dry_run:2332print("Would apply")2333 applied = []2334 last =len(commits) -12335for i, commit inenumerate(commits):2336if self.dry_run:2337print(" ",read_pipe(["git","show","-s",2338"--format=format:%h%s", commit]))2339 ok =True2340else:2341 ok = self.applyCommit(commit)2342if ok:2343 applied.append(commit)2344else:2345if self.prepare_p4_only and i < last:2346print("Processing only the first commit due to option" \2347" --prepare-p4-only")2348break2349if i < last:2350 quit =False2351while True:2352# prompt for what to do, or use the option/variable2353if self.conflict_behavior =="ask":2354print("What do you want to do?")2355 response =raw_input("[s]kip this commit but apply"2356" the rest, or [q]uit? ")2357if not response:2358continue2359elif self.conflict_behavior =="skip":2360 response ="s"2361elif self.conflict_behavior =="quit":2362 response ="q"2363else:2364die("Unknown conflict_behavior '%s'"%2365 self.conflict_behavior)23662367if response[0] =="s":2368print("Skipping this commit, but applying the rest")2369break2370if response[0] =="q":2371print("Quitting")2372 quit =True2373break2374if quit:2375break23762377chdir(self.oldWorkingDirectory)2378 shelved_applied ="shelved"if self.shelve else"applied"2379if self.dry_run:2380pass2381elif self.prepare_p4_only:2382pass2383eliflen(commits) ==len(applied):2384print("All commits{0}!".format(shelved_applied))23852386 sync =P4Sync()2387if self.branch:2388 sync.branch = self.branch2389if self.disable_p4sync:2390 sync.sync_origin_only()2391else:2392 sync.run([])23932394if not self.disable_rebase:2395 rebase =P4Rebase()2396 rebase.rebase()23972398else:2399iflen(applied) ==0:2400print("No commits{0}.".format(shelved_applied))2401else:2402print("{0}only the commits marked with '*':".format(shelved_applied.capitalize()))2403for c in commits:2404if c in applied:2405 star ="*"2406else:2407 star =" "2408print(star,read_pipe(["git","show","-s",2409"--format=format:%h%s", c]))2410print("You will have to do 'git p4 sync' and rebase.")24112412ifgitConfigBool("git-p4.exportLabels"):2413 self.exportLabels =True24142415if self.exportLabels:2416 p4Labels =getP4Labels(self.depotPath)2417 gitTags =getGitTags()24182419 missingGitTags = gitTags - p4Labels2420 self.exportGitTags(missingGitTags)24212422# exit with error unless everything applied perfectly2423iflen(commits) !=len(applied):2424 sys.exit(1)24252426return True24272428classView(object):2429"""Represent a p4 view ("p4 help views"), and map files in a2430 repo according to the view."""24312432def__init__(self, client_name):2433 self.mappings = []2434 self.client_prefix ="//%s/"% client_name2435# cache results of "p4 where" to lookup client file locations2436 self.client_spec_path_cache = {}24372438defappend(self, view_line):2439"""Parse a view line, splitting it into depot and client2440 sides. Append to self.mappings, preserving order. This2441 is only needed for tag creation."""24422443# Split the view line into exactly two words. P4 enforces2444# structure on these lines that simplifies this quite a bit.2445#2446# Either or both words may be double-quoted.2447# Single quotes do not matter.2448# Double-quote marks cannot occur inside the words.2449# A + or - prefix is also inside the quotes.2450# There are no quotes unless they contain a space.2451# The line is already white-space stripped.2452# The two words are separated by a single space.2453#2454if view_line[0] =='"':2455# First word is double quoted. Find its end.2456 close_quote_index = view_line.find('"',1)2457if close_quote_index <=0:2458die("No first-word closing quote found:%s"% view_line)2459 depot_side = view_line[1:close_quote_index]2460# skip closing quote and space2461 rhs_index = close_quote_index +1+12462else:2463 space_index = view_line.find(" ")2464if space_index <=0:2465die("No word-splitting space found:%s"% view_line)2466 depot_side = view_line[0:space_index]2467 rhs_index = space_index +124682469# prefix + means overlay on previous mapping2470if depot_side.startswith("+"):2471 depot_side = depot_side[1:]24722473# prefix - means exclude this path, leave out of mappings2474 exclude =False2475if depot_side.startswith("-"):2476 exclude =True2477 depot_side = depot_side[1:]24782479if not exclude:2480 self.mappings.append(depot_side)24812482defconvert_client_path(self, clientFile):2483# chop off //client/ part to make it relative2484if not clientFile.startswith(self.client_prefix):2485die("No prefix '%s' on clientFile '%s'"%2486(self.client_prefix, clientFile))2487return clientFile[len(self.client_prefix):]24882489defupdate_client_spec_path_cache(self, files):2490""" Caching file paths by "p4 where" batch query """24912492# List depot file paths exclude that already cached2493 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]24942495iflen(fileArgs) ==0:2496return# All files in cache24972498 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)2499for res in where_result:2500if"code"in res and res["code"] =="error":2501# assume error is "... file(s) not in client view"2502continue2503if"clientFile"not in res:2504die("No clientFile in 'p4 where' output")2505if"unmap"in res:2506# it will list all of them, but only one not unmap-ped2507continue2508ifgitConfigBool("core.ignorecase"):2509 res['depotFile'] = res['depotFile'].lower()2510 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])25112512# not found files or unmap files set to ""2513for depotFile in fileArgs:2514ifgitConfigBool("core.ignorecase"):2515 depotFile = depotFile.lower()2516if depotFile not in self.client_spec_path_cache:2517 self.client_spec_path_cache[depotFile] =""25182519defmap_in_client(self, depot_path):2520"""Return the relative location in the client where this2521 depot file should live. Returns "" if the file should2522 not be mapped in the client."""25232524ifgitConfigBool("core.ignorecase"):2525 depot_path = depot_path.lower()25262527if depot_path in self.client_spec_path_cache:2528return self.client_spec_path_cache[depot_path]25292530die("Error:%sis not found in client spec path"% depot_path )2531return""25322533defcloneExcludeCallback(option, opt_str, value, parser):2534# prepend "/" because the first "/" was consumed as part of the option itself.2535# ("-//depot/A/..." becomes "/depot/A/..." after option parsing)2536 parser.values.cloneExclude += ["/"+ re.sub(r"\.\.\.$","", value)]25372538classP4Sync(Command, P4UserMap):25392540def__init__(self):2541 Command.__init__(self)2542 P4UserMap.__init__(self)2543 self.options = [2544 optparse.make_option("--branch", dest="branch"),2545 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),2546 optparse.make_option("--changesfile", dest="changesFile"),2547 optparse.make_option("--silent", dest="silent", action="store_true"),2548 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),2549 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2550 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",2551help="Import into refs/heads/ , not refs/remotes"),2552 optparse.make_option("--max-changes", dest="maxChanges",2553help="Maximum number of changes to import"),2554 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",2555help="Internal block size to use when iteratively calling p4 changes"),2556 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2557help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2558 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2559help="Only sync files that are included in the Perforce Client Spec"),2560 optparse.make_option("-/", dest="cloneExclude",2561 action="callback", callback=cloneExcludeCallback,type="string",2562help="exclude depot path"),2563]2564 self.description ="""Imports from Perforce into a git repository.\n2565 example:2566 //depot/my/project/ -- to import the current head2567 //depot/my/project/@all -- to import everything2568 //depot/my/project/@1,6 -- to import only from revision 1 to 625692570 (a ... is not needed in the path p4 specification, it's added implicitly)"""25712572 self.usage +=" //depot/path[@revRange]"2573 self.silent =False2574 self.createdBranches =set()2575 self.committedChanges =set()2576 self.branch =""2577 self.detectBranches =False2578 self.detectLabels =False2579 self.importLabels =False2580 self.changesFile =""2581 self.syncWithOrigin =True2582 self.importIntoRemotes =True2583 self.maxChanges =""2584 self.changes_block_size =None2585 self.keepRepoPath =False2586 self.depotPaths =None2587 self.p4BranchesInGit = []2588 self.cloneExclude = []2589 self.useClientSpec =False2590 self.useClientSpec_from_options =False2591 self.clientSpecDirs =None2592 self.tempBranches = []2593 self.tempBranchLocation ="refs/git-p4-tmp"2594 self.largeFileSystem =None2595 self.suppress_meta_comment =False25962597ifgitConfig('git-p4.largeFileSystem'):2598 largeFileSystemConstructor =globals()[gitConfig('git-p4.largeFileSystem')]2599 self.largeFileSystem =largeFileSystemConstructor(2600lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)2601)26022603ifgitConfig("git-p4.syncFromOrigin") =="false":2604 self.syncWithOrigin =False26052606 self.depotPaths = []2607 self.changeRange =""2608 self.previousDepotPaths = []2609 self.hasOrigin =False26102611# map from branch depot path to parent branch2612 self.knownBranches = {}2613 self.initialParents = {}26142615 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))2616 self.labels = {}26172618# Force a checkpoint in fast-import and wait for it to finish2619defcheckpoint(self):2620 self.gitStream.write("checkpoint\n\n")2621 self.gitStream.write("progress checkpoint\n\n")2622 out = self.gitOutput.readline()2623if self.verbose:2624print("checkpoint finished: "+ out)26252626defisPathWanted(self, path):2627for p in self.cloneExclude:2628if p.endswith("/"):2629ifp4PathStartsWith(path, p):2630return False2631# "-//depot/file1" without a trailing "/" should only exclude "file1", but not "file111" or "file1_dir/file2"2632elif path.lower() == p.lower():2633return False2634for p in self.depotPaths:2635ifp4PathStartsWith(path, p):2636return True2637return False26382639defextractFilesFromCommit(self, commit, shelved=False, shelved_cl =0):2640 files = []2641 fnum =02642while"depotFile%s"% fnum in commit:2643 path = commit["depotFile%s"% fnum]2644 found = self.isPathWanted(path)2645if not found:2646 fnum = fnum +12647continue26482649file= {}2650file["path"] = path2651file["rev"] = commit["rev%s"% fnum]2652file["action"] = commit["action%s"% fnum]2653file["type"] = commit["type%s"% fnum]2654if shelved:2655file["shelved_cl"] =int(shelved_cl)2656 files.append(file)2657 fnum = fnum +12658return files26592660defextractJobsFromCommit(self, commit):2661 jobs = []2662 jnum =02663while"job%s"% jnum in commit:2664 job = commit["job%s"% jnum]2665 jobs.append(job)2666 jnum = jnum +12667return jobs26682669defstripRepoPath(self, path, prefixes):2670"""When streaming files, this is called to map a p4 depot path2671 to where it should go in git. The prefixes are either2672 self.depotPaths, or self.branchPrefixes in the case of2673 branch detection."""26742675if self.useClientSpec:2676# branch detection moves files up a level (the branch name)2677# from what client spec interpretation gives2678 path = self.clientSpecDirs.map_in_client(path)2679if self.detectBranches:2680for b in self.knownBranches:2681ifp4PathStartsWith(path, b +"/"):2682 path = path[len(b)+1:]26832684elif self.keepRepoPath:2685# Preserve everything in relative path name except leading2686# //depot/; just look at first prefix as they all should2687# be in the same depot.2688 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2689ifp4PathStartsWith(path, depot):2690 path = path[len(depot):]26912692else:2693for p in prefixes:2694ifp4PathStartsWith(path, p):2695 path = path[len(p):]2696break26972698 path =wildcard_decode(path)2699return path27002701defsplitFilesIntoBranches(self, commit):2702"""Look at each depotFile in the commit to figure out to what2703 branch it belongs."""27042705if self.clientSpecDirs:2706 files = self.extractFilesFromCommit(commit)2707 self.clientSpecDirs.update_client_spec_path_cache(files)27082709 branches = {}2710 fnum =02711while"depotFile%s"% fnum in commit:2712 path = commit["depotFile%s"% fnum]2713 found = self.isPathWanted(path)2714if not found:2715 fnum = fnum +12716continue27172718file= {}2719file["path"] = path2720file["rev"] = commit["rev%s"% fnum]2721file["action"] = commit["action%s"% fnum]2722file["type"] = commit["type%s"% fnum]2723 fnum = fnum +127242725# start with the full relative path where this file would2726# go in a p4 client2727if self.useClientSpec:2728 relPath = self.clientSpecDirs.map_in_client(path)2729else:2730 relPath = self.stripRepoPath(path, self.depotPaths)27312732for branch in self.knownBranches.keys():2733# add a trailing slash so that a commit into qt/4.2foo2734# doesn't end up in qt/4.2, e.g.2735ifp4PathStartsWith(relPath, branch +"/"):2736if branch not in branches:2737 branches[branch] = []2738 branches[branch].append(file)2739break27402741return branches27422743defwriteToGitStream(self, gitMode, relPath, contents):2744 self.gitStream.write('M%sinline%s\n'% (gitMode, relPath))2745 self.gitStream.write('data%d\n'%sum(len(d)for d in contents))2746for d in contents:2747 self.gitStream.write(d)2748 self.gitStream.write('\n')27492750defencodeWithUTF8(self, path):2751try:2752 path.decode('ascii')2753except:2754 encoding ='utf8'2755ifgitConfig('git-p4.pathEncoding'):2756 encoding =gitConfig('git-p4.pathEncoding')2757 path = path.decode(encoding,'replace').encode('utf8','replace')2758if self.verbose:2759print('Path with non-ASCII characters detected. Used%sto encode:%s'% (encoding, path))2760return path27612762# output one file from the P4 stream2763# - helper for streamP4Files27642765defstreamOneP4File(self,file, contents):2766 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2767 relPath = self.encodeWithUTF8(relPath)2768if verbose:2769if'fileSize'in self.stream_file:2770 size =int(self.stream_file['fileSize'])2771else:2772 size =0# deleted files don't get a fileSize apparently2773 sys.stdout.write('\r%s-->%s(%iMB)\n'% (file['depotFile'], relPath, size/1024/1024))2774 sys.stdout.flush()27752776(type_base, type_mods) =split_p4_type(file["type"])27772778 git_mode ="100644"2779if"x"in type_mods:2780 git_mode ="100755"2781if type_base =="symlink":2782 git_mode ="120000"2783# p4 print on a symlink sometimes contains "target\n";2784# if it does, remove the newline2785 data =''.join(contents)2786if not data:2787# Some version of p4 allowed creating a symlink that pointed2788# to nothing. This causes p4 errors when checking out such2789# a change, and errors here too. Work around it by ignoring2790# the bad symlink; hopefully a future change fixes it.2791print("\nIgnoring empty symlink in%s"%file['depotFile'])2792return2793elif data[-1] =='\n':2794 contents = [data[:-1]]2795else:2796 contents = [data]27972798if type_base =="utf16":2799# p4 delivers different text in the python output to -G2800# than it does when using "print -o", or normal p4 client2801# operations. utf16 is converted to ascii or utf8, perhaps.2802# But ascii text saved as -t utf16 is completely mangled.2803# Invoke print -o to get the real contents.2804#2805# On windows, the newlines will always be mangled by print, so put2806# them back too. This is not needed to the cygwin windows version,2807# just the native "NT" type.2808#2809try:2810 text =p4_read_pipe(['print','-q','-o','-','%s@%s'% (file['depotFile'],file['change'])])2811exceptExceptionas e:2812if'Translation of file content failed'instr(e):2813 type_base ='binary'2814else:2815raise e2816else:2817ifp4_version_string().find('/NT') >=0:2818 text = text.replace('\r\n','\n')2819 contents = [ text ]28202821if type_base =="apple":2822# Apple filetype files will be streamed as a concatenation of2823# its appledouble header and the contents. This is useless2824# on both macs and non-macs. If using "print -q -o xx", it2825# will create "xx" with the data, and "%xx" with the header.2826# This is also not very useful.2827#2828# Ideally, someday, this script can learn how to generate2829# appledouble files directly and import those to git, but2830# non-mac machines can never find a use for apple filetype.2831print("\nIgnoring apple filetype file%s"%file['depotFile'])2832return28332834# Note that we do not try to de-mangle keywords on utf16 files,2835# even though in theory somebody may want that.2836 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2837if pattern:2838 regexp = re.compile(pattern, re.VERBOSE)2839 text =''.join(contents)2840 text = regexp.sub(r'$\1$', text)2841 contents = [ text ]28422843if self.largeFileSystem:2844(git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)28452846 self.writeToGitStream(git_mode, relPath, contents)28472848defstreamOneP4Deletion(self,file):2849 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2850 relPath = self.encodeWithUTF8(relPath)2851if verbose:2852 sys.stdout.write("delete%s\n"% relPath)2853 sys.stdout.flush()2854 self.gitStream.write("D%s\n"% relPath)28552856if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):2857 self.largeFileSystem.removeLargeFile(relPath)28582859# handle another chunk of streaming data2860defstreamP4FilesCb(self, marshalled):28612862# catch p4 errors and complain2863 err =None2864if"code"in marshalled:2865if marshalled["code"] =="error":2866if"data"in marshalled:2867 err = marshalled["data"].rstrip()28682869if not err and'fileSize'in self.stream_file:2870 required_bytes =int((4*int(self.stream_file["fileSize"])) -calcDiskFree())2871if required_bytes >0:2872 err ='Not enough space left on%s! Free at least%iMB.'% (2873 os.getcwd(), required_bytes/1024/10242874)28752876if err:2877 f =None2878if self.stream_have_file_info:2879if"depotFile"in self.stream_file:2880 f = self.stream_file["depotFile"]2881# force a failure in fast-import, else an empty2882# commit will be made2883 self.gitStream.write("\n")2884 self.gitStream.write("die-now\n")2885 self.gitStream.close()2886# ignore errors, but make sure it exits first2887 self.importProcess.wait()2888if f:2889die("Error from p4 print for%s:%s"% (f, err))2890else:2891die("Error from p4 print:%s"% err)28922893if'depotFile'in marshalled and self.stream_have_file_info:2894# start of a new file - output the old one first2895 self.streamOneP4File(self.stream_file, self.stream_contents)2896 self.stream_file = {}2897 self.stream_contents = []2898 self.stream_have_file_info =False28992900# pick up the new file information... for the2901# 'data' field we need to append to our array2902for k in marshalled.keys():2903if k =='data':2904if'streamContentSize'not in self.stream_file:2905 self.stream_file['streamContentSize'] =02906 self.stream_file['streamContentSize'] +=len(marshalled['data'])2907 self.stream_contents.append(marshalled['data'])2908else:2909 self.stream_file[k] = marshalled[k]29102911if(verbose and2912'streamContentSize'in self.stream_file and2913'fileSize'in self.stream_file and2914'depotFile'in self.stream_file):2915 size =int(self.stream_file["fileSize"])2916if size >0:2917 progress =100*self.stream_file['streamContentSize']/size2918 sys.stdout.write('\r%s %d%%(%iMB)'% (self.stream_file['depotFile'], progress,int(size/1024/1024)))2919 sys.stdout.flush()29202921 self.stream_have_file_info =True29222923# Stream directly from "p4 files" into "git fast-import"2924defstreamP4Files(self, files):2925 filesForCommit = []2926 filesToRead = []2927 filesToDelete = []29282929for f in files:2930 filesForCommit.append(f)2931if f['action']in self.delete_actions:2932 filesToDelete.append(f)2933else:2934 filesToRead.append(f)29352936# deleted files...2937for f in filesToDelete:2938 self.streamOneP4Deletion(f)29392940iflen(filesToRead) >0:2941 self.stream_file = {}2942 self.stream_contents = []2943 self.stream_have_file_info =False29442945# curry self argument2946defstreamP4FilesCbSelf(entry):2947 self.streamP4FilesCb(entry)29482949 fileArgs = []2950for f in filesToRead:2951if'shelved_cl'in f:2952# Handle shelved CLs using the "p4 print file@=N" syntax to print2953# the contents2954 fileArg ='%s@=%d'% (f['path'], f['shelved_cl'])2955else:2956 fileArg ='%s#%s'% (f['path'], f['rev'])29572958 fileArgs.append(fileArg)29592960p4CmdList(["-x","-","print"],2961 stdin=fileArgs,2962 cb=streamP4FilesCbSelf)29632964# do the last chunk2965if'depotFile'in self.stream_file:2966 self.streamOneP4File(self.stream_file, self.stream_contents)29672968defmake_email(self, userid):2969if userid in self.users:2970return self.users[userid]2971else:2972return"%s<a@b>"% userid29732974defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2975""" Stream a p4 tag.2976 commit is either a git commit, or a fast-import mark, ":<p4commit>"2977 """29782979if verbose:2980print("writing tag%sfor commit%s"% (labelName, commit))2981 gitStream.write("tag%s\n"% labelName)2982 gitStream.write("from%s\n"% commit)29832984if'Owner'in labelDetails:2985 owner = labelDetails["Owner"]2986else:2987 owner =None29882989# Try to use the owner of the p4 label, or failing that,2990# the current p4 user id.2991if owner:2992 email = self.make_email(owner)2993else:2994 email = self.make_email(self.p4UserId())2995 tagger ="%s %s %s"% (email, epoch, self.tz)29962997 gitStream.write("tagger%s\n"% tagger)29982999print("labelDetails=",labelDetails)3000if'Description'in labelDetails:3001 description = labelDetails['Description']3002else:3003 description ='Label from git p4'30043005 gitStream.write("data%d\n"%len(description))3006 gitStream.write(description)3007 gitStream.write("\n")30083009definClientSpec(self, path):3010if not self.clientSpecDirs:3011return True3012 inClientSpec = self.clientSpecDirs.map_in_client(path)3013if not inClientSpec and self.verbose:3014print('Ignoring file outside of client spec:{0}'.format(path))3015return inClientSpec30163017defhasBranchPrefix(self, path):3018if not self.branchPrefixes:3019return True3020 hasPrefix = [p for p in self.branchPrefixes3021ifp4PathStartsWith(path, p)]3022if not hasPrefix and self.verbose:3023print('Ignoring file outside of prefix:{0}'.format(path))3024return hasPrefix30253026defcommit(self, details, files, branch, parent ="", allow_empty=False):3027 epoch = details["time"]3028 author = details["user"]3029 jobs = self.extractJobsFromCommit(details)30303031if self.verbose:3032print('commit into{0}'.format(branch))30333034if self.clientSpecDirs:3035 self.clientSpecDirs.update_client_spec_path_cache(files)30363037 files = [f for f in files3038if self.inClientSpec(f['path'])and self.hasBranchPrefix(f['path'])]30393040ifgitConfigBool('git-p4.keepEmptyCommits'):3041 allow_empty =True30423043if not files and not allow_empty:3044print('Ignoring revision{0}as it would produce an empty commit.'3045.format(details['change']))3046return30473048 self.gitStream.write("commit%s\n"% branch)3049 self.gitStream.write("mark :%s\n"% details["change"])3050 self.committedChanges.add(int(details["change"]))3051 committer =""3052if author not in self.users:3053 self.getUserMapFromPerforceServer()3054 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)30553056 self.gitStream.write("committer%s\n"% committer)30573058 self.gitStream.write("data <<EOT\n")3059 self.gitStream.write(details["desc"])3060iflen(jobs) >0:3061 self.gitStream.write("\nJobs:%s"% (' '.join(jobs)))30623063if not self.suppress_meta_comment:3064 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%3065(','.join(self.branchPrefixes), details["change"]))3066iflen(details['options']) >0:3067 self.gitStream.write(": options =%s"% details['options'])3068 self.gitStream.write("]\n")30693070 self.gitStream.write("EOT\n\n")30713072iflen(parent) >0:3073if self.verbose:3074print("parent%s"% parent)3075 self.gitStream.write("from%s\n"% parent)30763077 self.streamP4Files(files)3078 self.gitStream.write("\n")30793080 change =int(details["change"])30813082if change in self.labels:3083 label = self.labels[change]3084 labelDetails = label[0]3085 labelRevisions = label[1]3086if self.verbose:3087print("Change%sis labelled%s"% (change, labelDetails))30883089 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)3090for p in self.branchPrefixes])30913092iflen(files) ==len(labelRevisions):30933094 cleanedFiles = {}3095for info in files:3096if info["action"]in self.delete_actions:3097continue3098 cleanedFiles[info["depotFile"]] = info["rev"]30993100if cleanedFiles == labelRevisions:3101 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)31023103else:3104if not self.silent:3105print("Tag%sdoes not match with change%s: files do not match."3106% (labelDetails["label"], change))31073108else:3109if not self.silent:3110print("Tag%sdoes not match with change%s: file count is different."3111% (labelDetails["label"], change))31123113# Build a dictionary of changelists and labels, for "detect-labels" option.3114defgetLabels(self):3115 self.labels = {}31163117 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])3118iflen(l) >0and not self.silent:3119print("Finding files belonging to labels in%s"% self.depotPaths)31203121for output in l:3122 label = output["label"]3123 revisions = {}3124 newestChange =03125if self.verbose:3126print("Querying files for label%s"% label)3127forfileinp4CmdList(["files"] +3128["%s...@%s"% (p, label)3129for p in self.depotPaths]):3130 revisions[file["depotFile"]] =file["rev"]3131 change =int(file["change"])3132if change > newestChange:3133 newestChange = change31343135 self.labels[newestChange] = [output, revisions]31363137if self.verbose:3138print("Label changes:%s"% self.labels.keys())31393140# Import p4 labels as git tags. A direct mapping does not3141# exist, so assume that if all the files are at the same revision3142# then we can use that, or it's something more complicated we should3143# just ignore.3144defimportP4Labels(self, stream, p4Labels):3145if verbose:3146print("import p4 labels: "+' '.join(p4Labels))31473148 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")3149 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")3150iflen(validLabelRegexp) ==0:3151 validLabelRegexp = defaultLabelRegexp3152 m = re.compile(validLabelRegexp)31533154for name in p4Labels:3155 commitFound =False31563157if not m.match(name):3158if verbose:3159print("label%sdoes not match regexp%s"% (name,validLabelRegexp))3160continue31613162if name in ignoredP4Labels:3163continue31643165 labelDetails =p4CmdList(['label',"-o", name])[0]31663167# get the most recent changelist for each file in this label3168 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)3169for p in self.depotPaths])31703171if'change'in change:3172# find the corresponding git commit; take the oldest commit3173 changelist =int(change['change'])3174if changelist in self.committedChanges:3175 gitCommit =":%d"% changelist # use a fast-import mark3176 commitFound =True3177else:3178 gitCommit =read_pipe(["git","rev-list","--max-count=1",3179"--reverse",":/\[git-p4:.*change =%d\]"% changelist], ignore_error=True)3180iflen(gitCommit) ==0:3181print("importing label%s: could not find git commit for changelist%d"% (name, changelist))3182else:3183 commitFound =True3184 gitCommit = gitCommit.strip()31853186if commitFound:3187# Convert from p4 time format3188try:3189 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")3190exceptValueError:3191print("Could not convert label time%s"% labelDetails['Update'])3192 tmwhen =131933194 when =int(time.mktime(tmwhen))3195 self.streamTag(stream, name, labelDetails, gitCommit, when)3196if verbose:3197print("p4 label%smapped to git commit%s"% (name, gitCommit))3198else:3199if verbose:3200print("Label%shas no changelists - possibly deleted?"% name)32013202if not commitFound:3203# We can't import this label; don't try again as it will get very3204# expensive repeatedly fetching all the files for labels that will3205# never be imported. If the label is moved in the future, the3206# ignore will need to be removed manually.3207system(["git","config","--add","git-p4.ignoredP4Labels", name])32083209defguessProjectName(self):3210for p in self.depotPaths:3211if p.endswith("/"):3212 p = p[:-1]3213 p = p[p.strip().rfind("/") +1:]3214if not p.endswith("/"):3215 p +="/"3216return p32173218defgetBranchMapping(self):3219 lostAndFoundBranches =set()32203221 user =gitConfig("git-p4.branchUser")3222iflen(user) >0:3223 command ="branches -u%s"% user3224else:3225 command ="branches"32263227for info inp4CmdList(command):3228 details =p4Cmd(["branch","-o", info["branch"]])3229 viewIdx =03230while"View%s"% viewIdx in details:3231 paths = details["View%s"% viewIdx].split(" ")3232 viewIdx = viewIdx +13233# require standard //depot/foo/... //depot/bar/... mapping3234iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):3235continue3236 source = paths[0]3237 destination = paths[1]3238## HACK3239ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):3240 source = source[len(self.depotPaths[0]):-4]3241 destination = destination[len(self.depotPaths[0]):-4]32423243if destination in self.knownBranches:3244if not self.silent:3245print("p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination))3246print("but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination))3247continue32483249 self.knownBranches[destination] = source32503251 lostAndFoundBranches.discard(destination)32523253if source not in self.knownBranches:3254 lostAndFoundBranches.add(source)32553256# Perforce does not strictly require branches to be defined, so we also3257# check git config for a branch list.3258#3259# Example of branch definition in git config file:3260# [git-p4]3261# branchList=main:branchA3262# branchList=main:branchB3263# branchList=branchA:branchC3264 configBranches =gitConfigList("git-p4.branchList")3265for branch in configBranches:3266if branch:3267(source, destination) = branch.split(":")3268 self.knownBranches[destination] = source32693270 lostAndFoundBranches.discard(destination)32713272if source not in self.knownBranches:3273 lostAndFoundBranches.add(source)327432753276for branch in lostAndFoundBranches:3277 self.knownBranches[branch] = branch32783279defgetBranchMappingFromGitBranches(self):3280 branches =p4BranchesInGit(self.importIntoRemotes)3281for branch in branches.keys():3282if branch =="master":3283 branch ="main"3284else:3285 branch = branch[len(self.projectName):]3286 self.knownBranches[branch] = branch32873288defupdateOptionDict(self, d):3289 option_keys = {}3290if self.keepRepoPath:3291 option_keys['keepRepoPath'] =132923293 d["options"] =' '.join(sorted(option_keys.keys()))32943295defreadOptions(self, d):3296 self.keepRepoPath = ('options'in d3297and('keepRepoPath'in d['options']))32983299defgitRefForBranch(self, branch):3300if branch =="main":3301return self.refPrefix +"master"33023303iflen(branch) <=0:3304return branch33053306return self.refPrefix + self.projectName + branch33073308defgitCommitByP4Change(self, ref, change):3309if self.verbose:3310print("looking in ref "+ ref +" for change%susing bisect..."% change)33113312 earliestCommit =""3313 latestCommit =parseRevision(ref)33143315while True:3316if self.verbose:3317print("trying: earliest%slatest%s"% (earliestCommit, latestCommit))3318 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()3319iflen(next) ==0:3320if self.verbose:3321print("argh")3322return""3323 log =extractLogMessageFromGitCommit(next)3324 settings =extractSettingsGitLog(log)3325 currentChange =int(settings['change'])3326if self.verbose:3327print("current change%s"% currentChange)33283329if currentChange == change:3330if self.verbose:3331print("found%s"% next)3332return next33333334if currentChange < change:3335 earliestCommit ="^%s"% next3336else:3337if next == latestCommit:3338die("Infinite loop while looking in ref%sfor change%s. Check your branch mappings"% (ref, change))3339 latestCommit ="%s^@"% next33403341return""33423343defimportNewBranch(self, branch, maxChange):3344# make fast-import flush all changes to disk and update the refs using the checkpoint3345# command so that we can try to find the branch parent in the git history3346 self.gitStream.write("checkpoint\n\n");3347 self.gitStream.flush();3348 branchPrefix = self.depotPaths[0] + branch +"/"3349range="@1,%s"% maxChange3350#print "prefix" + branchPrefix3351 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)3352iflen(changes) <=0:3353return False3354 firstChange = changes[0]3355#print "first change in branch: %s" % firstChange3356 sourceBranch = self.knownBranches[branch]3357 sourceDepotPath = self.depotPaths[0] + sourceBranch3358 sourceRef = self.gitRefForBranch(sourceBranch)3359#print "source " + sourceBranch33603361 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])3362#print "branch parent: %s" % branchParentChange3363 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)3364iflen(gitParent) >0:3365 self.initialParents[self.gitRefForBranch(branch)] = gitParent3366#print "parent git commit: %s" % gitParent33673368 self.importChanges(changes)3369return True33703371defsearchParent(self, parent, branch, target):3372 parentFound =False3373for blob inread_pipe_lines(["git","rev-list","--reverse",3374"--no-merges", parent]):3375 blob = blob.strip()3376iflen(read_pipe(["git","diff-tree", blob, target])) ==0:3377 parentFound =True3378if self.verbose:3379print("Found parent of%sin commit%s"% (branch, blob))3380break3381if parentFound:3382return blob3383else:3384return None33853386defimportChanges(self, changes, origin_revision=0):3387 cnt =13388for change in changes:3389 description =p4_describe(change)3390 self.updateOptionDict(description)33913392if not self.silent:3393 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))3394 sys.stdout.flush()3395 cnt = cnt +133963397try:3398if self.detectBranches:3399 branches = self.splitFilesIntoBranches(description)3400for branch in branches.keys():3401## HACK --hwn3402 branchPrefix = self.depotPaths[0] + branch +"/"3403 self.branchPrefixes = [ branchPrefix ]34043405 parent =""34063407 filesForCommit = branches[branch]34083409if self.verbose:3410print("branch is%s"% branch)34113412 self.updatedBranches.add(branch)34133414if branch not in self.createdBranches:3415 self.createdBranches.add(branch)3416 parent = self.knownBranches[branch]3417if parent == branch:3418 parent =""3419else:3420 fullBranch = self.projectName + branch3421if fullBranch not in self.p4BranchesInGit:3422if not self.silent:3423print("\nImporting new branch%s"% fullBranch);3424if self.importNewBranch(branch, change -1):3425 parent =""3426 self.p4BranchesInGit.append(fullBranch)3427if not self.silent:3428print("\nResuming with change%s"% change);34293430if self.verbose:3431print("parent determined through known branches:%s"% parent)34323433 branch = self.gitRefForBranch(branch)3434 parent = self.gitRefForBranch(parent)34353436if self.verbose:3437print("looking for initial parent for%s; current parent is%s"% (branch, parent))34383439iflen(parent) ==0and branch in self.initialParents:3440 parent = self.initialParents[branch]3441del self.initialParents[branch]34423443 blob =None3444iflen(parent) >0:3445 tempBranch ="%s/%d"% (self.tempBranchLocation, change)3446if self.verbose:3447print("Creating temporary branch: "+ tempBranch)3448 self.commit(description, filesForCommit, tempBranch)3449 self.tempBranches.append(tempBranch)3450 self.checkpoint()3451 blob = self.searchParent(parent, branch, tempBranch)3452if blob:3453 self.commit(description, filesForCommit, branch, blob)3454else:3455if self.verbose:3456print("Parent of%snot found. Committing into head of%s"% (branch, parent))3457 self.commit(description, filesForCommit, branch, parent)3458else:3459 files = self.extractFilesFromCommit(description)3460 self.commit(description, files, self.branch,3461 self.initialParent)3462# only needed once, to connect to the previous commit3463 self.initialParent =""3464exceptIOError:3465print(self.gitError.read())3466 sys.exit(1)34673468defsync_origin_only(self):3469if self.syncWithOrigin:3470 self.hasOrigin =originP4BranchesExist()3471if self.hasOrigin:3472if not self.silent:3473print('Syncing with origin first, using "git fetch origin"')3474system("git fetch origin")34753476defimportHeadRevision(self, revision):3477print("Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch))34783479 details = {}3480 details["user"] ="git perforce import user"3481 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"3482% (' '.join(self.depotPaths), revision))3483 details["change"] = revision3484 newestRevision =034853486 fileCnt =03487 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]34883489for info inp4CmdList(["files"] + fileArgs):34903491if'code'in info and info['code'] =='error':3492 sys.stderr.write("p4 returned an error:%s\n"3493% info['data'])3494if info['data'].find("must refer to client") >=0:3495 sys.stderr.write("This particular p4 error is misleading.\n")3496 sys.stderr.write("Perhaps the depot path was misspelled.\n");3497 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))3498 sys.exit(1)3499if'p4ExitCode'in info:3500 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])3501 sys.exit(1)350235033504 change =int(info["change"])3505if change > newestRevision:3506 newestRevision = change35073508if info["action"]in self.delete_actions:3509# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!3510#fileCnt = fileCnt + 13511continue35123513for prop in["depotFile","rev","action","type"]:3514 details["%s%s"% (prop, fileCnt)] = info[prop]35153516 fileCnt = fileCnt +135173518 details["change"] = newestRevision35193520# Use time from top-most change so that all git p4 clones of3521# the same p4 repo have the same commit SHA1s.3522 res =p4_describe(newestRevision)3523 details["time"] = res["time"]35243525 self.updateOptionDict(details)3526try:3527 self.commit(details, self.extractFilesFromCommit(details), self.branch)3528exceptIOError:3529print("IO error with git fast-import. Is your git version recent enough?")3530print(self.gitError.read())35313532defopenStreams(self):3533 self.importProcess = subprocess.Popen(["git","fast-import"],3534 stdin=subprocess.PIPE,3535 stdout=subprocess.PIPE,3536 stderr=subprocess.PIPE);3537 self.gitOutput = self.importProcess.stdout3538 self.gitStream = self.importProcess.stdin3539 self.gitError = self.importProcess.stderr35403541defcloseStreams(self):3542 self.gitStream.close()3543if self.importProcess.wait() !=0:3544die("fast-import failed:%s"% self.gitError.read())3545 self.gitOutput.close()3546 self.gitError.close()35473548defrun(self, args):3549if self.importIntoRemotes:3550 self.refPrefix ="refs/remotes/p4/"3551else:3552 self.refPrefix ="refs/heads/p4/"35533554 self.sync_origin_only()35553556 branch_arg_given =bool(self.branch)3557iflen(self.branch) ==0:3558 self.branch = self.refPrefix +"master"3559ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:3560system("git update-ref%srefs/heads/p4"% self.branch)3561system("git branch -D p4")35623563# accept either the command-line option, or the configuration variable3564if self.useClientSpec:3565# will use this after clone to set the variable3566 self.useClientSpec_from_options =True3567else:3568ifgitConfigBool("git-p4.useclientspec"):3569 self.useClientSpec =True3570if self.useClientSpec:3571 self.clientSpecDirs =getClientSpec()35723573# TODO: should always look at previous commits,3574# merge with previous imports, if possible.3575if args == []:3576if self.hasOrigin:3577createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)35783579# branches holds mapping from branch name to sha13580 branches =p4BranchesInGit(self.importIntoRemotes)35813582# restrict to just this one, disabling detect-branches3583if branch_arg_given:3584 short = self.branch.split("/")[-1]3585if short in branches:3586 self.p4BranchesInGit = [ short ]3587else:3588 self.p4BranchesInGit = branches.keys()35893590iflen(self.p4BranchesInGit) >1:3591if not self.silent:3592print("Importing from/into multiple branches")3593 self.detectBranches =True3594for branch in branches.keys():3595 self.initialParents[self.refPrefix + branch] = \3596 branches[branch]35973598if self.verbose:3599print("branches:%s"% self.p4BranchesInGit)36003601 p4Change =03602for branch in self.p4BranchesInGit:3603 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)36043605 settings =extractSettingsGitLog(logMsg)36063607 self.readOptions(settings)3608if('depot-paths'in settings3609and'change'in settings):3610 change =int(settings['change']) +13611 p4Change =max(p4Change, change)36123613 depotPaths =sorted(settings['depot-paths'])3614if self.previousDepotPaths == []:3615 self.previousDepotPaths = depotPaths3616else:3617 paths = []3618for(prev, cur)inzip(self.previousDepotPaths, depotPaths):3619 prev_list = prev.split("/")3620 cur_list = cur.split("/")3621for i inrange(0,min(len(cur_list),len(prev_list))):3622if cur_list[i] != prev_list[i]:3623 i = i -13624break36253626 paths.append("/".join(cur_list[:i +1]))36273628 self.previousDepotPaths = paths36293630if p4Change >0:3631 self.depotPaths =sorted(self.previousDepotPaths)3632 self.changeRange ="@%s,#head"% p4Change3633if not self.silent and not self.detectBranches:3634print("Performing incremental import into%sgit branch"% self.branch)36353636# accept multiple ref name abbreviations:3637# refs/foo/bar/branch -> use it exactly3638# p4/branch -> prepend refs/remotes/ or refs/heads/3639# branch -> prepend refs/remotes/p4/ or refs/heads/p4/3640if not self.branch.startswith("refs/"):3641if self.importIntoRemotes:3642 prepend ="refs/remotes/"3643else:3644 prepend ="refs/heads/"3645if not self.branch.startswith("p4/"):3646 prepend +="p4/"3647 self.branch = prepend + self.branch36483649iflen(args) ==0and self.depotPaths:3650if not self.silent:3651print("Depot paths:%s"%' '.join(self.depotPaths))3652else:3653if self.depotPaths and self.depotPaths != args:3654print("previous import used depot path%sand now%swas specified. "3655"This doesn't work!"% (' '.join(self.depotPaths),3656' '.join(args)))3657 sys.exit(1)36583659 self.depotPaths =sorted(args)36603661 revision =""3662 self.users = {}36633664# Make sure no revision specifiers are used when --changesfile3665# is specified.3666 bad_changesfile =False3667iflen(self.changesFile) >0:3668for p in self.depotPaths:3669if p.find("@") >=0or p.find("#") >=0:3670 bad_changesfile =True3671break3672if bad_changesfile:3673die("Option --changesfile is incompatible with revision specifiers")36743675 newPaths = []3676for p in self.depotPaths:3677if p.find("@") != -1:3678 atIdx = p.index("@")3679 self.changeRange = p[atIdx:]3680if self.changeRange =="@all":3681 self.changeRange =""3682elif','not in self.changeRange:3683 revision = self.changeRange3684 self.changeRange =""3685 p = p[:atIdx]3686elif p.find("#") != -1:3687 hashIdx = p.index("#")3688 revision = p[hashIdx:]3689 p = p[:hashIdx]3690elif self.previousDepotPaths == []:3691# pay attention to changesfile, if given, else import3692# the entire p4 tree at the head revision3693iflen(self.changesFile) ==0:3694 revision ="#head"36953696 p = re.sub("\.\.\.$","", p)3697if not p.endswith("/"):3698 p +="/"36993700 newPaths.append(p)37013702 self.depotPaths = newPaths37033704# --detect-branches may change this for each branch3705 self.branchPrefixes = self.depotPaths37063707 self.loadUserMapFromCache()3708 self.labels = {}3709if self.detectLabels:3710 self.getLabels();37113712if self.detectBranches:3713## FIXME - what's a P4 projectName ?3714 self.projectName = self.guessProjectName()37153716if self.hasOrigin:3717 self.getBranchMappingFromGitBranches()3718else:3719 self.getBranchMapping()3720if self.verbose:3721print("p4-git branches:%s"% self.p4BranchesInGit)3722print("initial parents:%s"% self.initialParents)3723for b in self.p4BranchesInGit:3724if b !="master":37253726## FIXME3727 b = b[len(self.projectName):]3728 self.createdBranches.add(b)37293730 self.openStreams()37313732if revision:3733 self.importHeadRevision(revision)3734else:3735 changes = []37363737iflen(self.changesFile) >0:3738 output =open(self.changesFile).readlines()3739 changeSet =set()3740for line in output:3741 changeSet.add(int(line))37423743for change in changeSet:3744 changes.append(change)37453746 changes.sort()3747else:3748# catch "git p4 sync" with no new branches, in a repo that3749# does not have any existing p4 branches3750iflen(args) ==0:3751if not self.p4BranchesInGit:3752die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")37533754# The default branch is master, unless --branch is used to3755# specify something else. Make sure it exists, or complain3756# nicely about how to use --branch.3757if not self.detectBranches:3758if notbranch_exists(self.branch):3759if branch_arg_given:3760die("Error: branch%sdoes not exist."% self.branch)3761else:3762die("Error: no branch%s; perhaps specify one with --branch."%3763 self.branch)37643765if self.verbose:3766print("Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3767 self.changeRange))3768 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)37693770iflen(self.maxChanges) >0:3771 changes = changes[:min(int(self.maxChanges),len(changes))]37723773iflen(changes) ==0:3774if not self.silent:3775print("No changes to import!")3776else:3777if not self.silent and not self.detectBranches:3778print("Import destination:%s"% self.branch)37793780 self.updatedBranches =set()37813782if not self.detectBranches:3783if args:3784# start a new branch3785 self.initialParent =""3786else:3787# build on a previous revision3788 self.initialParent =parseRevision(self.branch)37893790 self.importChanges(changes)37913792if not self.silent:3793print("")3794iflen(self.updatedBranches) >0:3795 sys.stdout.write("Updated branches: ")3796for b in self.updatedBranches:3797 sys.stdout.write("%s"% b)3798 sys.stdout.write("\n")37993800ifgitConfigBool("git-p4.importLabels"):3801 self.importLabels =True38023803if self.importLabels:3804 p4Labels =getP4Labels(self.depotPaths)3805 gitTags =getGitTags()38063807 missingP4Labels = p4Labels - gitTags3808 self.importP4Labels(self.gitStream, missingP4Labels)38093810 self.closeStreams()38113812# Cleanup temporary branches created during import3813if self.tempBranches != []:3814for branch in self.tempBranches:3815read_pipe("git update-ref -d%s"% branch)3816 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))38173818# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3819# a convenient shortcut refname "p4".3820if self.importIntoRemotes:3821 head_ref = self.refPrefix +"HEAD"3822if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3823system(["git","symbolic-ref", head_ref, self.branch])38243825return True38263827classP4Rebase(Command):3828def__init__(self):3829 Command.__init__(self)3830 self.options = [3831 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3832]3833 self.importLabels =False3834 self.description = ("Fetches the latest revision from perforce and "3835+"rebases the current work (branch) against it")38363837defrun(self, args):3838 sync =P4Sync()3839 sync.importLabels = self.importLabels3840 sync.run([])38413842return self.rebase()38433844defrebase(self):3845if os.system("git update-index --refresh") !=0:3846die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up to date or stash away all your changes with git stash.");3847iflen(read_pipe("git diff-index HEAD --")) >0:3848die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");38493850[upstream, settings] =findUpstreamBranchPoint()3851iflen(upstream) ==0:3852die("Cannot find upstream branchpoint for rebase")38533854# the branchpoint may be p4/foo~3, so strip off the parent3855 upstream = re.sub("~[0-9]+$","", upstream)38563857print("Rebasing the current branch onto%s"% upstream)3858 oldHead =read_pipe("git rev-parse HEAD").strip()3859system("git rebase%s"% upstream)3860system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3861return True38623863classP4Clone(P4Sync):3864def__init__(self):3865 P4Sync.__init__(self)3866 self.description ="Creates a new git repository and imports from Perforce into it"3867 self.usage ="usage: %prog [options] //depot/path[@revRange]"3868 self.options += [3869 optparse.make_option("--destination", dest="cloneDestination",3870 action='store', default=None,3871help="where to leave result of the clone"),3872 optparse.make_option("--bare", dest="cloneBare",3873 action="store_true", default=False),3874]3875 self.cloneDestination =None3876 self.needsGit =False3877 self.cloneBare =False38783879defdefaultDestination(self, args):3880## TODO: use common prefix of args?3881 depotPath = args[0]3882 depotDir = re.sub("(@[^@]*)$","", depotPath)3883 depotDir = re.sub("(#[^#]*)$","", depotDir)3884 depotDir = re.sub(r"\.\.\.$","", depotDir)3885 depotDir = re.sub(r"/$","", depotDir)3886return os.path.split(depotDir)[1]38873888defrun(self, args):3889iflen(args) <1:3890return False38913892if self.keepRepoPath and not self.cloneDestination:3893 sys.stderr.write("Must specify destination for --keep-path\n")3894 sys.exit(1)38953896 depotPaths = args38973898if not self.cloneDestination andlen(depotPaths) >1:3899 self.cloneDestination = depotPaths[-1]3900 depotPaths = depotPaths[:-1]39013902for p in depotPaths:3903if not p.startswith("//"):3904 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3905return False39063907if not self.cloneDestination:3908 self.cloneDestination = self.defaultDestination(args)39093910print("Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination))39113912if not os.path.exists(self.cloneDestination):3913 os.makedirs(self.cloneDestination)3914chdir(self.cloneDestination)39153916 init_cmd = ["git","init"]3917if self.cloneBare:3918 init_cmd.append("--bare")3919 retcode = subprocess.call(init_cmd)3920if retcode:3921raiseCalledProcessError(retcode, init_cmd)39223923if not P4Sync.run(self, depotPaths):3924return False39253926# create a master branch and check out a work tree3927ifgitBranchExists(self.branch):3928system(["git","branch","master", self.branch ])3929if not self.cloneBare:3930system(["git","checkout","-f"])3931else:3932print('Not checking out any branch, use ' \3933'"git checkout -q -b master <branch>"')39343935# auto-set this variable if invoked with --use-client-spec3936if self.useClientSpec_from_options:3937system("git config --bool git-p4.useclientspec true")39383939return True39403941classP4Unshelve(Command):3942def__init__(self):3943 Command.__init__(self)3944 self.options = []3945 self.origin ="HEAD"3946 self.description ="Unshelve a P4 changelist into a git commit"3947 self.usage ="usage: %prog [options] changelist"3948 self.options += [3949 optparse.make_option("--origin", dest="origin",3950help="Use this base revision instead of the default (%s)"% self.origin),3951]3952 self.verbose =False3953 self.noCommit =False3954 self.destbranch ="refs/remotes/p4-unshelved"39553956defrenameBranch(self, branch_name):3957""" Rename the existing branch to branch_name.N3958 """39593960 found =True3961for i inrange(0,1000):3962 backup_branch_name ="{0}.{1}".format(branch_name, i)3963if notgitBranchExists(backup_branch_name):3964gitUpdateRef(backup_branch_name, branch_name)# copy ref to backup3965gitDeleteRef(branch_name)3966 found =True3967print("renamed old unshelve branch to{0}".format(backup_branch_name))3968break39693970if not found:3971 sys.exit("gave up trying to rename existing branch{0}".format(sync.branch))39723973deffindLastP4Revision(self, starting_point):3974""" Look back from starting_point for the first commit created by git-p43975 to find the P4 commit we are based on, and the depot-paths.3976 """39773978for parent in(range(65535)):3979 log =extractLogMessageFromGitCommit("{0}^{1}".format(starting_point, parent))3980 settings =extractSettingsGitLog(log)3981if'change'in settings:3982return settings39833984 sys.exit("could not find git-p4 commits in{0}".format(self.origin))39853986defcreateShelveParent(self, change, branch_name, sync, origin):3987""" Create a commit matching the parent of the shelved changelist 'change'3988 """3989 parent_description =p4_describe(change, shelved=True)3990 parent_description['desc'] ='parent for shelved changelist {}\n'.format(change)3991 files = sync.extractFilesFromCommit(parent_description, shelved=False, shelved_cl=change)39923993 parent_files = []3994for f in files:3995# if it was added in the shelved changelist, it won't exist in the parent3996if f['action']in self.add_actions:3997continue39983999# if it was deleted in the shelved changelist it must not be deleted4000# in the parent - we might even need to create it if the origin branch4001# does not have it4002if f['action']in self.delete_actions:4003 f['action'] ='add'40044005 parent_files.append(f)40064007 sync.commit(parent_description, parent_files, branch_name,4008 parent=origin, allow_empty=True)4009print("created parent commit for{0}based on{1}in{2}".format(4010 change, self.origin, branch_name))40114012defrun(self, args):4013iflen(args) !=1:4014return False40154016if notgitBranchExists(self.origin):4017 sys.exit("origin branch{0}does not exist".format(self.origin))40184019 sync =P4Sync()4020 changes = args40214022# only one change at a time4023 change = changes[0]40244025# if the target branch already exists, rename it4026 branch_name ="{0}/{1}".format(self.destbranch, change)4027ifgitBranchExists(branch_name):4028 self.renameBranch(branch_name)4029 sync.branch = branch_name40304031 sync.verbose = self.verbose4032 sync.suppress_meta_comment =True40334034 settings = self.findLastP4Revision(self.origin)4035 sync.depotPaths = settings['depot-paths']4036 sync.branchPrefixes = sync.depotPaths40374038 sync.openStreams()4039 sync.loadUserMapFromCache()4040 sync.silent =True40414042# create a commit for the parent of the shelved changelist4043 self.createShelveParent(change, branch_name, sync, self.origin)40444045# create the commit for the shelved changelist itself4046 description =p4_describe(change,True)4047 files = sync.extractFilesFromCommit(description,True, change)40484049 sync.commit(description, files, branch_name,"")4050 sync.closeStreams()40514052print("unshelved changelist{0}into{1}".format(change, branch_name))40534054return True40554056classP4Branches(Command):4057def__init__(self):4058 Command.__init__(self)4059 self.options = [ ]4060 self.description = ("Shows the git branches that hold imports and their "4061+"corresponding perforce depot paths")4062 self.verbose =False40634064defrun(self, args):4065iforiginP4BranchesExist():4066createOrUpdateBranchesFromOrigin()40674068 cmdline ="git rev-parse --symbolic "4069 cmdline +=" --remotes"40704071for line inread_pipe_lines(cmdline):4072 line = line.strip()40734074if not line.startswith('p4/')or line =="p4/HEAD":4075continue4076 branch = line40774078 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)4079 settings =extractSettingsGitLog(log)40804081print("%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"]))4082return True40834084classHelpFormatter(optparse.IndentedHelpFormatter):4085def__init__(self):4086 optparse.IndentedHelpFormatter.__init__(self)40874088defformat_description(self, description):4089if description:4090return description +"\n"4091else:4092return""40934094defprintUsage(commands):4095print("usage:%s<command> [options]"% sys.argv[0])4096print("")4097print("valid commands:%s"%", ".join(commands))4098print("")4099print("Try%s<command> --help for command specific help."% sys.argv[0])4100print("")41014102commands = {4103"debug": P4Debug,4104"submit": P4Submit,4105"commit": P4Submit,4106"sync": P4Sync,4107"rebase": P4Rebase,4108"clone": P4Clone,4109"rollback": P4RollBack,4110"branches": P4Branches,4111"unshelve": P4Unshelve,4112}411341144115defmain():4116iflen(sys.argv[1:]) ==0:4117printUsage(commands.keys())4118 sys.exit(2)41194120 cmdName = sys.argv[1]4121try:4122 klass = commands[cmdName]4123 cmd =klass()4124exceptKeyError:4125print("unknown command%s"% cmdName)4126print("")4127printUsage(commands.keys())4128 sys.exit(2)41294130 options = cmd.options4131 cmd.gitdir = os.environ.get("GIT_DIR",None)41324133 args = sys.argv[2:]41344135 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))4136if cmd.needsGit:4137 options.append(optparse.make_option("--git-dir", dest="gitdir"))41384139 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),4140 options,4141 description = cmd.description,4142 formatter =HelpFormatter())41434144(cmd, args) = parser.parse_args(sys.argv[2:], cmd);4145global verbose4146 verbose = cmd.verbose4147if cmd.needsGit:4148if cmd.gitdir ==None:4149 cmd.gitdir = os.path.abspath(".git")4150if notisValidGitDir(cmd.gitdir):4151# "rev-parse --git-dir" without arguments will try $PWD/.git4152 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()4153if os.path.exists(cmd.gitdir):4154 cdup =read_pipe("git rev-parse --show-cdup").strip()4155iflen(cdup) >0:4156chdir(cdup);41574158if notisValidGitDir(cmd.gitdir):4159ifisValidGitDir(cmd.gitdir +"/.git"):4160 cmd.gitdir +="/.git"4161else:4162die("fatal: cannot locate git repository at%s"% cmd.gitdir)41634164# so git commands invoked from the P4 workspace will succeed4165 os.environ["GIT_DIR"] = cmd.gitdir41664167if not cmd.run(args):4168 parser.print_help()4169 sys.exit(2)417041714172if __name__ =='__main__':4173main()