1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25import zipfile 26import zlib 27import ctypes 28 29try: 30from subprocess import CalledProcessError 31exceptImportError: 32# from python2.7:subprocess.py 33# Exception classes used by this module. 34classCalledProcessError(Exception): 35"""This exception is raised when a process run by check_call() returns 36 a non-zero exit status. The exit status will be stored in the 37 returncode attribute.""" 38def__init__(self, returncode, cmd): 39 self.returncode = returncode 40 self.cmd = cmd 41def__str__(self): 42return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 43 44verbose =False 45 46# Only labels/tags matching this will be imported/exported 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 48 49# Grab changes in blocks of this many revisions, unless otherwise requested 50defaultBlockSize =512 51 52defp4_build_cmd(cmd): 53"""Build a suitable p4 command line. 54 55 This consolidates building and returning a p4 command line into one 56 location. It means that hooking into the environment, or other configuration 57 can be done more easily. 58 """ 59 real_cmd = ["p4"] 60 61 user =gitConfig("git-p4.user") 62iflen(user) >0: 63 real_cmd += ["-u",user] 64 65 password =gitConfig("git-p4.password") 66iflen(password) >0: 67 real_cmd += ["-P", password] 68 69 port =gitConfig("git-p4.port") 70iflen(port) >0: 71 real_cmd += ["-p", port] 72 73 host =gitConfig("git-p4.host") 74iflen(host) >0: 75 real_cmd += ["-H", host] 76 77 client =gitConfig("git-p4.client") 78iflen(client) >0: 79 real_cmd += ["-c", client] 80 81 82ifisinstance(cmd,basestring): 83 real_cmd =' '.join(real_cmd) +' '+ cmd 84else: 85 real_cmd += cmd 86return real_cmd 87 88defchdir(path, is_client_path=False): 89"""Do chdir to the given path, and set the PWD environment 90 variable for use by P4. It does not look at getcwd() output. 91 Since we're not using the shell, it is necessary to set the 92 PWD environment variable explicitly. 93 94 Normally, expand the path to force it to be absolute. This 95 addresses the use of relative path names inside P4 settings, 96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 97 as given; it looks for .p4config using PWD. 98 99 If is_client_path, the path was handed to us directly by p4, 100 and may be a symbolic link. Do not call os.getcwd() in this 101 case, because it will cause p4 to think that PWD is not inside 102 the client path. 103 """ 104 105 os.chdir(path) 106if not is_client_path: 107 path = os.getcwd() 108 os.environ['PWD'] = path 109 110defcalcDiskFree(): 111"""Return free space in bytes on the disk of the given dirname.""" 112if platform.system() =='Windows': 113 free_bytes = ctypes.c_ulonglong(0) 114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()),None,None, ctypes.pointer(free_bytes)) 115return free_bytes.value 116else: 117 st = os.statvfs(os.getcwd()) 118return st.f_bavail * st.f_frsize 119 120defdie(msg): 121if verbose: 122raiseException(msg) 123else: 124 sys.stderr.write(msg +"\n") 125 sys.exit(1) 126 127defwrite_pipe(c, stdin): 128if verbose: 129 sys.stderr.write('Writing pipe:%s\n'%str(c)) 130 131 expand =isinstance(c,basestring) 132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 133 pipe = p.stdin 134 val = pipe.write(stdin) 135 pipe.close() 136if p.wait(): 137die('Command failed:%s'%str(c)) 138 139return val 140 141defp4_write_pipe(c, stdin): 142 real_cmd =p4_build_cmd(c) 143returnwrite_pipe(real_cmd, stdin) 144 145defread_pipe(c, ignore_error=False): 146if verbose: 147 sys.stderr.write('Reading pipe:%s\n'%str(c)) 148 149 expand =isinstance(c,basestring) 150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) 151(out, err) = p.communicate() 152if p.returncode !=0and not ignore_error: 153die('Command failed:%s\nError:%s'% (str(c), err)) 154return out 155 156defp4_read_pipe(c, ignore_error=False): 157 real_cmd =p4_build_cmd(c) 158returnread_pipe(real_cmd, ignore_error) 159 160defread_pipe_lines(c): 161if verbose: 162 sys.stderr.write('Reading pipe:%s\n'%str(c)) 163 164 expand =isinstance(c, basestring) 165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 166 pipe = p.stdout 167 val = pipe.readlines() 168if pipe.close()or p.wait(): 169die('Command failed:%s'%str(c)) 170 171return val 172 173defp4_read_pipe_lines(c): 174"""Specifically invoke p4 on the command supplied. """ 175 real_cmd =p4_build_cmd(c) 176returnread_pipe_lines(real_cmd) 177 178defp4_has_command(cmd): 179"""Ask p4 for help on this command. If it returns an error, the 180 command does not exist in this version of p4.""" 181 real_cmd =p4_build_cmd(["help", cmd]) 182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 183 stderr=subprocess.PIPE) 184 p.communicate() 185return p.returncode ==0 186 187defp4_has_move_command(): 188"""See if the move command exists, that it supports -k, and that 189 it has not been administratively disabled. The arguments 190 must be correct, but the filenames do not have to exist. Use 191 ones with wildcards so even if they exist, it will fail.""" 192 193if notp4_has_command("move"): 194return False 195 cmd =p4_build_cmd(["move","-k","@from","@to"]) 196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 197(out, err) = p.communicate() 198# return code will be 1 in either case 199if err.find("Invalid option") >=0: 200return False 201if err.find("disabled") >=0: 202return False 203# assume it failed because @... was invalid changelist 204return True 205 206defsystem(cmd, ignore_error=False): 207 expand =isinstance(cmd,basestring) 208if verbose: 209 sys.stderr.write("executing%s\n"%str(cmd)) 210 retcode = subprocess.call(cmd, shell=expand) 211if retcode and not ignore_error: 212raiseCalledProcessError(retcode, cmd) 213 214return retcode 215 216defp4_system(cmd): 217"""Specifically invoke p4 as the system command. """ 218 real_cmd =p4_build_cmd(cmd) 219 expand =isinstance(real_cmd, basestring) 220 retcode = subprocess.call(real_cmd, shell=expand) 221if retcode: 222raiseCalledProcessError(retcode, real_cmd) 223 224_p4_version_string =None 225defp4_version_string(): 226"""Read the version string, showing just the last line, which 227 hopefully is the interesting version bit. 228 229 $ p4 -V 230 Perforce - The Fast Software Configuration Management System. 231 Copyright 1995-2011 Perforce Software. All rights reserved. 232 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 233 """ 234global _p4_version_string 235if not _p4_version_string: 236 a =p4_read_pipe_lines(["-V"]) 237 _p4_version_string = a[-1].rstrip() 238return _p4_version_string 239 240defp4_integrate(src, dest): 241p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 242 243defp4_sync(f, *options): 244p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 245 246defp4_add(f): 247# forcibly add file names with wildcards 248ifwildcard_present(f): 249p4_system(["add","-f", f]) 250else: 251p4_system(["add", f]) 252 253defp4_delete(f): 254p4_system(["delete",wildcard_encode(f)]) 255 256defp4_edit(f, *options): 257p4_system(["edit"] +list(options) + [wildcard_encode(f)]) 258 259defp4_revert(f): 260p4_system(["revert",wildcard_encode(f)]) 261 262defp4_reopen(type, f): 263p4_system(["reopen","-t",type,wildcard_encode(f)]) 264 265defp4_reopen_in_change(changelist, files): 266 cmd = ["reopen","-c",str(changelist)] + files 267p4_system(cmd) 268 269defp4_move(src, dest): 270p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 271 272defp4_last_change(): 273 results =p4CmdList(["changes","-m","1"]) 274returnint(results[0]['change']) 275 276defp4_describe(change): 277"""Make sure it returns a valid result by checking for 278 the presence of field "time". Return a dict of the 279 results.""" 280 281 ds =p4CmdList(["describe","-s",str(change)]) 282iflen(ds) !=1: 283die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 284 285 d = ds[0] 286 287if"p4ExitCode"in d: 288die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 289str(d))) 290if"code"in d: 291if d["code"] =="error": 292die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 293 294if"time"not in d: 295die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 296 297return d 298 299# 300# Canonicalize the p4 type and return a tuple of the 301# base type, plus any modifiers. See "p4 help filetypes" 302# for a list and explanation. 303# 304defsplit_p4_type(p4type): 305 306 p4_filetypes_historical = { 307"ctempobj":"binary+Sw", 308"ctext":"text+C", 309"cxtext":"text+Cx", 310"ktext":"text+k", 311"kxtext":"text+kx", 312"ltext":"text+F", 313"tempobj":"binary+FSw", 314"ubinary":"binary+F", 315"uresource":"resource+F", 316"uxbinary":"binary+Fx", 317"xbinary":"binary+x", 318"xltext":"text+Fx", 319"xtempobj":"binary+Swx", 320"xtext":"text+x", 321"xunicode":"unicode+x", 322"xutf16":"utf16+x", 323} 324if p4type in p4_filetypes_historical: 325 p4type = p4_filetypes_historical[p4type] 326 mods ="" 327 s = p4type.split("+") 328 base = s[0] 329 mods ="" 330iflen(s) >1: 331 mods = s[1] 332return(base, mods) 333 334# 335# return the raw p4 type of a file (text, text+ko, etc) 336# 337defp4_type(f): 338 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 339return results[0]['headType'] 340 341# 342# Given a type base and modifier, return a regexp matching 343# the keywords that can be expanded in the file 344# 345defp4_keywords_regexp_for_type(base, type_mods): 346if base in("text","unicode","binary"): 347 kwords =None 348if"ko"in type_mods: 349 kwords ='Id|Header' 350elif"k"in type_mods: 351 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 352else: 353return None 354 pattern = r""" 355 \$ # Starts with a dollar, followed by... 356 (%s) # one of the keywords, followed by... 357 (:[^$\n]+)? # possibly an old expansion, followed by... 358 \$ # another dollar 359 """% kwords 360return pattern 361else: 362return None 363 364# 365# Given a file, return a regexp matching the possible 366# RCS keywords that will be expanded, or None for files 367# with kw expansion turned off. 368# 369defp4_keywords_regexp_for_file(file): 370if not os.path.exists(file): 371return None 372else: 373(type_base, type_mods) =split_p4_type(p4_type(file)) 374returnp4_keywords_regexp_for_type(type_base, type_mods) 375 376defsetP4ExecBit(file, mode): 377# Reopens an already open file and changes the execute bit to match 378# the execute bit setting in the passed in mode. 379 380 p4Type ="+x" 381 382if notisModeExec(mode): 383 p4Type =getP4OpenedType(file) 384 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 385 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 386if p4Type[-1] =="+": 387 p4Type = p4Type[0:-1] 388 389p4_reopen(p4Type,file) 390 391defgetP4OpenedType(file): 392# Returns the perforce file type for the given file. 393 394 result =p4_read_pipe(["opened",wildcard_encode(file)]) 395 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 396if match: 397return match.group(1) 398else: 399die("Could not determine file type for%s(result: '%s')"% (file, result)) 400 401# Return the set of all p4 labels 402defgetP4Labels(depotPaths): 403 labels =set() 404ifisinstance(depotPaths,basestring): 405 depotPaths = [depotPaths] 406 407for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 408 label = l['label'] 409 labels.add(label) 410 411return labels 412 413# Return the set of all git tags 414defgetGitTags(): 415 gitTags =set() 416for line inread_pipe_lines(["git","tag"]): 417 tag = line.strip() 418 gitTags.add(tag) 419return gitTags 420 421defdiffTreePattern(): 422# This is a simple generator for the diff tree regex pattern. This could be 423# a class variable if this and parseDiffTreeEntry were a part of a class. 424 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 425while True: 426yield pattern 427 428defparseDiffTreeEntry(entry): 429"""Parses a single diff tree entry into its component elements. 430 431 See git-diff-tree(1) manpage for details about the format of the diff 432 output. This method returns a dictionary with the following elements: 433 434 src_mode - The mode of the source file 435 dst_mode - The mode of the destination file 436 src_sha1 - The sha1 for the source file 437 dst_sha1 - The sha1 fr the destination file 438 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 439 status_score - The score for the status (applicable for 'C' and 'R' 440 statuses). This is None if there is no score. 441 src - The path for the source file. 442 dst - The path for the destination file. This is only present for 443 copy or renames. If it is not present, this is None. 444 445 If the pattern is not matched, None is returned.""" 446 447 match =diffTreePattern().next().match(entry) 448if match: 449return{ 450'src_mode': match.group(1), 451'dst_mode': match.group(2), 452'src_sha1': match.group(3), 453'dst_sha1': match.group(4), 454'status': match.group(5), 455'status_score': match.group(6), 456'src': match.group(7), 457'dst': match.group(10) 458} 459return None 460 461defisModeExec(mode): 462# Returns True if the given git mode represents an executable file, 463# otherwise False. 464return mode[-3:] =="755" 465 466defisModeExecChanged(src_mode, dst_mode): 467returnisModeExec(src_mode) !=isModeExec(dst_mode) 468 469defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 470 471ifisinstance(cmd,basestring): 472 cmd ="-G "+ cmd 473 expand =True 474else: 475 cmd = ["-G"] + cmd 476 expand =False 477 478 cmd =p4_build_cmd(cmd) 479if verbose: 480 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 481 482# Use a temporary file to avoid deadlocks without 483# subprocess.communicate(), which would put another copy 484# of stdout into memory. 485 stdin_file =None 486if stdin is not None: 487 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 488ifisinstance(stdin,basestring): 489 stdin_file.write(stdin) 490else: 491for i in stdin: 492 stdin_file.write(i +'\n') 493 stdin_file.flush() 494 stdin_file.seek(0) 495 496 p4 = subprocess.Popen(cmd, 497 shell=expand, 498 stdin=stdin_file, 499 stdout=subprocess.PIPE) 500 501 result = [] 502try: 503while True: 504 entry = marshal.load(p4.stdout) 505if cb is not None: 506cb(entry) 507else: 508 result.append(entry) 509exceptEOFError: 510pass 511 exitCode = p4.wait() 512if exitCode !=0: 513 entry = {} 514 entry["p4ExitCode"] = exitCode 515 result.append(entry) 516 517return result 518 519defp4Cmd(cmd): 520list=p4CmdList(cmd) 521 result = {} 522for entry inlist: 523 result.update(entry) 524return result; 525 526defp4Where(depotPath): 527if not depotPath.endswith("/"): 528 depotPath +="/" 529 depotPathLong = depotPath +"..." 530 outputList =p4CmdList(["where", depotPathLong]) 531 output =None 532for entry in outputList: 533if"depotFile"in entry: 534# Search for the base client side depot path, as long as it starts with the branch's P4 path. 535# The base path always ends with "/...". 536if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 537 output = entry 538break 539elif"data"in entry: 540 data = entry.get("data") 541 space = data.find(" ") 542if data[:space] == depotPath: 543 output = entry 544break 545if output ==None: 546return"" 547if output["code"] =="error": 548return"" 549 clientPath ="" 550if"path"in output: 551 clientPath = output.get("path") 552elif"data"in output: 553 data = output.get("data") 554 lastSpace = data.rfind(" ") 555 clientPath = data[lastSpace +1:] 556 557if clientPath.endswith("..."): 558 clientPath = clientPath[:-3] 559return clientPath 560 561defcurrentGitBranch(): 562 retcode =system(["git","symbolic-ref","-q","HEAD"], ignore_error=True) 563if retcode !=0: 564# on a detached head 565return None 566else: 567returnread_pipe(["git","name-rev","HEAD"]).split(" ")[1].strip() 568 569defisValidGitDir(path): 570if(os.path.exists(path +"/HEAD") 571and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 572return True; 573return False 574 575defparseRevision(ref): 576returnread_pipe("git rev-parse%s"% ref).strip() 577 578defbranchExists(ref): 579 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 580 ignore_error=True) 581returnlen(rev) >0 582 583defextractLogMessageFromGitCommit(commit): 584 logMessage ="" 585 586## fixme: title is first line of commit, not 1st paragraph. 587 foundTitle =False 588for log inread_pipe_lines("git cat-file commit%s"% commit): 589if not foundTitle: 590iflen(log) ==1: 591 foundTitle =True 592continue 593 594 logMessage += log 595return logMessage 596 597defextractSettingsGitLog(log): 598 values = {} 599for line in log.split("\n"): 600 line = line.strip() 601 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 602if not m: 603continue 604 605 assignments = m.group(1).split(':') 606for a in assignments: 607 vals = a.split('=') 608 key = vals[0].strip() 609 val = ('='.join(vals[1:])).strip() 610if val.endswith('\"')and val.startswith('"'): 611 val = val[1:-1] 612 613 values[key] = val 614 615 paths = values.get("depot-paths") 616if not paths: 617 paths = values.get("depot-path") 618if paths: 619 values['depot-paths'] = paths.split(',') 620return values 621 622defgitBranchExists(branch): 623 proc = subprocess.Popen(["git","rev-parse", branch], 624 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 625return proc.wait() ==0; 626 627_gitConfig = {} 628 629defgitConfig(key, typeSpecifier=None): 630if not _gitConfig.has_key(key): 631 cmd = ["git","config"] 632if typeSpecifier: 633 cmd += [ typeSpecifier ] 634 cmd += [ key ] 635 s =read_pipe(cmd, ignore_error=True) 636 _gitConfig[key] = s.strip() 637return _gitConfig[key] 638 639defgitConfigBool(key): 640"""Return a bool, using git config --bool. It is True only if the 641 variable is set to true, and False if set to false or not present 642 in the config.""" 643 644if not _gitConfig.has_key(key): 645 _gitConfig[key] =gitConfig(key,'--bool') =="true" 646return _gitConfig[key] 647 648defgitConfigInt(key): 649if not _gitConfig.has_key(key): 650 cmd = ["git","config","--int", key ] 651 s =read_pipe(cmd, ignore_error=True) 652 v = s.strip() 653try: 654 _gitConfig[key] =int(gitConfig(key,'--int')) 655exceptValueError: 656 _gitConfig[key] =None 657return _gitConfig[key] 658 659defgitConfigList(key): 660if not _gitConfig.has_key(key): 661 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 662 _gitConfig[key] = s.strip().split(os.linesep) 663if _gitConfig[key] == ['']: 664 _gitConfig[key] = [] 665return _gitConfig[key] 666 667defp4BranchesInGit(branchesAreInRemotes=True): 668"""Find all the branches whose names start with "p4/", looking 669 in remotes or heads as specified by the argument. Return 670 a dictionary of{ branch: revision }for each one found. 671 The branch names are the short names, without any 672 "p4/" prefix.""" 673 674 branches = {} 675 676 cmdline ="git rev-parse --symbolic " 677if branchesAreInRemotes: 678 cmdline +="--remotes" 679else: 680 cmdline +="--branches" 681 682for line inread_pipe_lines(cmdline): 683 line = line.strip() 684 685# only import to p4/ 686if not line.startswith('p4/'): 687continue 688# special symbolic ref to p4/master 689if line =="p4/HEAD": 690continue 691 692# strip off p4/ prefix 693 branch = line[len("p4/"):] 694 695 branches[branch] =parseRevision(line) 696 697return branches 698 699defbranch_exists(branch): 700"""Make sure that the given ref name really exists.""" 701 702 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 703 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 704 out, _ = p.communicate() 705if p.returncode: 706return False 707# expect exactly one line of output: the branch name 708return out.rstrip() == branch 709 710deffindUpstreamBranchPoint(head ="HEAD"): 711 branches =p4BranchesInGit() 712# map from depot-path to branch name 713 branchByDepotPath = {} 714for branch in branches.keys(): 715 tip = branches[branch] 716 log =extractLogMessageFromGitCommit(tip) 717 settings =extractSettingsGitLog(log) 718if settings.has_key("depot-paths"): 719 paths =",".join(settings["depot-paths"]) 720 branchByDepotPath[paths] ="remotes/p4/"+ branch 721 722 settings =None 723 parent =0 724while parent <65535: 725 commit = head +"~%s"% parent 726 log =extractLogMessageFromGitCommit(commit) 727 settings =extractSettingsGitLog(log) 728if settings.has_key("depot-paths"): 729 paths =",".join(settings["depot-paths"]) 730if branchByDepotPath.has_key(paths): 731return[branchByDepotPath[paths], settings] 732 733 parent = parent +1 734 735return["", settings] 736 737defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 738if not silent: 739print("Creating/updating branch(es) in%sbased on origin branch(es)" 740% localRefPrefix) 741 742 originPrefix ="origin/p4/" 743 744for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 745 line = line.strip() 746if(not line.startswith(originPrefix))or line.endswith("HEAD"): 747continue 748 749 headName = line[len(originPrefix):] 750 remoteHead = localRefPrefix + headName 751 originHead = line 752 753 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 754if(not original.has_key('depot-paths') 755or not original.has_key('change')): 756continue 757 758 update =False 759if notgitBranchExists(remoteHead): 760if verbose: 761print"creating%s"% remoteHead 762 update =True 763else: 764 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 765if settings.has_key('change') >0: 766if settings['depot-paths'] == original['depot-paths']: 767 originP4Change =int(original['change']) 768 p4Change =int(settings['change']) 769if originP4Change > p4Change: 770print("%s(%s) is newer than%s(%s). " 771"Updating p4 branch from origin." 772% (originHead, originP4Change, 773 remoteHead, p4Change)) 774 update =True 775else: 776print("Ignoring:%swas imported from%swhile " 777"%swas imported from%s" 778% (originHead,','.join(original['depot-paths']), 779 remoteHead,','.join(settings['depot-paths']))) 780 781if update: 782system("git update-ref%s %s"% (remoteHead, originHead)) 783 784deforiginP4BranchesExist(): 785returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 786 787 788defp4ParseNumericChangeRange(parts): 789 changeStart =int(parts[0][1:]) 790if parts[1] =='#head': 791 changeEnd =p4_last_change() 792else: 793 changeEnd =int(parts[1]) 794 795return(changeStart, changeEnd) 796 797defchooseBlockSize(blockSize): 798if blockSize: 799return blockSize 800else: 801return defaultBlockSize 802 803defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 804assert depotPaths 805 806# Parse the change range into start and end. Try to find integer 807# revision ranges as these can be broken up into blocks to avoid 808# hitting server-side limits (maxrows, maxscanresults). But if 809# that doesn't work, fall back to using the raw revision specifier 810# strings, without using block mode. 811 812if changeRange is None or changeRange =='': 813 changeStart =1 814 changeEnd =p4_last_change() 815 block_size =chooseBlockSize(requestedBlockSize) 816else: 817 parts = changeRange.split(',') 818assertlen(parts) ==2 819try: 820(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 821 block_size =chooseBlockSize(requestedBlockSize) 822except: 823 changeStart = parts[0][1:] 824 changeEnd = parts[1] 825if requestedBlockSize: 826die("cannot use --changes-block-size with non-numeric revisions") 827 block_size =None 828 829 changes = [] 830 831# Retrieve changes a block at a time, to prevent running 832# into a MaxResults/MaxScanRows error from the server. 833 834while True: 835 cmd = ['changes'] 836 837if block_size: 838 end =min(changeEnd, changeStart + block_size) 839 revisionRange ="%d,%d"% (changeStart, end) 840else: 841 revisionRange ="%s,%s"% (changeStart, changeEnd) 842 843for p in depotPaths: 844 cmd += ["%s...@%s"% (p, revisionRange)] 845 846# Insert changes in chronological order 847for line inreversed(p4_read_pipe_lines(cmd)): 848 changes.append(int(line.split(" ")[1])) 849 850if not block_size: 851break 852 853if end >= changeEnd: 854break 855 856 changeStart = end +1 857 858 changes =sorted(changes) 859return changes 860 861defp4PathStartsWith(path, prefix): 862# This method tries to remedy a potential mixed-case issue: 863# 864# If UserA adds //depot/DirA/file1 865# and UserB adds //depot/dira/file2 866# 867# we may or may not have a problem. If you have core.ignorecase=true, 868# we treat DirA and dira as the same directory 869ifgitConfigBool("core.ignorecase"): 870return path.lower().startswith(prefix.lower()) 871return path.startswith(prefix) 872 873defgetClientSpec(): 874"""Look at the p4 client spec, create a View() object that contains 875 all the mappings, and return it.""" 876 877 specList =p4CmdList("client -o") 878iflen(specList) !=1: 879die('Output from "client -o" is%dlines, expecting 1'% 880len(specList)) 881 882# dictionary of all client parameters 883 entry = specList[0] 884 885# the //client/ name 886 client_name = entry["Client"] 887 888# just the keys that start with "View" 889 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 890 891# hold this new View 892 view =View(client_name) 893 894# append the lines, in order, to the view 895for view_num inrange(len(view_keys)): 896 k ="View%d"% view_num 897if k not in view_keys: 898die("Expected view key%smissing"% k) 899 view.append(entry[k]) 900 901return view 902 903defgetClientRoot(): 904"""Grab the client directory.""" 905 906 output =p4CmdList("client -o") 907iflen(output) !=1: 908die('Output from "client -o" is%dlines, expecting 1'%len(output)) 909 910 entry = output[0] 911if"Root"not in entry: 912die('Client has no "Root"') 913 914return entry["Root"] 915 916# 917# P4 wildcards are not allowed in filenames. P4 complains 918# if you simply add them, but you can force it with "-f", in 919# which case it translates them into %xx encoding internally. 920# 921defwildcard_decode(path): 922# Search for and fix just these four characters. Do % last so 923# that fixing it does not inadvertently create new %-escapes. 924# Cannot have * in a filename in windows; untested as to 925# what p4 would do in such a case. 926if not platform.system() =="Windows": 927 path = path.replace("%2A","*") 928 path = path.replace("%23","#") \ 929.replace("%40","@") \ 930.replace("%25","%") 931return path 932 933defwildcard_encode(path): 934# do % first to avoid double-encoding the %s introduced here 935 path = path.replace("%","%25") \ 936.replace("*","%2A") \ 937.replace("#","%23") \ 938.replace("@","%40") 939return path 940 941defwildcard_present(path): 942 m = re.search("[*#@%]", path) 943return m is not None 944 945classLargeFileSystem(object): 946"""Base class for large file system support.""" 947 948def__init__(self, writeToGitStream): 949 self.largeFiles =set() 950 self.writeToGitStream = writeToGitStream 951 952defgeneratePointer(self, cloneDestination, contentFile): 953"""Return the content of a pointer file that is stored in Git instead of 954 the actual content.""" 955assert False,"Method 'generatePointer' required in "+ self.__class__.__name__ 956 957defpushFile(self, localLargeFile): 958"""Push the actual content which is not stored in the Git repository to 959 a server.""" 960assert False,"Method 'pushFile' required in "+ self.__class__.__name__ 961 962defhasLargeFileExtension(self, relPath): 963returnreduce( 964lambda a, b: a or b, 965[relPath.endswith('.'+ e)for e ingitConfigList('git-p4.largeFileExtensions')], 966False 967) 968 969defgenerateTempFile(self, contents): 970 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 971for d in contents: 972 contentFile.write(d) 973 contentFile.close() 974return contentFile.name 975 976defexceedsLargeFileThreshold(self, relPath, contents): 977ifgitConfigInt('git-p4.largeFileThreshold'): 978 contentsSize =sum(len(d)for d in contents) 979if contentsSize >gitConfigInt('git-p4.largeFileThreshold'): 980return True 981ifgitConfigInt('git-p4.largeFileCompressedThreshold'): 982 contentsSize =sum(len(d)for d in contents) 983if contentsSize <=gitConfigInt('git-p4.largeFileCompressedThreshold'): 984return False 985 contentTempFile = self.generateTempFile(contents) 986 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 987 zf = zipfile.ZipFile(compressedContentFile.name, mode='w') 988 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) 989 zf.close() 990 compressedContentsSize = zf.infolist()[0].compress_size 991 os.remove(contentTempFile) 992 os.remove(compressedContentFile.name) 993if compressedContentsSize >gitConfigInt('git-p4.largeFileCompressedThreshold'): 994return True 995return False 996 997defaddLargeFile(self, relPath): 998 self.largeFiles.add(relPath) 9991000defremoveLargeFile(self, relPath):1001 self.largeFiles.remove(relPath)10021003defisLargeFile(self, relPath):1004return relPath in self.largeFiles10051006defprocessContent(self, git_mode, relPath, contents):1007"""Processes the content of git fast import. This method decides if a1008 file is stored in the large file system and handles all necessary1009 steps."""1010if self.exceedsLargeFileThreshold(relPath, contents)or self.hasLargeFileExtension(relPath):1011 contentTempFile = self.generateTempFile(contents)1012(git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)10131014# Move temp file to final location in large file system1015 largeFileDir = os.path.dirname(localLargeFile)1016if not os.path.isdir(largeFileDir):1017 os.makedirs(largeFileDir)1018 shutil.move(contentTempFile, localLargeFile)1019 self.addLargeFile(relPath)1020ifgitConfigBool('git-p4.largeFilePush'):1021 self.pushFile(localLargeFile)1022if verbose:1023 sys.stderr.write("%smoved to large file system (%s)\n"% (relPath, localLargeFile))1024return(git_mode, contents)10251026classMockLFS(LargeFileSystem):1027"""Mock large file system for testing."""10281029defgeneratePointer(self, contentFile):1030"""The pointer content is the original content prefixed with "pointer-".1031 The local filename of the large file storage is derived from the file content.1032 """1033withopen(contentFile,'r')as f:1034 content =next(f)1035 gitMode ='100644'1036 pointerContents ='pointer-'+ content1037 localLargeFile = os.path.join(os.getcwd(),'.git','mock-storage','local', content[:-1])1038return(gitMode, pointerContents, localLargeFile)10391040defpushFile(self, localLargeFile):1041"""The remote filename of the large file storage is the same as the local1042 one but in a different directory.1043 """1044 remotePath = os.path.join(os.path.dirname(localLargeFile),'..','remote')1045if not os.path.exists(remotePath):1046 os.makedirs(remotePath)1047 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))10481049classGitLFS(LargeFileSystem):1050"""Git LFS as backend for the git-p4 large file system.1051 See https://git-lfs.github.com/ for details."""10521053def__init__(self, *args):1054 LargeFileSystem.__init__(self, *args)1055 self.baseGitAttributes = []10561057defgeneratePointer(self, contentFile):1058"""Generate a Git LFS pointer for the content. Return LFS Pointer file1059 mode and content which is stored in the Git repository instead of1060 the actual content. Return also the new location of the actual1061 content.1062 """1063 pointerProcess = subprocess.Popen(1064['git','lfs','pointer','--file='+ contentFile],1065 stdout=subprocess.PIPE1066)1067 pointerFile = pointerProcess.stdout.read()1068if pointerProcess.wait():1069 os.remove(contentFile)1070die('git-lfs pointer command failed. Did you install the extension?')10711072# Git LFS removed the preamble in the output of the 'pointer' command1073# starting from version 1.2.0. Check for the preamble here to support1074# earlier versions.1075# c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b431076if pointerFile.startswith('Git LFS pointer for'):1077 pointerFile = re.sub(r'Git LFS pointer for.*\n\n','', pointerFile)10781079 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)1080 localLargeFile = os.path.join(1081 os.getcwd(),1082'.git','lfs','objects', oid[:2], oid[2:4],1083 oid,1084)1085# LFS Spec states that pointer files should not have the executable bit set.1086 gitMode ='100644'1087return(gitMode, pointerFile, localLargeFile)10881089defpushFile(self, localLargeFile):1090 uploadProcess = subprocess.Popen(1091['git','lfs','push','--object-id','origin', os.path.basename(localLargeFile)]1092)1093if uploadProcess.wait():1094die('git-lfs push command failed. Did you define a remote?')10951096defgenerateGitAttributes(self):1097return(1098 self.baseGitAttributes +1099[1100'\n',1101'#\n',1102'# Git LFS (see https://git-lfs.github.com/)\n',1103'#\n',1104] +1105['*.'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1106for f insorted(gitConfigList('git-p4.largeFileExtensions'))1107] +1108['/'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1109for f insorted(self.largeFiles)if not self.hasLargeFileExtension(f)1110]1111)11121113defaddLargeFile(self, relPath):1114 LargeFileSystem.addLargeFile(self, relPath)1115 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11161117defremoveLargeFile(self, relPath):1118 LargeFileSystem.removeLargeFile(self, relPath)1119 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11201121defprocessContent(self, git_mode, relPath, contents):1122if relPath =='.gitattributes':1123 self.baseGitAttributes = contents1124return(git_mode, self.generateGitAttributes())1125else:1126return LargeFileSystem.processContent(self, git_mode, relPath, contents)11271128class Command:1129def__init__(self):1130 self.usage ="usage: %prog [options]"1131 self.needsGit =True1132 self.verbose =False11331134class P4UserMap:1135def__init__(self):1136 self.userMapFromPerforceServer =False1137 self.myP4UserId =None11381139defp4UserId(self):1140if self.myP4UserId:1141return self.myP4UserId11421143 results =p4CmdList("user -o")1144for r in results:1145if r.has_key('User'):1146 self.myP4UserId = r['User']1147return r['User']1148die("Could not find your p4 user id")11491150defp4UserIsMe(self, p4User):1151# return True if the given p4 user is actually me1152 me = self.p4UserId()1153if not p4User or p4User != me:1154return False1155else:1156return True11571158defgetUserCacheFilename(self):1159 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))1160return home +"/.gitp4-usercache.txt"11611162defgetUserMapFromPerforceServer(self):1163if self.userMapFromPerforceServer:1164return1165 self.users = {}1166 self.emails = {}11671168for output inp4CmdList("users"):1169if not output.has_key("User"):1170continue1171 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">"1172 self.emails[output["Email"]] = output["User"]11731174 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)1175for mapUserConfig ingitConfigList("git-p4.mapUser"):1176 mapUser = mapUserConfigRegex.findall(mapUserConfig)1177if mapUser andlen(mapUser[0]) ==3:1178 user = mapUser[0][0]1179 fullname = mapUser[0][1]1180 email = mapUser[0][2]1181 self.users[user] = fullname +" <"+ email +">"1182 self.emails[email] = user11831184 s =''1185for(key, val)in self.users.items():1186 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1))11871188open(self.getUserCacheFilename(),"wb").write(s)1189 self.userMapFromPerforceServer =True11901191defloadUserMapFromCache(self):1192 self.users = {}1193 self.userMapFromPerforceServer =False1194try:1195 cache =open(self.getUserCacheFilename(),"rb")1196 lines = cache.readlines()1197 cache.close()1198for line in lines:1199 entry = line.strip().split("\t")1200 self.users[entry[0]] = entry[1]1201exceptIOError:1202 self.getUserMapFromPerforceServer()12031204classP4Debug(Command):1205def__init__(self):1206 Command.__init__(self)1207 self.options = []1208 self.description ="A tool to debug the output of p4 -G."1209 self.needsGit =False12101211defrun(self, args):1212 j =01213for output inp4CmdList(args):1214print'Element:%d'% j1215 j +=11216print output1217return True12181219classP4RollBack(Command):1220def__init__(self):1221 Command.__init__(self)1222 self.options = [1223 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1224]1225 self.description ="A tool to debug the multi-branch import. Don't use :)"1226 self.rollbackLocalBranches =False12271228defrun(self, args):1229iflen(args) !=1:1230return False1231 maxChange =int(args[0])12321233if"p4ExitCode"inp4Cmd("changes -m 1"):1234die("Problems executing p4");12351236if self.rollbackLocalBranches:1237 refPrefix ="refs/heads/"1238 lines =read_pipe_lines("git rev-parse --symbolic --branches")1239else:1240 refPrefix ="refs/remotes/"1241 lines =read_pipe_lines("git rev-parse --symbolic --remotes")12421243for line in lines:1244if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1245 line = line.strip()1246 ref = refPrefix + line1247 log =extractLogMessageFromGitCommit(ref)1248 settings =extractSettingsGitLog(log)12491250 depotPaths = settings['depot-paths']1251 change = settings['change']12521253 changed =False12541255iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1256for p in depotPaths]))) ==0:1257print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange)1258system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1259continue12601261while change andint(change) > maxChange:1262 changed =True1263if self.verbose:1264print"%sis at%s; rewinding towards%s"% (ref, change, maxChange)1265system("git update-ref%s\"%s^\""% (ref, ref))1266 log =extractLogMessageFromGitCommit(ref)1267 settings =extractSettingsGitLog(log)126812691270 depotPaths = settings['depot-paths']1271 change = settings['change']12721273if changed:1274print"%srewound to%s"% (ref, change)12751276return True12771278classP4Submit(Command, P4UserMap):12791280 conflict_behavior_choices = ("ask","skip","quit")12811282def__init__(self):1283 Command.__init__(self)1284 P4UserMap.__init__(self)1285 self.options = [1286 optparse.make_option("--origin", dest="origin"),1287 optparse.make_option("-M", dest="detectRenames", action="store_true"),1288# preserve the user, requires relevant p4 permissions1289 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1290 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1291 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1292 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1293 optparse.make_option("--conflict", dest="conflict_behavior",1294 choices=self.conflict_behavior_choices),1295 optparse.make_option("--branch", dest="branch"),1296 optparse.make_option("--shelve", dest="shelve", action="store_true",1297help="Shelve instead of submit. Shelved files are reverted, "1298"restoring the workspace to the state before the shelve"),1299 optparse.make_option("--update-shelve", dest="update_shelve", action="store",type="int",1300 metavar="CHANGELIST",1301help="update an existing shelved changelist, implies --shelve")1302]1303 self.description ="Submit changes from git to the perforce depot."1304 self.usage +=" [name of git branch to submit into perforce depot]"1305 self.origin =""1306 self.detectRenames =False1307 self.preserveUser =gitConfigBool("git-p4.preserveUser")1308 self.dry_run =False1309 self.shelve =False1310 self.update_shelve =None1311 self.prepare_p4_only =False1312 self.conflict_behavior =None1313 self.isWindows = (platform.system() =="Windows")1314 self.exportLabels =False1315 self.p4HasMoveCommand =p4_has_move_command()1316 self.branch =None13171318ifgitConfig('git-p4.largeFileSystem'):1319die("Large file system not supported for git-p4 submit command. Please remove it from config.")13201321defcheck(self):1322iflen(p4CmdList("opened ...")) >0:1323die("You have files opened with perforce! Close them before starting the sync.")13241325defseparate_jobs_from_description(self, message):1326"""Extract and return a possible Jobs field in the commit1327 message. It goes into a separate section in the p4 change1328 specification.13291330 A jobs line starts with "Jobs:" and looks like a new field1331 in a form. Values are white-space separated on the same1332 line or on following lines that start with a tab.13331334 This does not parse and extract the full git commit message1335 like a p4 form. It just sees the Jobs: line as a marker1336 to pass everything from then on directly into the p4 form,1337 but outside the description section.13381339 Return a tuple (stripped log message, jobs string)."""13401341 m = re.search(r'^Jobs:', message, re.MULTILINE)1342if m is None:1343return(message,None)13441345 jobtext = message[m.start():]1346 stripped_message = message[:m.start()].rstrip()1347return(stripped_message, jobtext)13481349defprepareLogMessage(self, template, message, jobs):1350"""Edits the template returned from "p4 change -o" to insert1351 the message in the Description field, and the jobs text in1352 the Jobs field."""1353 result =""13541355 inDescriptionSection =False13561357for line in template.split("\n"):1358if line.startswith("#"):1359 result += line +"\n"1360continue13611362if inDescriptionSection:1363if line.startswith("Files:")or line.startswith("Jobs:"):1364 inDescriptionSection =False1365# insert Jobs section1366if jobs:1367 result += jobs +"\n"1368else:1369continue1370else:1371if line.startswith("Description:"):1372 inDescriptionSection =True1373 line +="\n"1374for messageLine in message.split("\n"):1375 line +="\t"+ messageLine +"\n"13761377 result += line +"\n"13781379return result13801381defpatchRCSKeywords(self,file, pattern):1382# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1383(handle, outFileName) = tempfile.mkstemp(dir='.')1384try:1385 outFile = os.fdopen(handle,"w+")1386 inFile =open(file,"r")1387 regexp = re.compile(pattern, re.VERBOSE)1388for line in inFile.readlines():1389 line = regexp.sub(r'$\1$', line)1390 outFile.write(line)1391 inFile.close()1392 outFile.close()1393# Forcibly overwrite the original file1394 os.unlink(file)1395 shutil.move(outFileName,file)1396except:1397# cleanup our temporary file1398 os.unlink(outFileName)1399print"Failed to strip RCS keywords in%s"%file1400raise14011402print"Patched up RCS keywords in%s"%file14031404defp4UserForCommit(self,id):1405# Return the tuple (perforce user,git email) for a given git commit id1406 self.getUserMapFromPerforceServer()1407 gitEmail =read_pipe(["git","log","--max-count=1",1408"--format=%ae",id])1409 gitEmail = gitEmail.strip()1410if not self.emails.has_key(gitEmail):1411return(None,gitEmail)1412else:1413return(self.emails[gitEmail],gitEmail)14141415defcheckValidP4Users(self,commits):1416# check if any git authors cannot be mapped to p4 users1417foridin commits:1418(user,email) = self.p4UserForCommit(id)1419if not user:1420 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1421ifgitConfigBool("git-p4.allowMissingP4Users"):1422print"%s"% msg1423else:1424die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)14251426deflastP4Changelist(self):1427# Get back the last changelist number submitted in this client spec. This1428# then gets used to patch up the username in the change. If the same1429# client spec is being used by multiple processes then this might go1430# wrong.1431 results =p4CmdList("client -o")# find the current client1432 client =None1433for r in results:1434if r.has_key('Client'):1435 client = r['Client']1436break1437if not client:1438die("could not get client spec")1439 results =p4CmdList(["changes","-c", client,"-m","1"])1440for r in results:1441if r.has_key('change'):1442return r['change']1443die("Could not get changelist number for last submit - cannot patch up user details")14441445defmodifyChangelistUser(self, changelist, newUser):1446# fixup the user field of a changelist after it has been submitted.1447 changes =p4CmdList("change -o%s"% changelist)1448iflen(changes) !=1:1449die("Bad output from p4 change modifying%sto user%s"%1450(changelist, newUser))14511452 c = changes[0]1453if c['User'] == newUser:return# nothing to do1454 c['User'] = newUser1455input= marshal.dumps(c)14561457 result =p4CmdList("change -f -i", stdin=input)1458for r in result:1459if r.has_key('code'):1460if r['code'] =='error':1461die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1462if r.has_key('data'):1463print("Updated user field for changelist%sto%s"% (changelist, newUser))1464return1465die("Could not modify user field of changelist%sto%s"% (changelist, newUser))14661467defcanChangeChangelists(self):1468# check to see if we have p4 admin or super-user permissions, either of1469# which are required to modify changelists.1470 results =p4CmdList(["protects", self.depotPath])1471for r in results:1472if r.has_key('perm'):1473if r['perm'] =='admin':1474return11475if r['perm'] =='super':1476return11477return014781479defprepareSubmitTemplate(self, changelist=None):1480"""Run "p4 change -o" to grab a change specification template.1481 This does not use "p4 -G", as it is nice to keep the submission1482 template in original order, since a human might edit it.14831484 Remove lines in the Files section that show changes to files1485 outside the depot path we're committing into."""14861487[upstream, settings] =findUpstreamBranchPoint()14881489 template =""1490 inFilesSection =False1491 args = ['change','-o']1492if changelist:1493 args.append(str(changelist))14941495for line inp4_read_pipe_lines(args):1496if line.endswith("\r\n"):1497 line = line[:-2] +"\n"1498if inFilesSection:1499if line.startswith("\t"):1500# path starts and ends with a tab1501 path = line[1:]1502 lastTab = path.rfind("\t")1503if lastTab != -1:1504 path = path[:lastTab]1505if settings.has_key('depot-paths'):1506if not[p for p in settings['depot-paths']1507ifp4PathStartsWith(path, p)]:1508continue1509else:1510if notp4PathStartsWith(path, self.depotPath):1511continue1512else:1513 inFilesSection =False1514else:1515if line.startswith("Files:"):1516 inFilesSection =True15171518 template += line15191520return template15211522defedit_template(self, template_file):1523"""Invoke the editor to let the user change the submission1524 message. Return true if okay to continue with the submit."""15251526# if configured to skip the editing part, just submit1527ifgitConfigBool("git-p4.skipSubmitEdit"):1528return True15291530# look at the modification time, to check later if the user saved1531# the file1532 mtime = os.stat(template_file).st_mtime15331534# invoke the editor1535if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1536 editor = os.environ.get("P4EDITOR")1537else:1538 editor =read_pipe("git var GIT_EDITOR").strip()1539system(["sh","-c", ('%s"$@"'% editor), editor, template_file])15401541# If the file was not saved, prompt to see if this patch should1542# be skipped. But skip this verification step if configured so.1543ifgitConfigBool("git-p4.skipSubmitEditCheck"):1544return True15451546# modification time updated means user saved the file1547if os.stat(template_file).st_mtime > mtime:1548return True15491550while True:1551 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1552if response =='y':1553return True1554if response =='n':1555return False15561557defget_diff_description(self, editedFiles, filesToAdd):1558# diff1559if os.environ.has_key("P4DIFF"):1560del(os.environ["P4DIFF"])1561 diff =""1562for editedFile in editedFiles:1563 diff +=p4_read_pipe(['diff','-du',1564wildcard_encode(editedFile)])15651566# new file diff1567 newdiff =""1568for newFile in filesToAdd:1569 newdiff +="==== new file ====\n"1570 newdiff +="--- /dev/null\n"1571 newdiff +="+++%s\n"% newFile1572 f =open(newFile,"r")1573for line in f.readlines():1574 newdiff +="+"+ line1575 f.close()15761577return(diff + newdiff).replace('\r\n','\n')15781579defapplyCommit(self,id):1580"""Apply one commit, return True if it succeeded."""15811582print"Applying",read_pipe(["git","show","-s",1583"--format=format:%h%s",id])15841585(p4User, gitEmail) = self.p4UserForCommit(id)15861587 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1588 filesToAdd =set()1589 filesToChangeType =set()1590 filesToDelete =set()1591 editedFiles =set()1592 pureRenameCopy =set()1593 filesToChangeExecBit = {}1594 all_files =list()15951596for line in diff:1597 diff =parseDiffTreeEntry(line)1598 modifier = diff['status']1599 path = diff['src']1600 all_files.append(path)16011602if modifier =="M":1603p4_edit(path)1604ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1605 filesToChangeExecBit[path] = diff['dst_mode']1606 editedFiles.add(path)1607elif modifier =="A":1608 filesToAdd.add(path)1609 filesToChangeExecBit[path] = diff['dst_mode']1610if path in filesToDelete:1611 filesToDelete.remove(path)1612elif modifier =="D":1613 filesToDelete.add(path)1614if path in filesToAdd:1615 filesToAdd.remove(path)1616elif modifier =="C":1617 src, dest = diff['src'], diff['dst']1618p4_integrate(src, dest)1619 pureRenameCopy.add(dest)1620if diff['src_sha1'] != diff['dst_sha1']:1621p4_edit(dest)1622 pureRenameCopy.discard(dest)1623ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1624p4_edit(dest)1625 pureRenameCopy.discard(dest)1626 filesToChangeExecBit[dest] = diff['dst_mode']1627if self.isWindows:1628# turn off read-only attribute1629 os.chmod(dest, stat.S_IWRITE)1630 os.unlink(dest)1631 editedFiles.add(dest)1632elif modifier =="R":1633 src, dest = diff['src'], diff['dst']1634if self.p4HasMoveCommand:1635p4_edit(src)# src must be open before move1636p4_move(src, dest)# opens for (move/delete, move/add)1637else:1638p4_integrate(src, dest)1639if diff['src_sha1'] != diff['dst_sha1']:1640p4_edit(dest)1641else:1642 pureRenameCopy.add(dest)1643ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1644if not self.p4HasMoveCommand:1645p4_edit(dest)# with move: already open, writable1646 filesToChangeExecBit[dest] = diff['dst_mode']1647if not self.p4HasMoveCommand:1648if self.isWindows:1649 os.chmod(dest, stat.S_IWRITE)1650 os.unlink(dest)1651 filesToDelete.add(src)1652 editedFiles.add(dest)1653elif modifier =="T":1654 filesToChangeType.add(path)1655else:1656die("unknown modifier%sfor%s"% (modifier, path))16571658 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1659 patchcmd = diffcmd +" | git apply "1660 tryPatchCmd = patchcmd +"--check -"1661 applyPatchCmd = patchcmd +"--check --apply -"1662 patch_succeeded =True16631664if os.system(tryPatchCmd) !=0:1665 fixed_rcs_keywords =False1666 patch_succeeded =False1667print"Unfortunately applying the change failed!"16681669# Patch failed, maybe it's just RCS keyword woes. Look through1670# the patch to see if that's possible.1671ifgitConfigBool("git-p4.attemptRCSCleanup"):1672file=None1673 pattern =None1674 kwfiles = {}1675forfilein editedFiles | filesToDelete:1676# did this file's delta contain RCS keywords?1677 pattern =p4_keywords_regexp_for_file(file)16781679if pattern:1680# this file is a possibility...look for RCS keywords.1681 regexp = re.compile(pattern, re.VERBOSE)1682for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1683if regexp.search(line):1684if verbose:1685print"got keyword match on%sin%sin%s"% (pattern, line,file)1686 kwfiles[file] = pattern1687break16881689forfilein kwfiles:1690if verbose:1691print"zapping%swith%s"% (line,pattern)1692# File is being deleted, so not open in p4. Must1693# disable the read-only bit on windows.1694if self.isWindows andfilenot in editedFiles:1695 os.chmod(file, stat.S_IWRITE)1696 self.patchRCSKeywords(file, kwfiles[file])1697 fixed_rcs_keywords =True16981699if fixed_rcs_keywords:1700print"Retrying the patch with RCS keywords cleaned up"1701if os.system(tryPatchCmd) ==0:1702 patch_succeeded =True17031704if not patch_succeeded:1705for f in editedFiles:1706p4_revert(f)1707return False17081709#1710# Apply the patch for real, and do add/delete/+x handling.1711#1712system(applyPatchCmd)17131714for f in filesToChangeType:1715p4_edit(f,"-t","auto")1716for f in filesToAdd:1717p4_add(f)1718for f in filesToDelete:1719p4_revert(f)1720p4_delete(f)17211722# Set/clear executable bits1723for f in filesToChangeExecBit.keys():1724 mode = filesToChangeExecBit[f]1725setP4ExecBit(f, mode)17261727if self.update_shelve:1728print("all_files =%s"%str(all_files))1729p4_reopen_in_change(self.update_shelve, all_files)17301731#1732# Build p4 change description, starting with the contents1733# of the git commit message.1734#1735 logMessage =extractLogMessageFromGitCommit(id)1736 logMessage = logMessage.strip()1737(logMessage, jobs) = self.separate_jobs_from_description(logMessage)17381739 template = self.prepareSubmitTemplate(self.update_shelve)1740 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)17411742if self.preserveUser:1743 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User17441745if self.checkAuthorship and not self.p4UserIsMe(p4User):1746 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1747 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1748 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"17491750 separatorLine ="######## everything below this line is just the diff #######\n"1751if not self.prepare_p4_only:1752 submitTemplate += separatorLine1753 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)17541755(handle, fileName) = tempfile.mkstemp()1756 tmpFile = os.fdopen(handle,"w+b")1757if self.isWindows:1758 submitTemplate = submitTemplate.replace("\n","\r\n")1759 tmpFile.write(submitTemplate)1760 tmpFile.close()17611762if self.prepare_p4_only:1763#1764# Leave the p4 tree prepared, and the submit template around1765# and let the user decide what to do next1766#1767print1768print"P4 workspace prepared for submission."1769print"To submit or revert, go to client workspace"1770print" "+ self.clientPath1771print1772print"To submit, use\"p4 submit\"to write a new description,"1773print"or\"p4 submit -i <%s\"to use the one prepared by" \1774"\"git p4\"."% fileName1775print"You can delete the file\"%s\"when finished."% fileName17761777if self.preserveUser and p4User and not self.p4UserIsMe(p4User):1778print"To preserve change ownership by user%s, you must\n" \1779"do\"p4 change -f <change>\"after submitting and\n" \1780"edit the User field."1781if pureRenameCopy:1782print"After submitting, renamed files must be re-synced."1783print"Invoke\"p4 sync -f\"on each of these files:"1784for f in pureRenameCopy:1785print" "+ f17861787print1788print"To revert the changes, use\"p4 revert ...\", and delete"1789print"the submit template file\"%s\""% fileName1790if filesToAdd:1791print"Since the commit adds new files, they must be deleted:"1792for f in filesToAdd:1793print" "+ f1794print1795return True17961797#1798# Let the user edit the change description, then submit it.1799#1800 submitted =False18011802try:1803if self.edit_template(fileName):1804# read the edited message and submit1805 tmpFile =open(fileName,"rb")1806 message = tmpFile.read()1807 tmpFile.close()1808if self.isWindows:1809 message = message.replace("\r\n","\n")1810 submitTemplate = message[:message.index(separatorLine)]18111812if self.update_shelve:1813p4_write_pipe(['shelve','-r','-i'], submitTemplate)1814elif self.shelve:1815p4_write_pipe(['shelve','-i'], submitTemplate)1816else:1817p4_write_pipe(['submit','-i'], submitTemplate)1818# The rename/copy happened by applying a patch that created a1819# new file. This leaves it writable, which confuses p4.1820for f in pureRenameCopy:1821p4_sync(f,"-f")18221823if self.preserveUser:1824if p4User:1825# Get last changelist number. Cannot easily get it from1826# the submit command output as the output is1827# unmarshalled.1828 changelist = self.lastP4Changelist()1829 self.modifyChangelistUser(changelist, p4User)18301831 submitted =True18321833finally:1834# skip this patch1835if not submitted or self.shelve:1836if self.shelve:1837print("Reverting shelved files.")1838else:1839print("Submission cancelled, undoing p4 changes.")1840for f in editedFiles | filesToDelete:1841p4_revert(f)1842for f in filesToAdd:1843p4_revert(f)1844 os.remove(f)18451846 os.remove(fileName)1847return submitted18481849# Export git tags as p4 labels. Create a p4 label and then tag1850# with that.1851defexportGitTags(self, gitTags):1852 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1853iflen(validLabelRegexp) ==0:1854 validLabelRegexp = defaultLabelRegexp1855 m = re.compile(validLabelRegexp)18561857for name in gitTags:18581859if not m.match(name):1860if verbose:1861print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1862continue18631864# Get the p4 commit this corresponds to1865 logMessage =extractLogMessageFromGitCommit(name)1866 values =extractSettingsGitLog(logMessage)18671868if not values.has_key('change'):1869# a tag pointing to something not sent to p4; ignore1870if verbose:1871print"git tag%sdoes not give a p4 commit"% name1872continue1873else:1874 changelist = values['change']18751876# Get the tag details.1877 inHeader =True1878 isAnnotated =False1879 body = []1880for l inread_pipe_lines(["git","cat-file","-p", name]):1881 l = l.strip()1882if inHeader:1883if re.match(r'tag\s+', l):1884 isAnnotated =True1885elif re.match(r'\s*$', l):1886 inHeader =False1887continue1888else:1889 body.append(l)18901891if not isAnnotated:1892 body = ["lightweight tag imported by git p4\n"]18931894# Create the label - use the same view as the client spec we are using1895 clientSpec =getClientSpec()18961897 labelTemplate ="Label:%s\n"% name1898 labelTemplate +="Description:\n"1899for b in body:1900 labelTemplate +="\t"+ b +"\n"1901 labelTemplate +="View:\n"1902for depot_side in clientSpec.mappings:1903 labelTemplate +="\t%s\n"% depot_side19041905if self.dry_run:1906print"Would create p4 label%sfor tag"% name1907elif self.prepare_p4_only:1908print"Not creating p4 label%sfor tag due to option" \1909" --prepare-p4-only"% name1910else:1911p4_write_pipe(["label","-i"], labelTemplate)19121913# Use the label1914p4_system(["tag","-l", name] +1915["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])19161917if verbose:1918print"created p4 label for tag%s"% name19191920defrun(self, args):1921iflen(args) ==0:1922 self.master =currentGitBranch()1923eliflen(args) ==1:1924 self.master = args[0]1925if notbranchExists(self.master):1926die("Branch%sdoes not exist"% self.master)1927else:1928return False19291930if self.master:1931 allowSubmit =gitConfig("git-p4.allowSubmit")1932iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1933die("%sis not in git-p4.allowSubmit"% self.master)19341935[upstream, settings] =findUpstreamBranchPoint()1936 self.depotPath = settings['depot-paths'][0]1937iflen(self.origin) ==0:1938 self.origin = upstream19391940if self.update_shelve:1941 self.shelve =True19421943if self.preserveUser:1944if not self.canChangeChangelists():1945die("Cannot preserve user names without p4 super-user or admin permissions")19461947# if not set from the command line, try the config file1948if self.conflict_behavior is None:1949 val =gitConfig("git-p4.conflict")1950if val:1951if val not in self.conflict_behavior_choices:1952die("Invalid value '%s' for config git-p4.conflict"% val)1953else:1954 val ="ask"1955 self.conflict_behavior = val19561957if self.verbose:1958print"Origin branch is "+ self.origin19591960iflen(self.depotPath) ==0:1961print"Internal error: cannot locate perforce depot path from existing branches"1962 sys.exit(128)19631964 self.useClientSpec =False1965ifgitConfigBool("git-p4.useclientspec"):1966 self.useClientSpec =True1967if self.useClientSpec:1968 self.clientSpecDirs =getClientSpec()19691970# Check for the existence of P4 branches1971 branchesDetected = (len(p4BranchesInGit().keys()) >1)19721973if self.useClientSpec and not branchesDetected:1974# all files are relative to the client spec1975 self.clientPath =getClientRoot()1976else:1977 self.clientPath =p4Where(self.depotPath)19781979if self.clientPath =="":1980die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)19811982print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1983 self.oldWorkingDirectory = os.getcwd()19841985# ensure the clientPath exists1986 new_client_dir =False1987if not os.path.exists(self.clientPath):1988 new_client_dir =True1989 os.makedirs(self.clientPath)19901991chdir(self.clientPath, is_client_path=True)1992if self.dry_run:1993print"Would synchronize p4 checkout in%s"% self.clientPath1994else:1995print"Synchronizing p4 checkout..."1996if new_client_dir:1997# old one was destroyed, and maybe nobody told p41998p4_sync("...","-f")1999else:2000p4_sync("...")2001 self.check()20022003 commits = []2004if self.master:2005 commitish = self.master2006else:2007 commitish ='HEAD'20082009for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, commitish)]):2010 commits.append(line.strip())2011 commits.reverse()20122013if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):2014 self.checkAuthorship =False2015else:2016 self.checkAuthorship =True20172018if self.preserveUser:2019 self.checkValidP4Users(commits)20202021#2022# Build up a set of options to be passed to diff when2023# submitting each commit to p4.2024#2025if self.detectRenames:2026# command-line -M arg2027 self.diffOpts ="-M"2028else:2029# If not explicitly set check the config variable2030 detectRenames =gitConfig("git-p4.detectRenames")20312032if detectRenames.lower() =="false"or detectRenames =="":2033 self.diffOpts =""2034elif detectRenames.lower() =="true":2035 self.diffOpts ="-M"2036else:2037 self.diffOpts ="-M%s"% detectRenames20382039# no command-line arg for -C or --find-copies-harder, just2040# config variables2041 detectCopies =gitConfig("git-p4.detectCopies")2042if detectCopies.lower() =="false"or detectCopies =="":2043pass2044elif detectCopies.lower() =="true":2045 self.diffOpts +=" -C"2046else:2047 self.diffOpts +=" -C%s"% detectCopies20482049ifgitConfigBool("git-p4.detectCopiesHarder"):2050 self.diffOpts +=" --find-copies-harder"20512052#2053# Apply the commits, one at a time. On failure, ask if should2054# continue to try the rest of the patches, or quit.2055#2056if self.dry_run:2057print"Would apply"2058 applied = []2059 last =len(commits) -12060for i, commit inenumerate(commits):2061if self.dry_run:2062print" ",read_pipe(["git","show","-s",2063"--format=format:%h%s", commit])2064 ok =True2065else:2066 ok = self.applyCommit(commit)2067if ok:2068 applied.append(commit)2069else:2070if self.prepare_p4_only and i < last:2071print"Processing only the first commit due to option" \2072" --prepare-p4-only"2073break2074if i < last:2075 quit =False2076while True:2077# prompt for what to do, or use the option/variable2078if self.conflict_behavior =="ask":2079print"What do you want to do?"2080 response =raw_input("[s]kip this commit but apply"2081" the rest, or [q]uit? ")2082if not response:2083continue2084elif self.conflict_behavior =="skip":2085 response ="s"2086elif self.conflict_behavior =="quit":2087 response ="q"2088else:2089die("Unknown conflict_behavior '%s'"%2090 self.conflict_behavior)20912092if response[0] =="s":2093print"Skipping this commit, but applying the rest"2094break2095if response[0] =="q":2096print"Quitting"2097 quit =True2098break2099if quit:2100break21012102chdir(self.oldWorkingDirectory)2103 shelved_applied ="shelved"if self.shelve else"applied"2104if self.dry_run:2105pass2106elif self.prepare_p4_only:2107pass2108eliflen(commits) ==len(applied):2109print("All commits{0}!".format(shelved_applied))21102111 sync =P4Sync()2112if self.branch:2113 sync.branch = self.branch2114 sync.run([])21152116 rebase =P4Rebase()2117 rebase.rebase()21182119else:2120iflen(applied) ==0:2121print("No commits{0}.".format(shelved_applied))2122else:2123print("{0}only the commits marked with '*':".format(shelved_applied.capitalize()))2124for c in commits:2125if c in applied:2126 star ="*"2127else:2128 star =" "2129print star,read_pipe(["git","show","-s",2130"--format=format:%h%s", c])2131print"You will have to do 'git p4 sync' and rebase."21322133ifgitConfigBool("git-p4.exportLabels"):2134 self.exportLabels =True21352136if self.exportLabels:2137 p4Labels =getP4Labels(self.depotPath)2138 gitTags =getGitTags()21392140 missingGitTags = gitTags - p4Labels2141 self.exportGitTags(missingGitTags)21422143# exit with error unless everything applied perfectly2144iflen(commits) !=len(applied):2145 sys.exit(1)21462147return True21482149classView(object):2150"""Represent a p4 view ("p4 help views"), and map files in a2151 repo according to the view."""21522153def__init__(self, client_name):2154 self.mappings = []2155 self.client_prefix ="//%s/"% client_name2156# cache results of "p4 where" to lookup client file locations2157 self.client_spec_path_cache = {}21582159defappend(self, view_line):2160"""Parse a view line, splitting it into depot and client2161 sides. Append to self.mappings, preserving order. This2162 is only needed for tag creation."""21632164# Split the view line into exactly two words. P4 enforces2165# structure on these lines that simplifies this quite a bit.2166#2167# Either or both words may be double-quoted.2168# Single quotes do not matter.2169# Double-quote marks cannot occur inside the words.2170# A + or - prefix is also inside the quotes.2171# There are no quotes unless they contain a space.2172# The line is already white-space stripped.2173# The two words are separated by a single space.2174#2175if view_line[0] =='"':2176# First word is double quoted. Find its end.2177 close_quote_index = view_line.find('"',1)2178if close_quote_index <=0:2179die("No first-word closing quote found:%s"% view_line)2180 depot_side = view_line[1:close_quote_index]2181# skip closing quote and space2182 rhs_index = close_quote_index +1+12183else:2184 space_index = view_line.find(" ")2185if space_index <=0:2186die("No word-splitting space found:%s"% view_line)2187 depot_side = view_line[0:space_index]2188 rhs_index = space_index +121892190# prefix + means overlay on previous mapping2191if depot_side.startswith("+"):2192 depot_side = depot_side[1:]21932194# prefix - means exclude this path, leave out of mappings2195 exclude =False2196if depot_side.startswith("-"):2197 exclude =True2198 depot_side = depot_side[1:]21992200if not exclude:2201 self.mappings.append(depot_side)22022203defconvert_client_path(self, clientFile):2204# chop off //client/ part to make it relative2205if not clientFile.startswith(self.client_prefix):2206die("No prefix '%s' on clientFile '%s'"%2207(self.client_prefix, clientFile))2208return clientFile[len(self.client_prefix):]22092210defupdate_client_spec_path_cache(self, files):2211""" Caching file paths by "p4 where" batch query """22122213# List depot file paths exclude that already cached2214 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]22152216iflen(fileArgs) ==0:2217return# All files in cache22182219 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)2220for res in where_result:2221if"code"in res and res["code"] =="error":2222# assume error is "... file(s) not in client view"2223continue2224if"clientFile"not in res:2225die("No clientFile in 'p4 where' output")2226if"unmap"in res:2227# it will list all of them, but only one not unmap-ped2228continue2229ifgitConfigBool("core.ignorecase"):2230 res['depotFile'] = res['depotFile'].lower()2231 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])22322233# not found files or unmap files set to ""2234for depotFile in fileArgs:2235ifgitConfigBool("core.ignorecase"):2236 depotFile = depotFile.lower()2237if depotFile not in self.client_spec_path_cache:2238 self.client_spec_path_cache[depotFile] =""22392240defmap_in_client(self, depot_path):2241"""Return the relative location in the client where this2242 depot file should live. Returns "" if the file should2243 not be mapped in the client."""22442245ifgitConfigBool("core.ignorecase"):2246 depot_path = depot_path.lower()22472248if depot_path in self.client_spec_path_cache:2249return self.client_spec_path_cache[depot_path]22502251die("Error:%sis not found in client spec path"% depot_path )2252return""22532254classP4Sync(Command, P4UserMap):2255 delete_actions = ("delete","move/delete","purge")22562257def__init__(self):2258 Command.__init__(self)2259 P4UserMap.__init__(self)2260 self.options = [2261 optparse.make_option("--branch", dest="branch"),2262 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),2263 optparse.make_option("--changesfile", dest="changesFile"),2264 optparse.make_option("--silent", dest="silent", action="store_true"),2265 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),2266 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2267 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",2268help="Import into refs/heads/ , not refs/remotes"),2269 optparse.make_option("--max-changes", dest="maxChanges",2270help="Maximum number of changes to import"),2271 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",2272help="Internal block size to use when iteratively calling p4 changes"),2273 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2274help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2275 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2276help="Only sync files that are included in the Perforce Client Spec"),2277 optparse.make_option("-/", dest="cloneExclude",2278 action="append",type="string",2279help="exclude depot path"),2280]2281 self.description ="""Imports from Perforce into a git repository.\n2282 example:2283 //depot/my/project/ -- to import the current head2284 //depot/my/project/@all -- to import everything2285 //depot/my/project/@1,6 -- to import only from revision 1 to 622862287 (a ... is not needed in the path p4 specification, it's added implicitly)"""22882289 self.usage +=" //depot/path[@revRange]"2290 self.silent =False2291 self.createdBranches =set()2292 self.committedChanges =set()2293 self.branch =""2294 self.detectBranches =False2295 self.detectLabels =False2296 self.importLabels =False2297 self.changesFile =""2298 self.syncWithOrigin =True2299 self.importIntoRemotes =True2300 self.maxChanges =""2301 self.changes_block_size =None2302 self.keepRepoPath =False2303 self.depotPaths =None2304 self.p4BranchesInGit = []2305 self.cloneExclude = []2306 self.useClientSpec =False2307 self.useClientSpec_from_options =False2308 self.clientSpecDirs =None2309 self.tempBranches = []2310 self.tempBranchLocation ="refs/git-p4-tmp"2311 self.largeFileSystem =None23122313ifgitConfig('git-p4.largeFileSystem'):2314 largeFileSystemConstructor =globals()[gitConfig('git-p4.largeFileSystem')]2315 self.largeFileSystem =largeFileSystemConstructor(2316lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)2317)23182319ifgitConfig("git-p4.syncFromOrigin") =="false":2320 self.syncWithOrigin =False23212322# This is required for the "append" cloneExclude action2323defensure_value(self, attr, value):2324if nothasattr(self, attr)orgetattr(self, attr)is None:2325setattr(self, attr, value)2326returngetattr(self, attr)23272328# Force a checkpoint in fast-import and wait for it to finish2329defcheckpoint(self):2330 self.gitStream.write("checkpoint\n\n")2331 self.gitStream.write("progress checkpoint\n\n")2332 out = self.gitOutput.readline()2333if self.verbose:2334print"checkpoint finished: "+ out23352336defextractFilesFromCommit(self, commit):2337 self.cloneExclude = [re.sub(r"\.\.\.$","", path)2338for path in self.cloneExclude]2339 files = []2340 fnum =02341while commit.has_key("depotFile%s"% fnum):2342 path = commit["depotFile%s"% fnum]23432344if[p for p in self.cloneExclude2345ifp4PathStartsWith(path, p)]:2346 found =False2347else:2348 found = [p for p in self.depotPaths2349ifp4PathStartsWith(path, p)]2350if not found:2351 fnum = fnum +12352continue23532354file= {}2355file["path"] = path2356file["rev"] = commit["rev%s"% fnum]2357file["action"] = commit["action%s"% fnum]2358file["type"] = commit["type%s"% fnum]2359 files.append(file)2360 fnum = fnum +12361return files23622363defextractJobsFromCommit(self, commit):2364 jobs = []2365 jnum =02366while commit.has_key("job%s"% jnum):2367 job = commit["job%s"% jnum]2368 jobs.append(job)2369 jnum = jnum +12370return jobs23712372defstripRepoPath(self, path, prefixes):2373"""When streaming files, this is called to map a p4 depot path2374 to where it should go in git. The prefixes are either2375 self.depotPaths, or self.branchPrefixes in the case of2376 branch detection."""23772378if self.useClientSpec:2379# branch detection moves files up a level (the branch name)2380# from what client spec interpretation gives2381 path = self.clientSpecDirs.map_in_client(path)2382if self.detectBranches:2383for b in self.knownBranches:2384if path.startswith(b +"/"):2385 path = path[len(b)+1:]23862387elif self.keepRepoPath:2388# Preserve everything in relative path name except leading2389# //depot/; just look at first prefix as they all should2390# be in the same depot.2391 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2392ifp4PathStartsWith(path, depot):2393 path = path[len(depot):]23942395else:2396for p in prefixes:2397ifp4PathStartsWith(path, p):2398 path = path[len(p):]2399break24002401 path =wildcard_decode(path)2402return path24032404defsplitFilesIntoBranches(self, commit):2405"""Look at each depotFile in the commit to figure out to what2406 branch it belongs."""24072408if self.clientSpecDirs:2409 files = self.extractFilesFromCommit(commit)2410 self.clientSpecDirs.update_client_spec_path_cache(files)24112412 branches = {}2413 fnum =02414while commit.has_key("depotFile%s"% fnum):2415 path = commit["depotFile%s"% fnum]2416 found = [p for p in self.depotPaths2417ifp4PathStartsWith(path, p)]2418if not found:2419 fnum = fnum +12420continue24212422file= {}2423file["path"] = path2424file["rev"] = commit["rev%s"% fnum]2425file["action"] = commit["action%s"% fnum]2426file["type"] = commit["type%s"% fnum]2427 fnum = fnum +124282429# start with the full relative path where this file would2430# go in a p4 client2431if self.useClientSpec:2432 relPath = self.clientSpecDirs.map_in_client(path)2433else:2434 relPath = self.stripRepoPath(path, self.depotPaths)24352436for branch in self.knownBranches.keys():2437# add a trailing slash so that a commit into qt/4.2foo2438# doesn't end up in qt/4.2, e.g.2439if relPath.startswith(branch +"/"):2440if branch not in branches:2441 branches[branch] = []2442 branches[branch].append(file)2443break24442445return branches24462447defwriteToGitStream(self, gitMode, relPath, contents):2448 self.gitStream.write('M%sinline%s\n'% (gitMode, relPath))2449 self.gitStream.write('data%d\n'%sum(len(d)for d in contents))2450for d in contents:2451 self.gitStream.write(d)2452 self.gitStream.write('\n')24532454# output one file from the P4 stream2455# - helper for streamP4Files24562457defstreamOneP4File(self,file, contents):2458 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2459if verbose:2460 size =int(self.stream_file['fileSize'])2461 sys.stdout.write('\r%s-->%s(%iMB)\n'% (file['depotFile'], relPath, size/1024/1024))2462 sys.stdout.flush()24632464(type_base, type_mods) =split_p4_type(file["type"])24652466 git_mode ="100644"2467if"x"in type_mods:2468 git_mode ="100755"2469if type_base =="symlink":2470 git_mode ="120000"2471# p4 print on a symlink sometimes contains "target\n";2472# if it does, remove the newline2473 data =''.join(contents)2474if not data:2475# Some version of p4 allowed creating a symlink that pointed2476# to nothing. This causes p4 errors when checking out such2477# a change, and errors here too. Work around it by ignoring2478# the bad symlink; hopefully a future change fixes it.2479print"\nIgnoring empty symlink in%s"%file['depotFile']2480return2481elif data[-1] =='\n':2482 contents = [data[:-1]]2483else:2484 contents = [data]24852486if type_base =="utf16":2487# p4 delivers different text in the python output to -G2488# than it does when using "print -o", or normal p4 client2489# operations. utf16 is converted to ascii or utf8, perhaps.2490# But ascii text saved as -t utf16 is completely mangled.2491# Invoke print -o to get the real contents.2492#2493# On windows, the newlines will always be mangled by print, so put2494# them back too. This is not needed to the cygwin windows version,2495# just the native "NT" type.2496#2497try:2498 text =p4_read_pipe(['print','-q','-o','-','%s@%s'% (file['depotFile'],file['change'])])2499exceptExceptionas e:2500if'Translation of file content failed'instr(e):2501 type_base ='binary'2502else:2503raise e2504else:2505ifp4_version_string().find('/NT') >=0:2506 text = text.replace('\r\n','\n')2507 contents = [ text ]25082509if type_base =="apple":2510# Apple filetype files will be streamed as a concatenation of2511# its appledouble header and the contents. This is useless2512# on both macs and non-macs. If using "print -q -o xx", it2513# will create "xx" with the data, and "%xx" with the header.2514# This is also not very useful.2515#2516# Ideally, someday, this script can learn how to generate2517# appledouble files directly and import those to git, but2518# non-mac machines can never find a use for apple filetype.2519print"\nIgnoring apple filetype file%s"%file['depotFile']2520return25212522# Note that we do not try to de-mangle keywords on utf16 files,2523# even though in theory somebody may want that.2524 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2525if pattern:2526 regexp = re.compile(pattern, re.VERBOSE)2527 text =''.join(contents)2528 text = regexp.sub(r'$\1$', text)2529 contents = [ text ]25302531try:2532 relPath.decode('ascii')2533except:2534 encoding ='utf8'2535ifgitConfig('git-p4.pathEncoding'):2536 encoding =gitConfig('git-p4.pathEncoding')2537 relPath = relPath.decode(encoding,'replace').encode('utf8','replace')2538if self.verbose:2539print'Path with non-ASCII characters detected. Used%sto encode:%s'% (encoding, relPath)25402541if self.largeFileSystem:2542(git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)25432544 self.writeToGitStream(git_mode, relPath, contents)25452546defstreamOneP4Deletion(self,file):2547 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2548if verbose:2549 sys.stdout.write("delete%s\n"% relPath)2550 sys.stdout.flush()2551 self.gitStream.write("D%s\n"% relPath)25522553if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):2554 self.largeFileSystem.removeLargeFile(relPath)25552556# handle another chunk of streaming data2557defstreamP4FilesCb(self, marshalled):25582559# catch p4 errors and complain2560 err =None2561if"code"in marshalled:2562if marshalled["code"] =="error":2563if"data"in marshalled:2564 err = marshalled["data"].rstrip()25652566if not err and'fileSize'in self.stream_file:2567 required_bytes =int((4*int(self.stream_file["fileSize"])) -calcDiskFree())2568if required_bytes >0:2569 err ='Not enough space left on%s! Free at least%iMB.'% (2570 os.getcwd(), required_bytes/1024/10242571)25722573if err:2574 f =None2575if self.stream_have_file_info:2576if"depotFile"in self.stream_file:2577 f = self.stream_file["depotFile"]2578# force a failure in fast-import, else an empty2579# commit will be made2580 self.gitStream.write("\n")2581 self.gitStream.write("die-now\n")2582 self.gitStream.close()2583# ignore errors, but make sure it exits first2584 self.importProcess.wait()2585if f:2586die("Error from p4 print for%s:%s"% (f, err))2587else:2588die("Error from p4 print:%s"% err)25892590if marshalled.has_key('depotFile')and self.stream_have_file_info:2591# start of a new file - output the old one first2592 self.streamOneP4File(self.stream_file, self.stream_contents)2593 self.stream_file = {}2594 self.stream_contents = []2595 self.stream_have_file_info =False25962597# pick up the new file information... for the2598# 'data' field we need to append to our array2599for k in marshalled.keys():2600if k =='data':2601if'streamContentSize'not in self.stream_file:2602 self.stream_file['streamContentSize'] =02603 self.stream_file['streamContentSize'] +=len(marshalled['data'])2604 self.stream_contents.append(marshalled['data'])2605else:2606 self.stream_file[k] = marshalled[k]26072608if(verbose and2609'streamContentSize'in self.stream_file and2610'fileSize'in self.stream_file and2611'depotFile'in self.stream_file):2612 size =int(self.stream_file["fileSize"])2613if size >0:2614 progress =100*self.stream_file['streamContentSize']/size2615 sys.stdout.write('\r%s %d%%(%iMB)'% (self.stream_file['depotFile'], progress,int(size/1024/1024)))2616 sys.stdout.flush()26172618 self.stream_have_file_info =True26192620# Stream directly from "p4 files" into "git fast-import"2621defstreamP4Files(self, files):2622 filesForCommit = []2623 filesToRead = []2624 filesToDelete = []26252626for f in files:2627 filesForCommit.append(f)2628if f['action']in self.delete_actions:2629 filesToDelete.append(f)2630else:2631 filesToRead.append(f)26322633# deleted files...2634for f in filesToDelete:2635 self.streamOneP4Deletion(f)26362637iflen(filesToRead) >0:2638 self.stream_file = {}2639 self.stream_contents = []2640 self.stream_have_file_info =False26412642# curry self argument2643defstreamP4FilesCbSelf(entry):2644 self.streamP4FilesCb(entry)26452646 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]26472648p4CmdList(["-x","-","print"],2649 stdin=fileArgs,2650 cb=streamP4FilesCbSelf)26512652# do the last chunk2653if self.stream_file.has_key('depotFile'):2654 self.streamOneP4File(self.stream_file, self.stream_contents)26552656defmake_email(self, userid):2657if userid in self.users:2658return self.users[userid]2659else:2660return"%s<a@b>"% userid26612662defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2663""" Stream a p4 tag.2664 commit is either a git commit, or a fast-import mark, ":<p4commit>"2665 """26662667if verbose:2668print"writing tag%sfor commit%s"% (labelName, commit)2669 gitStream.write("tag%s\n"% labelName)2670 gitStream.write("from%s\n"% commit)26712672if labelDetails.has_key('Owner'):2673 owner = labelDetails["Owner"]2674else:2675 owner =None26762677# Try to use the owner of the p4 label, or failing that,2678# the current p4 user id.2679if owner:2680 email = self.make_email(owner)2681else:2682 email = self.make_email(self.p4UserId())2683 tagger ="%s %s %s"% (email, epoch, self.tz)26842685 gitStream.write("tagger%s\n"% tagger)26862687print"labelDetails=",labelDetails2688if labelDetails.has_key('Description'):2689 description = labelDetails['Description']2690else:2691 description ='Label from git p4'26922693 gitStream.write("data%d\n"%len(description))2694 gitStream.write(description)2695 gitStream.write("\n")26962697definClientSpec(self, path):2698if not self.clientSpecDirs:2699return True2700 inClientSpec = self.clientSpecDirs.map_in_client(path)2701if not inClientSpec and self.verbose:2702print('Ignoring file outside of client spec:{0}'.format(path))2703return inClientSpec27042705defhasBranchPrefix(self, path):2706if not self.branchPrefixes:2707return True2708 hasPrefix = [p for p in self.branchPrefixes2709ifp4PathStartsWith(path, p)]2710if not hasPrefix and self.verbose:2711print('Ignoring file outside of prefix:{0}'.format(path))2712return hasPrefix27132714defcommit(self, details, files, branch, parent =""):2715 epoch = details["time"]2716 author = details["user"]2717 jobs = self.extractJobsFromCommit(details)27182719if self.verbose:2720print('commit into{0}'.format(branch))27212722if self.clientSpecDirs:2723 self.clientSpecDirs.update_client_spec_path_cache(files)27242725 files = [f for f in files2726if self.inClientSpec(f['path'])and self.hasBranchPrefix(f['path'])]27272728if not files and notgitConfigBool('git-p4.keepEmptyCommits'):2729print('Ignoring revision{0}as it would produce an empty commit.'2730.format(details['change']))2731return27322733 self.gitStream.write("commit%s\n"% branch)2734 self.gitStream.write("mark :%s\n"% details["change"])2735 self.committedChanges.add(int(details["change"]))2736 committer =""2737if author not in self.users:2738 self.getUserMapFromPerforceServer()2739 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)27402741 self.gitStream.write("committer%s\n"% committer)27422743 self.gitStream.write("data <<EOT\n")2744 self.gitStream.write(details["desc"])2745iflen(jobs) >0:2746 self.gitStream.write("\nJobs:%s"% (' '.join(jobs)))2747 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2748(','.join(self.branchPrefixes), details["change"]))2749iflen(details['options']) >0:2750 self.gitStream.write(": options =%s"% details['options'])2751 self.gitStream.write("]\nEOT\n\n")27522753iflen(parent) >0:2754if self.verbose:2755print"parent%s"% parent2756 self.gitStream.write("from%s\n"% parent)27572758 self.streamP4Files(files)2759 self.gitStream.write("\n")27602761 change =int(details["change"])27622763if self.labels.has_key(change):2764 label = self.labels[change]2765 labelDetails = label[0]2766 labelRevisions = label[1]2767if self.verbose:2768print"Change%sis labelled%s"% (change, labelDetails)27692770 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2771for p in self.branchPrefixes])27722773iflen(files) ==len(labelRevisions):27742775 cleanedFiles = {}2776for info in files:2777if info["action"]in self.delete_actions:2778continue2779 cleanedFiles[info["depotFile"]] = info["rev"]27802781if cleanedFiles == labelRevisions:2782 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)27832784else:2785if not self.silent:2786print("Tag%sdoes not match with change%s: files do not match."2787% (labelDetails["label"], change))27882789else:2790if not self.silent:2791print("Tag%sdoes not match with change%s: file count is different."2792% (labelDetails["label"], change))27932794# Build a dictionary of changelists and labels, for "detect-labels" option.2795defgetLabels(self):2796 self.labels = {}27972798 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2799iflen(l) >0and not self.silent:2800print"Finding files belonging to labels in%s"% `self.depotPaths`28012802for output in l:2803 label = output["label"]2804 revisions = {}2805 newestChange =02806if self.verbose:2807print"Querying files for label%s"% label2808forfileinp4CmdList(["files"] +2809["%s...@%s"% (p, label)2810for p in self.depotPaths]):2811 revisions[file["depotFile"]] =file["rev"]2812 change =int(file["change"])2813if change > newestChange:2814 newestChange = change28152816 self.labels[newestChange] = [output, revisions]28172818if self.verbose:2819print"Label changes:%s"% self.labels.keys()28202821# Import p4 labels as git tags. A direct mapping does not2822# exist, so assume that if all the files are at the same revision2823# then we can use that, or it's something more complicated we should2824# just ignore.2825defimportP4Labels(self, stream, p4Labels):2826if verbose:2827print"import p4 labels: "+' '.join(p4Labels)28282829 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2830 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2831iflen(validLabelRegexp) ==0:2832 validLabelRegexp = defaultLabelRegexp2833 m = re.compile(validLabelRegexp)28342835for name in p4Labels:2836 commitFound =False28372838if not m.match(name):2839if verbose:2840print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2841continue28422843if name in ignoredP4Labels:2844continue28452846 labelDetails =p4CmdList(['label',"-o", name])[0]28472848# get the most recent changelist for each file in this label2849 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2850for p in self.depotPaths])28512852if change.has_key('change'):2853# find the corresponding git commit; take the oldest commit2854 changelist =int(change['change'])2855if changelist in self.committedChanges:2856 gitCommit =":%d"% changelist # use a fast-import mark2857 commitFound =True2858else:2859 gitCommit =read_pipe(["git","rev-list","--max-count=1",2860"--reverse",":/\[git-p4:.*change =%d\]"% changelist], ignore_error=True)2861iflen(gitCommit) ==0:2862print"importing label%s: could not find git commit for changelist%d"% (name, changelist)2863else:2864 commitFound =True2865 gitCommit = gitCommit.strip()28662867if commitFound:2868# Convert from p4 time format2869try:2870 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2871exceptValueError:2872print"Could not convert label time%s"% labelDetails['Update']2873 tmwhen =128742875 when =int(time.mktime(tmwhen))2876 self.streamTag(stream, name, labelDetails, gitCommit, when)2877if verbose:2878print"p4 label%smapped to git commit%s"% (name, gitCommit)2879else:2880if verbose:2881print"Label%shas no changelists - possibly deleted?"% name28822883if not commitFound:2884# We can't import this label; don't try again as it will get very2885# expensive repeatedly fetching all the files for labels that will2886# never be imported. If the label is moved in the future, the2887# ignore will need to be removed manually.2888system(["git","config","--add","git-p4.ignoredP4Labels", name])28892890defguessProjectName(self):2891for p in self.depotPaths:2892if p.endswith("/"):2893 p = p[:-1]2894 p = p[p.strip().rfind("/") +1:]2895if not p.endswith("/"):2896 p +="/"2897return p28982899defgetBranchMapping(self):2900 lostAndFoundBranches =set()29012902 user =gitConfig("git-p4.branchUser")2903iflen(user) >0:2904 command ="branches -u%s"% user2905else:2906 command ="branches"29072908for info inp4CmdList(command):2909 details =p4Cmd(["branch","-o", info["branch"]])2910 viewIdx =02911while details.has_key("View%s"% viewIdx):2912 paths = details["View%s"% viewIdx].split(" ")2913 viewIdx = viewIdx +12914# require standard //depot/foo/... //depot/bar/... mapping2915iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2916continue2917 source = paths[0]2918 destination = paths[1]2919## HACK2920ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2921 source = source[len(self.depotPaths[0]):-4]2922 destination = destination[len(self.depotPaths[0]):-4]29232924if destination in self.knownBranches:2925if not self.silent:2926print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2927print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2928continue29292930 self.knownBranches[destination] = source29312932 lostAndFoundBranches.discard(destination)29332934if source not in self.knownBranches:2935 lostAndFoundBranches.add(source)29362937# Perforce does not strictly require branches to be defined, so we also2938# check git config for a branch list.2939#2940# Example of branch definition in git config file:2941# [git-p4]2942# branchList=main:branchA2943# branchList=main:branchB2944# branchList=branchA:branchC2945 configBranches =gitConfigList("git-p4.branchList")2946for branch in configBranches:2947if branch:2948(source, destination) = branch.split(":")2949 self.knownBranches[destination] = source29502951 lostAndFoundBranches.discard(destination)29522953if source not in self.knownBranches:2954 lostAndFoundBranches.add(source)295529562957for branch in lostAndFoundBranches:2958 self.knownBranches[branch] = branch29592960defgetBranchMappingFromGitBranches(self):2961 branches =p4BranchesInGit(self.importIntoRemotes)2962for branch in branches.keys():2963if branch =="master":2964 branch ="main"2965else:2966 branch = branch[len(self.projectName):]2967 self.knownBranches[branch] = branch29682969defupdateOptionDict(self, d):2970 option_keys = {}2971if self.keepRepoPath:2972 option_keys['keepRepoPath'] =129732974 d["options"] =' '.join(sorted(option_keys.keys()))29752976defreadOptions(self, d):2977 self.keepRepoPath = (d.has_key('options')2978and('keepRepoPath'in d['options']))29792980defgitRefForBranch(self, branch):2981if branch =="main":2982return self.refPrefix +"master"29832984iflen(branch) <=0:2985return branch29862987return self.refPrefix + self.projectName + branch29882989defgitCommitByP4Change(self, ref, change):2990if self.verbose:2991print"looking in ref "+ ref +" for change%susing bisect..."% change29922993 earliestCommit =""2994 latestCommit =parseRevision(ref)29952996while True:2997if self.verbose:2998print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2999 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()3000iflen(next) ==0:3001if self.verbose:3002print"argh"3003return""3004 log =extractLogMessageFromGitCommit(next)3005 settings =extractSettingsGitLog(log)3006 currentChange =int(settings['change'])3007if self.verbose:3008print"current change%s"% currentChange30093010if currentChange == change:3011if self.verbose:3012print"found%s"% next3013return next30143015if currentChange < change:3016 earliestCommit ="^%s"% next3017else:3018 latestCommit ="%s"% next30193020return""30213022defimportNewBranch(self, branch, maxChange):3023# make fast-import flush all changes to disk and update the refs using the checkpoint3024# command so that we can try to find the branch parent in the git history3025 self.gitStream.write("checkpoint\n\n");3026 self.gitStream.flush();3027 branchPrefix = self.depotPaths[0] + branch +"/"3028range="@1,%s"% maxChange3029#print "prefix" + branchPrefix3030 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)3031iflen(changes) <=0:3032return False3033 firstChange = changes[0]3034#print "first change in branch: %s" % firstChange3035 sourceBranch = self.knownBranches[branch]3036 sourceDepotPath = self.depotPaths[0] + sourceBranch3037 sourceRef = self.gitRefForBranch(sourceBranch)3038#print "source " + sourceBranch30393040 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])3041#print "branch parent: %s" % branchParentChange3042 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)3043iflen(gitParent) >0:3044 self.initialParents[self.gitRefForBranch(branch)] = gitParent3045#print "parent git commit: %s" % gitParent30463047 self.importChanges(changes)3048return True30493050defsearchParent(self, parent, branch, target):3051 parentFound =False3052for blob inread_pipe_lines(["git","rev-list","--reverse",3053"--no-merges", parent]):3054 blob = blob.strip()3055iflen(read_pipe(["git","diff-tree", blob, target])) ==0:3056 parentFound =True3057if self.verbose:3058print"Found parent of%sin commit%s"% (branch, blob)3059break3060if parentFound:3061return blob3062else:3063return None30643065defimportChanges(self, changes):3066 cnt =13067for change in changes:3068 description =p4_describe(change)3069 self.updateOptionDict(description)30703071if not self.silent:3072 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))3073 sys.stdout.flush()3074 cnt = cnt +130753076try:3077if self.detectBranches:3078 branches = self.splitFilesIntoBranches(description)3079for branch in branches.keys():3080## HACK --hwn3081 branchPrefix = self.depotPaths[0] + branch +"/"3082 self.branchPrefixes = [ branchPrefix ]30833084 parent =""30853086 filesForCommit = branches[branch]30873088if self.verbose:3089print"branch is%s"% branch30903091 self.updatedBranches.add(branch)30923093if branch not in self.createdBranches:3094 self.createdBranches.add(branch)3095 parent = self.knownBranches[branch]3096if parent == branch:3097 parent =""3098else:3099 fullBranch = self.projectName + branch3100if fullBranch not in self.p4BranchesInGit:3101if not self.silent:3102print("\nImporting new branch%s"% fullBranch);3103if self.importNewBranch(branch, change -1):3104 parent =""3105 self.p4BranchesInGit.append(fullBranch)3106if not self.silent:3107print("\nResuming with change%s"% change);31083109if self.verbose:3110print"parent determined through known branches:%s"% parent31113112 branch = self.gitRefForBranch(branch)3113 parent = self.gitRefForBranch(parent)31143115if self.verbose:3116print"looking for initial parent for%s; current parent is%s"% (branch, parent)31173118iflen(parent) ==0and branch in self.initialParents:3119 parent = self.initialParents[branch]3120del self.initialParents[branch]31213122 blob =None3123iflen(parent) >0:3124 tempBranch ="%s/%d"% (self.tempBranchLocation, change)3125if self.verbose:3126print"Creating temporary branch: "+ tempBranch3127 self.commit(description, filesForCommit, tempBranch)3128 self.tempBranches.append(tempBranch)3129 self.checkpoint()3130 blob = self.searchParent(parent, branch, tempBranch)3131if blob:3132 self.commit(description, filesForCommit, branch, blob)3133else:3134if self.verbose:3135print"Parent of%snot found. Committing into head of%s"% (branch, parent)3136 self.commit(description, filesForCommit, branch, parent)3137else:3138 files = self.extractFilesFromCommit(description)3139 self.commit(description, files, self.branch,3140 self.initialParent)3141# only needed once, to connect to the previous commit3142 self.initialParent =""3143exceptIOError:3144print self.gitError.read()3145 sys.exit(1)31463147defimportHeadRevision(self, revision):3148print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)31493150 details = {}3151 details["user"] ="git perforce import user"3152 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"3153% (' '.join(self.depotPaths), revision))3154 details["change"] = revision3155 newestRevision =031563157 fileCnt =03158 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]31593160for info inp4CmdList(["files"] + fileArgs):31613162if'code'in info and info['code'] =='error':3163 sys.stderr.write("p4 returned an error:%s\n"3164% info['data'])3165if info['data'].find("must refer to client") >=0:3166 sys.stderr.write("This particular p4 error is misleading.\n")3167 sys.stderr.write("Perhaps the depot path was misspelled.\n");3168 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))3169 sys.exit(1)3170if'p4ExitCode'in info:3171 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])3172 sys.exit(1)317331743175 change =int(info["change"])3176if change > newestRevision:3177 newestRevision = change31783179if info["action"]in self.delete_actions:3180# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!3181#fileCnt = fileCnt + 13182continue31833184for prop in["depotFile","rev","action","type"]:3185 details["%s%s"% (prop, fileCnt)] = info[prop]31863187 fileCnt = fileCnt +131883189 details["change"] = newestRevision31903191# Use time from top-most change so that all git p4 clones of3192# the same p4 repo have the same commit SHA1s.3193 res =p4_describe(newestRevision)3194 details["time"] = res["time"]31953196 self.updateOptionDict(details)3197try:3198 self.commit(details, self.extractFilesFromCommit(details), self.branch)3199exceptIOError:3200print"IO error with git fast-import. Is your git version recent enough?"3201print self.gitError.read()320232033204defrun(self, args):3205 self.depotPaths = []3206 self.changeRange =""3207 self.previousDepotPaths = []3208 self.hasOrigin =False32093210# map from branch depot path to parent branch3211 self.knownBranches = {}3212 self.initialParents = {}32133214if self.importIntoRemotes:3215 self.refPrefix ="refs/remotes/p4/"3216else:3217 self.refPrefix ="refs/heads/p4/"32183219if self.syncWithOrigin:3220 self.hasOrigin =originP4BranchesExist()3221if self.hasOrigin:3222if not self.silent:3223print'Syncing with origin first, using "git fetch origin"'3224system("git fetch origin")32253226 branch_arg_given =bool(self.branch)3227iflen(self.branch) ==0:3228 self.branch = self.refPrefix +"master"3229ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:3230system("git update-ref%srefs/heads/p4"% self.branch)3231system("git branch -D p4")32323233# accept either the command-line option, or the configuration variable3234if self.useClientSpec:3235# will use this after clone to set the variable3236 self.useClientSpec_from_options =True3237else:3238ifgitConfigBool("git-p4.useclientspec"):3239 self.useClientSpec =True3240if self.useClientSpec:3241 self.clientSpecDirs =getClientSpec()32423243# TODO: should always look at previous commits,3244# merge with previous imports, if possible.3245if args == []:3246if self.hasOrigin:3247createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)32483249# branches holds mapping from branch name to sha13250 branches =p4BranchesInGit(self.importIntoRemotes)32513252# restrict to just this one, disabling detect-branches3253if branch_arg_given:3254 short = self.branch.split("/")[-1]3255if short in branches:3256 self.p4BranchesInGit = [ short ]3257else:3258 self.p4BranchesInGit = branches.keys()32593260iflen(self.p4BranchesInGit) >1:3261if not self.silent:3262print"Importing from/into multiple branches"3263 self.detectBranches =True3264for branch in branches.keys():3265 self.initialParents[self.refPrefix + branch] = \3266 branches[branch]32673268if self.verbose:3269print"branches:%s"% self.p4BranchesInGit32703271 p4Change =03272for branch in self.p4BranchesInGit:3273 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)32743275 settings =extractSettingsGitLog(logMsg)32763277 self.readOptions(settings)3278if(settings.has_key('depot-paths')3279and settings.has_key('change')):3280 change =int(settings['change']) +13281 p4Change =max(p4Change, change)32823283 depotPaths =sorted(settings['depot-paths'])3284if self.previousDepotPaths == []:3285 self.previousDepotPaths = depotPaths3286else:3287 paths = []3288for(prev, cur)inzip(self.previousDepotPaths, depotPaths):3289 prev_list = prev.split("/")3290 cur_list = cur.split("/")3291for i inrange(0,min(len(cur_list),len(prev_list))):3292if cur_list[i] <> prev_list[i]:3293 i = i -13294break32953296 paths.append("/".join(cur_list[:i +1]))32973298 self.previousDepotPaths = paths32993300if p4Change >0:3301 self.depotPaths =sorted(self.previousDepotPaths)3302 self.changeRange ="@%s,#head"% p4Change3303if not self.silent and not self.detectBranches:3304print"Performing incremental import into%sgit branch"% self.branch33053306# accept multiple ref name abbreviations:3307# refs/foo/bar/branch -> use it exactly3308# p4/branch -> prepend refs/remotes/ or refs/heads/3309# branch -> prepend refs/remotes/p4/ or refs/heads/p4/3310if not self.branch.startswith("refs/"):3311if self.importIntoRemotes:3312 prepend ="refs/remotes/"3313else:3314 prepend ="refs/heads/"3315if not self.branch.startswith("p4/"):3316 prepend +="p4/"3317 self.branch = prepend + self.branch33183319iflen(args) ==0and self.depotPaths:3320if not self.silent:3321print"Depot paths:%s"%' '.join(self.depotPaths)3322else:3323if self.depotPaths and self.depotPaths != args:3324print("previous import used depot path%sand now%swas specified. "3325"This doesn't work!"% (' '.join(self.depotPaths),3326' '.join(args)))3327 sys.exit(1)33283329 self.depotPaths =sorted(args)33303331 revision =""3332 self.users = {}33333334# Make sure no revision specifiers are used when --changesfile3335# is specified.3336 bad_changesfile =False3337iflen(self.changesFile) >0:3338for p in self.depotPaths:3339if p.find("@") >=0or p.find("#") >=0:3340 bad_changesfile =True3341break3342if bad_changesfile:3343die("Option --changesfile is incompatible with revision specifiers")33443345 newPaths = []3346for p in self.depotPaths:3347if p.find("@") != -1:3348 atIdx = p.index("@")3349 self.changeRange = p[atIdx:]3350if self.changeRange =="@all":3351 self.changeRange =""3352elif','not in self.changeRange:3353 revision = self.changeRange3354 self.changeRange =""3355 p = p[:atIdx]3356elif p.find("#") != -1:3357 hashIdx = p.index("#")3358 revision = p[hashIdx:]3359 p = p[:hashIdx]3360elif self.previousDepotPaths == []:3361# pay attention to changesfile, if given, else import3362# the entire p4 tree at the head revision3363iflen(self.changesFile) ==0:3364 revision ="#head"33653366 p = re.sub("\.\.\.$","", p)3367if not p.endswith("/"):3368 p +="/"33693370 newPaths.append(p)33713372 self.depotPaths = newPaths33733374# --detect-branches may change this for each branch3375 self.branchPrefixes = self.depotPaths33763377 self.loadUserMapFromCache()3378 self.labels = {}3379if self.detectLabels:3380 self.getLabels();33813382if self.detectBranches:3383## FIXME - what's a P4 projectName ?3384 self.projectName = self.guessProjectName()33853386if self.hasOrigin:3387 self.getBranchMappingFromGitBranches()3388else:3389 self.getBranchMapping()3390if self.verbose:3391print"p4-git branches:%s"% self.p4BranchesInGit3392print"initial parents:%s"% self.initialParents3393for b in self.p4BranchesInGit:3394if b !="master":33953396## FIXME3397 b = b[len(self.projectName):]3398 self.createdBranches.add(b)33993400 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))34013402 self.importProcess = subprocess.Popen(["git","fast-import"],3403 stdin=subprocess.PIPE,3404 stdout=subprocess.PIPE,3405 stderr=subprocess.PIPE);3406 self.gitOutput = self.importProcess.stdout3407 self.gitStream = self.importProcess.stdin3408 self.gitError = self.importProcess.stderr34093410if revision:3411 self.importHeadRevision(revision)3412else:3413 changes = []34143415iflen(self.changesFile) >0:3416 output =open(self.changesFile).readlines()3417 changeSet =set()3418for line in output:3419 changeSet.add(int(line))34203421for change in changeSet:3422 changes.append(change)34233424 changes.sort()3425else:3426# catch "git p4 sync" with no new branches, in a repo that3427# does not have any existing p4 branches3428iflen(args) ==0:3429if not self.p4BranchesInGit:3430die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")34313432# The default branch is master, unless --branch is used to3433# specify something else. Make sure it exists, or complain3434# nicely about how to use --branch.3435if not self.detectBranches:3436if notbranch_exists(self.branch):3437if branch_arg_given:3438die("Error: branch%sdoes not exist."% self.branch)3439else:3440die("Error: no branch%s; perhaps specify one with --branch."%3441 self.branch)34423443if self.verbose:3444print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3445 self.changeRange)3446 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)34473448iflen(self.maxChanges) >0:3449 changes = changes[:min(int(self.maxChanges),len(changes))]34503451iflen(changes) ==0:3452if not self.silent:3453print"No changes to import!"3454else:3455if not self.silent and not self.detectBranches:3456print"Import destination:%s"% self.branch34573458 self.updatedBranches =set()34593460if not self.detectBranches:3461if args:3462# start a new branch3463 self.initialParent =""3464else:3465# build on a previous revision3466 self.initialParent =parseRevision(self.branch)34673468 self.importChanges(changes)34693470if not self.silent:3471print""3472iflen(self.updatedBranches) >0:3473 sys.stdout.write("Updated branches: ")3474for b in self.updatedBranches:3475 sys.stdout.write("%s"% b)3476 sys.stdout.write("\n")34773478ifgitConfigBool("git-p4.importLabels"):3479 self.importLabels =True34803481if self.importLabels:3482 p4Labels =getP4Labels(self.depotPaths)3483 gitTags =getGitTags()34843485 missingP4Labels = p4Labels - gitTags3486 self.importP4Labels(self.gitStream, missingP4Labels)34873488 self.gitStream.close()3489if self.importProcess.wait() !=0:3490die("fast-import failed:%s"% self.gitError.read())3491 self.gitOutput.close()3492 self.gitError.close()34933494# Cleanup temporary branches created during import3495if self.tempBranches != []:3496for branch in self.tempBranches:3497read_pipe("git update-ref -d%s"% branch)3498 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))34993500# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3501# a convenient shortcut refname "p4".3502if self.importIntoRemotes:3503 head_ref = self.refPrefix +"HEAD"3504if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3505system(["git","symbolic-ref", head_ref, self.branch])35063507return True35083509classP4Rebase(Command):3510def__init__(self):3511 Command.__init__(self)3512 self.options = [3513 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3514]3515 self.importLabels =False3516 self.description = ("Fetches the latest revision from perforce and "3517+"rebases the current work (branch) against it")35183519defrun(self, args):3520 sync =P4Sync()3521 sync.importLabels = self.importLabels3522 sync.run([])35233524return self.rebase()35253526defrebase(self):3527if os.system("git update-index --refresh") !=0:3528die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");3529iflen(read_pipe("git diff-index HEAD --")) >0:3530die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");35313532[upstream, settings] =findUpstreamBranchPoint()3533iflen(upstream) ==0:3534die("Cannot find upstream branchpoint for rebase")35353536# the branchpoint may be p4/foo~3, so strip off the parent3537 upstream = re.sub("~[0-9]+$","", upstream)35383539print"Rebasing the current branch onto%s"% upstream3540 oldHead =read_pipe("git rev-parse HEAD").strip()3541system("git rebase%s"% upstream)3542system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3543return True35443545classP4Clone(P4Sync):3546def__init__(self):3547 P4Sync.__init__(self)3548 self.description ="Creates a new git repository and imports from Perforce into it"3549 self.usage ="usage: %prog [options] //depot/path[@revRange]"3550 self.options += [3551 optparse.make_option("--destination", dest="cloneDestination",3552 action='store', default=None,3553help="where to leave result of the clone"),3554 optparse.make_option("--bare", dest="cloneBare",3555 action="store_true", default=False),3556]3557 self.cloneDestination =None3558 self.needsGit =False3559 self.cloneBare =False35603561defdefaultDestination(self, args):3562## TODO: use common prefix of args?3563 depotPath = args[0]3564 depotDir = re.sub("(@[^@]*)$","", depotPath)3565 depotDir = re.sub("(#[^#]*)$","", depotDir)3566 depotDir = re.sub(r"\.\.\.$","", depotDir)3567 depotDir = re.sub(r"/$","", depotDir)3568return os.path.split(depotDir)[1]35693570defrun(self, args):3571iflen(args) <1:3572return False35733574if self.keepRepoPath and not self.cloneDestination:3575 sys.stderr.write("Must specify destination for --keep-path\n")3576 sys.exit(1)35773578 depotPaths = args35793580if not self.cloneDestination andlen(depotPaths) >1:3581 self.cloneDestination = depotPaths[-1]3582 depotPaths = depotPaths[:-1]35833584 self.cloneExclude = ["/"+p for p in self.cloneExclude]3585for p in depotPaths:3586if not p.startswith("//"):3587 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3588return False35893590if not self.cloneDestination:3591 self.cloneDestination = self.defaultDestination(args)35923593print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)35943595if not os.path.exists(self.cloneDestination):3596 os.makedirs(self.cloneDestination)3597chdir(self.cloneDestination)35983599 init_cmd = ["git","init"]3600if self.cloneBare:3601 init_cmd.append("--bare")3602 retcode = subprocess.call(init_cmd)3603if retcode:3604raiseCalledProcessError(retcode, init_cmd)36053606if not P4Sync.run(self, depotPaths):3607return False36083609# create a master branch and check out a work tree3610ifgitBranchExists(self.branch):3611system(["git","branch","master", self.branch ])3612if not self.cloneBare:3613system(["git","checkout","-f"])3614else:3615print'Not checking out any branch, use ' \3616'"git checkout -q -b master <branch>"'36173618# auto-set this variable if invoked with --use-client-spec3619if self.useClientSpec_from_options:3620system("git config --bool git-p4.useclientspec true")36213622return True36233624classP4Branches(Command):3625def__init__(self):3626 Command.__init__(self)3627 self.options = [ ]3628 self.description = ("Shows the git branches that hold imports and their "3629+"corresponding perforce depot paths")3630 self.verbose =False36313632defrun(self, args):3633iforiginP4BranchesExist():3634createOrUpdateBranchesFromOrigin()36353636 cmdline ="git rev-parse --symbolic "3637 cmdline +=" --remotes"36383639for line inread_pipe_lines(cmdline):3640 line = line.strip()36413642if not line.startswith('p4/')or line =="p4/HEAD":3643continue3644 branch = line36453646 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)3647 settings =extractSettingsGitLog(log)36483649print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])3650return True36513652classHelpFormatter(optparse.IndentedHelpFormatter):3653def__init__(self):3654 optparse.IndentedHelpFormatter.__init__(self)36553656defformat_description(self, description):3657if description:3658return description +"\n"3659else:3660return""36613662defprintUsage(commands):3663print"usage:%s<command> [options]"% sys.argv[0]3664print""3665print"valid commands:%s"%", ".join(commands)3666print""3667print"Try%s<command> --help for command specific help."% sys.argv[0]3668print""36693670commands = {3671"debug": P4Debug,3672"submit": P4Submit,3673"commit": P4Submit,3674"sync": P4Sync,3675"rebase": P4Rebase,3676"clone": P4Clone,3677"rollback": P4RollBack,3678"branches": P4Branches3679}368036813682defmain():3683iflen(sys.argv[1:]) ==0:3684printUsage(commands.keys())3685 sys.exit(2)36863687 cmdName = sys.argv[1]3688try:3689 klass = commands[cmdName]3690 cmd =klass()3691exceptKeyError:3692print"unknown command%s"% cmdName3693print""3694printUsage(commands.keys())3695 sys.exit(2)36963697 options = cmd.options3698 cmd.gitdir = os.environ.get("GIT_DIR",None)36993700 args = sys.argv[2:]37013702 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))3703if cmd.needsGit:3704 options.append(optparse.make_option("--git-dir", dest="gitdir"))37053706 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3707 options,3708 description = cmd.description,3709 formatter =HelpFormatter())37103711(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3712global verbose3713 verbose = cmd.verbose3714if cmd.needsGit:3715if cmd.gitdir ==None:3716 cmd.gitdir = os.path.abspath(".git")3717if notisValidGitDir(cmd.gitdir):3718 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3719if os.path.exists(cmd.gitdir):3720 cdup =read_pipe("git rev-parse --show-cdup").strip()3721iflen(cdup) >0:3722chdir(cdup);37233724if notisValidGitDir(cmd.gitdir):3725ifisValidGitDir(cmd.gitdir +"/.git"):3726 cmd.gitdir +="/.git"3727else:3728die("fatal: cannot locate git repository at%s"% cmd.gitdir)37293730 os.environ["GIT_DIR"] = cmd.gitdir37313732if not cmd.run(args):3733 parser.print_help()3734 sys.exit(2)373537363737if __name__ =='__main__':3738main()