1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25import zipfile 26import zlib 27import ctypes 28 29try: 30from subprocess import CalledProcessError 31exceptImportError: 32# from python2.7:subprocess.py 33# Exception classes used by this module. 34classCalledProcessError(Exception): 35"""This exception is raised when a process run by check_call() returns 36 a non-zero exit status. The exit status will be stored in the 37 returncode attribute.""" 38def__init__(self, returncode, cmd): 39 self.returncode = returncode 40 self.cmd = cmd 41def__str__(self): 42return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 43 44verbose =False 45 46# Only labels/tags matching this will be imported/exported 47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 48 49# Grab changes in blocks of this many revisions, unless otherwise requested 50defaultBlockSize =512 51 52defp4_build_cmd(cmd): 53"""Build a suitable p4 command line. 54 55 This consolidates building and returning a p4 command line into one 56 location. It means that hooking into the environment, or other configuration 57 can be done more easily. 58 """ 59 real_cmd = ["p4"] 60 61 user =gitConfig("git-p4.user") 62iflen(user) >0: 63 real_cmd += ["-u",user] 64 65 password =gitConfig("git-p4.password") 66iflen(password) >0: 67 real_cmd += ["-P", password] 68 69 port =gitConfig("git-p4.port") 70iflen(port) >0: 71 real_cmd += ["-p", port] 72 73 host =gitConfig("git-p4.host") 74iflen(host) >0: 75 real_cmd += ["-H", host] 76 77 client =gitConfig("git-p4.client") 78iflen(client) >0: 79 real_cmd += ["-c", client] 80 81 82ifisinstance(cmd,basestring): 83 real_cmd =' '.join(real_cmd) +' '+ cmd 84else: 85 real_cmd += cmd 86return real_cmd 87 88defchdir(path, is_client_path=False): 89"""Do chdir to the given path, and set the PWD environment 90 variable for use by P4. It does not look at getcwd() output. 91 Since we're not using the shell, it is necessary to set the 92 PWD environment variable explicitly. 93 94 Normally, expand the path to force it to be absolute. This 95 addresses the use of relative path names inside P4 settings, 96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 97 as given; it looks for .p4config using PWD. 98 99 If is_client_path, the path was handed to us directly by p4, 100 and may be a symbolic link. Do not call os.getcwd() in this 101 case, because it will cause p4 to think that PWD is not inside 102 the client path. 103 """ 104 105 os.chdir(path) 106if not is_client_path: 107 path = os.getcwd() 108 os.environ['PWD'] = path 109 110defcalcDiskFree(): 111"""Return free space in bytes on the disk of the given dirname.""" 112if platform.system() =='Windows': 113 free_bytes = ctypes.c_ulonglong(0) 114 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()),None,None, ctypes.pointer(free_bytes)) 115return free_bytes.value 116else: 117 st = os.statvfs(os.getcwd()) 118return st.f_bavail * st.f_frsize 119 120defdie(msg): 121if verbose: 122raiseException(msg) 123else: 124 sys.stderr.write(msg +"\n") 125 sys.exit(1) 126 127defwrite_pipe(c, stdin): 128if verbose: 129 sys.stderr.write('Writing pipe:%s\n'%str(c)) 130 131 expand =isinstance(c,basestring) 132 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 133 pipe = p.stdin 134 val = pipe.write(stdin) 135 pipe.close() 136if p.wait(): 137die('Command failed:%s'%str(c)) 138 139return val 140 141defp4_write_pipe(c, stdin): 142 real_cmd =p4_build_cmd(c) 143returnwrite_pipe(real_cmd, stdin) 144 145defread_pipe(c, ignore_error=False): 146if verbose: 147 sys.stderr.write('Reading pipe:%s\n'%str(c)) 148 149 expand =isinstance(c,basestring) 150 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) 151(out, err) = p.communicate() 152if p.returncode !=0and not ignore_error: 153die('Command failed:%s\nError:%s'% (str(c), err)) 154return out 155 156defp4_read_pipe(c, ignore_error=False): 157 real_cmd =p4_build_cmd(c) 158returnread_pipe(real_cmd, ignore_error) 159 160defread_pipe_lines(c): 161if verbose: 162 sys.stderr.write('Reading pipe:%s\n'%str(c)) 163 164 expand =isinstance(c, basestring) 165 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 166 pipe = p.stdout 167 val = pipe.readlines() 168if pipe.close()or p.wait(): 169die('Command failed:%s'%str(c)) 170 171return val 172 173defp4_read_pipe_lines(c): 174"""Specifically invoke p4 on the command supplied. """ 175 real_cmd =p4_build_cmd(c) 176returnread_pipe_lines(real_cmd) 177 178defp4_has_command(cmd): 179"""Ask p4 for help on this command. If it returns an error, the 180 command does not exist in this version of p4.""" 181 real_cmd =p4_build_cmd(["help", cmd]) 182 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 183 stderr=subprocess.PIPE) 184 p.communicate() 185return p.returncode ==0 186 187defp4_has_move_command(): 188"""See if the move command exists, that it supports -k, and that 189 it has not been administratively disabled. The arguments 190 must be correct, but the filenames do not have to exist. Use 191 ones with wildcards so even if they exist, it will fail.""" 192 193if notp4_has_command("move"): 194return False 195 cmd =p4_build_cmd(["move","-k","@from","@to"]) 196 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 197(out, err) = p.communicate() 198# return code will be 1 in either case 199if err.find("Invalid option") >=0: 200return False 201if err.find("disabled") >=0: 202return False 203# assume it failed because @... was invalid changelist 204return True 205 206defsystem(cmd, ignore_error=False): 207 expand =isinstance(cmd,basestring) 208if verbose: 209 sys.stderr.write("executing%s\n"%str(cmd)) 210 retcode = subprocess.call(cmd, shell=expand) 211if retcode and not ignore_error: 212raiseCalledProcessError(retcode, cmd) 213 214return retcode 215 216defp4_system(cmd): 217"""Specifically invoke p4 as the system command. """ 218 real_cmd =p4_build_cmd(cmd) 219 expand =isinstance(real_cmd, basestring) 220 retcode = subprocess.call(real_cmd, shell=expand) 221if retcode: 222raiseCalledProcessError(retcode, real_cmd) 223 224_p4_version_string =None 225defp4_version_string(): 226"""Read the version string, showing just the last line, which 227 hopefully is the interesting version bit. 228 229 $ p4 -V 230 Perforce - The Fast Software Configuration Management System. 231 Copyright 1995-2011 Perforce Software. All rights reserved. 232 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 233 """ 234global _p4_version_string 235if not _p4_version_string: 236 a =p4_read_pipe_lines(["-V"]) 237 _p4_version_string = a[-1].rstrip() 238return _p4_version_string 239 240defp4_integrate(src, dest): 241p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 242 243defp4_sync(f, *options): 244p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 245 246defp4_add(f): 247# forcibly add file names with wildcards 248ifwildcard_present(f): 249p4_system(["add","-f", f]) 250else: 251p4_system(["add", f]) 252 253defp4_delete(f): 254p4_system(["delete",wildcard_encode(f)]) 255 256defp4_edit(f, *options): 257p4_system(["edit"] +list(options) + [wildcard_encode(f)]) 258 259defp4_revert(f): 260p4_system(["revert",wildcard_encode(f)]) 261 262defp4_reopen(type, f): 263p4_system(["reopen","-t",type,wildcard_encode(f)]) 264 265defp4_move(src, dest): 266p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 267 268defp4_last_change(): 269 results =p4CmdList(["changes","-m","1"]) 270returnint(results[0]['change']) 271 272defp4_describe(change): 273"""Make sure it returns a valid result by checking for 274 the presence of field "time". Return a dict of the 275 results.""" 276 277 ds =p4CmdList(["describe","-s",str(change)]) 278iflen(ds) !=1: 279die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 280 281 d = ds[0] 282 283if"p4ExitCode"in d: 284die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 285str(d))) 286if"code"in d: 287if d["code"] =="error": 288die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 289 290if"time"not in d: 291die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 292 293return d 294 295# 296# Canonicalize the p4 type and return a tuple of the 297# base type, plus any modifiers. See "p4 help filetypes" 298# for a list and explanation. 299# 300defsplit_p4_type(p4type): 301 302 p4_filetypes_historical = { 303"ctempobj":"binary+Sw", 304"ctext":"text+C", 305"cxtext":"text+Cx", 306"ktext":"text+k", 307"kxtext":"text+kx", 308"ltext":"text+F", 309"tempobj":"binary+FSw", 310"ubinary":"binary+F", 311"uresource":"resource+F", 312"uxbinary":"binary+Fx", 313"xbinary":"binary+x", 314"xltext":"text+Fx", 315"xtempobj":"binary+Swx", 316"xtext":"text+x", 317"xunicode":"unicode+x", 318"xutf16":"utf16+x", 319} 320if p4type in p4_filetypes_historical: 321 p4type = p4_filetypes_historical[p4type] 322 mods ="" 323 s = p4type.split("+") 324 base = s[0] 325 mods ="" 326iflen(s) >1: 327 mods = s[1] 328return(base, mods) 329 330# 331# return the raw p4 type of a file (text, text+ko, etc) 332# 333defp4_type(f): 334 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 335return results[0]['headType'] 336 337# 338# Given a type base and modifier, return a regexp matching 339# the keywords that can be expanded in the file 340# 341defp4_keywords_regexp_for_type(base, type_mods): 342if base in("text","unicode","binary"): 343 kwords =None 344if"ko"in type_mods: 345 kwords ='Id|Header' 346elif"k"in type_mods: 347 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 348else: 349return None 350 pattern = r""" 351 \$ # Starts with a dollar, followed by... 352 (%s) # one of the keywords, followed by... 353 (:[^$\n]+)? # possibly an old expansion, followed by... 354 \$ # another dollar 355 """% kwords 356return pattern 357else: 358return None 359 360# 361# Given a file, return a regexp matching the possible 362# RCS keywords that will be expanded, or None for files 363# with kw expansion turned off. 364# 365defp4_keywords_regexp_for_file(file): 366if not os.path.exists(file): 367return None 368else: 369(type_base, type_mods) =split_p4_type(p4_type(file)) 370returnp4_keywords_regexp_for_type(type_base, type_mods) 371 372defsetP4ExecBit(file, mode): 373# Reopens an already open file and changes the execute bit to match 374# the execute bit setting in the passed in mode. 375 376 p4Type ="+x" 377 378if notisModeExec(mode): 379 p4Type =getP4OpenedType(file) 380 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 381 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 382if p4Type[-1] =="+": 383 p4Type = p4Type[0:-1] 384 385p4_reopen(p4Type,file) 386 387defgetP4OpenedType(file): 388# Returns the perforce file type for the given file. 389 390 result =p4_read_pipe(["opened",wildcard_encode(file)]) 391 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 392if match: 393return match.group(1) 394else: 395die("Could not determine file type for%s(result: '%s')"% (file, result)) 396 397# Return the set of all p4 labels 398defgetP4Labels(depotPaths): 399 labels =set() 400ifisinstance(depotPaths,basestring): 401 depotPaths = [depotPaths] 402 403for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 404 label = l['label'] 405 labels.add(label) 406 407return labels 408 409# Return the set of all git tags 410defgetGitTags(): 411 gitTags =set() 412for line inread_pipe_lines(["git","tag"]): 413 tag = line.strip() 414 gitTags.add(tag) 415return gitTags 416 417defdiffTreePattern(): 418# This is a simple generator for the diff tree regex pattern. This could be 419# a class variable if this and parseDiffTreeEntry were a part of a class. 420 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 421while True: 422yield pattern 423 424defparseDiffTreeEntry(entry): 425"""Parses a single diff tree entry into its component elements. 426 427 See git-diff-tree(1) manpage for details about the format of the diff 428 output. This method returns a dictionary with the following elements: 429 430 src_mode - The mode of the source file 431 dst_mode - The mode of the destination file 432 src_sha1 - The sha1 for the source file 433 dst_sha1 - The sha1 fr the destination file 434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 435 status_score - The score for the status (applicable for 'C' and 'R' 436 statuses). This is None if there is no score. 437 src - The path for the source file. 438 dst - The path for the destination file. This is only present for 439 copy or renames. If it is not present, this is None. 440 441 If the pattern is not matched, None is returned.""" 442 443 match =diffTreePattern().next().match(entry) 444if match: 445return{ 446'src_mode': match.group(1), 447'dst_mode': match.group(2), 448'src_sha1': match.group(3), 449'dst_sha1': match.group(4), 450'status': match.group(5), 451'status_score': match.group(6), 452'src': match.group(7), 453'dst': match.group(10) 454} 455return None 456 457defisModeExec(mode): 458# Returns True if the given git mode represents an executable file, 459# otherwise False. 460return mode[-3:] =="755" 461 462defisModeExecChanged(src_mode, dst_mode): 463returnisModeExec(src_mode) !=isModeExec(dst_mode) 464 465defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 466 467ifisinstance(cmd,basestring): 468 cmd ="-G "+ cmd 469 expand =True 470else: 471 cmd = ["-G"] + cmd 472 expand =False 473 474 cmd =p4_build_cmd(cmd) 475if verbose: 476 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 477 478# Use a temporary file to avoid deadlocks without 479# subprocess.communicate(), which would put another copy 480# of stdout into memory. 481 stdin_file =None 482if stdin is not None: 483 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 484ifisinstance(stdin,basestring): 485 stdin_file.write(stdin) 486else: 487for i in stdin: 488 stdin_file.write(i +'\n') 489 stdin_file.flush() 490 stdin_file.seek(0) 491 492 p4 = subprocess.Popen(cmd, 493 shell=expand, 494 stdin=stdin_file, 495 stdout=subprocess.PIPE) 496 497 result = [] 498try: 499while True: 500 entry = marshal.load(p4.stdout) 501if cb is not None: 502cb(entry) 503else: 504 result.append(entry) 505exceptEOFError: 506pass 507 exitCode = p4.wait() 508if exitCode !=0: 509 entry = {} 510 entry["p4ExitCode"] = exitCode 511 result.append(entry) 512 513return result 514 515defp4Cmd(cmd): 516list=p4CmdList(cmd) 517 result = {} 518for entry inlist: 519 result.update(entry) 520return result; 521 522defp4Where(depotPath): 523if not depotPath.endswith("/"): 524 depotPath +="/" 525 depotPathLong = depotPath +"..." 526 outputList =p4CmdList(["where", depotPathLong]) 527 output =None 528for entry in outputList: 529if"depotFile"in entry: 530# Search for the base client side depot path, as long as it starts with the branch's P4 path. 531# The base path always ends with "/...". 532if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 533 output = entry 534break 535elif"data"in entry: 536 data = entry.get("data") 537 space = data.find(" ") 538if data[:space] == depotPath: 539 output = entry 540break 541if output ==None: 542return"" 543if output["code"] =="error": 544return"" 545 clientPath ="" 546if"path"in output: 547 clientPath = output.get("path") 548elif"data"in output: 549 data = output.get("data") 550 lastSpace = data.rfind(" ") 551 clientPath = data[lastSpace +1:] 552 553if clientPath.endswith("..."): 554 clientPath = clientPath[:-3] 555return clientPath 556 557defcurrentGitBranch(): 558 retcode =system(["git","symbolic-ref","-q","HEAD"], ignore_error=True) 559if retcode !=0: 560# on a detached head 561return None 562else: 563returnread_pipe(["git","name-rev","HEAD"]).split(" ")[1].strip() 564 565defisValidGitDir(path): 566if(os.path.exists(path +"/HEAD") 567and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 568return True; 569return False 570 571defparseRevision(ref): 572returnread_pipe("git rev-parse%s"% ref).strip() 573 574defbranchExists(ref): 575 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 576 ignore_error=True) 577returnlen(rev) >0 578 579defextractLogMessageFromGitCommit(commit): 580 logMessage ="" 581 582## fixme: title is first line of commit, not 1st paragraph. 583 foundTitle =False 584for log inread_pipe_lines("git cat-file commit%s"% commit): 585if not foundTitle: 586iflen(log) ==1: 587 foundTitle =True 588continue 589 590 logMessage += log 591return logMessage 592 593defextractSettingsGitLog(log): 594 values = {} 595for line in log.split("\n"): 596 line = line.strip() 597 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 598if not m: 599continue 600 601 assignments = m.group(1).split(':') 602for a in assignments: 603 vals = a.split('=') 604 key = vals[0].strip() 605 val = ('='.join(vals[1:])).strip() 606if val.endswith('\"')and val.startswith('"'): 607 val = val[1:-1] 608 609 values[key] = val 610 611 paths = values.get("depot-paths") 612if not paths: 613 paths = values.get("depot-path") 614if paths: 615 values['depot-paths'] = paths.split(',') 616return values 617 618defgitBranchExists(branch): 619 proc = subprocess.Popen(["git","rev-parse", branch], 620 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 621return proc.wait() ==0; 622 623_gitConfig = {} 624 625defgitConfig(key, typeSpecifier=None): 626if not _gitConfig.has_key(key): 627 cmd = ["git","config"] 628if typeSpecifier: 629 cmd += [ typeSpecifier ] 630 cmd += [ key ] 631 s =read_pipe(cmd, ignore_error=True) 632 _gitConfig[key] = s.strip() 633return _gitConfig[key] 634 635defgitConfigBool(key): 636"""Return a bool, using git config --bool. It is True only if the 637 variable is set to true, and False if set to false or not present 638 in the config.""" 639 640if not _gitConfig.has_key(key): 641 _gitConfig[key] =gitConfig(key,'--bool') =="true" 642return _gitConfig[key] 643 644defgitConfigInt(key): 645if not _gitConfig.has_key(key): 646 cmd = ["git","config","--int", key ] 647 s =read_pipe(cmd, ignore_error=True) 648 v = s.strip() 649try: 650 _gitConfig[key] =int(gitConfig(key,'--int')) 651exceptValueError: 652 _gitConfig[key] =None 653return _gitConfig[key] 654 655defgitConfigList(key): 656if not _gitConfig.has_key(key): 657 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 658 _gitConfig[key] = s.strip().split(os.linesep) 659if _gitConfig[key] == ['']: 660 _gitConfig[key] = [] 661return _gitConfig[key] 662 663defp4BranchesInGit(branchesAreInRemotes=True): 664"""Find all the branches whose names start with "p4/", looking 665 in remotes or heads as specified by the argument. Return 666 a dictionary of{ branch: revision }for each one found. 667 The branch names are the short names, without any 668 "p4/" prefix.""" 669 670 branches = {} 671 672 cmdline ="git rev-parse --symbolic " 673if branchesAreInRemotes: 674 cmdline +="--remotes" 675else: 676 cmdline +="--branches" 677 678for line inread_pipe_lines(cmdline): 679 line = line.strip() 680 681# only import to p4/ 682if not line.startswith('p4/'): 683continue 684# special symbolic ref to p4/master 685if line =="p4/HEAD": 686continue 687 688# strip off p4/ prefix 689 branch = line[len("p4/"):] 690 691 branches[branch] =parseRevision(line) 692 693return branches 694 695defbranch_exists(branch): 696"""Make sure that the given ref name really exists.""" 697 698 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 699 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 700 out, _ = p.communicate() 701if p.returncode: 702return False 703# expect exactly one line of output: the branch name 704return out.rstrip() == branch 705 706deffindUpstreamBranchPoint(head ="HEAD"): 707 branches =p4BranchesInGit() 708# map from depot-path to branch name 709 branchByDepotPath = {} 710for branch in branches.keys(): 711 tip = branches[branch] 712 log =extractLogMessageFromGitCommit(tip) 713 settings =extractSettingsGitLog(log) 714if settings.has_key("depot-paths"): 715 paths =",".join(settings["depot-paths"]) 716 branchByDepotPath[paths] ="remotes/p4/"+ branch 717 718 settings =None 719 parent =0 720while parent <65535: 721 commit = head +"~%s"% parent 722 log =extractLogMessageFromGitCommit(commit) 723 settings =extractSettingsGitLog(log) 724if settings.has_key("depot-paths"): 725 paths =",".join(settings["depot-paths"]) 726if branchByDepotPath.has_key(paths): 727return[branchByDepotPath[paths], settings] 728 729 parent = parent +1 730 731return["", settings] 732 733defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 734if not silent: 735print("Creating/updating branch(es) in%sbased on origin branch(es)" 736% localRefPrefix) 737 738 originPrefix ="origin/p4/" 739 740for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 741 line = line.strip() 742if(not line.startswith(originPrefix))or line.endswith("HEAD"): 743continue 744 745 headName = line[len(originPrefix):] 746 remoteHead = localRefPrefix + headName 747 originHead = line 748 749 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 750if(not original.has_key('depot-paths') 751or not original.has_key('change')): 752continue 753 754 update =False 755if notgitBranchExists(remoteHead): 756if verbose: 757print"creating%s"% remoteHead 758 update =True 759else: 760 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 761if settings.has_key('change') >0: 762if settings['depot-paths'] == original['depot-paths']: 763 originP4Change =int(original['change']) 764 p4Change =int(settings['change']) 765if originP4Change > p4Change: 766print("%s(%s) is newer than%s(%s). " 767"Updating p4 branch from origin." 768% (originHead, originP4Change, 769 remoteHead, p4Change)) 770 update =True 771else: 772print("Ignoring:%swas imported from%swhile " 773"%swas imported from%s" 774% (originHead,','.join(original['depot-paths']), 775 remoteHead,','.join(settings['depot-paths']))) 776 777if update: 778system("git update-ref%s %s"% (remoteHead, originHead)) 779 780deforiginP4BranchesExist(): 781returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 782 783 784defp4ParseNumericChangeRange(parts): 785 changeStart =int(parts[0][1:]) 786if parts[1] =='#head': 787 changeEnd =p4_last_change() 788else: 789 changeEnd =int(parts[1]) 790 791return(changeStart, changeEnd) 792 793defchooseBlockSize(blockSize): 794if blockSize: 795return blockSize 796else: 797return defaultBlockSize 798 799defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 800assert depotPaths 801 802# Parse the change range into start and end. Try to find integer 803# revision ranges as these can be broken up into blocks to avoid 804# hitting server-side limits (maxrows, maxscanresults). But if 805# that doesn't work, fall back to using the raw revision specifier 806# strings, without using block mode. 807 808if changeRange is None or changeRange =='': 809 changeStart =1 810 changeEnd =p4_last_change() 811 block_size =chooseBlockSize(requestedBlockSize) 812else: 813 parts = changeRange.split(',') 814assertlen(parts) ==2 815try: 816(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 817 block_size =chooseBlockSize(requestedBlockSize) 818except: 819 changeStart = parts[0][1:] 820 changeEnd = parts[1] 821if requestedBlockSize: 822die("cannot use --changes-block-size with non-numeric revisions") 823 block_size =None 824 825 changes = [] 826 827# Retrieve changes a block at a time, to prevent running 828# into a MaxResults/MaxScanRows error from the server. 829 830while True: 831 cmd = ['changes'] 832 833if block_size: 834 end =min(changeEnd, changeStart + block_size) 835 revisionRange ="%d,%d"% (changeStart, end) 836else: 837 revisionRange ="%s,%s"% (changeStart, changeEnd) 838 839for p in depotPaths: 840 cmd += ["%s...@%s"% (p, revisionRange)] 841 842# Insert changes in chronological order 843for line inreversed(p4_read_pipe_lines(cmd)): 844 changes.append(int(line.split(" ")[1])) 845 846if not block_size: 847break 848 849if end >= changeEnd: 850break 851 852 changeStart = end +1 853 854 changes =sorted(changes) 855return changes 856 857defp4PathStartsWith(path, prefix): 858# This method tries to remedy a potential mixed-case issue: 859# 860# If UserA adds //depot/DirA/file1 861# and UserB adds //depot/dira/file2 862# 863# we may or may not have a problem. If you have core.ignorecase=true, 864# we treat DirA and dira as the same directory 865ifgitConfigBool("core.ignorecase"): 866return path.lower().startswith(prefix.lower()) 867return path.startswith(prefix) 868 869defgetClientSpec(): 870"""Look at the p4 client spec, create a View() object that contains 871 all the mappings, and return it.""" 872 873 specList =p4CmdList("client -o") 874iflen(specList) !=1: 875die('Output from "client -o" is%dlines, expecting 1'% 876len(specList)) 877 878# dictionary of all client parameters 879 entry = specList[0] 880 881# the //client/ name 882 client_name = entry["Client"] 883 884# just the keys that start with "View" 885 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 886 887# hold this new View 888 view =View(client_name) 889 890# append the lines, in order, to the view 891for view_num inrange(len(view_keys)): 892 k ="View%d"% view_num 893if k not in view_keys: 894die("Expected view key%smissing"% k) 895 view.append(entry[k]) 896 897return view 898 899defgetClientRoot(): 900"""Grab the client directory.""" 901 902 output =p4CmdList("client -o") 903iflen(output) !=1: 904die('Output from "client -o" is%dlines, expecting 1'%len(output)) 905 906 entry = output[0] 907if"Root"not in entry: 908die('Client has no "Root"') 909 910return entry["Root"] 911 912# 913# P4 wildcards are not allowed in filenames. P4 complains 914# if you simply add them, but you can force it with "-f", in 915# which case it translates them into %xx encoding internally. 916# 917defwildcard_decode(path): 918# Search for and fix just these four characters. Do % last so 919# that fixing it does not inadvertently create new %-escapes. 920# Cannot have * in a filename in windows; untested as to 921# what p4 would do in such a case. 922if not platform.system() =="Windows": 923 path = path.replace("%2A","*") 924 path = path.replace("%23","#") \ 925.replace("%40","@") \ 926.replace("%25","%") 927return path 928 929defwildcard_encode(path): 930# do % first to avoid double-encoding the %s introduced here 931 path = path.replace("%","%25") \ 932.replace("*","%2A") \ 933.replace("#","%23") \ 934.replace("@","%40") 935return path 936 937defwildcard_present(path): 938 m = re.search("[*#@%]", path) 939return m is not None 940 941classLargeFileSystem(object): 942"""Base class for large file system support.""" 943 944def__init__(self, writeToGitStream): 945 self.largeFiles =set() 946 self.writeToGitStream = writeToGitStream 947 948defgeneratePointer(self, cloneDestination, contentFile): 949"""Return the content of a pointer file that is stored in Git instead of 950 the actual content.""" 951assert False,"Method 'generatePointer' required in "+ self.__class__.__name__ 952 953defpushFile(self, localLargeFile): 954"""Push the actual content which is not stored in the Git repository to 955 a server.""" 956assert False,"Method 'pushFile' required in "+ self.__class__.__name__ 957 958defhasLargeFileExtension(self, relPath): 959returnreduce( 960lambda a, b: a or b, 961[relPath.endswith('.'+ e)for e ingitConfigList('git-p4.largeFileExtensions')], 962False 963) 964 965defgenerateTempFile(self, contents): 966 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 967for d in contents: 968 contentFile.write(d) 969 contentFile.close() 970return contentFile.name 971 972defexceedsLargeFileThreshold(self, relPath, contents): 973ifgitConfigInt('git-p4.largeFileThreshold'): 974 contentsSize =sum(len(d)for d in contents) 975if contentsSize >gitConfigInt('git-p4.largeFileThreshold'): 976return True 977ifgitConfigInt('git-p4.largeFileCompressedThreshold'): 978 contentsSize =sum(len(d)for d in contents) 979if contentsSize <=gitConfigInt('git-p4.largeFileCompressedThreshold'): 980return False 981 contentTempFile = self.generateTempFile(contents) 982 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 983 zf = zipfile.ZipFile(compressedContentFile.name, mode='w') 984 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) 985 zf.close() 986 compressedContentsSize = zf.infolist()[0].compress_size 987 os.remove(contentTempFile) 988 os.remove(compressedContentFile.name) 989if compressedContentsSize >gitConfigInt('git-p4.largeFileCompressedThreshold'): 990return True 991return False 992 993defaddLargeFile(self, relPath): 994 self.largeFiles.add(relPath) 995 996defremoveLargeFile(self, relPath): 997 self.largeFiles.remove(relPath) 998 999defisLargeFile(self, relPath):1000return relPath in self.largeFiles10011002defprocessContent(self, git_mode, relPath, contents):1003"""Processes the content of git fast import. This method decides if a1004 file is stored in the large file system and handles all necessary1005 steps."""1006if self.exceedsLargeFileThreshold(relPath, contents)or self.hasLargeFileExtension(relPath):1007 contentTempFile = self.generateTempFile(contents)1008(git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)10091010# Move temp file to final location in large file system1011 largeFileDir = os.path.dirname(localLargeFile)1012if not os.path.isdir(largeFileDir):1013 os.makedirs(largeFileDir)1014 shutil.move(contentTempFile, localLargeFile)1015 self.addLargeFile(relPath)1016ifgitConfigBool('git-p4.largeFilePush'):1017 self.pushFile(localLargeFile)1018if verbose:1019 sys.stderr.write("%smoved to large file system (%s)\n"% (relPath, localLargeFile))1020return(git_mode, contents)10211022classMockLFS(LargeFileSystem):1023"""Mock large file system for testing."""10241025defgeneratePointer(self, contentFile):1026"""The pointer content is the original content prefixed with "pointer-".1027 The local filename of the large file storage is derived from the file content.1028 """1029withopen(contentFile,'r')as f:1030 content =next(f)1031 gitMode ='100644'1032 pointerContents ='pointer-'+ content1033 localLargeFile = os.path.join(os.getcwd(),'.git','mock-storage','local', content[:-1])1034return(gitMode, pointerContents, localLargeFile)10351036defpushFile(self, localLargeFile):1037"""The remote filename of the large file storage is the same as the local1038 one but in a different directory.1039 """1040 remotePath = os.path.join(os.path.dirname(localLargeFile),'..','remote')1041if not os.path.exists(remotePath):1042 os.makedirs(remotePath)1043 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))10441045classGitLFS(LargeFileSystem):1046"""Git LFS as backend for the git-p4 large file system.1047 See https://git-lfs.github.com/ for details."""10481049def__init__(self, *args):1050 LargeFileSystem.__init__(self, *args)1051 self.baseGitAttributes = []10521053defgeneratePointer(self, contentFile):1054"""Generate a Git LFS pointer for the content. Return LFS Pointer file1055 mode and content which is stored in the Git repository instead of1056 the actual content. Return also the new location of the actual1057 content.1058 """1059 pointerProcess = subprocess.Popen(1060['git','lfs','pointer','--file='+ contentFile],1061 stdout=subprocess.PIPE1062)1063 pointerFile = pointerProcess.stdout.read()1064if pointerProcess.wait():1065 os.remove(contentFile)1066die('git-lfs pointer command failed. Did you install the extension?')1067 pointerContents = [i+'\n'for i in pointerFile.split('\n')[2:][:-1]]1068 oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]1069 localLargeFile = os.path.join(1070 os.getcwd(),1071'.git','lfs','objects', oid[:2], oid[2:4],1072 oid,1073)1074# LFS Spec states that pointer files should not have the executable bit set.1075 gitMode ='100644'1076return(gitMode, pointerContents, localLargeFile)10771078defpushFile(self, localLargeFile):1079 uploadProcess = subprocess.Popen(1080['git','lfs','push','--object-id','origin', os.path.basename(localLargeFile)]1081)1082if uploadProcess.wait():1083die('git-lfs push command failed. Did you define a remote?')10841085defgenerateGitAttributes(self):1086return(1087 self.baseGitAttributes +1088[1089'\n',1090'#\n',1091'# Git LFS (see https://git-lfs.github.com/)\n',1092'#\n',1093] +1094['*.'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1095for f insorted(gitConfigList('git-p4.largeFileExtensions'))1096] +1097['/'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1098for f insorted(self.largeFiles)if not self.hasLargeFileExtension(f)1099]1100)11011102defaddLargeFile(self, relPath):1103 LargeFileSystem.addLargeFile(self, relPath)1104 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11051106defremoveLargeFile(self, relPath):1107 LargeFileSystem.removeLargeFile(self, relPath)1108 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11091110defprocessContent(self, git_mode, relPath, contents):1111if relPath =='.gitattributes':1112 self.baseGitAttributes = contents1113return(git_mode, self.generateGitAttributes())1114else:1115return LargeFileSystem.processContent(self, git_mode, relPath, contents)11161117class Command:1118def__init__(self):1119 self.usage ="usage: %prog [options]"1120 self.needsGit =True1121 self.verbose =False11221123class P4UserMap:1124def__init__(self):1125 self.userMapFromPerforceServer =False1126 self.myP4UserId =None11271128defp4UserId(self):1129if self.myP4UserId:1130return self.myP4UserId11311132 results =p4CmdList("user -o")1133for r in results:1134if r.has_key('User'):1135 self.myP4UserId = r['User']1136return r['User']1137die("Could not find your p4 user id")11381139defp4UserIsMe(self, p4User):1140# return True if the given p4 user is actually me1141 me = self.p4UserId()1142if not p4User or p4User != me:1143return False1144else:1145return True11461147defgetUserCacheFilename(self):1148 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))1149return home +"/.gitp4-usercache.txt"11501151defgetUserMapFromPerforceServer(self):1152if self.userMapFromPerforceServer:1153return1154 self.users = {}1155 self.emails = {}11561157for output inp4CmdList("users"):1158if not output.has_key("User"):1159continue1160 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">"1161 self.emails[output["Email"]] = output["User"]11621163 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)1164for mapUserConfig ingitConfigList("git-p4.mapUser"):1165 mapUser = mapUserConfigRegex.findall(mapUserConfig)1166if mapUser andlen(mapUser[0]) ==3:1167 user = mapUser[0][0]1168 fullname = mapUser[0][1]1169 email = mapUser[0][2]1170 self.users[user] = fullname +" <"+ email +">"1171 self.emails[email] = user11721173 s =''1174for(key, val)in self.users.items():1175 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1))11761177open(self.getUserCacheFilename(),"wb").write(s)1178 self.userMapFromPerforceServer =True11791180defloadUserMapFromCache(self):1181 self.users = {}1182 self.userMapFromPerforceServer =False1183try:1184 cache =open(self.getUserCacheFilename(),"rb")1185 lines = cache.readlines()1186 cache.close()1187for line in lines:1188 entry = line.strip().split("\t")1189 self.users[entry[0]] = entry[1]1190exceptIOError:1191 self.getUserMapFromPerforceServer()11921193classP4Debug(Command):1194def__init__(self):1195 Command.__init__(self)1196 self.options = []1197 self.description ="A tool to debug the output of p4 -G."1198 self.needsGit =False11991200defrun(self, args):1201 j =01202for output inp4CmdList(args):1203print'Element:%d'% j1204 j +=11205print output1206return True12071208classP4RollBack(Command):1209def__init__(self):1210 Command.__init__(self)1211 self.options = [1212 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1213]1214 self.description ="A tool to debug the multi-branch import. Don't use :)"1215 self.rollbackLocalBranches =False12161217defrun(self, args):1218iflen(args) !=1:1219return False1220 maxChange =int(args[0])12211222if"p4ExitCode"inp4Cmd("changes -m 1"):1223die("Problems executing p4");12241225if self.rollbackLocalBranches:1226 refPrefix ="refs/heads/"1227 lines =read_pipe_lines("git rev-parse --symbolic --branches")1228else:1229 refPrefix ="refs/remotes/"1230 lines =read_pipe_lines("git rev-parse --symbolic --remotes")12311232for line in lines:1233if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1234 line = line.strip()1235 ref = refPrefix + line1236 log =extractLogMessageFromGitCommit(ref)1237 settings =extractSettingsGitLog(log)12381239 depotPaths = settings['depot-paths']1240 change = settings['change']12411242 changed =False12431244iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1245for p in depotPaths]))) ==0:1246print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange)1247system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1248continue12491250while change andint(change) > maxChange:1251 changed =True1252if self.verbose:1253print"%sis at%s; rewinding towards%s"% (ref, change, maxChange)1254system("git update-ref%s\"%s^\""% (ref, ref))1255 log =extractLogMessageFromGitCommit(ref)1256 settings =extractSettingsGitLog(log)125712581259 depotPaths = settings['depot-paths']1260 change = settings['change']12611262if changed:1263print"%srewound to%s"% (ref, change)12641265return True12661267classP4Submit(Command, P4UserMap):12681269 conflict_behavior_choices = ("ask","skip","quit")12701271def__init__(self):1272 Command.__init__(self)1273 P4UserMap.__init__(self)1274 self.options = [1275 optparse.make_option("--origin", dest="origin"),1276 optparse.make_option("-M", dest="detectRenames", action="store_true"),1277# preserve the user, requires relevant p4 permissions1278 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1279 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1280 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1281 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1282 optparse.make_option("--conflict", dest="conflict_behavior",1283 choices=self.conflict_behavior_choices),1284 optparse.make_option("--branch", dest="branch"),1285]1286 self.description ="Submit changes from git to the perforce depot."1287 self.usage +=" [name of git branch to submit into perforce depot]"1288 self.origin =""1289 self.detectRenames =False1290 self.preserveUser =gitConfigBool("git-p4.preserveUser")1291 self.dry_run =False1292 self.prepare_p4_only =False1293 self.conflict_behavior =None1294 self.isWindows = (platform.system() =="Windows")1295 self.exportLabels =False1296 self.p4HasMoveCommand =p4_has_move_command()1297 self.branch =None12981299ifgitConfig('git-p4.largeFileSystem'):1300die("Large file system not supported for git-p4 submit command. Please remove it from config.")13011302defcheck(self):1303iflen(p4CmdList("opened ...")) >0:1304die("You have files opened with perforce! Close them before starting the sync.")13051306defseparate_jobs_from_description(self, message):1307"""Extract and return a possible Jobs field in the commit1308 message. It goes into a separate section in the p4 change1309 specification.13101311 A jobs line starts with "Jobs:" and looks like a new field1312 in a form. Values are white-space separated on the same1313 line or on following lines that start with a tab.13141315 This does not parse and extract the full git commit message1316 like a p4 form. It just sees the Jobs: line as a marker1317 to pass everything from then on directly into the p4 form,1318 but outside the description section.13191320 Return a tuple (stripped log message, jobs string)."""13211322 m = re.search(r'^Jobs:', message, re.MULTILINE)1323if m is None:1324return(message,None)13251326 jobtext = message[m.start():]1327 stripped_message = message[:m.start()].rstrip()1328return(stripped_message, jobtext)13291330defprepareLogMessage(self, template, message, jobs):1331"""Edits the template returned from "p4 change -o" to insert1332 the message in the Description field, and the jobs text in1333 the Jobs field."""1334 result =""13351336 inDescriptionSection =False13371338for line in template.split("\n"):1339if line.startswith("#"):1340 result += line +"\n"1341continue13421343if inDescriptionSection:1344if line.startswith("Files:")or line.startswith("Jobs:"):1345 inDescriptionSection =False1346# insert Jobs section1347if jobs:1348 result += jobs +"\n"1349else:1350continue1351else:1352if line.startswith("Description:"):1353 inDescriptionSection =True1354 line +="\n"1355for messageLine in message.split("\n"):1356 line +="\t"+ messageLine +"\n"13571358 result += line +"\n"13591360return result13611362defpatchRCSKeywords(self,file, pattern):1363# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1364(handle, outFileName) = tempfile.mkstemp(dir='.')1365try:1366 outFile = os.fdopen(handle,"w+")1367 inFile =open(file,"r")1368 regexp = re.compile(pattern, re.VERBOSE)1369for line in inFile.readlines():1370 line = regexp.sub(r'$\1$', line)1371 outFile.write(line)1372 inFile.close()1373 outFile.close()1374# Forcibly overwrite the original file1375 os.unlink(file)1376 shutil.move(outFileName,file)1377except:1378# cleanup our temporary file1379 os.unlink(outFileName)1380print"Failed to strip RCS keywords in%s"%file1381raise13821383print"Patched up RCS keywords in%s"%file13841385defp4UserForCommit(self,id):1386# Return the tuple (perforce user,git email) for a given git commit id1387 self.getUserMapFromPerforceServer()1388 gitEmail =read_pipe(["git","log","--max-count=1",1389"--format=%ae",id])1390 gitEmail = gitEmail.strip()1391if not self.emails.has_key(gitEmail):1392return(None,gitEmail)1393else:1394return(self.emails[gitEmail],gitEmail)13951396defcheckValidP4Users(self,commits):1397# check if any git authors cannot be mapped to p4 users1398foridin commits:1399(user,email) = self.p4UserForCommit(id)1400if not user:1401 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1402ifgitConfigBool("git-p4.allowMissingP4Users"):1403print"%s"% msg1404else:1405die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)14061407deflastP4Changelist(self):1408# Get back the last changelist number submitted in this client spec. This1409# then gets used to patch up the username in the change. If the same1410# client spec is being used by multiple processes then this might go1411# wrong.1412 results =p4CmdList("client -o")# find the current client1413 client =None1414for r in results:1415if r.has_key('Client'):1416 client = r['Client']1417break1418if not client:1419die("could not get client spec")1420 results =p4CmdList(["changes","-c", client,"-m","1"])1421for r in results:1422if r.has_key('change'):1423return r['change']1424die("Could not get changelist number for last submit - cannot patch up user details")14251426defmodifyChangelistUser(self, changelist, newUser):1427# fixup the user field of a changelist after it has been submitted.1428 changes =p4CmdList("change -o%s"% changelist)1429iflen(changes) !=1:1430die("Bad output from p4 change modifying%sto user%s"%1431(changelist, newUser))14321433 c = changes[0]1434if c['User'] == newUser:return# nothing to do1435 c['User'] = newUser1436input= marshal.dumps(c)14371438 result =p4CmdList("change -f -i", stdin=input)1439for r in result:1440if r.has_key('code'):1441if r['code'] =='error':1442die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1443if r.has_key('data'):1444print("Updated user field for changelist%sto%s"% (changelist, newUser))1445return1446die("Could not modify user field of changelist%sto%s"% (changelist, newUser))14471448defcanChangeChangelists(self):1449# check to see if we have p4 admin or super-user permissions, either of1450# which are required to modify changelists.1451 results =p4CmdList(["protects", self.depotPath])1452for r in results:1453if r.has_key('perm'):1454if r['perm'] =='admin':1455return11456if r['perm'] =='super':1457return11458return014591460defprepareSubmitTemplate(self):1461"""Run "p4 change -o" to grab a change specification template.1462 This does not use "p4 -G", as it is nice to keep the submission1463 template in original order, since a human might edit it.14641465 Remove lines in the Files section that show changes to files1466 outside the depot path we're committing into."""14671468[upstream, settings] =findUpstreamBranchPoint()14691470 template =""1471 inFilesSection =False1472for line inp4_read_pipe_lines(['change','-o']):1473if line.endswith("\r\n"):1474 line = line[:-2] +"\n"1475if inFilesSection:1476if line.startswith("\t"):1477# path starts and ends with a tab1478 path = line[1:]1479 lastTab = path.rfind("\t")1480if lastTab != -1:1481 path = path[:lastTab]1482if settings.has_key('depot-paths'):1483if not[p for p in settings['depot-paths']1484ifp4PathStartsWith(path, p)]:1485continue1486else:1487if notp4PathStartsWith(path, self.depotPath):1488continue1489else:1490 inFilesSection =False1491else:1492if line.startswith("Files:"):1493 inFilesSection =True14941495 template += line14961497return template14981499defedit_template(self, template_file):1500"""Invoke the editor to let the user change the submission1501 message. Return true if okay to continue with the submit."""15021503# if configured to skip the editing part, just submit1504ifgitConfigBool("git-p4.skipSubmitEdit"):1505return True15061507# look at the modification time, to check later if the user saved1508# the file1509 mtime = os.stat(template_file).st_mtime15101511# invoke the editor1512if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1513 editor = os.environ.get("P4EDITOR")1514else:1515 editor =read_pipe("git var GIT_EDITOR").strip()1516system(["sh","-c", ('%s"$@"'% editor), editor, template_file])15171518# If the file was not saved, prompt to see if this patch should1519# be skipped. But skip this verification step if configured so.1520ifgitConfigBool("git-p4.skipSubmitEditCheck"):1521return True15221523# modification time updated means user saved the file1524if os.stat(template_file).st_mtime > mtime:1525return True15261527while True:1528 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1529if response =='y':1530return True1531if response =='n':1532return False15331534defget_diff_description(self, editedFiles, filesToAdd):1535# diff1536if os.environ.has_key("P4DIFF"):1537del(os.environ["P4DIFF"])1538 diff =""1539for editedFile in editedFiles:1540 diff +=p4_read_pipe(['diff','-du',1541wildcard_encode(editedFile)])15421543# new file diff1544 newdiff =""1545for newFile in filesToAdd:1546 newdiff +="==== new file ====\n"1547 newdiff +="--- /dev/null\n"1548 newdiff +="+++%s\n"% newFile1549 f =open(newFile,"r")1550for line in f.readlines():1551 newdiff +="+"+ line1552 f.close()15531554return(diff + newdiff).replace('\r\n','\n')15551556defapplyCommit(self,id):1557"""Apply one commit, return True if it succeeded."""15581559print"Applying",read_pipe(["git","show","-s",1560"--format=format:%h%s",id])15611562(p4User, gitEmail) = self.p4UserForCommit(id)15631564 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1565 filesToAdd =set()1566 filesToChangeType =set()1567 filesToDelete =set()1568 editedFiles =set()1569 pureRenameCopy =set()1570 filesToChangeExecBit = {}15711572for line in diff:1573 diff =parseDiffTreeEntry(line)1574 modifier = diff['status']1575 path = diff['src']1576if modifier =="M":1577p4_edit(path)1578ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1579 filesToChangeExecBit[path] = diff['dst_mode']1580 editedFiles.add(path)1581elif modifier =="A":1582 filesToAdd.add(path)1583 filesToChangeExecBit[path] = diff['dst_mode']1584if path in filesToDelete:1585 filesToDelete.remove(path)1586elif modifier =="D":1587 filesToDelete.add(path)1588if path in filesToAdd:1589 filesToAdd.remove(path)1590elif modifier =="C":1591 src, dest = diff['src'], diff['dst']1592p4_integrate(src, dest)1593 pureRenameCopy.add(dest)1594if diff['src_sha1'] != diff['dst_sha1']:1595p4_edit(dest)1596 pureRenameCopy.discard(dest)1597ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1598p4_edit(dest)1599 pureRenameCopy.discard(dest)1600 filesToChangeExecBit[dest] = diff['dst_mode']1601if self.isWindows:1602# turn off read-only attribute1603 os.chmod(dest, stat.S_IWRITE)1604 os.unlink(dest)1605 editedFiles.add(dest)1606elif modifier =="R":1607 src, dest = diff['src'], diff['dst']1608if self.p4HasMoveCommand:1609p4_edit(src)# src must be open before move1610p4_move(src, dest)# opens for (move/delete, move/add)1611else:1612p4_integrate(src, dest)1613if diff['src_sha1'] != diff['dst_sha1']:1614p4_edit(dest)1615else:1616 pureRenameCopy.add(dest)1617ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1618if not self.p4HasMoveCommand:1619p4_edit(dest)# with move: already open, writable1620 filesToChangeExecBit[dest] = diff['dst_mode']1621if not self.p4HasMoveCommand:1622if self.isWindows:1623 os.chmod(dest, stat.S_IWRITE)1624 os.unlink(dest)1625 filesToDelete.add(src)1626 editedFiles.add(dest)1627elif modifier =="T":1628 filesToChangeType.add(path)1629else:1630die("unknown modifier%sfor%s"% (modifier, path))16311632 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1633 patchcmd = diffcmd +" | git apply "1634 tryPatchCmd = patchcmd +"--check -"1635 applyPatchCmd = patchcmd +"--check --apply -"1636 patch_succeeded =True16371638if os.system(tryPatchCmd) !=0:1639 fixed_rcs_keywords =False1640 patch_succeeded =False1641print"Unfortunately applying the change failed!"16421643# Patch failed, maybe it's just RCS keyword woes. Look through1644# the patch to see if that's possible.1645ifgitConfigBool("git-p4.attemptRCSCleanup"):1646file=None1647 pattern =None1648 kwfiles = {}1649forfilein editedFiles | filesToDelete:1650# did this file's delta contain RCS keywords?1651 pattern =p4_keywords_regexp_for_file(file)16521653if pattern:1654# this file is a possibility...look for RCS keywords.1655 regexp = re.compile(pattern, re.VERBOSE)1656for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1657if regexp.search(line):1658if verbose:1659print"got keyword match on%sin%sin%s"% (pattern, line,file)1660 kwfiles[file] = pattern1661break16621663forfilein kwfiles:1664if verbose:1665print"zapping%swith%s"% (line,pattern)1666# File is being deleted, so not open in p4. Must1667# disable the read-only bit on windows.1668if self.isWindows andfilenot in editedFiles:1669 os.chmod(file, stat.S_IWRITE)1670 self.patchRCSKeywords(file, kwfiles[file])1671 fixed_rcs_keywords =True16721673if fixed_rcs_keywords:1674print"Retrying the patch with RCS keywords cleaned up"1675if os.system(tryPatchCmd) ==0:1676 patch_succeeded =True16771678if not patch_succeeded:1679for f in editedFiles:1680p4_revert(f)1681return False16821683#1684# Apply the patch for real, and do add/delete/+x handling.1685#1686system(applyPatchCmd)16871688for f in filesToChangeType:1689p4_edit(f,"-t","auto")1690for f in filesToAdd:1691p4_add(f)1692for f in filesToDelete:1693p4_revert(f)1694p4_delete(f)16951696# Set/clear executable bits1697for f in filesToChangeExecBit.keys():1698 mode = filesToChangeExecBit[f]1699setP4ExecBit(f, mode)17001701#1702# Build p4 change description, starting with the contents1703# of the git commit message.1704#1705 logMessage =extractLogMessageFromGitCommit(id)1706 logMessage = logMessage.strip()1707(logMessage, jobs) = self.separate_jobs_from_description(logMessage)17081709 template = self.prepareSubmitTemplate()1710 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)17111712if self.preserveUser:1713 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User17141715if self.checkAuthorship and not self.p4UserIsMe(p4User):1716 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1717 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1718 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"17191720 separatorLine ="######## everything below this line is just the diff #######\n"1721if not self.prepare_p4_only:1722 submitTemplate += separatorLine1723 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)17241725(handle, fileName) = tempfile.mkstemp()1726 tmpFile = os.fdopen(handle,"w+b")1727if self.isWindows:1728 submitTemplate = submitTemplate.replace("\n","\r\n")1729 tmpFile.write(submitTemplate)1730 tmpFile.close()17311732if self.prepare_p4_only:1733#1734# Leave the p4 tree prepared, and the submit template around1735# and let the user decide what to do next1736#1737print1738print"P4 workspace prepared for submission."1739print"To submit or revert, go to client workspace"1740print" "+ self.clientPath1741print1742print"To submit, use\"p4 submit\"to write a new description,"1743print"or\"p4 submit -i <%s\"to use the one prepared by" \1744"\"git p4\"."% fileName1745print"You can delete the file\"%s\"when finished."% fileName17461747if self.preserveUser and p4User and not self.p4UserIsMe(p4User):1748print"To preserve change ownership by user%s, you must\n" \1749"do\"p4 change -f <change>\"after submitting and\n" \1750"edit the User field."1751if pureRenameCopy:1752print"After submitting, renamed files must be re-synced."1753print"Invoke\"p4 sync -f\"on each of these files:"1754for f in pureRenameCopy:1755print" "+ f17561757print1758print"To revert the changes, use\"p4 revert ...\", and delete"1759print"the submit template file\"%s\""% fileName1760if filesToAdd:1761print"Since the commit adds new files, they must be deleted:"1762for f in filesToAdd:1763print" "+ f1764print1765return True17661767#1768# Let the user edit the change description, then submit it.1769#1770 submitted =False17711772try:1773if self.edit_template(fileName):1774# read the edited message and submit1775 tmpFile =open(fileName,"rb")1776 message = tmpFile.read()1777 tmpFile.close()1778if self.isWindows:1779 message = message.replace("\r\n","\n")1780 submitTemplate = message[:message.index(separatorLine)]1781p4_write_pipe(['submit','-i'], submitTemplate)17821783if self.preserveUser:1784if p4User:1785# Get last changelist number. Cannot easily get it from1786# the submit command output as the output is1787# unmarshalled.1788 changelist = self.lastP4Changelist()1789 self.modifyChangelistUser(changelist, p4User)17901791# The rename/copy happened by applying a patch that created a1792# new file. This leaves it writable, which confuses p4.1793for f in pureRenameCopy:1794p4_sync(f,"-f")1795 submitted =True17961797finally:1798# skip this patch1799if not submitted:1800print"Submission cancelled, undoing p4 changes."1801for f in editedFiles:1802p4_revert(f)1803for f in filesToAdd:1804p4_revert(f)1805 os.remove(f)1806for f in filesToDelete:1807p4_revert(f)18081809 os.remove(fileName)1810return submitted18111812# Export git tags as p4 labels. Create a p4 label and then tag1813# with that.1814defexportGitTags(self, gitTags):1815 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1816iflen(validLabelRegexp) ==0:1817 validLabelRegexp = defaultLabelRegexp1818 m = re.compile(validLabelRegexp)18191820for name in gitTags:18211822if not m.match(name):1823if verbose:1824print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1825continue18261827# Get the p4 commit this corresponds to1828 logMessage =extractLogMessageFromGitCommit(name)1829 values =extractSettingsGitLog(logMessage)18301831if not values.has_key('change'):1832# a tag pointing to something not sent to p4; ignore1833if verbose:1834print"git tag%sdoes not give a p4 commit"% name1835continue1836else:1837 changelist = values['change']18381839# Get the tag details.1840 inHeader =True1841 isAnnotated =False1842 body = []1843for l inread_pipe_lines(["git","cat-file","-p", name]):1844 l = l.strip()1845if inHeader:1846if re.match(r'tag\s+', l):1847 isAnnotated =True1848elif re.match(r'\s*$', l):1849 inHeader =False1850continue1851else:1852 body.append(l)18531854if not isAnnotated:1855 body = ["lightweight tag imported by git p4\n"]18561857# Create the label - use the same view as the client spec we are using1858 clientSpec =getClientSpec()18591860 labelTemplate ="Label:%s\n"% name1861 labelTemplate +="Description:\n"1862for b in body:1863 labelTemplate +="\t"+ b +"\n"1864 labelTemplate +="View:\n"1865for depot_side in clientSpec.mappings:1866 labelTemplate +="\t%s\n"% depot_side18671868if self.dry_run:1869print"Would create p4 label%sfor tag"% name1870elif self.prepare_p4_only:1871print"Not creating p4 label%sfor tag due to option" \1872" --prepare-p4-only"% name1873else:1874p4_write_pipe(["label","-i"], labelTemplate)18751876# Use the label1877p4_system(["tag","-l", name] +1878["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])18791880if verbose:1881print"created p4 label for tag%s"% name18821883defrun(self, args):1884iflen(args) ==0:1885 self.master =currentGitBranch()1886eliflen(args) ==1:1887 self.master = args[0]1888if notbranchExists(self.master):1889die("Branch%sdoes not exist"% self.master)1890else:1891return False18921893if self.master:1894 allowSubmit =gitConfig("git-p4.allowSubmit")1895iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1896die("%sis not in git-p4.allowSubmit"% self.master)18971898[upstream, settings] =findUpstreamBranchPoint()1899 self.depotPath = settings['depot-paths'][0]1900iflen(self.origin) ==0:1901 self.origin = upstream19021903if self.preserveUser:1904if not self.canChangeChangelists():1905die("Cannot preserve user names without p4 super-user or admin permissions")19061907# if not set from the command line, try the config file1908if self.conflict_behavior is None:1909 val =gitConfig("git-p4.conflict")1910if val:1911if val not in self.conflict_behavior_choices:1912die("Invalid value '%s' for config git-p4.conflict"% val)1913else:1914 val ="ask"1915 self.conflict_behavior = val19161917if self.verbose:1918print"Origin branch is "+ self.origin19191920iflen(self.depotPath) ==0:1921print"Internal error: cannot locate perforce depot path from existing branches"1922 sys.exit(128)19231924 self.useClientSpec =False1925ifgitConfigBool("git-p4.useclientspec"):1926 self.useClientSpec =True1927if self.useClientSpec:1928 self.clientSpecDirs =getClientSpec()19291930# Check for the existance of P4 branches1931 branchesDetected = (len(p4BranchesInGit().keys()) >1)19321933if self.useClientSpec and not branchesDetected:1934# all files are relative to the client spec1935 self.clientPath =getClientRoot()1936else:1937 self.clientPath =p4Where(self.depotPath)19381939if self.clientPath =="":1940die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)19411942print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1943 self.oldWorkingDirectory = os.getcwd()19441945# ensure the clientPath exists1946 new_client_dir =False1947if not os.path.exists(self.clientPath):1948 new_client_dir =True1949 os.makedirs(self.clientPath)19501951chdir(self.clientPath, is_client_path=True)1952if self.dry_run:1953print"Would synchronize p4 checkout in%s"% self.clientPath1954else:1955print"Synchronizing p4 checkout..."1956if new_client_dir:1957# old one was destroyed, and maybe nobody told p41958p4_sync("...","-f")1959else:1960p4_sync("...")1961 self.check()19621963 commits = []1964if self.master:1965 commitish = self.master1966else:1967 commitish ='HEAD'19681969for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, commitish)]):1970 commits.append(line.strip())1971 commits.reverse()19721973if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):1974 self.checkAuthorship =False1975else:1976 self.checkAuthorship =True19771978if self.preserveUser:1979 self.checkValidP4Users(commits)19801981#1982# Build up a set of options to be passed to diff when1983# submitting each commit to p4.1984#1985if self.detectRenames:1986# command-line -M arg1987 self.diffOpts ="-M"1988else:1989# If not explicitly set check the config variable1990 detectRenames =gitConfig("git-p4.detectRenames")19911992if detectRenames.lower() =="false"or detectRenames =="":1993 self.diffOpts =""1994elif detectRenames.lower() =="true":1995 self.diffOpts ="-M"1996else:1997 self.diffOpts ="-M%s"% detectRenames19981999# no command-line arg for -C or --find-copies-harder, just2000# config variables2001 detectCopies =gitConfig("git-p4.detectCopies")2002if detectCopies.lower() =="false"or detectCopies =="":2003pass2004elif detectCopies.lower() =="true":2005 self.diffOpts +=" -C"2006else:2007 self.diffOpts +=" -C%s"% detectCopies20082009ifgitConfigBool("git-p4.detectCopiesHarder"):2010 self.diffOpts +=" --find-copies-harder"20112012#2013# Apply the commits, one at a time. On failure, ask if should2014# continue to try the rest of the patches, or quit.2015#2016if self.dry_run:2017print"Would apply"2018 applied = []2019 last =len(commits) -12020for i, commit inenumerate(commits):2021if self.dry_run:2022print" ",read_pipe(["git","show","-s",2023"--format=format:%h%s", commit])2024 ok =True2025else:2026 ok = self.applyCommit(commit)2027if ok:2028 applied.append(commit)2029else:2030if self.prepare_p4_only and i < last:2031print"Processing only the first commit due to option" \2032" --prepare-p4-only"2033break2034if i < last:2035 quit =False2036while True:2037# prompt for what to do, or use the option/variable2038if self.conflict_behavior =="ask":2039print"What do you want to do?"2040 response =raw_input("[s]kip this commit but apply"2041" the rest, or [q]uit? ")2042if not response:2043continue2044elif self.conflict_behavior =="skip":2045 response ="s"2046elif self.conflict_behavior =="quit":2047 response ="q"2048else:2049die("Unknown conflict_behavior '%s'"%2050 self.conflict_behavior)20512052if response[0] =="s":2053print"Skipping this commit, but applying the rest"2054break2055if response[0] =="q":2056print"Quitting"2057 quit =True2058break2059if quit:2060break20612062chdir(self.oldWorkingDirectory)20632064if self.dry_run:2065pass2066elif self.prepare_p4_only:2067pass2068eliflen(commits) ==len(applied):2069print"All commits applied!"20702071 sync =P4Sync()2072if self.branch:2073 sync.branch = self.branch2074 sync.run([])20752076 rebase =P4Rebase()2077 rebase.rebase()20782079else:2080iflen(applied) ==0:2081print"No commits applied."2082else:2083print"Applied only the commits marked with '*':"2084for c in commits:2085if c in applied:2086 star ="*"2087else:2088 star =" "2089print star,read_pipe(["git","show","-s",2090"--format=format:%h%s", c])2091print"You will have to do 'git p4 sync' and rebase."20922093ifgitConfigBool("git-p4.exportLabels"):2094 self.exportLabels =True20952096if self.exportLabels:2097 p4Labels =getP4Labels(self.depotPath)2098 gitTags =getGitTags()20992100 missingGitTags = gitTags - p4Labels2101 self.exportGitTags(missingGitTags)21022103# exit with error unless everything applied perfectly2104iflen(commits) !=len(applied):2105 sys.exit(1)21062107return True21082109classView(object):2110"""Represent a p4 view ("p4 help views"), and map files in a2111 repo according to the view."""21122113def__init__(self, client_name):2114 self.mappings = []2115 self.client_prefix ="//%s/"% client_name2116# cache results of "p4 where" to lookup client file locations2117 self.client_spec_path_cache = {}21182119defappend(self, view_line):2120"""Parse a view line, splitting it into depot and client2121 sides. Append to self.mappings, preserving order. This2122 is only needed for tag creation."""21232124# Split the view line into exactly two words. P4 enforces2125# structure on these lines that simplifies this quite a bit.2126#2127# Either or both words may be double-quoted.2128# Single quotes do not matter.2129# Double-quote marks cannot occur inside the words.2130# A + or - prefix is also inside the quotes.2131# There are no quotes unless they contain a space.2132# The line is already white-space stripped.2133# The two words are separated by a single space.2134#2135if view_line[0] =='"':2136# First word is double quoted. Find its end.2137 close_quote_index = view_line.find('"',1)2138if close_quote_index <=0:2139die("No first-word closing quote found:%s"% view_line)2140 depot_side = view_line[1:close_quote_index]2141# skip closing quote and space2142 rhs_index = close_quote_index +1+12143else:2144 space_index = view_line.find(" ")2145if space_index <=0:2146die("No word-splitting space found:%s"% view_line)2147 depot_side = view_line[0:space_index]2148 rhs_index = space_index +121492150# prefix + means overlay on previous mapping2151if depot_side.startswith("+"):2152 depot_side = depot_side[1:]21532154# prefix - means exclude this path, leave out of mappings2155 exclude =False2156if depot_side.startswith("-"):2157 exclude =True2158 depot_side = depot_side[1:]21592160if not exclude:2161 self.mappings.append(depot_side)21622163defconvert_client_path(self, clientFile):2164# chop off //client/ part to make it relative2165if not clientFile.startswith(self.client_prefix):2166die("No prefix '%s' on clientFile '%s'"%2167(self.client_prefix, clientFile))2168return clientFile[len(self.client_prefix):]21692170defupdate_client_spec_path_cache(self, files):2171""" Caching file paths by "p4 where" batch query """21722173# List depot file paths exclude that already cached2174 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]21752176iflen(fileArgs) ==0:2177return# All files in cache21782179 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)2180for res in where_result:2181if"code"in res and res["code"] =="error":2182# assume error is "... file(s) not in client view"2183continue2184if"clientFile"not in res:2185die("No clientFile in 'p4 where' output")2186if"unmap"in res:2187# it will list all of them, but only one not unmap-ped2188continue2189ifgitConfigBool("core.ignorecase"):2190 res['depotFile'] = res['depotFile'].lower()2191 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])21922193# not found files or unmap files set to ""2194for depotFile in fileArgs:2195ifgitConfigBool("core.ignorecase"):2196 depotFile = depotFile.lower()2197if depotFile not in self.client_spec_path_cache:2198 self.client_spec_path_cache[depotFile] =""21992200defmap_in_client(self, depot_path):2201"""Return the relative location in the client where this2202 depot file should live. Returns "" if the file should2203 not be mapped in the client."""22042205ifgitConfigBool("core.ignorecase"):2206 depot_path = depot_path.lower()22072208if depot_path in self.client_spec_path_cache:2209return self.client_spec_path_cache[depot_path]22102211die("Error:%sis not found in client spec path"% depot_path )2212return""22132214classP4Sync(Command, P4UserMap):2215 delete_actions = ("delete","move/delete","purge")22162217def__init__(self):2218 Command.__init__(self)2219 P4UserMap.__init__(self)2220 self.options = [2221 optparse.make_option("--branch", dest="branch"),2222 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),2223 optparse.make_option("--changesfile", dest="changesFile"),2224 optparse.make_option("--silent", dest="silent", action="store_true"),2225 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),2226 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2227 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",2228help="Import into refs/heads/ , not refs/remotes"),2229 optparse.make_option("--max-changes", dest="maxChanges",2230help="Maximum number of changes to import"),2231 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",2232help="Internal block size to use when iteratively calling p4 changes"),2233 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2234help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2235 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2236help="Only sync files that are included in the Perforce Client Spec"),2237 optparse.make_option("-/", dest="cloneExclude",2238 action="append",type="string",2239help="exclude depot path"),2240]2241 self.description ="""Imports from Perforce into a git repository.\n2242 example:2243 //depot/my/project/ -- to import the current head2244 //depot/my/project/@all -- to import everything2245 //depot/my/project/@1,6 -- to import only from revision 1 to 622462247 (a ... is not needed in the path p4 specification, it's added implicitly)"""22482249 self.usage +=" //depot/path[@revRange]"2250 self.silent =False2251 self.createdBranches =set()2252 self.committedChanges =set()2253 self.branch =""2254 self.detectBranches =False2255 self.detectLabels =False2256 self.importLabels =False2257 self.changesFile =""2258 self.syncWithOrigin =True2259 self.importIntoRemotes =True2260 self.maxChanges =""2261 self.changes_block_size =None2262 self.keepRepoPath =False2263 self.depotPaths =None2264 self.p4BranchesInGit = []2265 self.cloneExclude = []2266 self.useClientSpec =False2267 self.useClientSpec_from_options =False2268 self.clientSpecDirs =None2269 self.tempBranches = []2270 self.tempBranchLocation ="git-p4-tmp"2271 self.largeFileSystem =None22722273ifgitConfig('git-p4.largeFileSystem'):2274 largeFileSystemConstructor =globals()[gitConfig('git-p4.largeFileSystem')]2275 self.largeFileSystem =largeFileSystemConstructor(2276lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)2277)22782279ifgitConfig("git-p4.syncFromOrigin") =="false":2280 self.syncWithOrigin =False22812282# This is required for the "append" cloneExclude action2283defensure_value(self, attr, value):2284if nothasattr(self, attr)orgetattr(self, attr)is None:2285setattr(self, attr, value)2286returngetattr(self, attr)22872288# Force a checkpoint in fast-import and wait for it to finish2289defcheckpoint(self):2290 self.gitStream.write("checkpoint\n\n")2291 self.gitStream.write("progress checkpoint\n\n")2292 out = self.gitOutput.readline()2293if self.verbose:2294print"checkpoint finished: "+ out22952296defextractFilesFromCommit(self, commit):2297 self.cloneExclude = [re.sub(r"\.\.\.$","", path)2298for path in self.cloneExclude]2299 files = []2300 fnum =02301while commit.has_key("depotFile%s"% fnum):2302 path = commit["depotFile%s"% fnum]23032304if[p for p in self.cloneExclude2305ifp4PathStartsWith(path, p)]:2306 found =False2307else:2308 found = [p for p in self.depotPaths2309ifp4PathStartsWith(path, p)]2310if not found:2311 fnum = fnum +12312continue23132314file= {}2315file["path"] = path2316file["rev"] = commit["rev%s"% fnum]2317file["action"] = commit["action%s"% fnum]2318file["type"] = commit["type%s"% fnum]2319 files.append(file)2320 fnum = fnum +12321return files23222323defextractJobsFromCommit(self, commit):2324 jobs = []2325 jnum =02326while commit.has_key("job%s"% jnum):2327 job = commit["job%s"% jnum]2328 jobs.append(job)2329 jnum = jnum +12330return jobs23312332defstripRepoPath(self, path, prefixes):2333"""When streaming files, this is called to map a p4 depot path2334 to where it should go in git. The prefixes are either2335 self.depotPaths, or self.branchPrefixes in the case of2336 branch detection."""23372338if self.useClientSpec:2339# branch detection moves files up a level (the branch name)2340# from what client spec interpretation gives2341 path = self.clientSpecDirs.map_in_client(path)2342if self.detectBranches:2343for b in self.knownBranches:2344if path.startswith(b +"/"):2345 path = path[len(b)+1:]23462347elif self.keepRepoPath:2348# Preserve everything in relative path name except leading2349# //depot/; just look at first prefix as they all should2350# be in the same depot.2351 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2352ifp4PathStartsWith(path, depot):2353 path = path[len(depot):]23542355else:2356for p in prefixes:2357ifp4PathStartsWith(path, p):2358 path = path[len(p):]2359break23602361 path =wildcard_decode(path)2362return path23632364defsplitFilesIntoBranches(self, commit):2365"""Look at each depotFile in the commit to figure out to what2366 branch it belongs."""23672368if self.clientSpecDirs:2369 files = self.extractFilesFromCommit(commit)2370 self.clientSpecDirs.update_client_spec_path_cache(files)23712372 branches = {}2373 fnum =02374while commit.has_key("depotFile%s"% fnum):2375 path = commit["depotFile%s"% fnum]2376 found = [p for p in self.depotPaths2377ifp4PathStartsWith(path, p)]2378if not found:2379 fnum = fnum +12380continue23812382file= {}2383file["path"] = path2384file["rev"] = commit["rev%s"% fnum]2385file["action"] = commit["action%s"% fnum]2386file["type"] = commit["type%s"% fnum]2387 fnum = fnum +123882389# start with the full relative path where this file would2390# go in a p4 client2391if self.useClientSpec:2392 relPath = self.clientSpecDirs.map_in_client(path)2393else:2394 relPath = self.stripRepoPath(path, self.depotPaths)23952396for branch in self.knownBranches.keys():2397# add a trailing slash so that a commit into qt/4.2foo2398# doesn't end up in qt/4.2, e.g.2399if relPath.startswith(branch +"/"):2400if branch not in branches:2401 branches[branch] = []2402 branches[branch].append(file)2403break24042405return branches24062407defwriteToGitStream(self, gitMode, relPath, contents):2408 self.gitStream.write('M%sinline%s\n'% (gitMode, relPath))2409 self.gitStream.write('data%d\n'%sum(len(d)for d in contents))2410for d in contents:2411 self.gitStream.write(d)2412 self.gitStream.write('\n')24132414# output one file from the P4 stream2415# - helper for streamP4Files24162417defstreamOneP4File(self,file, contents):2418 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2419if verbose:2420 size =int(self.stream_file['fileSize'])2421 sys.stdout.write('\r%s-->%s(%iMB)\n'% (file['depotFile'], relPath, size/1024/1024))2422 sys.stdout.flush()24232424(type_base, type_mods) =split_p4_type(file["type"])24252426 git_mode ="100644"2427if"x"in type_mods:2428 git_mode ="100755"2429if type_base =="symlink":2430 git_mode ="120000"2431# p4 print on a symlink sometimes contains "target\n";2432# if it does, remove the newline2433 data =''.join(contents)2434if not data:2435# Some version of p4 allowed creating a symlink that pointed2436# to nothing. This causes p4 errors when checking out such2437# a change, and errors here too. Work around it by ignoring2438# the bad symlink; hopefully a future change fixes it.2439print"\nIgnoring empty symlink in%s"%file['depotFile']2440return2441elif data[-1] =='\n':2442 contents = [data[:-1]]2443else:2444 contents = [data]24452446if type_base =="utf16":2447# p4 delivers different text in the python output to -G2448# than it does when using "print -o", or normal p4 client2449# operations. utf16 is converted to ascii or utf8, perhaps.2450# But ascii text saved as -t utf16 is completely mangled.2451# Invoke print -o to get the real contents.2452#2453# On windows, the newlines will always be mangled by print, so put2454# them back too. This is not needed to the cygwin windows version,2455# just the native "NT" type.2456#2457try:2458 text =p4_read_pipe(['print','-q','-o','-','%s@%s'% (file['depotFile'],file['change'])])2459exceptExceptionas e:2460if'Translation of file content failed'instr(e):2461 type_base ='binary'2462else:2463raise e2464else:2465ifp4_version_string().find('/NT') >=0:2466 text = text.replace('\r\n','\n')2467 contents = [ text ]24682469if type_base =="apple":2470# Apple filetype files will be streamed as a concatenation of2471# its appledouble header and the contents. This is useless2472# on both macs and non-macs. If using "print -q -o xx", it2473# will create "xx" with the data, and "%xx" with the header.2474# This is also not very useful.2475#2476# Ideally, someday, this script can learn how to generate2477# appledouble files directly and import those to git, but2478# non-mac machines can never find a use for apple filetype.2479print"\nIgnoring apple filetype file%s"%file['depotFile']2480return24812482# Note that we do not try to de-mangle keywords on utf16 files,2483# even though in theory somebody may want that.2484 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2485if pattern:2486 regexp = re.compile(pattern, re.VERBOSE)2487 text =''.join(contents)2488 text = regexp.sub(r'$\1$', text)2489 contents = [ text ]24902491try:2492 relPath.decode('ascii')2493except:2494 encoding ='utf8'2495ifgitConfig('git-p4.pathEncoding'):2496 encoding =gitConfig('git-p4.pathEncoding')2497 relPath = relPath.decode(encoding,'replace').encode('utf8','replace')2498if self.verbose:2499print'Path with non-ASCII characters detected. Used%sto encode:%s'% (encoding, relPath)25002501if self.largeFileSystem:2502(git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)25032504 self.writeToGitStream(git_mode, relPath, contents)25052506defstreamOneP4Deletion(self,file):2507 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2508if verbose:2509 sys.stdout.write("delete%s\n"% relPath)2510 sys.stdout.flush()2511 self.gitStream.write("D%s\n"% relPath)25122513if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):2514 self.largeFileSystem.removeLargeFile(relPath)25152516# handle another chunk of streaming data2517defstreamP4FilesCb(self, marshalled):25182519# catch p4 errors and complain2520 err =None2521if"code"in marshalled:2522if marshalled["code"] =="error":2523if"data"in marshalled:2524 err = marshalled["data"].rstrip()25252526if not err and'fileSize'in self.stream_file:2527 required_bytes =int((4*int(self.stream_file["fileSize"])) -calcDiskFree())2528if required_bytes >0:2529 err ='Not enough space left on%s! Free at least%iMB.'% (2530 os.getcwd(), required_bytes/1024/10242531)25322533if err:2534 f =None2535if self.stream_have_file_info:2536if"depotFile"in self.stream_file:2537 f = self.stream_file["depotFile"]2538# force a failure in fast-import, else an empty2539# commit will be made2540 self.gitStream.write("\n")2541 self.gitStream.write("die-now\n")2542 self.gitStream.close()2543# ignore errors, but make sure it exits first2544 self.importProcess.wait()2545if f:2546die("Error from p4 print for%s:%s"% (f, err))2547else:2548die("Error from p4 print:%s"% err)25492550if marshalled.has_key('depotFile')and self.stream_have_file_info:2551# start of a new file - output the old one first2552 self.streamOneP4File(self.stream_file, self.stream_contents)2553 self.stream_file = {}2554 self.stream_contents = []2555 self.stream_have_file_info =False25562557# pick up the new file information... for the2558# 'data' field we need to append to our array2559for k in marshalled.keys():2560if k =='data':2561if'streamContentSize'not in self.stream_file:2562 self.stream_file['streamContentSize'] =02563 self.stream_file['streamContentSize'] +=len(marshalled['data'])2564 self.stream_contents.append(marshalled['data'])2565else:2566 self.stream_file[k] = marshalled[k]25672568if(verbose and2569'streamContentSize'in self.stream_file and2570'fileSize'in self.stream_file and2571'depotFile'in self.stream_file):2572 size =int(self.stream_file["fileSize"])2573if size >0:2574 progress =100*self.stream_file['streamContentSize']/size2575 sys.stdout.write('\r%s %d%%(%iMB)'% (self.stream_file['depotFile'], progress,int(size/1024/1024)))2576 sys.stdout.flush()25772578 self.stream_have_file_info =True25792580# Stream directly from "p4 files" into "git fast-import"2581defstreamP4Files(self, files):2582 filesForCommit = []2583 filesToRead = []2584 filesToDelete = []25852586for f in files:2587 filesForCommit.append(f)2588if f['action']in self.delete_actions:2589 filesToDelete.append(f)2590else:2591 filesToRead.append(f)25922593# deleted files...2594for f in filesToDelete:2595 self.streamOneP4Deletion(f)25962597iflen(filesToRead) >0:2598 self.stream_file = {}2599 self.stream_contents = []2600 self.stream_have_file_info =False26012602# curry self argument2603defstreamP4FilesCbSelf(entry):2604 self.streamP4FilesCb(entry)26052606 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]26072608p4CmdList(["-x","-","print"],2609 stdin=fileArgs,2610 cb=streamP4FilesCbSelf)26112612# do the last chunk2613if self.stream_file.has_key('depotFile'):2614 self.streamOneP4File(self.stream_file, self.stream_contents)26152616defmake_email(self, userid):2617if userid in self.users:2618return self.users[userid]2619else:2620return"%s<a@b>"% userid26212622defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2623""" Stream a p4 tag.2624 commit is either a git commit, or a fast-import mark, ":<p4commit>"2625 """26262627if verbose:2628print"writing tag%sfor commit%s"% (labelName, commit)2629 gitStream.write("tag%s\n"% labelName)2630 gitStream.write("from%s\n"% commit)26312632if labelDetails.has_key('Owner'):2633 owner = labelDetails["Owner"]2634else:2635 owner =None26362637# Try to use the owner of the p4 label, or failing that,2638# the current p4 user id.2639if owner:2640 email = self.make_email(owner)2641else:2642 email = self.make_email(self.p4UserId())2643 tagger ="%s %s %s"% (email, epoch, self.tz)26442645 gitStream.write("tagger%s\n"% tagger)26462647print"labelDetails=",labelDetails2648if labelDetails.has_key('Description'):2649 description = labelDetails['Description']2650else:2651 description ='Label from git p4'26522653 gitStream.write("data%d\n"%len(description))2654 gitStream.write(description)2655 gitStream.write("\n")26562657definClientSpec(self, path):2658if not self.clientSpecDirs:2659return True2660 inClientSpec = self.clientSpecDirs.map_in_client(path)2661if not inClientSpec and self.verbose:2662print('Ignoring file outside of client spec:{0}'.format(path))2663return inClientSpec26642665defhasBranchPrefix(self, path):2666if not self.branchPrefixes:2667return True2668 hasPrefix = [p for p in self.branchPrefixes2669ifp4PathStartsWith(path, p)]2670if hasPrefix and self.verbose:2671print('Ignoring file outside of prefix:{0}'.format(path))2672return hasPrefix26732674defcommit(self, details, files, branch, parent =""):2675 epoch = details["time"]2676 author = details["user"]2677 jobs = self.extractJobsFromCommit(details)26782679if self.verbose:2680print('commit into{0}'.format(branch))26812682if self.clientSpecDirs:2683 self.clientSpecDirs.update_client_spec_path_cache(files)26842685 files = [f for f in files2686if self.inClientSpec(f['path'])and self.hasBranchPrefix(f['path'])]26872688if not files and notgitConfigBool('git-p4.keepEmptyCommits'):2689print('Ignoring revision{0}as it would produce an empty commit.'2690.format(details['change']))2691return26922693 self.gitStream.write("commit%s\n"% branch)2694 self.gitStream.write("mark :%s\n"% details["change"])2695 self.committedChanges.add(int(details["change"]))2696 committer =""2697if author not in self.users:2698 self.getUserMapFromPerforceServer()2699 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)27002701 self.gitStream.write("committer%s\n"% committer)27022703 self.gitStream.write("data <<EOT\n")2704 self.gitStream.write(details["desc"])2705iflen(jobs) >0:2706 self.gitStream.write("\nJobs:%s"% (' '.join(jobs)))2707 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2708(','.join(self.branchPrefixes), details["change"]))2709iflen(details['options']) >0:2710 self.gitStream.write(": options =%s"% details['options'])2711 self.gitStream.write("]\nEOT\n\n")27122713iflen(parent) >0:2714if self.verbose:2715print"parent%s"% parent2716 self.gitStream.write("from%s\n"% parent)27172718 self.streamP4Files(files)2719 self.gitStream.write("\n")27202721 change =int(details["change"])27222723if self.labels.has_key(change):2724 label = self.labels[change]2725 labelDetails = label[0]2726 labelRevisions = label[1]2727if self.verbose:2728print"Change%sis labelled%s"% (change, labelDetails)27292730 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2731for p in self.branchPrefixes])27322733iflen(files) ==len(labelRevisions):27342735 cleanedFiles = {}2736for info in files:2737if info["action"]in self.delete_actions:2738continue2739 cleanedFiles[info["depotFile"]] = info["rev"]27402741if cleanedFiles == labelRevisions:2742 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)27432744else:2745if not self.silent:2746print("Tag%sdoes not match with change%s: files do not match."2747% (labelDetails["label"], change))27482749else:2750if not self.silent:2751print("Tag%sdoes not match with change%s: file count is different."2752% (labelDetails["label"], change))27532754# Build a dictionary of changelists and labels, for "detect-labels" option.2755defgetLabels(self):2756 self.labels = {}27572758 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2759iflen(l) >0and not self.silent:2760print"Finding files belonging to labels in%s"% `self.depotPaths`27612762for output in l:2763 label = output["label"]2764 revisions = {}2765 newestChange =02766if self.verbose:2767print"Querying files for label%s"% label2768forfileinp4CmdList(["files"] +2769["%s...@%s"% (p, label)2770for p in self.depotPaths]):2771 revisions[file["depotFile"]] =file["rev"]2772 change =int(file["change"])2773if change > newestChange:2774 newestChange = change27752776 self.labels[newestChange] = [output, revisions]27772778if self.verbose:2779print"Label changes:%s"% self.labels.keys()27802781# Import p4 labels as git tags. A direct mapping does not2782# exist, so assume that if all the files are at the same revision2783# then we can use that, or it's something more complicated we should2784# just ignore.2785defimportP4Labels(self, stream, p4Labels):2786if verbose:2787print"import p4 labels: "+' '.join(p4Labels)27882789 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2790 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2791iflen(validLabelRegexp) ==0:2792 validLabelRegexp = defaultLabelRegexp2793 m = re.compile(validLabelRegexp)27942795for name in p4Labels:2796 commitFound =False27972798if not m.match(name):2799if verbose:2800print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2801continue28022803if name in ignoredP4Labels:2804continue28052806 labelDetails =p4CmdList(['label',"-o", name])[0]28072808# get the most recent changelist for each file in this label2809 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2810for p in self.depotPaths])28112812if change.has_key('change'):2813# find the corresponding git commit; take the oldest commit2814 changelist =int(change['change'])2815if changelist in self.committedChanges:2816 gitCommit =":%d"% changelist # use a fast-import mark2817 commitFound =True2818else:2819 gitCommit =read_pipe(["git","rev-list","--max-count=1",2820"--reverse",":/\[git-p4:.*change =%d\]"% changelist], ignore_error=True)2821iflen(gitCommit) ==0:2822print"importing label%s: could not find git commit for changelist%d"% (name, changelist)2823else:2824 commitFound =True2825 gitCommit = gitCommit.strip()28262827if commitFound:2828# Convert from p4 time format2829try:2830 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2831exceptValueError:2832print"Could not convert label time%s"% labelDetails['Update']2833 tmwhen =128342835 when =int(time.mktime(tmwhen))2836 self.streamTag(stream, name, labelDetails, gitCommit, when)2837if verbose:2838print"p4 label%smapped to git commit%s"% (name, gitCommit)2839else:2840if verbose:2841print"Label%shas no changelists - possibly deleted?"% name28422843if not commitFound:2844# We can't import this label; don't try again as it will get very2845# expensive repeatedly fetching all the files for labels that will2846# never be imported. If the label is moved in the future, the2847# ignore will need to be removed manually.2848system(["git","config","--add","git-p4.ignoredP4Labels", name])28492850defguessProjectName(self):2851for p in self.depotPaths:2852if p.endswith("/"):2853 p = p[:-1]2854 p = p[p.strip().rfind("/") +1:]2855if not p.endswith("/"):2856 p +="/"2857return p28582859defgetBranchMapping(self):2860 lostAndFoundBranches =set()28612862 user =gitConfig("git-p4.branchUser")2863iflen(user) >0:2864 command ="branches -u%s"% user2865else:2866 command ="branches"28672868for info inp4CmdList(command):2869 details =p4Cmd(["branch","-o", info["branch"]])2870 viewIdx =02871while details.has_key("View%s"% viewIdx):2872 paths = details["View%s"% viewIdx].split(" ")2873 viewIdx = viewIdx +12874# require standard //depot/foo/... //depot/bar/... mapping2875iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2876continue2877 source = paths[0]2878 destination = paths[1]2879## HACK2880ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2881 source = source[len(self.depotPaths[0]):-4]2882 destination = destination[len(self.depotPaths[0]):-4]28832884if destination in self.knownBranches:2885if not self.silent:2886print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2887print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2888continue28892890 self.knownBranches[destination] = source28912892 lostAndFoundBranches.discard(destination)28932894if source not in self.knownBranches:2895 lostAndFoundBranches.add(source)28962897# Perforce does not strictly require branches to be defined, so we also2898# check git config for a branch list.2899#2900# Example of branch definition in git config file:2901# [git-p4]2902# branchList=main:branchA2903# branchList=main:branchB2904# branchList=branchA:branchC2905 configBranches =gitConfigList("git-p4.branchList")2906for branch in configBranches:2907if branch:2908(source, destination) = branch.split(":")2909 self.knownBranches[destination] = source29102911 lostAndFoundBranches.discard(destination)29122913if source not in self.knownBranches:2914 lostAndFoundBranches.add(source)291529162917for branch in lostAndFoundBranches:2918 self.knownBranches[branch] = branch29192920defgetBranchMappingFromGitBranches(self):2921 branches =p4BranchesInGit(self.importIntoRemotes)2922for branch in branches.keys():2923if branch =="master":2924 branch ="main"2925else:2926 branch = branch[len(self.projectName):]2927 self.knownBranches[branch] = branch29282929defupdateOptionDict(self, d):2930 option_keys = {}2931if self.keepRepoPath:2932 option_keys['keepRepoPath'] =129332934 d["options"] =' '.join(sorted(option_keys.keys()))29352936defreadOptions(self, d):2937 self.keepRepoPath = (d.has_key('options')2938and('keepRepoPath'in d['options']))29392940defgitRefForBranch(self, branch):2941if branch =="main":2942return self.refPrefix +"master"29432944iflen(branch) <=0:2945return branch29462947return self.refPrefix + self.projectName + branch29482949defgitCommitByP4Change(self, ref, change):2950if self.verbose:2951print"looking in ref "+ ref +" for change%susing bisect..."% change29522953 earliestCommit =""2954 latestCommit =parseRevision(ref)29552956while True:2957if self.verbose:2958print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2959 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2960iflen(next) ==0:2961if self.verbose:2962print"argh"2963return""2964 log =extractLogMessageFromGitCommit(next)2965 settings =extractSettingsGitLog(log)2966 currentChange =int(settings['change'])2967if self.verbose:2968print"current change%s"% currentChange29692970if currentChange == change:2971if self.verbose:2972print"found%s"% next2973return next29742975if currentChange < change:2976 earliestCommit ="^%s"% next2977else:2978 latestCommit ="%s"% next29792980return""29812982defimportNewBranch(self, branch, maxChange):2983# make fast-import flush all changes to disk and update the refs using the checkpoint2984# command so that we can try to find the branch parent in the git history2985 self.gitStream.write("checkpoint\n\n");2986 self.gitStream.flush();2987 branchPrefix = self.depotPaths[0] + branch +"/"2988range="@1,%s"% maxChange2989#print "prefix" + branchPrefix2990 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)2991iflen(changes) <=0:2992return False2993 firstChange = changes[0]2994#print "first change in branch: %s" % firstChange2995 sourceBranch = self.knownBranches[branch]2996 sourceDepotPath = self.depotPaths[0] + sourceBranch2997 sourceRef = self.gitRefForBranch(sourceBranch)2998#print "source " + sourceBranch29993000 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])3001#print "branch parent: %s" % branchParentChange3002 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)3003iflen(gitParent) >0:3004 self.initialParents[self.gitRefForBranch(branch)] = gitParent3005#print "parent git commit: %s" % gitParent30063007 self.importChanges(changes)3008return True30093010defsearchParent(self, parent, branch, target):3011 parentFound =False3012for blob inread_pipe_lines(["git","rev-list","--reverse",3013"--no-merges", parent]):3014 blob = blob.strip()3015iflen(read_pipe(["git","diff-tree", blob, target])) ==0:3016 parentFound =True3017if self.verbose:3018print"Found parent of%sin commit%s"% (branch, blob)3019break3020if parentFound:3021return blob3022else:3023return None30243025defimportChanges(self, changes):3026 cnt =13027for change in changes:3028 description =p4_describe(change)3029 self.updateOptionDict(description)30303031if not self.silent:3032 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))3033 sys.stdout.flush()3034 cnt = cnt +130353036try:3037if self.detectBranches:3038 branches = self.splitFilesIntoBranches(description)3039for branch in branches.keys():3040## HACK --hwn3041 branchPrefix = self.depotPaths[0] + branch +"/"3042 self.branchPrefixes = [ branchPrefix ]30433044 parent =""30453046 filesForCommit = branches[branch]30473048if self.verbose:3049print"branch is%s"% branch30503051 self.updatedBranches.add(branch)30523053if branch not in self.createdBranches:3054 self.createdBranches.add(branch)3055 parent = self.knownBranches[branch]3056if parent == branch:3057 parent =""3058else:3059 fullBranch = self.projectName + branch3060if fullBranch not in self.p4BranchesInGit:3061if not self.silent:3062print("\nImporting new branch%s"% fullBranch);3063if self.importNewBranch(branch, change -1):3064 parent =""3065 self.p4BranchesInGit.append(fullBranch)3066if not self.silent:3067print("\nResuming with change%s"% change);30683069if self.verbose:3070print"parent determined through known branches:%s"% parent30713072 branch = self.gitRefForBranch(branch)3073 parent = self.gitRefForBranch(parent)30743075if self.verbose:3076print"looking for initial parent for%s; current parent is%s"% (branch, parent)30773078iflen(parent) ==0and branch in self.initialParents:3079 parent = self.initialParents[branch]3080del self.initialParents[branch]30813082 blob =None3083iflen(parent) >0:3084 tempBranch ="%s/%d"% (self.tempBranchLocation, change)3085if self.verbose:3086print"Creating temporary branch: "+ tempBranch3087 self.commit(description, filesForCommit, tempBranch)3088 self.tempBranches.append(tempBranch)3089 self.checkpoint()3090 blob = self.searchParent(parent, branch, tempBranch)3091if blob:3092 self.commit(description, filesForCommit, branch, blob)3093else:3094if self.verbose:3095print"Parent of%snot found. Committing into head of%s"% (branch, parent)3096 self.commit(description, filesForCommit, branch, parent)3097else:3098 files = self.extractFilesFromCommit(description)3099 self.commit(description, files, self.branch,3100 self.initialParent)3101# only needed once, to connect to the previous commit3102 self.initialParent =""3103exceptIOError:3104print self.gitError.read()3105 sys.exit(1)31063107defimportHeadRevision(self, revision):3108print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)31093110 details = {}3111 details["user"] ="git perforce import user"3112 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"3113% (' '.join(self.depotPaths), revision))3114 details["change"] = revision3115 newestRevision =031163117 fileCnt =03118 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]31193120for info inp4CmdList(["files"] + fileArgs):31213122if'code'in info and info['code'] =='error':3123 sys.stderr.write("p4 returned an error:%s\n"3124% info['data'])3125if info['data'].find("must refer to client") >=0:3126 sys.stderr.write("This particular p4 error is misleading.\n")3127 sys.stderr.write("Perhaps the depot path was misspelled.\n");3128 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))3129 sys.exit(1)3130if'p4ExitCode'in info:3131 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])3132 sys.exit(1)313331343135 change =int(info["change"])3136if change > newestRevision:3137 newestRevision = change31383139if info["action"]in self.delete_actions:3140# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!3141#fileCnt = fileCnt + 13142continue31433144for prop in["depotFile","rev","action","type"]:3145 details["%s%s"% (prop, fileCnt)] = info[prop]31463147 fileCnt = fileCnt +131483149 details["change"] = newestRevision31503151# Use time from top-most change so that all git p4 clones of3152# the same p4 repo have the same commit SHA1s.3153 res =p4_describe(newestRevision)3154 details["time"] = res["time"]31553156 self.updateOptionDict(details)3157try:3158 self.commit(details, self.extractFilesFromCommit(details), self.branch)3159exceptIOError:3160print"IO error with git fast-import. Is your git version recent enough?"3161print self.gitError.read()316231633164defrun(self, args):3165 self.depotPaths = []3166 self.changeRange =""3167 self.previousDepotPaths = []3168 self.hasOrigin =False31693170# map from branch depot path to parent branch3171 self.knownBranches = {}3172 self.initialParents = {}31733174if self.importIntoRemotes:3175 self.refPrefix ="refs/remotes/p4/"3176else:3177 self.refPrefix ="refs/heads/p4/"31783179if self.syncWithOrigin:3180 self.hasOrigin =originP4BranchesExist()3181if self.hasOrigin:3182if not self.silent:3183print'Syncing with origin first, using "git fetch origin"'3184system("git fetch origin")31853186 branch_arg_given =bool(self.branch)3187iflen(self.branch) ==0:3188 self.branch = self.refPrefix +"master"3189ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:3190system("git update-ref%srefs/heads/p4"% self.branch)3191system("git branch -D p4")31923193# accept either the command-line option, or the configuration variable3194if self.useClientSpec:3195# will use this after clone to set the variable3196 self.useClientSpec_from_options =True3197else:3198ifgitConfigBool("git-p4.useclientspec"):3199 self.useClientSpec =True3200if self.useClientSpec:3201 self.clientSpecDirs =getClientSpec()32023203# TODO: should always look at previous commits,3204# merge with previous imports, if possible.3205if args == []:3206if self.hasOrigin:3207createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)32083209# branches holds mapping from branch name to sha13210 branches =p4BranchesInGit(self.importIntoRemotes)32113212# restrict to just this one, disabling detect-branches3213if branch_arg_given:3214 short = self.branch.split("/")[-1]3215if short in branches:3216 self.p4BranchesInGit = [ short ]3217else:3218 self.p4BranchesInGit = branches.keys()32193220iflen(self.p4BranchesInGit) >1:3221if not self.silent:3222print"Importing from/into multiple branches"3223 self.detectBranches =True3224for branch in branches.keys():3225 self.initialParents[self.refPrefix + branch] = \3226 branches[branch]32273228if self.verbose:3229print"branches:%s"% self.p4BranchesInGit32303231 p4Change =03232for branch in self.p4BranchesInGit:3233 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)32343235 settings =extractSettingsGitLog(logMsg)32363237 self.readOptions(settings)3238if(settings.has_key('depot-paths')3239and settings.has_key('change')):3240 change =int(settings['change']) +13241 p4Change =max(p4Change, change)32423243 depotPaths =sorted(settings['depot-paths'])3244if self.previousDepotPaths == []:3245 self.previousDepotPaths = depotPaths3246else:3247 paths = []3248for(prev, cur)inzip(self.previousDepotPaths, depotPaths):3249 prev_list = prev.split("/")3250 cur_list = cur.split("/")3251for i inrange(0,min(len(cur_list),len(prev_list))):3252if cur_list[i] <> prev_list[i]:3253 i = i -13254break32553256 paths.append("/".join(cur_list[:i +1]))32573258 self.previousDepotPaths = paths32593260if p4Change >0:3261 self.depotPaths =sorted(self.previousDepotPaths)3262 self.changeRange ="@%s,#head"% p4Change3263if not self.silent and not self.detectBranches:3264print"Performing incremental import into%sgit branch"% self.branch32653266# accept multiple ref name abbreviations:3267# refs/foo/bar/branch -> use it exactly3268# p4/branch -> prepend refs/remotes/ or refs/heads/3269# branch -> prepend refs/remotes/p4/ or refs/heads/p4/3270if not self.branch.startswith("refs/"):3271if self.importIntoRemotes:3272 prepend ="refs/remotes/"3273else:3274 prepend ="refs/heads/"3275if not self.branch.startswith("p4/"):3276 prepend +="p4/"3277 self.branch = prepend + self.branch32783279iflen(args) ==0and self.depotPaths:3280if not self.silent:3281print"Depot paths:%s"%' '.join(self.depotPaths)3282else:3283if self.depotPaths and self.depotPaths != args:3284print("previous import used depot path%sand now%swas specified. "3285"This doesn't work!"% (' '.join(self.depotPaths),3286' '.join(args)))3287 sys.exit(1)32883289 self.depotPaths =sorted(args)32903291 revision =""3292 self.users = {}32933294# Make sure no revision specifiers are used when --changesfile3295# is specified.3296 bad_changesfile =False3297iflen(self.changesFile) >0:3298for p in self.depotPaths:3299if p.find("@") >=0or p.find("#") >=0:3300 bad_changesfile =True3301break3302if bad_changesfile:3303die("Option --changesfile is incompatible with revision specifiers")33043305 newPaths = []3306for p in self.depotPaths:3307if p.find("@") != -1:3308 atIdx = p.index("@")3309 self.changeRange = p[atIdx:]3310if self.changeRange =="@all":3311 self.changeRange =""3312elif','not in self.changeRange:3313 revision = self.changeRange3314 self.changeRange =""3315 p = p[:atIdx]3316elif p.find("#") != -1:3317 hashIdx = p.index("#")3318 revision = p[hashIdx:]3319 p = p[:hashIdx]3320elif self.previousDepotPaths == []:3321# pay attention to changesfile, if given, else import3322# the entire p4 tree at the head revision3323iflen(self.changesFile) ==0:3324 revision ="#head"33253326 p = re.sub("\.\.\.$","", p)3327if not p.endswith("/"):3328 p +="/"33293330 newPaths.append(p)33313332 self.depotPaths = newPaths33333334# --detect-branches may change this for each branch3335 self.branchPrefixes = self.depotPaths33363337 self.loadUserMapFromCache()3338 self.labels = {}3339if self.detectLabels:3340 self.getLabels();33413342if self.detectBranches:3343## FIXME - what's a P4 projectName ?3344 self.projectName = self.guessProjectName()33453346if self.hasOrigin:3347 self.getBranchMappingFromGitBranches()3348else:3349 self.getBranchMapping()3350if self.verbose:3351print"p4-git branches:%s"% self.p4BranchesInGit3352print"initial parents:%s"% self.initialParents3353for b in self.p4BranchesInGit:3354if b !="master":33553356## FIXME3357 b = b[len(self.projectName):]3358 self.createdBranches.add(b)33593360 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))33613362 self.importProcess = subprocess.Popen(["git","fast-import"],3363 stdin=subprocess.PIPE,3364 stdout=subprocess.PIPE,3365 stderr=subprocess.PIPE);3366 self.gitOutput = self.importProcess.stdout3367 self.gitStream = self.importProcess.stdin3368 self.gitError = self.importProcess.stderr33693370if revision:3371 self.importHeadRevision(revision)3372else:3373 changes = []33743375iflen(self.changesFile) >0:3376 output =open(self.changesFile).readlines()3377 changeSet =set()3378for line in output:3379 changeSet.add(int(line))33803381for change in changeSet:3382 changes.append(change)33833384 changes.sort()3385else:3386# catch "git p4 sync" with no new branches, in a repo that3387# does not have any existing p4 branches3388iflen(args) ==0:3389if not self.p4BranchesInGit:3390die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")33913392# The default branch is master, unless --branch is used to3393# specify something else. Make sure it exists, or complain3394# nicely about how to use --branch.3395if not self.detectBranches:3396if notbranch_exists(self.branch):3397if branch_arg_given:3398die("Error: branch%sdoes not exist."% self.branch)3399else:3400die("Error: no branch%s; perhaps specify one with --branch."%3401 self.branch)34023403if self.verbose:3404print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3405 self.changeRange)3406 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)34073408iflen(self.maxChanges) >0:3409 changes = changes[:min(int(self.maxChanges),len(changes))]34103411iflen(changes) ==0:3412if not self.silent:3413print"No changes to import!"3414else:3415if not self.silent and not self.detectBranches:3416print"Import destination:%s"% self.branch34173418 self.updatedBranches =set()34193420if not self.detectBranches:3421if args:3422# start a new branch3423 self.initialParent =""3424else:3425# build on a previous revision3426 self.initialParent =parseRevision(self.branch)34273428 self.importChanges(changes)34293430if not self.silent:3431print""3432iflen(self.updatedBranches) >0:3433 sys.stdout.write("Updated branches: ")3434for b in self.updatedBranches:3435 sys.stdout.write("%s"% b)3436 sys.stdout.write("\n")34373438ifgitConfigBool("git-p4.importLabels"):3439 self.importLabels =True34403441if self.importLabels:3442 p4Labels =getP4Labels(self.depotPaths)3443 gitTags =getGitTags()34443445 missingP4Labels = p4Labels - gitTags3446 self.importP4Labels(self.gitStream, missingP4Labels)34473448 self.gitStream.close()3449if self.importProcess.wait() !=0:3450die("fast-import failed:%s"% self.gitError.read())3451 self.gitOutput.close()3452 self.gitError.close()34533454# Cleanup temporary branches created during import3455if self.tempBranches != []:3456for branch in self.tempBranches:3457read_pipe("git update-ref -d%s"% branch)3458 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))34593460# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3461# a convenient shortcut refname "p4".3462if self.importIntoRemotes:3463 head_ref = self.refPrefix +"HEAD"3464if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3465system(["git","symbolic-ref", head_ref, self.branch])34663467return True34683469classP4Rebase(Command):3470def__init__(self):3471 Command.__init__(self)3472 self.options = [3473 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3474]3475 self.importLabels =False3476 self.description = ("Fetches the latest revision from perforce and "3477+"rebases the current work (branch) against it")34783479defrun(self, args):3480 sync =P4Sync()3481 sync.importLabels = self.importLabels3482 sync.run([])34833484return self.rebase()34853486defrebase(self):3487if os.system("git update-index --refresh") !=0:3488die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");3489iflen(read_pipe("git diff-index HEAD --")) >0:3490die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");34913492[upstream, settings] =findUpstreamBranchPoint()3493iflen(upstream) ==0:3494die("Cannot find upstream branchpoint for rebase")34953496# the branchpoint may be p4/foo~3, so strip off the parent3497 upstream = re.sub("~[0-9]+$","", upstream)34983499print"Rebasing the current branch onto%s"% upstream3500 oldHead =read_pipe("git rev-parse HEAD").strip()3501system("git rebase%s"% upstream)3502system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3503return True35043505classP4Clone(P4Sync):3506def__init__(self):3507 P4Sync.__init__(self)3508 self.description ="Creates a new git repository and imports from Perforce into it"3509 self.usage ="usage: %prog [options] //depot/path[@revRange]"3510 self.options += [3511 optparse.make_option("--destination", dest="cloneDestination",3512 action='store', default=None,3513help="where to leave result of the clone"),3514 optparse.make_option("--bare", dest="cloneBare",3515 action="store_true", default=False),3516]3517 self.cloneDestination =None3518 self.needsGit =False3519 self.cloneBare =False35203521defdefaultDestination(self, args):3522## TODO: use common prefix of args?3523 depotPath = args[0]3524 depotDir = re.sub("(@[^@]*)$","", depotPath)3525 depotDir = re.sub("(#[^#]*)$","", depotDir)3526 depotDir = re.sub(r"\.\.\.$","", depotDir)3527 depotDir = re.sub(r"/$","", depotDir)3528return os.path.split(depotDir)[1]35293530defrun(self, args):3531iflen(args) <1:3532return False35333534if self.keepRepoPath and not self.cloneDestination:3535 sys.stderr.write("Must specify destination for --keep-path\n")3536 sys.exit(1)35373538 depotPaths = args35393540if not self.cloneDestination andlen(depotPaths) >1:3541 self.cloneDestination = depotPaths[-1]3542 depotPaths = depotPaths[:-1]35433544 self.cloneExclude = ["/"+p for p in self.cloneExclude]3545for p in depotPaths:3546if not p.startswith("//"):3547 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3548return False35493550if not self.cloneDestination:3551 self.cloneDestination = self.defaultDestination(args)35523553print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)35543555if not os.path.exists(self.cloneDestination):3556 os.makedirs(self.cloneDestination)3557chdir(self.cloneDestination)35583559 init_cmd = ["git","init"]3560if self.cloneBare:3561 init_cmd.append("--bare")3562 retcode = subprocess.call(init_cmd)3563if retcode:3564raiseCalledProcessError(retcode, init_cmd)35653566if not P4Sync.run(self, depotPaths):3567return False35683569# create a master branch and check out a work tree3570ifgitBranchExists(self.branch):3571system(["git","branch","master", self.branch ])3572if not self.cloneBare:3573system(["git","checkout","-f"])3574else:3575print'Not checking out any branch, use ' \3576'"git checkout -q -b master <branch>"'35773578# auto-set this variable if invoked with --use-client-spec3579if self.useClientSpec_from_options:3580system("git config --bool git-p4.useclientspec true")35813582return True35833584classP4Branches(Command):3585def__init__(self):3586 Command.__init__(self)3587 self.options = [ ]3588 self.description = ("Shows the git branches that hold imports and their "3589+"corresponding perforce depot paths")3590 self.verbose =False35913592defrun(self, args):3593iforiginP4BranchesExist():3594createOrUpdateBranchesFromOrigin()35953596 cmdline ="git rev-parse --symbolic "3597 cmdline +=" --remotes"35983599for line inread_pipe_lines(cmdline):3600 line = line.strip()36013602if not line.startswith('p4/')or line =="p4/HEAD":3603continue3604 branch = line36053606 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)3607 settings =extractSettingsGitLog(log)36083609print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])3610return True36113612classHelpFormatter(optparse.IndentedHelpFormatter):3613def__init__(self):3614 optparse.IndentedHelpFormatter.__init__(self)36153616defformat_description(self, description):3617if description:3618return description +"\n"3619else:3620return""36213622defprintUsage(commands):3623print"usage:%s<command> [options]"% sys.argv[0]3624print""3625print"valid commands:%s"%", ".join(commands)3626print""3627print"Try%s<command> --help for command specific help."% sys.argv[0]3628print""36293630commands = {3631"debug": P4Debug,3632"submit": P4Submit,3633"commit": P4Submit,3634"sync": P4Sync,3635"rebase": P4Rebase,3636"clone": P4Clone,3637"rollback": P4RollBack,3638"branches": P4Branches3639}364036413642defmain():3643iflen(sys.argv[1:]) ==0:3644printUsage(commands.keys())3645 sys.exit(2)36463647 cmdName = sys.argv[1]3648try:3649 klass = commands[cmdName]3650 cmd =klass()3651exceptKeyError:3652print"unknown command%s"% cmdName3653print""3654printUsage(commands.keys())3655 sys.exit(2)36563657 options = cmd.options3658 cmd.gitdir = os.environ.get("GIT_DIR",None)36593660 args = sys.argv[2:]36613662 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))3663if cmd.needsGit:3664 options.append(optparse.make_option("--git-dir", dest="gitdir"))36653666 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3667 options,3668 description = cmd.description,3669 formatter =HelpFormatter())36703671(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3672global verbose3673 verbose = cmd.verbose3674if cmd.needsGit:3675if cmd.gitdir ==None:3676 cmd.gitdir = os.path.abspath(".git")3677if notisValidGitDir(cmd.gitdir):3678 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3679if os.path.exists(cmd.gitdir):3680 cdup =read_pipe("git rev-parse --show-cdup").strip()3681iflen(cdup) >0:3682chdir(cdup);36833684if notisValidGitDir(cmd.gitdir):3685ifisValidGitDir(cmd.gitdir +"/.git"):3686 cmd.gitdir +="/.git"3687else:3688die("fatal: cannot locate git repository at%s"% cmd.gitdir)36893690 os.environ["GIT_DIR"] = cmd.gitdir36913692if not cmd.run(args):3693 parser.print_help()3694 sys.exit(2)369536963697if __name__ =='__main__':3698main()