1#!/usr/bin/env python 2# 3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. 4# 5# Author: Simon Hausmann <simon@lst.de> 6# Copyright: 2007 Simon Hausmann <simon@lst.de> 7# 2007 Trolltech ASA 8# License: MIT <http://www.opensource.org/licenses/mit-license.php> 9# 10import sys 11if sys.hexversion <0x02040000: 12# The limiter is the subprocess module 13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n") 14 sys.exit(1) 15import os 16import optparse 17import marshal 18import subprocess 19import tempfile 20import time 21import platform 22import re 23import shutil 24import stat 25import zipfile 26import zlib 27import ctypes 28import errno 29 30try: 31from subprocess import CalledProcessError 32exceptImportError: 33# from python2.7:subprocess.py 34# Exception classes used by this module. 35classCalledProcessError(Exception): 36"""This exception is raised when a process run by check_call() returns 37 a non-zero exit status. The exit status will be stored in the 38 returncode attribute.""" 39def__init__(self, returncode, cmd): 40 self.returncode = returncode 41 self.cmd = cmd 42def__str__(self): 43return"Command '%s' returned non-zero exit status%d"% (self.cmd, self.returncode) 44 45verbose =False 46 47# Only labels/tags matching this will be imported/exported 48defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$' 49 50# Grab changes in blocks of this many revisions, unless otherwise requested 51defaultBlockSize =512 52 53defp4_build_cmd(cmd): 54"""Build a suitable p4 command line. 55 56 This consolidates building and returning a p4 command line into one 57 location. It means that hooking into the environment, or other configuration 58 can be done more easily. 59 """ 60 real_cmd = ["p4"] 61 62 user =gitConfig("git-p4.user") 63iflen(user) >0: 64 real_cmd += ["-u",user] 65 66 password =gitConfig("git-p4.password") 67iflen(password) >0: 68 real_cmd += ["-P", password] 69 70 port =gitConfig("git-p4.port") 71iflen(port) >0: 72 real_cmd += ["-p", port] 73 74 host =gitConfig("git-p4.host") 75iflen(host) >0: 76 real_cmd += ["-H", host] 77 78 client =gitConfig("git-p4.client") 79iflen(client) >0: 80 real_cmd += ["-c", client] 81 82 83ifisinstance(cmd,basestring): 84 real_cmd =' '.join(real_cmd) +' '+ cmd 85else: 86 real_cmd += cmd 87return real_cmd 88 89defchdir(path, is_client_path=False): 90"""Do chdir to the given path, and set the PWD environment 91 variable for use by P4. It does not look at getcwd() output. 92 Since we're not using the shell, it is necessary to set the 93 PWD environment variable explicitly. 94 95 Normally, expand the path to force it to be absolute. This 96 addresses the use of relative path names inside P4 settings, 97 e.g. P4CONFIG=.p4config. P4 does not simply open the filename 98 as given; it looks for .p4config using PWD. 99 100 If is_client_path, the path was handed to us directly by p4, 101 and may be a symbolic link. Do not call os.getcwd() in this 102 case, because it will cause p4 to think that PWD is not inside 103 the client path. 104 """ 105 106 os.chdir(path) 107if not is_client_path: 108 path = os.getcwd() 109 os.environ['PWD'] = path 110 111defcalcDiskFree(): 112"""Return free space in bytes on the disk of the given dirname.""" 113if platform.system() =='Windows': 114 free_bytes = ctypes.c_ulonglong(0) 115 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()),None,None, ctypes.pointer(free_bytes)) 116return free_bytes.value 117else: 118 st = os.statvfs(os.getcwd()) 119return st.f_bavail * st.f_frsize 120 121defdie(msg): 122if verbose: 123raiseException(msg) 124else: 125 sys.stderr.write(msg +"\n") 126 sys.exit(1) 127 128defwrite_pipe(c, stdin): 129if verbose: 130 sys.stderr.write('Writing pipe:%s\n'%str(c)) 131 132 expand =isinstance(c,basestring) 133 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) 134 pipe = p.stdin 135 val = pipe.write(stdin) 136 pipe.close() 137if p.wait(): 138die('Command failed:%s'%str(c)) 139 140return val 141 142defp4_write_pipe(c, stdin): 143 real_cmd =p4_build_cmd(c) 144returnwrite_pipe(real_cmd, stdin) 145 146defread_pipe(c, ignore_error=False): 147if verbose: 148 sys.stderr.write('Reading pipe:%s\n'%str(c)) 149 150 expand =isinstance(c,basestring) 151 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) 152(out, err) = p.communicate() 153if p.returncode !=0and not ignore_error: 154die('Command failed:%s\nError:%s'% (str(c), err)) 155return out 156 157defp4_read_pipe(c, ignore_error=False): 158 real_cmd =p4_build_cmd(c) 159returnread_pipe(real_cmd, ignore_error) 160 161defread_pipe_lines(c): 162if verbose: 163 sys.stderr.write('Reading pipe:%s\n'%str(c)) 164 165 expand =isinstance(c, basestring) 166 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) 167 pipe = p.stdout 168 val = pipe.readlines() 169if pipe.close()or p.wait(): 170die('Command failed:%s'%str(c)) 171 172return val 173 174defp4_read_pipe_lines(c): 175"""Specifically invoke p4 on the command supplied. """ 176 real_cmd =p4_build_cmd(c) 177returnread_pipe_lines(real_cmd) 178 179defp4_has_command(cmd): 180"""Ask p4 for help on this command. If it returns an error, the 181 command does not exist in this version of p4.""" 182 real_cmd =p4_build_cmd(["help", cmd]) 183 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, 184 stderr=subprocess.PIPE) 185 p.communicate() 186return p.returncode ==0 187 188defp4_has_move_command(): 189"""See if the move command exists, that it supports -k, and that 190 it has not been administratively disabled. The arguments 191 must be correct, but the filenames do not have to exist. Use 192 ones with wildcards so even if they exist, it will fail.""" 193 194if notp4_has_command("move"): 195return False 196 cmd =p4_build_cmd(["move","-k","@from","@to"]) 197 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 198(out, err) = p.communicate() 199# return code will be 1 in either case 200if err.find("Invalid option") >=0: 201return False 202if err.find("disabled") >=0: 203return False 204# assume it failed because @... was invalid changelist 205return True 206 207defsystem(cmd, ignore_error=False): 208 expand =isinstance(cmd,basestring) 209if verbose: 210 sys.stderr.write("executing%s\n"%str(cmd)) 211 retcode = subprocess.call(cmd, shell=expand) 212if retcode and not ignore_error: 213raiseCalledProcessError(retcode, cmd) 214 215return retcode 216 217defp4_system(cmd): 218"""Specifically invoke p4 as the system command. """ 219 real_cmd =p4_build_cmd(cmd) 220 expand =isinstance(real_cmd, basestring) 221 retcode = subprocess.call(real_cmd, shell=expand) 222if retcode: 223raiseCalledProcessError(retcode, real_cmd) 224 225_p4_version_string =None 226defp4_version_string(): 227"""Read the version string, showing just the last line, which 228 hopefully is the interesting version bit. 229 230 $ p4 -V 231 Perforce - The Fast Software Configuration Management System. 232 Copyright 1995-2011 Perforce Software. All rights reserved. 233 Rev. P4/NTX86/2011.1/393975 (2011/12/16). 234 """ 235global _p4_version_string 236if not _p4_version_string: 237 a =p4_read_pipe_lines(["-V"]) 238 _p4_version_string = a[-1].rstrip() 239return _p4_version_string 240 241defp4_integrate(src, dest): 242p4_system(["integrate","-Dt",wildcard_encode(src),wildcard_encode(dest)]) 243 244defp4_sync(f, *options): 245p4_system(["sync"] +list(options) + [wildcard_encode(f)]) 246 247defp4_add(f): 248# forcibly add file names with wildcards 249ifwildcard_present(f): 250p4_system(["add","-f", f]) 251else: 252p4_system(["add", f]) 253 254defp4_delete(f): 255p4_system(["delete",wildcard_encode(f)]) 256 257defp4_edit(f, *options): 258p4_system(["edit"] +list(options) + [wildcard_encode(f)]) 259 260defp4_revert(f): 261p4_system(["revert",wildcard_encode(f)]) 262 263defp4_reopen(type, f): 264p4_system(["reopen","-t",type,wildcard_encode(f)]) 265 266defp4_move(src, dest): 267p4_system(["move","-k",wildcard_encode(src),wildcard_encode(dest)]) 268 269defp4_last_change(): 270 results =p4CmdList(["changes","-m","1"]) 271returnint(results[0]['change']) 272 273defp4_describe(change): 274"""Make sure it returns a valid result by checking for 275 the presence of field "time". Return a dict of the 276 results.""" 277 278 ds =p4CmdList(["describe","-s",str(change)]) 279iflen(ds) !=1: 280die("p4 describe -s%ddid not return 1 result:%s"% (change,str(ds))) 281 282 d = ds[0] 283 284if"p4ExitCode"in d: 285die("p4 describe -s%dexited with%d:%s"% (change, d["p4ExitCode"], 286str(d))) 287if"code"in d: 288if d["code"] =="error": 289die("p4 describe -s%dreturned error code:%s"% (change,str(d))) 290 291if"time"not in d: 292die("p4 describe -s%dreturned no\"time\":%s"% (change,str(d))) 293 294return d 295 296# 297# Canonicalize the p4 type and return a tuple of the 298# base type, plus any modifiers. See "p4 help filetypes" 299# for a list and explanation. 300# 301defsplit_p4_type(p4type): 302 303 p4_filetypes_historical = { 304"ctempobj":"binary+Sw", 305"ctext":"text+C", 306"cxtext":"text+Cx", 307"ktext":"text+k", 308"kxtext":"text+kx", 309"ltext":"text+F", 310"tempobj":"binary+FSw", 311"ubinary":"binary+F", 312"uresource":"resource+F", 313"uxbinary":"binary+Fx", 314"xbinary":"binary+x", 315"xltext":"text+Fx", 316"xtempobj":"binary+Swx", 317"xtext":"text+x", 318"xunicode":"unicode+x", 319"xutf16":"utf16+x", 320} 321if p4type in p4_filetypes_historical: 322 p4type = p4_filetypes_historical[p4type] 323 mods ="" 324 s = p4type.split("+") 325 base = s[0] 326 mods ="" 327iflen(s) >1: 328 mods = s[1] 329return(base, mods) 330 331# 332# return the raw p4 type of a file (text, text+ko, etc) 333# 334defp4_type(f): 335 results =p4CmdList(["fstat","-T","headType",wildcard_encode(f)]) 336return results[0]['headType'] 337 338# 339# Given a type base and modifier, return a regexp matching 340# the keywords that can be expanded in the file 341# 342defp4_keywords_regexp_for_type(base, type_mods): 343if base in("text","unicode","binary"): 344 kwords =None 345if"ko"in type_mods: 346 kwords ='Id|Header' 347elif"k"in type_mods: 348 kwords ='Id|Header|Author|Date|DateTime|Change|File|Revision' 349else: 350return None 351 pattern = r""" 352 \$ # Starts with a dollar, followed by... 353 (%s) # one of the keywords, followed by... 354 (:[^$\n]+)? # possibly an old expansion, followed by... 355 \$ # another dollar 356 """% kwords 357return pattern 358else: 359return None 360 361# 362# Given a file, return a regexp matching the possible 363# RCS keywords that will be expanded, or None for files 364# with kw expansion turned off. 365# 366defp4_keywords_regexp_for_file(file): 367if not os.path.exists(file): 368return None 369else: 370(type_base, type_mods) =split_p4_type(p4_type(file)) 371returnp4_keywords_regexp_for_type(type_base, type_mods) 372 373defsetP4ExecBit(file, mode): 374# Reopens an already open file and changes the execute bit to match 375# the execute bit setting in the passed in mode. 376 377 p4Type ="+x" 378 379if notisModeExec(mode): 380 p4Type =getP4OpenedType(file) 381 p4Type = re.sub('^([cku]?)x(.*)','\\1\\2', p4Type) 382 p4Type = re.sub('(.*?\+.*?)x(.*?)','\\1\\2', p4Type) 383if p4Type[-1] =="+": 384 p4Type = p4Type[0:-1] 385 386p4_reopen(p4Type,file) 387 388defgetP4OpenedType(file): 389# Returns the perforce file type for the given file. 390 391 result =p4_read_pipe(["opened",wildcard_encode(file)]) 392 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result) 393if match: 394return match.group(1) 395else: 396die("Could not determine file type for%s(result: '%s')"% (file, result)) 397 398# Return the set of all p4 labels 399defgetP4Labels(depotPaths): 400 labels =set() 401ifisinstance(depotPaths,basestring): 402 depotPaths = [depotPaths] 403 404for l inp4CmdList(["labels"] + ["%s..."% p for p in depotPaths]): 405 label = l['label'] 406 labels.add(label) 407 408return labels 409 410# Return the set of all git tags 411defgetGitTags(): 412 gitTags =set() 413for line inread_pipe_lines(["git","tag"]): 414 tag = line.strip() 415 gitTags.add(tag) 416return gitTags 417 418defdiffTreePattern(): 419# This is a simple generator for the diff tree regex pattern. This could be 420# a class variable if this and parseDiffTreeEntry were a part of a class. 421 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)') 422while True: 423yield pattern 424 425defparseDiffTreeEntry(entry): 426"""Parses a single diff tree entry into its component elements. 427 428 See git-diff-tree(1) manpage for details about the format of the diff 429 output. This method returns a dictionary with the following elements: 430 431 src_mode - The mode of the source file 432 dst_mode - The mode of the destination file 433 src_sha1 - The sha1 for the source file 434 dst_sha1 - The sha1 fr the destination file 435 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc) 436 status_score - The score for the status (applicable for 'C' and 'R' 437 statuses). This is None if there is no score. 438 src - The path for the source file. 439 dst - The path for the destination file. This is only present for 440 copy or renames. If it is not present, this is None. 441 442 If the pattern is not matched, None is returned.""" 443 444 match =diffTreePattern().next().match(entry) 445if match: 446return{ 447'src_mode': match.group(1), 448'dst_mode': match.group(2), 449'src_sha1': match.group(3), 450'dst_sha1': match.group(4), 451'status': match.group(5), 452'status_score': match.group(6), 453'src': match.group(7), 454'dst': match.group(10) 455} 456return None 457 458defisModeExec(mode): 459# Returns True if the given git mode represents an executable file, 460# otherwise False. 461return mode[-3:] =="755" 462 463defisModeExecChanged(src_mode, dst_mode): 464returnisModeExec(src_mode) !=isModeExec(dst_mode) 465 466defp4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None): 467 468ifisinstance(cmd,basestring): 469 cmd ="-G "+ cmd 470 expand =True 471else: 472 cmd = ["-G"] + cmd 473 expand =False 474 475 cmd =p4_build_cmd(cmd) 476if verbose: 477 sys.stderr.write("Opening pipe:%s\n"%str(cmd)) 478 479# Use a temporary file to avoid deadlocks without 480# subprocess.communicate(), which would put another copy 481# of stdout into memory. 482 stdin_file =None 483if stdin is not None: 484 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) 485ifisinstance(stdin,basestring): 486 stdin_file.write(stdin) 487else: 488for i in stdin: 489 stdin_file.write(i +'\n') 490 stdin_file.flush() 491 stdin_file.seek(0) 492 493 p4 = subprocess.Popen(cmd, 494 shell=expand, 495 stdin=stdin_file, 496 stdout=subprocess.PIPE) 497 498 result = [] 499try: 500while True: 501 entry = marshal.load(p4.stdout) 502if cb is not None: 503cb(entry) 504else: 505 result.append(entry) 506exceptEOFError: 507pass 508 exitCode = p4.wait() 509if exitCode !=0: 510 entry = {} 511 entry["p4ExitCode"] = exitCode 512 result.append(entry) 513 514return result 515 516defp4Cmd(cmd): 517list=p4CmdList(cmd) 518 result = {} 519for entry inlist: 520 result.update(entry) 521return result; 522 523defp4Where(depotPath): 524if not depotPath.endswith("/"): 525 depotPath +="/" 526 depotPathLong = depotPath +"..." 527 outputList =p4CmdList(["where", depotPathLong]) 528 output =None 529for entry in outputList: 530if"depotFile"in entry: 531# Search for the base client side depot path, as long as it starts with the branch's P4 path. 532# The base path always ends with "/...". 533if entry["depotFile"].find(depotPath) ==0and entry["depotFile"][-4:] =="/...": 534 output = entry 535break 536elif"data"in entry: 537 data = entry.get("data") 538 space = data.find(" ") 539if data[:space] == depotPath: 540 output = entry 541break 542if output ==None: 543return"" 544if output["code"] =="error": 545return"" 546 clientPath ="" 547if"path"in output: 548 clientPath = output.get("path") 549elif"data"in output: 550 data = output.get("data") 551 lastSpace = data.rfind(" ") 552 clientPath = data[lastSpace +1:] 553 554if clientPath.endswith("..."): 555 clientPath = clientPath[:-3] 556return clientPath 557 558defcurrentGitBranch(): 559 retcode =system(["git","symbolic-ref","-q","HEAD"], ignore_error=True) 560if retcode !=0: 561# on a detached head 562return None 563else: 564returnread_pipe(["git","name-rev","HEAD"]).split(" ")[1].strip() 565 566defisValidGitDir(path): 567if(os.path.exists(path +"/HEAD") 568and os.path.exists(path +"/refs")and os.path.exists(path +"/objects")): 569return True; 570return False 571 572defparseRevision(ref): 573returnread_pipe("git rev-parse%s"% ref).strip() 574 575defbranchExists(ref): 576 rev =read_pipe(["git","rev-parse","-q","--verify", ref], 577 ignore_error=True) 578returnlen(rev) >0 579 580defextractLogMessageFromGitCommit(commit): 581 logMessage ="" 582 583## fixme: title is first line of commit, not 1st paragraph. 584 foundTitle =False 585for log inread_pipe_lines("git cat-file commit%s"% commit): 586if not foundTitle: 587iflen(log) ==1: 588 foundTitle =True 589continue 590 591 logMessage += log 592return logMessage 593 594defextractSettingsGitLog(log): 595 values = {} 596for line in log.split("\n"): 597 line = line.strip() 598 m = re.search(r"^ *\[git-p4: (.*)\]$", line) 599if not m: 600continue 601 602 assignments = m.group(1).split(':') 603for a in assignments: 604 vals = a.split('=') 605 key = vals[0].strip() 606 val = ('='.join(vals[1:])).strip() 607if val.endswith('\"')and val.startswith('"'): 608 val = val[1:-1] 609 610 values[key] = val 611 612 paths = values.get("depot-paths") 613if not paths: 614 paths = values.get("depot-path") 615if paths: 616 values['depot-paths'] = paths.split(',') 617return values 618 619defgitBranchExists(branch): 620 proc = subprocess.Popen(["git","rev-parse", branch], 621 stderr=subprocess.PIPE, stdout=subprocess.PIPE); 622return proc.wait() ==0; 623 624_gitConfig = {} 625 626defgitConfig(key, typeSpecifier=None): 627if not _gitConfig.has_key(key): 628 cmd = ["git","config"] 629if typeSpecifier: 630 cmd += [ typeSpecifier ] 631 cmd += [ key ] 632 s =read_pipe(cmd, ignore_error=True) 633 _gitConfig[key] = s.strip() 634return _gitConfig[key] 635 636defgitConfigBool(key): 637"""Return a bool, using git config --bool. It is True only if the 638 variable is set to true, and False if set to false or not present 639 in the config.""" 640 641if not _gitConfig.has_key(key): 642 _gitConfig[key] =gitConfig(key,'--bool') =="true" 643return _gitConfig[key] 644 645defgitConfigInt(key): 646if not _gitConfig.has_key(key): 647 cmd = ["git","config","--int", key ] 648 s =read_pipe(cmd, ignore_error=True) 649 v = s.strip() 650try: 651 _gitConfig[key] =int(gitConfig(key,'--int')) 652exceptValueError: 653 _gitConfig[key] =None 654return _gitConfig[key] 655 656defgitConfigList(key): 657if not _gitConfig.has_key(key): 658 s =read_pipe(["git","config","--get-all", key], ignore_error=True) 659 _gitConfig[key] = s.strip().split(os.linesep) 660if _gitConfig[key] == ['']: 661 _gitConfig[key] = [] 662return _gitConfig[key] 663 664defp4BranchesInGit(branchesAreInRemotes=True): 665"""Find all the branches whose names start with "p4/", looking 666 in remotes or heads as specified by the argument. Return 667 a dictionary of{ branch: revision }for each one found. 668 The branch names are the short names, without any 669 "p4/" prefix.""" 670 671 branches = {} 672 673 cmdline ="git rev-parse --symbolic " 674if branchesAreInRemotes: 675 cmdline +="--remotes" 676else: 677 cmdline +="--branches" 678 679for line inread_pipe_lines(cmdline): 680 line = line.strip() 681 682# only import to p4/ 683if not line.startswith('p4/'): 684continue 685# special symbolic ref to p4/master 686if line =="p4/HEAD": 687continue 688 689# strip off p4/ prefix 690 branch = line[len("p4/"):] 691 692 branches[branch] =parseRevision(line) 693 694return branches 695 696defbranch_exists(branch): 697"""Make sure that the given ref name really exists.""" 698 699 cmd = ["git","rev-parse","--symbolic","--verify", branch ] 700 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 701 out, _ = p.communicate() 702if p.returncode: 703return False 704# expect exactly one line of output: the branch name 705return out.rstrip() == branch 706 707deffindUpstreamBranchPoint(head ="HEAD"): 708 branches =p4BranchesInGit() 709# map from depot-path to branch name 710 branchByDepotPath = {} 711for branch in branches.keys(): 712 tip = branches[branch] 713 log =extractLogMessageFromGitCommit(tip) 714 settings =extractSettingsGitLog(log) 715if settings.has_key("depot-paths"): 716 paths =",".join(settings["depot-paths"]) 717 branchByDepotPath[paths] ="remotes/p4/"+ branch 718 719 settings =None 720 parent =0 721while parent <65535: 722 commit = head +"~%s"% parent 723 log =extractLogMessageFromGitCommit(commit) 724 settings =extractSettingsGitLog(log) 725if settings.has_key("depot-paths"): 726 paths =",".join(settings["depot-paths"]) 727if branchByDepotPath.has_key(paths): 728return[branchByDepotPath[paths], settings] 729 730 parent = parent +1 731 732return["", settings] 733 734defcreateOrUpdateBranchesFromOrigin(localRefPrefix ="refs/remotes/p4/", silent=True): 735if not silent: 736print("Creating/updating branch(es) in%sbased on origin branch(es)" 737% localRefPrefix) 738 739 originPrefix ="origin/p4/" 740 741for line inread_pipe_lines("git rev-parse --symbolic --remotes"): 742 line = line.strip() 743if(not line.startswith(originPrefix))or line.endswith("HEAD"): 744continue 745 746 headName = line[len(originPrefix):] 747 remoteHead = localRefPrefix + headName 748 originHead = line 749 750 original =extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) 751if(not original.has_key('depot-paths') 752or not original.has_key('change')): 753continue 754 755 update =False 756if notgitBranchExists(remoteHead): 757if verbose: 758print"creating%s"% remoteHead 759 update =True 760else: 761 settings =extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) 762if settings.has_key('change') >0: 763if settings['depot-paths'] == original['depot-paths']: 764 originP4Change =int(original['change']) 765 p4Change =int(settings['change']) 766if originP4Change > p4Change: 767print("%s(%s) is newer than%s(%s). " 768"Updating p4 branch from origin." 769% (originHead, originP4Change, 770 remoteHead, p4Change)) 771 update =True 772else: 773print("Ignoring:%swas imported from%swhile " 774"%swas imported from%s" 775% (originHead,','.join(original['depot-paths']), 776 remoteHead,','.join(settings['depot-paths']))) 777 778if update: 779system("git update-ref%s %s"% (remoteHead, originHead)) 780 781deforiginP4BranchesExist(): 782returngitBranchExists("origin")orgitBranchExists("origin/p4")orgitBranchExists("origin/p4/master") 783 784 785defp4ParseNumericChangeRange(parts): 786 changeStart =int(parts[0][1:]) 787if parts[1] =='#head': 788 changeEnd =p4_last_change() 789else: 790 changeEnd =int(parts[1]) 791 792return(changeStart, changeEnd) 793 794defchooseBlockSize(blockSize): 795if blockSize: 796return blockSize 797else: 798return defaultBlockSize 799 800defp4ChangesForPaths(depotPaths, changeRange, requestedBlockSize): 801assert depotPaths 802 803# Parse the change range into start and end. Try to find integer 804# revision ranges as these can be broken up into blocks to avoid 805# hitting server-side limits (maxrows, maxscanresults). But if 806# that doesn't work, fall back to using the raw revision specifier 807# strings, without using block mode. 808 809if changeRange is None or changeRange =='': 810 changeStart =1 811 changeEnd =p4_last_change() 812 block_size =chooseBlockSize(requestedBlockSize) 813else: 814 parts = changeRange.split(',') 815assertlen(parts) ==2 816try: 817(changeStart, changeEnd) =p4ParseNumericChangeRange(parts) 818 block_size =chooseBlockSize(requestedBlockSize) 819except: 820 changeStart = parts[0][1:] 821 changeEnd = parts[1] 822if requestedBlockSize: 823die("cannot use --changes-block-size with non-numeric revisions") 824 block_size =None 825 826 changes =set() 827 828# Retrieve changes a block at a time, to prevent running 829# into a MaxResults/MaxScanRows error from the server. 830 831while True: 832 cmd = ['changes'] 833 834if block_size: 835 end =min(changeEnd, changeStart + block_size) 836 revisionRange ="%d,%d"% (changeStart, end) 837else: 838 revisionRange ="%s,%s"% (changeStart, changeEnd) 839 840for p in depotPaths: 841 cmd += ["%s...@%s"% (p, revisionRange)] 842 843# Insert changes in chronological order 844for line inreversed(p4_read_pipe_lines(cmd)): 845 changes.add(int(line.split(" ")[1])) 846 847if not block_size: 848break 849 850if end >= changeEnd: 851break 852 853 changeStart = end +1 854 855 changes =sorted(changes) 856return changes 857 858defp4PathStartsWith(path, prefix): 859# This method tries to remedy a potential mixed-case issue: 860# 861# If UserA adds //depot/DirA/file1 862# and UserB adds //depot/dira/file2 863# 864# we may or may not have a problem. If you have core.ignorecase=true, 865# we treat DirA and dira as the same directory 866ifgitConfigBool("core.ignorecase"): 867return path.lower().startswith(prefix.lower()) 868return path.startswith(prefix) 869 870defgetClientSpec(): 871"""Look at the p4 client spec, create a View() object that contains 872 all the mappings, and return it.""" 873 874 specList =p4CmdList("client -o") 875iflen(specList) !=1: 876die('Output from "client -o" is%dlines, expecting 1'% 877len(specList)) 878 879# dictionary of all client parameters 880 entry = specList[0] 881 882# the //client/ name 883 client_name = entry["Client"] 884 885# just the keys that start with "View" 886 view_keys = [ k for k in entry.keys()if k.startswith("View") ] 887 888# hold this new View 889 view =View(client_name) 890 891# append the lines, in order, to the view 892for view_num inrange(len(view_keys)): 893 k ="View%d"% view_num 894if k not in view_keys: 895die("Expected view key%smissing"% k) 896 view.append(entry[k]) 897 898return view 899 900defgetClientRoot(): 901"""Grab the client directory.""" 902 903 output =p4CmdList("client -o") 904iflen(output) !=1: 905die('Output from "client -o" is%dlines, expecting 1'%len(output)) 906 907 entry = output[0] 908if"Root"not in entry: 909die('Client has no "Root"') 910 911return entry["Root"] 912 913# 914# P4 wildcards are not allowed in filenames. P4 complains 915# if you simply add them, but you can force it with "-f", in 916# which case it translates them into %xx encoding internally. 917# 918defwildcard_decode(path): 919# Search for and fix just these four characters. Do % last so 920# that fixing it does not inadvertently create new %-escapes. 921# Cannot have * in a filename in windows; untested as to 922# what p4 would do in such a case. 923if not platform.system() =="Windows": 924 path = path.replace("%2A","*") 925 path = path.replace("%23","#") \ 926.replace("%40","@") \ 927.replace("%25","%") 928return path 929 930defwildcard_encode(path): 931# do % first to avoid double-encoding the %s introduced here 932 path = path.replace("%","%25") \ 933.replace("*","%2A") \ 934.replace("#","%23") \ 935.replace("@","%40") 936return path 937 938defwildcard_present(path): 939 m = re.search("[*#@%]", path) 940return m is not None 941 942classLargeFileSystem(object): 943"""Base class for large file system support.""" 944 945def__init__(self, writeToGitStream): 946 self.largeFiles =set() 947 self.writeToGitStream = writeToGitStream 948 949defgeneratePointer(self, cloneDestination, contentFile): 950"""Return the content of a pointer file that is stored in Git instead of 951 the actual content.""" 952assert False,"Method 'generatePointer' required in "+ self.__class__.__name__ 953 954defpushFile(self, localLargeFile): 955"""Push the actual content which is not stored in the Git repository to 956 a server.""" 957assert False,"Method 'pushFile' required in "+ self.__class__.__name__ 958 959defhasLargeFileExtension(self, relPath): 960returnreduce( 961lambda a, b: a or b, 962[relPath.endswith('.'+ e)for e ingitConfigList('git-p4.largeFileExtensions')], 963False 964) 965 966defgenerateTempFile(self, contents): 967 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 968for d in contents: 969 contentFile.write(d) 970 contentFile.close() 971return contentFile.name 972 973defexceedsLargeFileThreshold(self, relPath, contents): 974ifgitConfigInt('git-p4.largeFileThreshold'): 975 contentsSize =sum(len(d)for d in contents) 976if contentsSize >gitConfigInt('git-p4.largeFileThreshold'): 977return True 978ifgitConfigInt('git-p4.largeFileCompressedThreshold'): 979 contentsSize =sum(len(d)for d in contents) 980if contentsSize <=gitConfigInt('git-p4.largeFileCompressedThreshold'): 981return False 982 contentTempFile = self.generateTempFile(contents) 983 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False) 984 zf = zipfile.ZipFile(compressedContentFile.name, mode='w') 985 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED) 986 zf.close() 987 compressedContentsSize = zf.infolist()[0].compress_size 988 os.remove(contentTempFile) 989 os.remove(compressedContentFile.name) 990if compressedContentsSize >gitConfigInt('git-p4.largeFileCompressedThreshold'): 991return True 992return False 993 994defaddLargeFile(self, relPath): 995 self.largeFiles.add(relPath) 996 997defremoveLargeFile(self, relPath): 998 self.largeFiles.remove(relPath) 9991000defisLargeFile(self, relPath):1001return relPath in self.largeFiles10021003defprocessContent(self, git_mode, relPath, contents):1004"""Processes the content of git fast import. This method decides if a1005 file is stored in the large file system and handles all necessary1006 steps."""1007if self.exceedsLargeFileThreshold(relPath, contents)or self.hasLargeFileExtension(relPath):1008 contentTempFile = self.generateTempFile(contents)1009(pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)1010if pointer_git_mode:1011 git_mode = pointer_git_mode1012if localLargeFile:1013# Move temp file to final location in large file system1014 largeFileDir = os.path.dirname(localLargeFile)1015if not os.path.isdir(largeFileDir):1016 os.makedirs(largeFileDir)1017 shutil.move(contentTempFile, localLargeFile)1018 self.addLargeFile(relPath)1019ifgitConfigBool('git-p4.largeFilePush'):1020 self.pushFile(localLargeFile)1021if verbose:1022 sys.stderr.write("%smoved to large file system (%s)\n"% (relPath, localLargeFile))1023return(git_mode, contents)10241025classMockLFS(LargeFileSystem):1026"""Mock large file system for testing."""10271028defgeneratePointer(self, contentFile):1029"""The pointer content is the original content prefixed with "pointer-".1030 The local filename of the large file storage is derived from the file content.1031 """1032withopen(contentFile,'r')as f:1033 content =next(f)1034 gitMode ='100644'1035 pointerContents ='pointer-'+ content1036 localLargeFile = os.path.join(os.getcwd(),'.git','mock-storage','local', content[:-1])1037return(gitMode, pointerContents, localLargeFile)10381039defpushFile(self, localLargeFile):1040"""The remote filename of the large file storage is the same as the local1041 one but in a different directory.1042 """1043 remotePath = os.path.join(os.path.dirname(localLargeFile),'..','remote')1044if not os.path.exists(remotePath):1045 os.makedirs(remotePath)1046 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))10471048classGitLFS(LargeFileSystem):1049"""Git LFS as backend for the git-p4 large file system.1050 See https://git-lfs.github.com/ for details."""10511052def__init__(self, *args):1053 LargeFileSystem.__init__(self, *args)1054 self.baseGitAttributes = []10551056defgeneratePointer(self, contentFile):1057"""Generate a Git LFS pointer for the content. Return LFS Pointer file1058 mode and content which is stored in the Git repository instead of1059 the actual content. Return also the new location of the actual1060 content.1061 """1062if os.path.getsize(contentFile) ==0:1063return(None,'',None)10641065 pointerProcess = subprocess.Popen(1066['git','lfs','pointer','--file='+ contentFile],1067 stdout=subprocess.PIPE1068)1069 pointerFile = pointerProcess.stdout.read()1070if pointerProcess.wait():1071 os.remove(contentFile)1072die('git-lfs pointer command failed. Did you install the extension?')10731074# Git LFS removed the preamble in the output of the 'pointer' command1075# starting from version 1.2.0. Check for the preamble here to support1076# earlier versions.1077# c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b431078if pointerFile.startswith('Git LFS pointer for'):1079 pointerFile = re.sub(r'Git LFS pointer for.*\n\n','', pointerFile)10801081 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)1082 localLargeFile = os.path.join(1083 os.getcwd(),1084'.git','lfs','objects', oid[:2], oid[2:4],1085 oid,1086)1087# LFS Spec states that pointer files should not have the executable bit set.1088 gitMode ='100644'1089return(gitMode, pointerFile, localLargeFile)10901091defpushFile(self, localLargeFile):1092 uploadProcess = subprocess.Popen(1093['git','lfs','push','--object-id','origin', os.path.basename(localLargeFile)]1094)1095if uploadProcess.wait():1096die('git-lfs push command failed. Did you define a remote?')10971098defgenerateGitAttributes(self):1099return(1100 self.baseGitAttributes +1101[1102'\n',1103'#\n',1104'# Git LFS (see https://git-lfs.github.com/)\n',1105'#\n',1106] +1107['*.'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1108for f insorted(gitConfigList('git-p4.largeFileExtensions'))1109] +1110['/'+ f.replace(' ','[[:space:]]') +' filter=lfs -text\n'1111for f insorted(self.largeFiles)if not self.hasLargeFileExtension(f)1112]1113)11141115defaddLargeFile(self, relPath):1116 LargeFileSystem.addLargeFile(self, relPath)1117 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11181119defremoveLargeFile(self, relPath):1120 LargeFileSystem.removeLargeFile(self, relPath)1121 self.writeToGitStream('100644','.gitattributes', self.generateGitAttributes())11221123defprocessContent(self, git_mode, relPath, contents):1124if relPath =='.gitattributes':1125 self.baseGitAttributes = contents1126return(git_mode, self.generateGitAttributes())1127else:1128return LargeFileSystem.processContent(self, git_mode, relPath, contents)11291130class Command:1131def__init__(self):1132 self.usage ="usage: %prog [options]"1133 self.needsGit =True1134 self.verbose =False11351136class P4UserMap:1137def__init__(self):1138 self.userMapFromPerforceServer =False1139 self.myP4UserId =None11401141defp4UserId(self):1142if self.myP4UserId:1143return self.myP4UserId11441145 results =p4CmdList("user -o")1146for r in results:1147if r.has_key('User'):1148 self.myP4UserId = r['User']1149return r['User']1150die("Could not find your p4 user id")11511152defp4UserIsMe(self, p4User):1153# return True if the given p4 user is actually me1154 me = self.p4UserId()1155if not p4User or p4User != me:1156return False1157else:1158return True11591160defgetUserCacheFilename(self):1161 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))1162return home +"/.gitp4-usercache.txt"11631164defgetUserMapFromPerforceServer(self):1165if self.userMapFromPerforceServer:1166return1167 self.users = {}1168 self.emails = {}11691170for output inp4CmdList("users"):1171if not output.has_key("User"):1172continue1173 self.users[output["User"]] = output["FullName"] +" <"+ output["Email"] +">"1174 self.emails[output["Email"]] = output["User"]11751176 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)1177for mapUserConfig ingitConfigList("git-p4.mapUser"):1178 mapUser = mapUserConfigRegex.findall(mapUserConfig)1179if mapUser andlen(mapUser[0]) ==3:1180 user = mapUser[0][0]1181 fullname = mapUser[0][1]1182 email = mapUser[0][2]1183 self.users[user] = fullname +" <"+ email +">"1184 self.emails[email] = user11851186 s =''1187for(key, val)in self.users.items():1188 s +="%s\t%s\n"% (key.expandtabs(1), val.expandtabs(1))11891190open(self.getUserCacheFilename(),"wb").write(s)1191 self.userMapFromPerforceServer =True11921193defloadUserMapFromCache(self):1194 self.users = {}1195 self.userMapFromPerforceServer =False1196try:1197 cache =open(self.getUserCacheFilename(),"rb")1198 lines = cache.readlines()1199 cache.close()1200for line in lines:1201 entry = line.strip().split("\t")1202 self.users[entry[0]] = entry[1]1203exceptIOError:1204 self.getUserMapFromPerforceServer()12051206classP4Debug(Command):1207def__init__(self):1208 Command.__init__(self)1209 self.options = []1210 self.description ="A tool to debug the output of p4 -G."1211 self.needsGit =False12121213defrun(self, args):1214 j =01215for output inp4CmdList(args):1216print'Element:%d'% j1217 j +=11218print output1219return True12201221classP4RollBack(Command):1222def__init__(self):1223 Command.__init__(self)1224 self.options = [1225 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")1226]1227 self.description ="A tool to debug the multi-branch import. Don't use :)"1228 self.rollbackLocalBranches =False12291230defrun(self, args):1231iflen(args) !=1:1232return False1233 maxChange =int(args[0])12341235if"p4ExitCode"inp4Cmd("changes -m 1"):1236die("Problems executing p4");12371238if self.rollbackLocalBranches:1239 refPrefix ="refs/heads/"1240 lines =read_pipe_lines("git rev-parse --symbolic --branches")1241else:1242 refPrefix ="refs/remotes/"1243 lines =read_pipe_lines("git rev-parse --symbolic --remotes")12441245for line in lines:1246if self.rollbackLocalBranches or(line.startswith("p4/")and line !="p4/HEAD\n"):1247 line = line.strip()1248 ref = refPrefix + line1249 log =extractLogMessageFromGitCommit(ref)1250 settings =extractSettingsGitLog(log)12511252 depotPaths = settings['depot-paths']1253 change = settings['change']12541255 changed =False12561257iflen(p4Cmd("changes -m 1 "+' '.join(['%s...@%s'% (p, maxChange)1258for p in depotPaths]))) ==0:1259print"Branch%sdid not exist at change%s, deleting."% (ref, maxChange)1260system("git update-ref -d%s`git rev-parse%s`"% (ref, ref))1261continue12621263while change andint(change) > maxChange:1264 changed =True1265if self.verbose:1266print"%sis at%s; rewinding towards%s"% (ref, change, maxChange)1267system("git update-ref%s\"%s^\""% (ref, ref))1268 log =extractLogMessageFromGitCommit(ref)1269 settings =extractSettingsGitLog(log)127012711272 depotPaths = settings['depot-paths']1273 change = settings['change']12741275if changed:1276print"%srewound to%s"% (ref, change)12771278return True12791280classP4Submit(Command, P4UserMap):12811282 conflict_behavior_choices = ("ask","skip","quit")12831284def__init__(self):1285 Command.__init__(self)1286 P4UserMap.__init__(self)1287 self.options = [1288 optparse.make_option("--origin", dest="origin"),1289 optparse.make_option("-M", dest="detectRenames", action="store_true"),1290# preserve the user, requires relevant p4 permissions1291 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),1292 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),1293 optparse.make_option("--dry-run","-n", dest="dry_run", action="store_true"),1294 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),1295 optparse.make_option("--conflict", dest="conflict_behavior",1296 choices=self.conflict_behavior_choices),1297 optparse.make_option("--branch", dest="branch"),1298]1299 self.description ="Submit changes from git to the perforce depot."1300 self.usage +=" [name of git branch to submit into perforce depot]"1301 self.origin =""1302 self.detectRenames =False1303 self.preserveUser =gitConfigBool("git-p4.preserveUser")1304 self.dry_run =False1305 self.prepare_p4_only =False1306 self.conflict_behavior =None1307 self.isWindows = (platform.system() =="Windows")1308 self.exportLabels =False1309 self.p4HasMoveCommand =p4_has_move_command()1310 self.branch =None13111312ifgitConfig('git-p4.largeFileSystem'):1313die("Large file system not supported for git-p4 submit command. Please remove it from config.")13141315defcheck(self):1316iflen(p4CmdList("opened ...")) >0:1317die("You have files opened with perforce! Close them before starting the sync.")13181319defseparate_jobs_from_description(self, message):1320"""Extract and return a possible Jobs field in the commit1321 message. It goes into a separate section in the p4 change1322 specification.13231324 A jobs line starts with "Jobs:" and looks like a new field1325 in a form. Values are white-space separated on the same1326 line or on following lines that start with a tab.13271328 This does not parse and extract the full git commit message1329 like a p4 form. It just sees the Jobs: line as a marker1330 to pass everything from then on directly into the p4 form,1331 but outside the description section.13321333 Return a tuple (stripped log message, jobs string)."""13341335 m = re.search(r'^Jobs:', message, re.MULTILINE)1336if m is None:1337return(message,None)13381339 jobtext = message[m.start():]1340 stripped_message = message[:m.start()].rstrip()1341return(stripped_message, jobtext)13421343defprepareLogMessage(self, template, message, jobs):1344"""Edits the template returned from "p4 change -o" to insert1345 the message in the Description field, and the jobs text in1346 the Jobs field."""1347 result =""13481349 inDescriptionSection =False13501351for line in template.split("\n"):1352if line.startswith("#"):1353 result += line +"\n"1354continue13551356if inDescriptionSection:1357if line.startswith("Files:")or line.startswith("Jobs:"):1358 inDescriptionSection =False1359# insert Jobs section1360if jobs:1361 result += jobs +"\n"1362else:1363continue1364else:1365if line.startswith("Description:"):1366 inDescriptionSection =True1367 line +="\n"1368for messageLine in message.split("\n"):1369 line +="\t"+ messageLine +"\n"13701371 result += line +"\n"13721373return result13741375defpatchRCSKeywords(self,file, pattern):1376# Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern1377(handle, outFileName) = tempfile.mkstemp(dir='.')1378try:1379 outFile = os.fdopen(handle,"w+")1380 inFile =open(file,"r")1381 regexp = re.compile(pattern, re.VERBOSE)1382for line in inFile.readlines():1383 line = regexp.sub(r'$\1$', line)1384 outFile.write(line)1385 inFile.close()1386 outFile.close()1387# Forcibly overwrite the original file1388 os.unlink(file)1389 shutil.move(outFileName,file)1390except:1391# cleanup our temporary file1392 os.unlink(outFileName)1393print"Failed to strip RCS keywords in%s"%file1394raise13951396print"Patched up RCS keywords in%s"%file13971398defp4UserForCommit(self,id):1399# Return the tuple (perforce user,git email) for a given git commit id1400 self.getUserMapFromPerforceServer()1401 gitEmail =read_pipe(["git","log","--max-count=1",1402"--format=%ae",id])1403 gitEmail = gitEmail.strip()1404if not self.emails.has_key(gitEmail):1405return(None,gitEmail)1406else:1407return(self.emails[gitEmail],gitEmail)14081409defcheckValidP4Users(self,commits):1410# check if any git authors cannot be mapped to p4 users1411foridin commits:1412(user,email) = self.p4UserForCommit(id)1413if not user:1414 msg ="Cannot find p4 user for email%sin commit%s."% (email,id)1415ifgitConfigBool("git-p4.allowMissingP4Users"):1416print"%s"% msg1417else:1418die("Error:%s\nSet git-p4.allowMissingP4Users to true to allow this."% msg)14191420deflastP4Changelist(self):1421# Get back the last changelist number submitted in this client spec. This1422# then gets used to patch up the username in the change. If the same1423# client spec is being used by multiple processes then this might go1424# wrong.1425 results =p4CmdList("client -o")# find the current client1426 client =None1427for r in results:1428if r.has_key('Client'):1429 client = r['Client']1430break1431if not client:1432die("could not get client spec")1433 results =p4CmdList(["changes","-c", client,"-m","1"])1434for r in results:1435if r.has_key('change'):1436return r['change']1437die("Could not get changelist number for last submit - cannot patch up user details")14381439defmodifyChangelistUser(self, changelist, newUser):1440# fixup the user field of a changelist after it has been submitted.1441 changes =p4CmdList("change -o%s"% changelist)1442iflen(changes) !=1:1443die("Bad output from p4 change modifying%sto user%s"%1444(changelist, newUser))14451446 c = changes[0]1447if c['User'] == newUser:return# nothing to do1448 c['User'] = newUser1449input= marshal.dumps(c)14501451 result =p4CmdList("change -f -i", stdin=input)1452for r in result:1453if r.has_key('code'):1454if r['code'] =='error':1455die("Could not modify user field of changelist%sto%s:%s"% (changelist, newUser, r['data']))1456if r.has_key('data'):1457print("Updated user field for changelist%sto%s"% (changelist, newUser))1458return1459die("Could not modify user field of changelist%sto%s"% (changelist, newUser))14601461defcanChangeChangelists(self):1462# check to see if we have p4 admin or super-user permissions, either of1463# which are required to modify changelists.1464 results =p4CmdList(["protects", self.depotPath])1465for r in results:1466if r.has_key('perm'):1467if r['perm'] =='admin':1468return11469if r['perm'] =='super':1470return11471return014721473defprepareSubmitTemplate(self):1474"""Run "p4 change -o" to grab a change specification template.1475 This does not use "p4 -G", as it is nice to keep the submission1476 template in original order, since a human might edit it.14771478 Remove lines in the Files section that show changes to files1479 outside the depot path we're committing into."""14801481[upstream, settings] =findUpstreamBranchPoint()14821483 template =""1484 inFilesSection =False1485for line inp4_read_pipe_lines(['change','-o']):1486if line.endswith("\r\n"):1487 line = line[:-2] +"\n"1488if inFilesSection:1489if line.startswith("\t"):1490# path starts and ends with a tab1491 path = line[1:]1492 lastTab = path.rfind("\t")1493if lastTab != -1:1494 path = path[:lastTab]1495if settings.has_key('depot-paths'):1496if not[p for p in settings['depot-paths']1497ifp4PathStartsWith(path, p)]:1498continue1499else:1500if notp4PathStartsWith(path, self.depotPath):1501continue1502else:1503 inFilesSection =False1504else:1505if line.startswith("Files:"):1506 inFilesSection =True15071508 template += line15091510return template15111512defedit_template(self, template_file):1513"""Invoke the editor to let the user change the submission1514 message. Return true if okay to continue with the submit."""15151516# if configured to skip the editing part, just submit1517ifgitConfigBool("git-p4.skipSubmitEdit"):1518return True15191520# look at the modification time, to check later if the user saved1521# the file1522 mtime = os.stat(template_file).st_mtime15231524# invoke the editor1525if os.environ.has_key("P4EDITOR")and(os.environ.get("P4EDITOR") !=""):1526 editor = os.environ.get("P4EDITOR")1527else:1528 editor =read_pipe("git var GIT_EDITOR").strip()1529system(["sh","-c", ('%s"$@"'% editor), editor, template_file])15301531# If the file was not saved, prompt to see if this patch should1532# be skipped. But skip this verification step if configured so.1533ifgitConfigBool("git-p4.skipSubmitEditCheck"):1534return True15351536# modification time updated means user saved the file1537if os.stat(template_file).st_mtime > mtime:1538return True15391540while True:1541 response =raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")1542if response =='y':1543return True1544if response =='n':1545return False15461547defget_diff_description(self, editedFiles, filesToAdd, symlinks):1548# diff1549if os.environ.has_key("P4DIFF"):1550del(os.environ["P4DIFF"])1551 diff =""1552for editedFile in editedFiles:1553 diff +=p4_read_pipe(['diff','-du',1554wildcard_encode(editedFile)])15551556# new file diff1557 newdiff =""1558for newFile in filesToAdd:1559 newdiff +="==== new file ====\n"1560 newdiff +="--- /dev/null\n"1561 newdiff +="+++%s\n"% newFile15621563 is_link = os.path.islink(newFile)1564 expect_link = newFile in symlinks15651566if is_link and expect_link:1567 newdiff +="+%s\n"% os.readlink(newFile)1568else:1569 f =open(newFile,"r")1570for line in f.readlines():1571 newdiff +="+"+ line1572 f.close()15731574return(diff + newdiff).replace('\r\n','\n')15751576defapplyCommit(self,id):1577"""Apply one commit, return True if it succeeded."""15781579print"Applying",read_pipe(["git","show","-s",1580"--format=format:%h%s",id])15811582(p4User, gitEmail) = self.p4UserForCommit(id)15831584 diff =read_pipe_lines("git diff-tree -r%s\"%s^\" \"%s\""% (self.diffOpts,id,id))1585 filesToAdd =set()1586 filesToChangeType =set()1587 filesToDelete =set()1588 editedFiles =set()1589 pureRenameCopy =set()1590 symlinks =set()1591 filesToChangeExecBit = {}15921593for line in diff:1594 diff =parseDiffTreeEntry(line)1595 modifier = diff['status']1596 path = diff['src']1597if modifier =="M":1598p4_edit(path)1599ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1600 filesToChangeExecBit[path] = diff['dst_mode']1601 editedFiles.add(path)1602elif modifier =="A":1603 filesToAdd.add(path)1604 filesToChangeExecBit[path] = diff['dst_mode']1605if path in filesToDelete:1606 filesToDelete.remove(path)16071608 dst_mode =int(diff['dst_mode'],8)1609if dst_mode ==0120000:1610 symlinks.add(path)16111612elif modifier =="D":1613 filesToDelete.add(path)1614if path in filesToAdd:1615 filesToAdd.remove(path)1616elif modifier =="C":1617 src, dest = diff['src'], diff['dst']1618p4_integrate(src, dest)1619 pureRenameCopy.add(dest)1620if diff['src_sha1'] != diff['dst_sha1']:1621p4_edit(dest)1622 pureRenameCopy.discard(dest)1623ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1624p4_edit(dest)1625 pureRenameCopy.discard(dest)1626 filesToChangeExecBit[dest] = diff['dst_mode']1627if self.isWindows:1628# turn off read-only attribute1629 os.chmod(dest, stat.S_IWRITE)1630 os.unlink(dest)1631 editedFiles.add(dest)1632elif modifier =="R":1633 src, dest = diff['src'], diff['dst']1634if self.p4HasMoveCommand:1635p4_edit(src)# src must be open before move1636p4_move(src, dest)# opens for (move/delete, move/add)1637else:1638p4_integrate(src, dest)1639if diff['src_sha1'] != diff['dst_sha1']:1640p4_edit(dest)1641else:1642 pureRenameCopy.add(dest)1643ifisModeExecChanged(diff['src_mode'], diff['dst_mode']):1644if not self.p4HasMoveCommand:1645p4_edit(dest)# with move: already open, writable1646 filesToChangeExecBit[dest] = diff['dst_mode']1647if not self.p4HasMoveCommand:1648if self.isWindows:1649 os.chmod(dest, stat.S_IWRITE)1650 os.unlink(dest)1651 filesToDelete.add(src)1652 editedFiles.add(dest)1653elif modifier =="T":1654 filesToChangeType.add(path)1655else:1656die("unknown modifier%sfor%s"% (modifier, path))16571658 diffcmd ="git diff-tree --full-index -p\"%s\""% (id)1659 patchcmd = diffcmd +" | git apply "1660 tryPatchCmd = patchcmd +"--check -"1661 applyPatchCmd = patchcmd +"--check --apply -"1662 patch_succeeded =True16631664if os.system(tryPatchCmd) !=0:1665 fixed_rcs_keywords =False1666 patch_succeeded =False1667print"Unfortunately applying the change failed!"16681669# Patch failed, maybe it's just RCS keyword woes. Look through1670# the patch to see if that's possible.1671ifgitConfigBool("git-p4.attemptRCSCleanup"):1672file=None1673 pattern =None1674 kwfiles = {}1675forfilein editedFiles | filesToDelete:1676# did this file's delta contain RCS keywords?1677 pattern =p4_keywords_regexp_for_file(file)16781679if pattern:1680# this file is a possibility...look for RCS keywords.1681 regexp = re.compile(pattern, re.VERBOSE)1682for line inread_pipe_lines(["git","diff","%s^..%s"% (id,id),file]):1683if regexp.search(line):1684if verbose:1685print"got keyword match on%sin%sin%s"% (pattern, line,file)1686 kwfiles[file] = pattern1687break16881689forfilein kwfiles:1690if verbose:1691print"zapping%swith%s"% (line,pattern)1692# File is being deleted, so not open in p4. Must1693# disable the read-only bit on windows.1694if self.isWindows andfilenot in editedFiles:1695 os.chmod(file, stat.S_IWRITE)1696 self.patchRCSKeywords(file, kwfiles[file])1697 fixed_rcs_keywords =True16981699if fixed_rcs_keywords:1700print"Retrying the patch with RCS keywords cleaned up"1701if os.system(tryPatchCmd) ==0:1702 patch_succeeded =True17031704if not patch_succeeded:1705for f in editedFiles:1706p4_revert(f)1707return False17081709#1710# Apply the patch for real, and do add/delete/+x handling.1711#1712system(applyPatchCmd)17131714for f in filesToChangeType:1715p4_edit(f,"-t","auto")1716for f in filesToAdd:1717p4_add(f)1718for f in filesToDelete:1719p4_revert(f)1720p4_delete(f)17211722# Set/clear executable bits1723for f in filesToChangeExecBit.keys():1724 mode = filesToChangeExecBit[f]1725setP4ExecBit(f, mode)17261727#1728# Build p4 change description, starting with the contents1729# of the git commit message.1730#1731 logMessage =extractLogMessageFromGitCommit(id)1732 logMessage = logMessage.strip()1733(logMessage, jobs) = self.separate_jobs_from_description(logMessage)17341735 template = self.prepareSubmitTemplate()1736 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)17371738if self.preserveUser:1739 submitTemplate +="\n######## Actual user%s, modified after commit\n"% p4User17401741if self.checkAuthorship and not self.p4UserIsMe(p4User):1742 submitTemplate +="######## git author%sdoes not match your p4 account.\n"% gitEmail1743 submitTemplate +="######## Use option --preserve-user to modify authorship.\n"1744 submitTemplate +="######## Variable git-p4.skipUserNameCheck hides this message.\n"17451746 separatorLine ="######## everything below this line is just the diff #######\n"1747if not self.prepare_p4_only:1748 submitTemplate += separatorLine1749 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)17501751(handle, fileName) = tempfile.mkstemp()1752 tmpFile = os.fdopen(handle,"w+b")1753if self.isWindows:1754 submitTemplate = submitTemplate.replace("\n","\r\n")1755 tmpFile.write(submitTemplate)1756 tmpFile.close()17571758if self.prepare_p4_only:1759#1760# Leave the p4 tree prepared, and the submit template around1761# and let the user decide what to do next1762#1763print1764print"P4 workspace prepared for submission."1765print"To submit or revert, go to client workspace"1766print" "+ self.clientPath1767print1768print"To submit, use\"p4 submit\"to write a new description,"1769print"or\"p4 submit -i <%s\"to use the one prepared by" \1770"\"git p4\"."% fileName1771print"You can delete the file\"%s\"when finished."% fileName17721773if self.preserveUser and p4User and not self.p4UserIsMe(p4User):1774print"To preserve change ownership by user%s, you must\n" \1775"do\"p4 change -f <change>\"after submitting and\n" \1776"edit the User field."1777if pureRenameCopy:1778print"After submitting, renamed files must be re-synced."1779print"Invoke\"p4 sync -f\"on each of these files:"1780for f in pureRenameCopy:1781print" "+ f17821783print1784print"To revert the changes, use\"p4 revert ...\", and delete"1785print"the submit template file\"%s\""% fileName1786if filesToAdd:1787print"Since the commit adds new files, they must be deleted:"1788for f in filesToAdd:1789print" "+ f1790print1791return True17921793#1794# Let the user edit the change description, then submit it.1795#1796 submitted =False17971798try:1799if self.edit_template(fileName):1800# read the edited message and submit1801 tmpFile =open(fileName,"rb")1802 message = tmpFile.read()1803 tmpFile.close()1804if self.isWindows:1805 message = message.replace("\r\n","\n")1806 submitTemplate = message[:message.index(separatorLine)]1807p4_write_pipe(['submit','-i'], submitTemplate)18081809if self.preserveUser:1810if p4User:1811# Get last changelist number. Cannot easily get it from1812# the submit command output as the output is1813# unmarshalled.1814 changelist = self.lastP4Changelist()1815 self.modifyChangelistUser(changelist, p4User)18161817# The rename/copy happened by applying a patch that created a1818# new file. This leaves it writable, which confuses p4.1819for f in pureRenameCopy:1820p4_sync(f,"-f")1821 submitted =True18221823finally:1824# skip this patch1825if not submitted:1826print"Submission cancelled, undoing p4 changes."1827for f in editedFiles:1828p4_revert(f)1829for f in filesToAdd:1830p4_revert(f)1831 os.remove(f)1832for f in filesToDelete:1833p4_revert(f)18341835 os.remove(fileName)1836return submitted18371838# Export git tags as p4 labels. Create a p4 label and then tag1839# with that.1840defexportGitTags(self, gitTags):1841 validLabelRegexp =gitConfig("git-p4.labelExportRegexp")1842iflen(validLabelRegexp) ==0:1843 validLabelRegexp = defaultLabelRegexp1844 m = re.compile(validLabelRegexp)18451846for name in gitTags:18471848if not m.match(name):1849if verbose:1850print"tag%sdoes not match regexp%s"% (name, validLabelRegexp)1851continue18521853# Get the p4 commit this corresponds to1854 logMessage =extractLogMessageFromGitCommit(name)1855 values =extractSettingsGitLog(logMessage)18561857if not values.has_key('change'):1858# a tag pointing to something not sent to p4; ignore1859if verbose:1860print"git tag%sdoes not give a p4 commit"% name1861continue1862else:1863 changelist = values['change']18641865# Get the tag details.1866 inHeader =True1867 isAnnotated =False1868 body = []1869for l inread_pipe_lines(["git","cat-file","-p", name]):1870 l = l.strip()1871if inHeader:1872if re.match(r'tag\s+', l):1873 isAnnotated =True1874elif re.match(r'\s*$', l):1875 inHeader =False1876continue1877else:1878 body.append(l)18791880if not isAnnotated:1881 body = ["lightweight tag imported by git p4\n"]18821883# Create the label - use the same view as the client spec we are using1884 clientSpec =getClientSpec()18851886 labelTemplate ="Label:%s\n"% name1887 labelTemplate +="Description:\n"1888for b in body:1889 labelTemplate +="\t"+ b +"\n"1890 labelTemplate +="View:\n"1891for depot_side in clientSpec.mappings:1892 labelTemplate +="\t%s\n"% depot_side18931894if self.dry_run:1895print"Would create p4 label%sfor tag"% name1896elif self.prepare_p4_only:1897print"Not creating p4 label%sfor tag due to option" \1898" --prepare-p4-only"% name1899else:1900p4_write_pipe(["label","-i"], labelTemplate)19011902# Use the label1903p4_system(["tag","-l", name] +1904["%s@%s"% (depot_side, changelist)for depot_side in clientSpec.mappings])19051906if verbose:1907print"created p4 label for tag%s"% name19081909defrun(self, args):1910iflen(args) ==0:1911 self.master =currentGitBranch()1912eliflen(args) ==1:1913 self.master = args[0]1914if notbranchExists(self.master):1915die("Branch%sdoes not exist"% self.master)1916else:1917return False19181919if self.master:1920 allowSubmit =gitConfig("git-p4.allowSubmit")1921iflen(allowSubmit) >0and not self.master in allowSubmit.split(","):1922die("%sis not in git-p4.allowSubmit"% self.master)19231924[upstream, settings] =findUpstreamBranchPoint()1925 self.depotPath = settings['depot-paths'][0]1926iflen(self.origin) ==0:1927 self.origin = upstream19281929if self.preserveUser:1930if not self.canChangeChangelists():1931die("Cannot preserve user names without p4 super-user or admin permissions")19321933# if not set from the command line, try the config file1934if self.conflict_behavior is None:1935 val =gitConfig("git-p4.conflict")1936if val:1937if val not in self.conflict_behavior_choices:1938die("Invalid value '%s' for config git-p4.conflict"% val)1939else:1940 val ="ask"1941 self.conflict_behavior = val19421943if self.verbose:1944print"Origin branch is "+ self.origin19451946iflen(self.depotPath) ==0:1947print"Internal error: cannot locate perforce depot path from existing branches"1948 sys.exit(128)19491950 self.useClientSpec =False1951ifgitConfigBool("git-p4.useclientspec"):1952 self.useClientSpec =True1953if self.useClientSpec:1954 self.clientSpecDirs =getClientSpec()19551956# Check for the existence of P4 branches1957 branchesDetected = (len(p4BranchesInGit().keys()) >1)19581959if self.useClientSpec and not branchesDetected:1960# all files are relative to the client spec1961 self.clientPath =getClientRoot()1962else:1963 self.clientPath =p4Where(self.depotPath)19641965if self.clientPath =="":1966die("Error: Cannot locate perforce checkout of%sin client view"% self.depotPath)19671968print"Perforce checkout for depot path%slocated at%s"% (self.depotPath, self.clientPath)1969 self.oldWorkingDirectory = os.getcwd()19701971# ensure the clientPath exists1972 new_client_dir =False1973if not os.path.exists(self.clientPath):1974 new_client_dir =True1975 os.makedirs(self.clientPath)19761977chdir(self.clientPath, is_client_path=True)1978if self.dry_run:1979print"Would synchronize p4 checkout in%s"% self.clientPath1980else:1981print"Synchronizing p4 checkout..."1982if new_client_dir:1983# old one was destroyed, and maybe nobody told p41984p4_sync("...","-f")1985else:1986p4_sync("...")1987 self.check()19881989 commits = []1990if self.master:1991 commitish = self.master1992else:1993 commitish ='HEAD'19941995for line inread_pipe_lines(["git","rev-list","--no-merges","%s..%s"% (self.origin, commitish)]):1996 commits.append(line.strip())1997 commits.reverse()19981999if self.preserveUser orgitConfigBool("git-p4.skipUserNameCheck"):2000 self.checkAuthorship =False2001else:2002 self.checkAuthorship =True20032004if self.preserveUser:2005 self.checkValidP4Users(commits)20062007#2008# Build up a set of options to be passed to diff when2009# submitting each commit to p4.2010#2011if self.detectRenames:2012# command-line -M arg2013 self.diffOpts ="-M"2014else:2015# If not explicitly set check the config variable2016 detectRenames =gitConfig("git-p4.detectRenames")20172018if detectRenames.lower() =="false"or detectRenames =="":2019 self.diffOpts =""2020elif detectRenames.lower() =="true":2021 self.diffOpts ="-M"2022else:2023 self.diffOpts ="-M%s"% detectRenames20242025# no command-line arg for -C or --find-copies-harder, just2026# config variables2027 detectCopies =gitConfig("git-p4.detectCopies")2028if detectCopies.lower() =="false"or detectCopies =="":2029pass2030elif detectCopies.lower() =="true":2031 self.diffOpts +=" -C"2032else:2033 self.diffOpts +=" -C%s"% detectCopies20342035ifgitConfigBool("git-p4.detectCopiesHarder"):2036 self.diffOpts +=" --find-copies-harder"20372038#2039# Apply the commits, one at a time. On failure, ask if should2040# continue to try the rest of the patches, or quit.2041#2042if self.dry_run:2043print"Would apply"2044 applied = []2045 last =len(commits) -12046for i, commit inenumerate(commits):2047if self.dry_run:2048print" ",read_pipe(["git","show","-s",2049"--format=format:%h%s", commit])2050 ok =True2051else:2052 ok = self.applyCommit(commit)2053if ok:2054 applied.append(commit)2055else:2056if self.prepare_p4_only and i < last:2057print"Processing only the first commit due to option" \2058" --prepare-p4-only"2059break2060if i < last:2061 quit =False2062while True:2063# prompt for what to do, or use the option/variable2064if self.conflict_behavior =="ask":2065print"What do you want to do?"2066 response =raw_input("[s]kip this commit but apply"2067" the rest, or [q]uit? ")2068if not response:2069continue2070elif self.conflict_behavior =="skip":2071 response ="s"2072elif self.conflict_behavior =="quit":2073 response ="q"2074else:2075die("Unknown conflict_behavior '%s'"%2076 self.conflict_behavior)20772078if response[0] =="s":2079print"Skipping this commit, but applying the rest"2080break2081if response[0] =="q":2082print"Quitting"2083 quit =True2084break2085if quit:2086break20872088chdir(self.oldWorkingDirectory)20892090if self.dry_run:2091pass2092elif self.prepare_p4_only:2093pass2094eliflen(commits) ==len(applied):2095print"All commits applied!"20962097 sync =P4Sync()2098if self.branch:2099 sync.branch = self.branch2100 sync.run([])21012102 rebase =P4Rebase()2103 rebase.rebase()21042105else:2106iflen(applied) ==0:2107print"No commits applied."2108else:2109print"Applied only the commits marked with '*':"2110for c in commits:2111if c in applied:2112 star ="*"2113else:2114 star =" "2115print star,read_pipe(["git","show","-s",2116"--format=format:%h%s", c])2117print"You will have to do 'git p4 sync' and rebase."21182119ifgitConfigBool("git-p4.exportLabels"):2120 self.exportLabels =True21212122if self.exportLabels:2123 p4Labels =getP4Labels(self.depotPath)2124 gitTags =getGitTags()21252126 missingGitTags = gitTags - p4Labels2127 self.exportGitTags(missingGitTags)21282129# exit with error unless everything applied perfectly2130iflen(commits) !=len(applied):2131 sys.exit(1)21322133return True21342135classView(object):2136"""Represent a p4 view ("p4 help views"), and map files in a2137 repo according to the view."""21382139def__init__(self, client_name):2140 self.mappings = []2141 self.client_prefix ="//%s/"% client_name2142# cache results of "p4 where" to lookup client file locations2143 self.client_spec_path_cache = {}21442145defappend(self, view_line):2146"""Parse a view line, splitting it into depot and client2147 sides. Append to self.mappings, preserving order. This2148 is only needed for tag creation."""21492150# Split the view line into exactly two words. P4 enforces2151# structure on these lines that simplifies this quite a bit.2152#2153# Either or both words may be double-quoted.2154# Single quotes do not matter.2155# Double-quote marks cannot occur inside the words.2156# A + or - prefix is also inside the quotes.2157# There are no quotes unless they contain a space.2158# The line is already white-space stripped.2159# The two words are separated by a single space.2160#2161if view_line[0] =='"':2162# First word is double quoted. Find its end.2163 close_quote_index = view_line.find('"',1)2164if close_quote_index <=0:2165die("No first-word closing quote found:%s"% view_line)2166 depot_side = view_line[1:close_quote_index]2167# skip closing quote and space2168 rhs_index = close_quote_index +1+12169else:2170 space_index = view_line.find(" ")2171if space_index <=0:2172die("No word-splitting space found:%s"% view_line)2173 depot_side = view_line[0:space_index]2174 rhs_index = space_index +121752176# prefix + means overlay on previous mapping2177if depot_side.startswith("+"):2178 depot_side = depot_side[1:]21792180# prefix - means exclude this path, leave out of mappings2181 exclude =False2182if depot_side.startswith("-"):2183 exclude =True2184 depot_side = depot_side[1:]21852186if not exclude:2187 self.mappings.append(depot_side)21882189defconvert_client_path(self, clientFile):2190# chop off //client/ part to make it relative2191if not clientFile.startswith(self.client_prefix):2192die("No prefix '%s' on clientFile '%s'"%2193(self.client_prefix, clientFile))2194return clientFile[len(self.client_prefix):]21952196defupdate_client_spec_path_cache(self, files):2197""" Caching file paths by "p4 where" batch query """21982199# List depot file paths exclude that already cached2200 fileArgs = [f['path']for f in files if f['path']not in self.client_spec_path_cache]22012202iflen(fileArgs) ==0:2203return# All files in cache22042205 where_result =p4CmdList(["-x","-","where"], stdin=fileArgs)2206for res in where_result:2207if"code"in res and res["code"] =="error":2208# assume error is "... file(s) not in client view"2209continue2210if"clientFile"not in res:2211die("No clientFile in 'p4 where' output")2212if"unmap"in res:2213# it will list all of them, but only one not unmap-ped2214continue2215ifgitConfigBool("core.ignorecase"):2216 res['depotFile'] = res['depotFile'].lower()2217 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])22182219# not found files or unmap files set to ""2220for depotFile in fileArgs:2221ifgitConfigBool("core.ignorecase"):2222 depotFile = depotFile.lower()2223if depotFile not in self.client_spec_path_cache:2224 self.client_spec_path_cache[depotFile] =""22252226defmap_in_client(self, depot_path):2227"""Return the relative location in the client where this2228 depot file should live. Returns "" if the file should2229 not be mapped in the client."""22302231ifgitConfigBool("core.ignorecase"):2232 depot_path = depot_path.lower()22332234if depot_path in self.client_spec_path_cache:2235return self.client_spec_path_cache[depot_path]22362237die("Error:%sis not found in client spec path"% depot_path )2238return""22392240classP4Sync(Command, P4UserMap):2241 delete_actions = ("delete","move/delete","purge")22422243def__init__(self):2244 Command.__init__(self)2245 P4UserMap.__init__(self)2246 self.options = [2247 optparse.make_option("--branch", dest="branch"),2248 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),2249 optparse.make_option("--changesfile", dest="changesFile"),2250 optparse.make_option("--silent", dest="silent", action="store_true"),2251 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),2252 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),2253 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",2254help="Import into refs/heads/ , not refs/remotes"),2255 optparse.make_option("--max-changes", dest="maxChanges",2256help="Maximum number of changes to import"),2257 optparse.make_option("--changes-block-size", dest="changes_block_size",type="int",2258help="Internal block size to use when iteratively calling p4 changes"),2259 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',2260help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),2261 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',2262help="Only sync files that are included in the Perforce Client Spec"),2263 optparse.make_option("-/", dest="cloneExclude",2264 action="append",type="string",2265help="exclude depot path"),2266]2267 self.description ="""Imports from Perforce into a git repository.\n2268 example:2269 //depot/my/project/ -- to import the current head2270 //depot/my/project/@all -- to import everything2271 //depot/my/project/@1,6 -- to import only from revision 1 to 622722273 (a ... is not needed in the path p4 specification, it's added implicitly)"""22742275 self.usage +=" //depot/path[@revRange]"2276 self.silent =False2277 self.createdBranches =set()2278 self.committedChanges =set()2279 self.branch =""2280 self.detectBranches =False2281 self.detectLabels =False2282 self.importLabels =False2283 self.changesFile =""2284 self.syncWithOrigin =True2285 self.importIntoRemotes =True2286 self.maxChanges =""2287 self.changes_block_size =None2288 self.keepRepoPath =False2289 self.depotPaths =None2290 self.p4BranchesInGit = []2291 self.cloneExclude = []2292 self.useClientSpec =False2293 self.useClientSpec_from_options =False2294 self.clientSpecDirs =None2295 self.tempBranches = []2296 self.tempBranchLocation ="refs/git-p4-tmp"2297 self.largeFileSystem =None22982299ifgitConfig('git-p4.largeFileSystem'):2300 largeFileSystemConstructor =globals()[gitConfig('git-p4.largeFileSystem')]2301 self.largeFileSystem =largeFileSystemConstructor(2302lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)2303)23042305ifgitConfig("git-p4.syncFromOrigin") =="false":2306 self.syncWithOrigin =False23072308# This is required for the "append" cloneExclude action2309defensure_value(self, attr, value):2310if nothasattr(self, attr)orgetattr(self, attr)is None:2311setattr(self, attr, value)2312returngetattr(self, attr)23132314# Force a checkpoint in fast-import and wait for it to finish2315defcheckpoint(self):2316 self.gitStream.write("checkpoint\n\n")2317 self.gitStream.write("progress checkpoint\n\n")2318 out = self.gitOutput.readline()2319if self.verbose:2320print"checkpoint finished: "+ out23212322defextractFilesFromCommit(self, commit):2323 self.cloneExclude = [re.sub(r"\.\.\.$","", path)2324for path in self.cloneExclude]2325 files = []2326 fnum =02327while commit.has_key("depotFile%s"% fnum):2328 path = commit["depotFile%s"% fnum]23292330if[p for p in self.cloneExclude2331ifp4PathStartsWith(path, p)]:2332 found =False2333else:2334 found = [p for p in self.depotPaths2335ifp4PathStartsWith(path, p)]2336if not found:2337 fnum = fnum +12338continue23392340file= {}2341file["path"] = path2342file["rev"] = commit["rev%s"% fnum]2343file["action"] = commit["action%s"% fnum]2344file["type"] = commit["type%s"% fnum]2345 files.append(file)2346 fnum = fnum +12347return files23482349defextractJobsFromCommit(self, commit):2350 jobs = []2351 jnum =02352while commit.has_key("job%s"% jnum):2353 job = commit["job%s"% jnum]2354 jobs.append(job)2355 jnum = jnum +12356return jobs23572358defstripRepoPath(self, path, prefixes):2359"""When streaming files, this is called to map a p4 depot path2360 to where it should go in git. The prefixes are either2361 self.depotPaths, or self.branchPrefixes in the case of2362 branch detection."""23632364if self.useClientSpec:2365# branch detection moves files up a level (the branch name)2366# from what client spec interpretation gives2367 path = self.clientSpecDirs.map_in_client(path)2368if self.detectBranches:2369for b in self.knownBranches:2370if path.startswith(b +"/"):2371 path = path[len(b)+1:]23722373elif self.keepRepoPath:2374# Preserve everything in relative path name except leading2375# //depot/; just look at first prefix as they all should2376# be in the same depot.2377 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])2378ifp4PathStartsWith(path, depot):2379 path = path[len(depot):]23802381else:2382for p in prefixes:2383ifp4PathStartsWith(path, p):2384 path = path[len(p):]2385break23862387 path =wildcard_decode(path)2388return path23892390defsplitFilesIntoBranches(self, commit):2391"""Look at each depotFile in the commit to figure out to what2392 branch it belongs."""23932394if self.clientSpecDirs:2395 files = self.extractFilesFromCommit(commit)2396 self.clientSpecDirs.update_client_spec_path_cache(files)23972398 branches = {}2399 fnum =02400while commit.has_key("depotFile%s"% fnum):2401 path = commit["depotFile%s"% fnum]2402 found = [p for p in self.depotPaths2403ifp4PathStartsWith(path, p)]2404if not found:2405 fnum = fnum +12406continue24072408file= {}2409file["path"] = path2410file["rev"] = commit["rev%s"% fnum]2411file["action"] = commit["action%s"% fnum]2412file["type"] = commit["type%s"% fnum]2413 fnum = fnum +124142415# start with the full relative path where this file would2416# go in a p4 client2417if self.useClientSpec:2418 relPath = self.clientSpecDirs.map_in_client(path)2419else:2420 relPath = self.stripRepoPath(path, self.depotPaths)24212422for branch in self.knownBranches.keys():2423# add a trailing slash so that a commit into qt/4.2foo2424# doesn't end up in qt/4.2, e.g.2425if relPath.startswith(branch +"/"):2426if branch not in branches:2427 branches[branch] = []2428 branches[branch].append(file)2429break24302431return branches24322433defwriteToGitStream(self, gitMode, relPath, contents):2434 self.gitStream.write('M%sinline%s\n'% (gitMode, relPath))2435 self.gitStream.write('data%d\n'%sum(len(d)for d in contents))2436for d in contents:2437 self.gitStream.write(d)2438 self.gitStream.write('\n')24392440# output one file from the P4 stream2441# - helper for streamP4Files24422443defstreamOneP4File(self,file, contents):2444 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)2445if verbose:2446 size =int(self.stream_file['fileSize'])2447 sys.stdout.write('\r%s-->%s(%iMB)\n'% (file['depotFile'], relPath, size/1024/1024))2448 sys.stdout.flush()24492450(type_base, type_mods) =split_p4_type(file["type"])24512452 git_mode ="100644"2453if"x"in type_mods:2454 git_mode ="100755"2455if type_base =="symlink":2456 git_mode ="120000"2457# p4 print on a symlink sometimes contains "target\n";2458# if it does, remove the newline2459 data =''.join(contents)2460if not data:2461# Some version of p4 allowed creating a symlink that pointed2462# to nothing. This causes p4 errors when checking out such2463# a change, and errors here too. Work around it by ignoring2464# the bad symlink; hopefully a future change fixes it.2465print"\nIgnoring empty symlink in%s"%file['depotFile']2466return2467elif data[-1] =='\n':2468 contents = [data[:-1]]2469else:2470 contents = [data]24712472if type_base =="utf16":2473# p4 delivers different text in the python output to -G2474# than it does when using "print -o", or normal p4 client2475# operations. utf16 is converted to ascii or utf8, perhaps.2476# But ascii text saved as -t utf16 is completely mangled.2477# Invoke print -o to get the real contents.2478#2479# On windows, the newlines will always be mangled by print, so put2480# them back too. This is not needed to the cygwin windows version,2481# just the native "NT" type.2482#2483try:2484 text =p4_read_pipe(['print','-q','-o','-','%s@%s'% (file['depotFile'],file['change'])])2485exceptExceptionas e:2486if'Translation of file content failed'instr(e):2487 type_base ='binary'2488else:2489raise e2490else:2491ifp4_version_string().find('/NT') >=0:2492 text = text.replace('\r\n','\n')2493 contents = [ text ]24942495if type_base =="apple":2496# Apple filetype files will be streamed as a concatenation of2497# its appledouble header and the contents. This is useless2498# on both macs and non-macs. If using "print -q -o xx", it2499# will create "xx" with the data, and "%xx" with the header.2500# This is also not very useful.2501#2502# Ideally, someday, this script can learn how to generate2503# appledouble files directly and import those to git, but2504# non-mac machines can never find a use for apple filetype.2505print"\nIgnoring apple filetype file%s"%file['depotFile']2506return25072508# Note that we do not try to de-mangle keywords on utf16 files,2509# even though in theory somebody may want that.2510 pattern =p4_keywords_regexp_for_type(type_base, type_mods)2511if pattern:2512 regexp = re.compile(pattern, re.VERBOSE)2513 text =''.join(contents)2514 text = regexp.sub(r'$\1$', text)2515 contents = [ text ]25162517try:2518 relPath.decode('ascii')2519except:2520 encoding ='utf8'2521ifgitConfig('git-p4.pathEncoding'):2522 encoding =gitConfig('git-p4.pathEncoding')2523 relPath = relPath.decode(encoding,'replace').encode('utf8','replace')2524if self.verbose:2525print'Path with non-ASCII characters detected. Used%sto encode:%s'% (encoding, relPath)25262527if self.largeFileSystem:2528(git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)25292530 self.writeToGitStream(git_mode, relPath, contents)25312532defstreamOneP4Deletion(self,file):2533 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)2534if verbose:2535 sys.stdout.write("delete%s\n"% relPath)2536 sys.stdout.flush()2537 self.gitStream.write("D%s\n"% relPath)25382539if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):2540 self.largeFileSystem.removeLargeFile(relPath)25412542# handle another chunk of streaming data2543defstreamP4FilesCb(self, marshalled):25442545# catch p4 errors and complain2546 err =None2547if"code"in marshalled:2548if marshalled["code"] =="error":2549if"data"in marshalled:2550 err = marshalled["data"].rstrip()25512552if not err and'fileSize'in self.stream_file:2553 required_bytes =int((4*int(self.stream_file["fileSize"])) -calcDiskFree())2554if required_bytes >0:2555 err ='Not enough space left on%s! Free at least%iMB.'% (2556 os.getcwd(), required_bytes/1024/10242557)25582559if err:2560 f =None2561if self.stream_have_file_info:2562if"depotFile"in self.stream_file:2563 f = self.stream_file["depotFile"]2564# force a failure in fast-import, else an empty2565# commit will be made2566 self.gitStream.write("\n")2567 self.gitStream.write("die-now\n")2568 self.gitStream.close()2569# ignore errors, but make sure it exits first2570 self.importProcess.wait()2571if f:2572die("Error from p4 print for%s:%s"% (f, err))2573else:2574die("Error from p4 print:%s"% err)25752576if marshalled.has_key('depotFile')and self.stream_have_file_info:2577# start of a new file - output the old one first2578 self.streamOneP4File(self.stream_file, self.stream_contents)2579 self.stream_file = {}2580 self.stream_contents = []2581 self.stream_have_file_info =False25822583# pick up the new file information... for the2584# 'data' field we need to append to our array2585for k in marshalled.keys():2586if k =='data':2587if'streamContentSize'not in self.stream_file:2588 self.stream_file['streamContentSize'] =02589 self.stream_file['streamContentSize'] +=len(marshalled['data'])2590 self.stream_contents.append(marshalled['data'])2591else:2592 self.stream_file[k] = marshalled[k]25932594if(verbose and2595'streamContentSize'in self.stream_file and2596'fileSize'in self.stream_file and2597'depotFile'in self.stream_file):2598 size =int(self.stream_file["fileSize"])2599if size >0:2600 progress =100*self.stream_file['streamContentSize']/size2601 sys.stdout.write('\r%s %d%%(%iMB)'% (self.stream_file['depotFile'], progress,int(size/1024/1024)))2602 sys.stdout.flush()26032604 self.stream_have_file_info =True26052606# Stream directly from "p4 files" into "git fast-import"2607defstreamP4Files(self, files):2608 filesForCommit = []2609 filesToRead = []2610 filesToDelete = []26112612for f in files:2613 filesForCommit.append(f)2614if f['action']in self.delete_actions:2615 filesToDelete.append(f)2616else:2617 filesToRead.append(f)26182619# deleted files...2620for f in filesToDelete:2621 self.streamOneP4Deletion(f)26222623iflen(filesToRead) >0:2624 self.stream_file = {}2625 self.stream_contents = []2626 self.stream_have_file_info =False26272628# curry self argument2629defstreamP4FilesCbSelf(entry):2630 self.streamP4FilesCb(entry)26312632 fileArgs = ['%s#%s'% (f['path'], f['rev'])for f in filesToRead]26332634p4CmdList(["-x","-","print"],2635 stdin=fileArgs,2636 cb=streamP4FilesCbSelf)26372638# do the last chunk2639if self.stream_file.has_key('depotFile'):2640 self.streamOneP4File(self.stream_file, self.stream_contents)26412642defmake_email(self, userid):2643if userid in self.users:2644return self.users[userid]2645else:2646return"%s<a@b>"% userid26472648defstreamTag(self, gitStream, labelName, labelDetails, commit, epoch):2649""" Stream a p4 tag.2650 commit is either a git commit, or a fast-import mark, ":<p4commit>"2651 """26522653if verbose:2654print"writing tag%sfor commit%s"% (labelName, commit)2655 gitStream.write("tag%s\n"% labelName)2656 gitStream.write("from%s\n"% commit)26572658if labelDetails.has_key('Owner'):2659 owner = labelDetails["Owner"]2660else:2661 owner =None26622663# Try to use the owner of the p4 label, or failing that,2664# the current p4 user id.2665if owner:2666 email = self.make_email(owner)2667else:2668 email = self.make_email(self.p4UserId())2669 tagger ="%s %s %s"% (email, epoch, self.tz)26702671 gitStream.write("tagger%s\n"% tagger)26722673print"labelDetails=",labelDetails2674if labelDetails.has_key('Description'):2675 description = labelDetails['Description']2676else:2677 description ='Label from git p4'26782679 gitStream.write("data%d\n"%len(description))2680 gitStream.write(description)2681 gitStream.write("\n")26822683definClientSpec(self, path):2684if not self.clientSpecDirs:2685return True2686 inClientSpec = self.clientSpecDirs.map_in_client(path)2687if not inClientSpec and self.verbose:2688print('Ignoring file outside of client spec:{0}'.format(path))2689return inClientSpec26902691defhasBranchPrefix(self, path):2692if not self.branchPrefixes:2693return True2694 hasPrefix = [p for p in self.branchPrefixes2695ifp4PathStartsWith(path, p)]2696if not hasPrefix and self.verbose:2697print('Ignoring file outside of prefix:{0}'.format(path))2698return hasPrefix26992700defcommit(self, details, files, branch, parent =""):2701 epoch = details["time"]2702 author = details["user"]2703 jobs = self.extractJobsFromCommit(details)27042705if self.verbose:2706print('commit into{0}'.format(branch))27072708if self.clientSpecDirs:2709 self.clientSpecDirs.update_client_spec_path_cache(files)27102711 files = [f for f in files2712if self.inClientSpec(f['path'])and self.hasBranchPrefix(f['path'])]27132714if not files and notgitConfigBool('git-p4.keepEmptyCommits'):2715print('Ignoring revision{0}as it would produce an empty commit.'2716.format(details['change']))2717return27182719 self.gitStream.write("commit%s\n"% branch)2720 self.gitStream.write("mark :%s\n"% details["change"])2721 self.committedChanges.add(int(details["change"]))2722 committer =""2723if author not in self.users:2724 self.getUserMapFromPerforceServer()2725 committer ="%s %s %s"% (self.make_email(author), epoch, self.tz)27262727 self.gitStream.write("committer%s\n"% committer)27282729 self.gitStream.write("data <<EOT\n")2730 self.gitStream.write(details["desc"])2731iflen(jobs) >0:2732 self.gitStream.write("\nJobs:%s"% (' '.join(jobs)))2733 self.gitStream.write("\n[git-p4: depot-paths =\"%s\": change =%s"%2734(','.join(self.branchPrefixes), details["change"]))2735iflen(details['options']) >0:2736 self.gitStream.write(": options =%s"% details['options'])2737 self.gitStream.write("]\nEOT\n\n")27382739iflen(parent) >0:2740if self.verbose:2741print"parent%s"% parent2742 self.gitStream.write("from%s\n"% parent)27432744 self.streamP4Files(files)2745 self.gitStream.write("\n")27462747 change =int(details["change"])27482749if self.labels.has_key(change):2750 label = self.labels[change]2751 labelDetails = label[0]2752 labelRevisions = label[1]2753if self.verbose:2754print"Change%sis labelled%s"% (change, labelDetails)27552756 files =p4CmdList(["files"] + ["%s...@%s"% (p, change)2757for p in self.branchPrefixes])27582759iflen(files) ==len(labelRevisions):27602761 cleanedFiles = {}2762for info in files:2763if info["action"]in self.delete_actions:2764continue2765 cleanedFiles[info["depotFile"]] = info["rev"]27662767if cleanedFiles == labelRevisions:2768 self.streamTag(self.gitStream,'tag_%s'% labelDetails['label'], labelDetails, branch, epoch)27692770else:2771if not self.silent:2772print("Tag%sdoes not match with change%s: files do not match."2773% (labelDetails["label"], change))27742775else:2776if not self.silent:2777print("Tag%sdoes not match with change%s: file count is different."2778% (labelDetails["label"], change))27792780# Build a dictionary of changelists and labels, for "detect-labels" option.2781defgetLabels(self):2782 self.labels = {}27832784 l =p4CmdList(["labels"] + ["%s..."% p for p in self.depotPaths])2785iflen(l) >0and not self.silent:2786print"Finding files belonging to labels in%s"% `self.depotPaths`27872788for output in l:2789 label = output["label"]2790 revisions = {}2791 newestChange =02792if self.verbose:2793print"Querying files for label%s"% label2794forfileinp4CmdList(["files"] +2795["%s...@%s"% (p, label)2796for p in self.depotPaths]):2797 revisions[file["depotFile"]] =file["rev"]2798 change =int(file["change"])2799if change > newestChange:2800 newestChange = change28012802 self.labels[newestChange] = [output, revisions]28032804if self.verbose:2805print"Label changes:%s"% self.labels.keys()28062807# Import p4 labels as git tags. A direct mapping does not2808# exist, so assume that if all the files are at the same revision2809# then we can use that, or it's something more complicated we should2810# just ignore.2811defimportP4Labels(self, stream, p4Labels):2812if verbose:2813print"import p4 labels: "+' '.join(p4Labels)28142815 ignoredP4Labels =gitConfigList("git-p4.ignoredP4Labels")2816 validLabelRegexp =gitConfig("git-p4.labelImportRegexp")2817iflen(validLabelRegexp) ==0:2818 validLabelRegexp = defaultLabelRegexp2819 m = re.compile(validLabelRegexp)28202821for name in p4Labels:2822 commitFound =False28232824if not m.match(name):2825if verbose:2826print"label%sdoes not match regexp%s"% (name,validLabelRegexp)2827continue28282829if name in ignoredP4Labels:2830continue28312832 labelDetails =p4CmdList(['label',"-o", name])[0]28332834# get the most recent changelist for each file in this label2835 change =p4Cmd(["changes","-m","1"] + ["%s...@%s"% (p, name)2836for p in self.depotPaths])28372838if change.has_key('change'):2839# find the corresponding git commit; take the oldest commit2840 changelist =int(change['change'])2841if changelist in self.committedChanges:2842 gitCommit =":%d"% changelist # use a fast-import mark2843 commitFound =True2844else:2845 gitCommit =read_pipe(["git","rev-list","--max-count=1",2846"--reverse",":/\[git-p4:.*change =%d\]"% changelist], ignore_error=True)2847iflen(gitCommit) ==0:2848print"importing label%s: could not find git commit for changelist%d"% (name, changelist)2849else:2850 commitFound =True2851 gitCommit = gitCommit.strip()28522853if commitFound:2854# Convert from p4 time format2855try:2856 tmwhen = time.strptime(labelDetails['Update'],"%Y/%m/%d%H:%M:%S")2857exceptValueError:2858print"Could not convert label time%s"% labelDetails['Update']2859 tmwhen =128602861 when =int(time.mktime(tmwhen))2862 self.streamTag(stream, name, labelDetails, gitCommit, when)2863if verbose:2864print"p4 label%smapped to git commit%s"% (name, gitCommit)2865else:2866if verbose:2867print"Label%shas no changelists - possibly deleted?"% name28682869if not commitFound:2870# We can't import this label; don't try again as it will get very2871# expensive repeatedly fetching all the files for labels that will2872# never be imported. If the label is moved in the future, the2873# ignore will need to be removed manually.2874system(["git","config","--add","git-p4.ignoredP4Labels", name])28752876defguessProjectName(self):2877for p in self.depotPaths:2878if p.endswith("/"):2879 p = p[:-1]2880 p = p[p.strip().rfind("/") +1:]2881if not p.endswith("/"):2882 p +="/"2883return p28842885defgetBranchMapping(self):2886 lostAndFoundBranches =set()28872888 user =gitConfig("git-p4.branchUser")2889iflen(user) >0:2890 command ="branches -u%s"% user2891else:2892 command ="branches"28932894for info inp4CmdList(command):2895 details =p4Cmd(["branch","-o", info["branch"]])2896 viewIdx =02897while details.has_key("View%s"% viewIdx):2898 paths = details["View%s"% viewIdx].split(" ")2899 viewIdx = viewIdx +12900# require standard //depot/foo/... //depot/bar/... mapping2901iflen(paths) !=2or not paths[0].endswith("/...")or not paths[1].endswith("/..."):2902continue2903 source = paths[0]2904 destination = paths[1]2905## HACK2906ifp4PathStartsWith(source, self.depotPaths[0])andp4PathStartsWith(destination, self.depotPaths[0]):2907 source = source[len(self.depotPaths[0]):-4]2908 destination = destination[len(self.depotPaths[0]):-4]29092910if destination in self.knownBranches:2911if not self.silent:2912print"p4 branch%sdefines a mapping from%sto%s"% (info["branch"], source, destination)2913print"but there exists another mapping from%sto%salready!"% (self.knownBranches[destination], destination)2914continue29152916 self.knownBranches[destination] = source29172918 lostAndFoundBranches.discard(destination)29192920if source not in self.knownBranches:2921 lostAndFoundBranches.add(source)29222923# Perforce does not strictly require branches to be defined, so we also2924# check git config for a branch list.2925#2926# Example of branch definition in git config file:2927# [git-p4]2928# branchList=main:branchA2929# branchList=main:branchB2930# branchList=branchA:branchC2931 configBranches =gitConfigList("git-p4.branchList")2932for branch in configBranches:2933if branch:2934(source, destination) = branch.split(":")2935 self.knownBranches[destination] = source29362937 lostAndFoundBranches.discard(destination)29382939if source not in self.knownBranches:2940 lostAndFoundBranches.add(source)294129422943for branch in lostAndFoundBranches:2944 self.knownBranches[branch] = branch29452946defgetBranchMappingFromGitBranches(self):2947 branches =p4BranchesInGit(self.importIntoRemotes)2948for branch in branches.keys():2949if branch =="master":2950 branch ="main"2951else:2952 branch = branch[len(self.projectName):]2953 self.knownBranches[branch] = branch29542955defupdateOptionDict(self, d):2956 option_keys = {}2957if self.keepRepoPath:2958 option_keys['keepRepoPath'] =129592960 d["options"] =' '.join(sorted(option_keys.keys()))29612962defreadOptions(self, d):2963 self.keepRepoPath = (d.has_key('options')2964and('keepRepoPath'in d['options']))29652966defgitRefForBranch(self, branch):2967if branch =="main":2968return self.refPrefix +"master"29692970iflen(branch) <=0:2971return branch29722973return self.refPrefix + self.projectName + branch29742975defgitCommitByP4Change(self, ref, change):2976if self.verbose:2977print"looking in ref "+ ref +" for change%susing bisect..."% change29782979 earliestCommit =""2980 latestCommit =parseRevision(ref)29812982while True:2983if self.verbose:2984print"trying: earliest%slatest%s"% (earliestCommit, latestCommit)2985 next =read_pipe("git rev-list --bisect%s %s"% (latestCommit, earliestCommit)).strip()2986iflen(next) ==0:2987if self.verbose:2988print"argh"2989return""2990 log =extractLogMessageFromGitCommit(next)2991 settings =extractSettingsGitLog(log)2992 currentChange =int(settings['change'])2993if self.verbose:2994print"current change%s"% currentChange29952996if currentChange == change:2997if self.verbose:2998print"found%s"% next2999return next30003001if currentChange < change:3002 earliestCommit ="^%s"% next3003else:3004 latestCommit ="%s"% next30053006return""30073008defimportNewBranch(self, branch, maxChange):3009# make fast-import flush all changes to disk and update the refs using the checkpoint3010# command so that we can try to find the branch parent in the git history3011 self.gitStream.write("checkpoint\n\n");3012 self.gitStream.flush();3013 branchPrefix = self.depotPaths[0] + branch +"/"3014range="@1,%s"% maxChange3015#print "prefix" + branchPrefix3016 changes =p4ChangesForPaths([branchPrefix],range, self.changes_block_size)3017iflen(changes) <=0:3018return False3019 firstChange = changes[0]3020#print "first change in branch: %s" % firstChange3021 sourceBranch = self.knownBranches[branch]3022 sourceDepotPath = self.depotPaths[0] + sourceBranch3023 sourceRef = self.gitRefForBranch(sourceBranch)3024#print "source " + sourceBranch30253026 branchParentChange =int(p4Cmd(["changes","-m","1","%s...@1,%s"% (sourceDepotPath, firstChange)])["change"])3027#print "branch parent: %s" % branchParentChange3028 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)3029iflen(gitParent) >0:3030 self.initialParents[self.gitRefForBranch(branch)] = gitParent3031#print "parent git commit: %s" % gitParent30323033 self.importChanges(changes)3034return True30353036defsearchParent(self, parent, branch, target):3037 parentFound =False3038for blob inread_pipe_lines(["git","rev-list","--reverse",3039"--no-merges", parent]):3040 blob = blob.strip()3041iflen(read_pipe(["git","diff-tree", blob, target])) ==0:3042 parentFound =True3043if self.verbose:3044print"Found parent of%sin commit%s"% (branch, blob)3045break3046if parentFound:3047return blob3048else:3049return None30503051defimportChanges(self, changes):3052 cnt =13053for change in changes:3054 description =p4_describe(change)3055 self.updateOptionDict(description)30563057if not self.silent:3058 sys.stdout.write("\rImporting revision%s(%s%%)"% (change, cnt *100/len(changes)))3059 sys.stdout.flush()3060 cnt = cnt +130613062try:3063if self.detectBranches:3064 branches = self.splitFilesIntoBranches(description)3065for branch in branches.keys():3066## HACK --hwn3067 branchPrefix = self.depotPaths[0] + branch +"/"3068 self.branchPrefixes = [ branchPrefix ]30693070 parent =""30713072 filesForCommit = branches[branch]30733074if self.verbose:3075print"branch is%s"% branch30763077 self.updatedBranches.add(branch)30783079if branch not in self.createdBranches:3080 self.createdBranches.add(branch)3081 parent = self.knownBranches[branch]3082if parent == branch:3083 parent =""3084else:3085 fullBranch = self.projectName + branch3086if fullBranch not in self.p4BranchesInGit:3087if not self.silent:3088print("\nImporting new branch%s"% fullBranch);3089if self.importNewBranch(branch, change -1):3090 parent =""3091 self.p4BranchesInGit.append(fullBranch)3092if not self.silent:3093print("\nResuming with change%s"% change);30943095if self.verbose:3096print"parent determined through known branches:%s"% parent30973098 branch = self.gitRefForBranch(branch)3099 parent = self.gitRefForBranch(parent)31003101if self.verbose:3102print"looking for initial parent for%s; current parent is%s"% (branch, parent)31033104iflen(parent) ==0and branch in self.initialParents:3105 parent = self.initialParents[branch]3106del self.initialParents[branch]31073108 blob =None3109iflen(parent) >0:3110 tempBranch ="%s/%d"% (self.tempBranchLocation, change)3111if self.verbose:3112print"Creating temporary branch: "+ tempBranch3113 self.commit(description, filesForCommit, tempBranch)3114 self.tempBranches.append(tempBranch)3115 self.checkpoint()3116 blob = self.searchParent(parent, branch, tempBranch)3117if blob:3118 self.commit(description, filesForCommit, branch, blob)3119else:3120if self.verbose:3121print"Parent of%snot found. Committing into head of%s"% (branch, parent)3122 self.commit(description, filesForCommit, branch, parent)3123else:3124 files = self.extractFilesFromCommit(description)3125 self.commit(description, files, self.branch,3126 self.initialParent)3127# only needed once, to connect to the previous commit3128 self.initialParent =""3129exceptIOError:3130print self.gitError.read()3131 sys.exit(1)31323133defimportHeadRevision(self, revision):3134print"Doing initial import of%sfrom revision%sinto%s"% (' '.join(self.depotPaths), revision, self.branch)31353136 details = {}3137 details["user"] ="git perforce import user"3138 details["desc"] = ("Initial import of%sfrom the state at revision%s\n"3139% (' '.join(self.depotPaths), revision))3140 details["change"] = revision3141 newestRevision =031423143 fileCnt =03144 fileArgs = ["%s...%s"% (p,revision)for p in self.depotPaths]31453146for info inp4CmdList(["files"] + fileArgs):31473148if'code'in info and info['code'] =='error':3149 sys.stderr.write("p4 returned an error:%s\n"3150% info['data'])3151if info['data'].find("must refer to client") >=0:3152 sys.stderr.write("This particular p4 error is misleading.\n")3153 sys.stderr.write("Perhaps the depot path was misspelled.\n");3154 sys.stderr.write("Depot path:%s\n"%" ".join(self.depotPaths))3155 sys.exit(1)3156if'p4ExitCode'in info:3157 sys.stderr.write("p4 exitcode:%s\n"% info['p4ExitCode'])3158 sys.exit(1)315931603161 change =int(info["change"])3162if change > newestRevision:3163 newestRevision = change31643165if info["action"]in self.delete_actions:3166# don't increase the file cnt, otherwise details["depotFile123"] will have gaps!3167#fileCnt = fileCnt + 13168continue31693170for prop in["depotFile","rev","action","type"]:3171 details["%s%s"% (prop, fileCnt)] = info[prop]31723173 fileCnt = fileCnt +131743175 details["change"] = newestRevision31763177# Use time from top-most change so that all git p4 clones of3178# the same p4 repo have the same commit SHA1s.3179 res =p4_describe(newestRevision)3180 details["time"] = res["time"]31813182 self.updateOptionDict(details)3183try:3184 self.commit(details, self.extractFilesFromCommit(details), self.branch)3185exceptIOError:3186print"IO error with git fast-import. Is your git version recent enough?"3187print self.gitError.read()318831893190defrun(self, args):3191 self.depotPaths = []3192 self.changeRange =""3193 self.previousDepotPaths = []3194 self.hasOrigin =False31953196# map from branch depot path to parent branch3197 self.knownBranches = {}3198 self.initialParents = {}31993200if self.importIntoRemotes:3201 self.refPrefix ="refs/remotes/p4/"3202else:3203 self.refPrefix ="refs/heads/p4/"32043205if self.syncWithOrigin:3206 self.hasOrigin =originP4BranchesExist()3207if self.hasOrigin:3208if not self.silent:3209print'Syncing with origin first, using "git fetch origin"'3210system("git fetch origin")32113212 branch_arg_given =bool(self.branch)3213iflen(self.branch) ==0:3214 self.branch = self.refPrefix +"master"3215ifgitBranchExists("refs/heads/p4")and self.importIntoRemotes:3216system("git update-ref%srefs/heads/p4"% self.branch)3217system("git branch -D p4")32183219# accept either the command-line option, or the configuration variable3220if self.useClientSpec:3221# will use this after clone to set the variable3222 self.useClientSpec_from_options =True3223else:3224ifgitConfigBool("git-p4.useclientspec"):3225 self.useClientSpec =True3226if self.useClientSpec:3227 self.clientSpecDirs =getClientSpec()32283229# TODO: should always look at previous commits,3230# merge with previous imports, if possible.3231if args == []:3232if self.hasOrigin:3233createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)32343235# branches holds mapping from branch name to sha13236 branches =p4BranchesInGit(self.importIntoRemotes)32373238# restrict to just this one, disabling detect-branches3239if branch_arg_given:3240 short = self.branch.split("/")[-1]3241if short in branches:3242 self.p4BranchesInGit = [ short ]3243else:3244 self.p4BranchesInGit = branches.keys()32453246iflen(self.p4BranchesInGit) >1:3247if not self.silent:3248print"Importing from/into multiple branches"3249 self.detectBranches =True3250for branch in branches.keys():3251 self.initialParents[self.refPrefix + branch] = \3252 branches[branch]32533254if self.verbose:3255print"branches:%s"% self.p4BranchesInGit32563257 p4Change =03258for branch in self.p4BranchesInGit:3259 logMsg =extractLogMessageFromGitCommit(self.refPrefix + branch)32603261 settings =extractSettingsGitLog(logMsg)32623263 self.readOptions(settings)3264if(settings.has_key('depot-paths')3265and settings.has_key('change')):3266 change =int(settings['change']) +13267 p4Change =max(p4Change, change)32683269 depotPaths =sorted(settings['depot-paths'])3270if self.previousDepotPaths == []:3271 self.previousDepotPaths = depotPaths3272else:3273 paths = []3274for(prev, cur)inzip(self.previousDepotPaths, depotPaths):3275 prev_list = prev.split("/")3276 cur_list = cur.split("/")3277for i inrange(0,min(len(cur_list),len(prev_list))):3278if cur_list[i] <> prev_list[i]:3279 i = i -13280break32813282 paths.append("/".join(cur_list[:i +1]))32833284 self.previousDepotPaths = paths32853286if p4Change >0:3287 self.depotPaths =sorted(self.previousDepotPaths)3288 self.changeRange ="@%s,#head"% p4Change3289if not self.silent and not self.detectBranches:3290print"Performing incremental import into%sgit branch"% self.branch32913292# accept multiple ref name abbreviations:3293# refs/foo/bar/branch -> use it exactly3294# p4/branch -> prepend refs/remotes/ or refs/heads/3295# branch -> prepend refs/remotes/p4/ or refs/heads/p4/3296if not self.branch.startswith("refs/"):3297if self.importIntoRemotes:3298 prepend ="refs/remotes/"3299else:3300 prepend ="refs/heads/"3301if not self.branch.startswith("p4/"):3302 prepend +="p4/"3303 self.branch = prepend + self.branch33043305iflen(args) ==0and self.depotPaths:3306if not self.silent:3307print"Depot paths:%s"%' '.join(self.depotPaths)3308else:3309if self.depotPaths and self.depotPaths != args:3310print("previous import used depot path%sand now%swas specified. "3311"This doesn't work!"% (' '.join(self.depotPaths),3312' '.join(args)))3313 sys.exit(1)33143315 self.depotPaths =sorted(args)33163317 revision =""3318 self.users = {}33193320# Make sure no revision specifiers are used when --changesfile3321# is specified.3322 bad_changesfile =False3323iflen(self.changesFile) >0:3324for p in self.depotPaths:3325if p.find("@") >=0or p.find("#") >=0:3326 bad_changesfile =True3327break3328if bad_changesfile:3329die("Option --changesfile is incompatible with revision specifiers")33303331 newPaths = []3332for p in self.depotPaths:3333if p.find("@") != -1:3334 atIdx = p.index("@")3335 self.changeRange = p[atIdx:]3336if self.changeRange =="@all":3337 self.changeRange =""3338elif','not in self.changeRange:3339 revision = self.changeRange3340 self.changeRange =""3341 p = p[:atIdx]3342elif p.find("#") != -1:3343 hashIdx = p.index("#")3344 revision = p[hashIdx:]3345 p = p[:hashIdx]3346elif self.previousDepotPaths == []:3347# pay attention to changesfile, if given, else import3348# the entire p4 tree at the head revision3349iflen(self.changesFile) ==0:3350 revision ="#head"33513352 p = re.sub("\.\.\.$","", p)3353if not p.endswith("/"):3354 p +="/"33553356 newPaths.append(p)33573358 self.depotPaths = newPaths33593360# --detect-branches may change this for each branch3361 self.branchPrefixes = self.depotPaths33623363 self.loadUserMapFromCache()3364 self.labels = {}3365if self.detectLabels:3366 self.getLabels();33673368if self.detectBranches:3369## FIXME - what's a P4 projectName ?3370 self.projectName = self.guessProjectName()33713372if self.hasOrigin:3373 self.getBranchMappingFromGitBranches()3374else:3375 self.getBranchMapping()3376if self.verbose:3377print"p4-git branches:%s"% self.p4BranchesInGit3378print"initial parents:%s"% self.initialParents3379for b in self.p4BranchesInGit:3380if b !="master":33813382## FIXME3383 b = b[len(self.projectName):]3384 self.createdBranches.add(b)33853386 self.tz ="%+03d%02d"% (- time.timezone /3600, ((- time.timezone %3600) /60))33873388 self.importProcess = subprocess.Popen(["git","fast-import"],3389 stdin=subprocess.PIPE,3390 stdout=subprocess.PIPE,3391 stderr=subprocess.PIPE);3392 self.gitOutput = self.importProcess.stdout3393 self.gitStream = self.importProcess.stdin3394 self.gitError = self.importProcess.stderr33953396if revision:3397 self.importHeadRevision(revision)3398else:3399 changes = []34003401iflen(self.changesFile) >0:3402 output =open(self.changesFile).readlines()3403 changeSet =set()3404for line in output:3405 changeSet.add(int(line))34063407for change in changeSet:3408 changes.append(change)34093410 changes.sort()3411else:3412# catch "git p4 sync" with no new branches, in a repo that3413# does not have any existing p4 branches3414iflen(args) ==0:3415if not self.p4BranchesInGit:3416die("No remote p4 branches. Perhaps you never did\"git p4 clone\"in here.")34173418# The default branch is master, unless --branch is used to3419# specify something else. Make sure it exists, or complain3420# nicely about how to use --branch.3421if not self.detectBranches:3422if notbranch_exists(self.branch):3423if branch_arg_given:3424die("Error: branch%sdoes not exist."% self.branch)3425else:3426die("Error: no branch%s; perhaps specify one with --branch."%3427 self.branch)34283429if self.verbose:3430print"Getting p4 changes for%s...%s"% (', '.join(self.depotPaths),3431 self.changeRange)3432 changes =p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)34333434iflen(self.maxChanges) >0:3435 changes = changes[:min(int(self.maxChanges),len(changes))]34363437iflen(changes) ==0:3438if not self.silent:3439print"No changes to import!"3440else:3441if not self.silent and not self.detectBranches:3442print"Import destination:%s"% self.branch34433444 self.updatedBranches =set()34453446if not self.detectBranches:3447if args:3448# start a new branch3449 self.initialParent =""3450else:3451# build on a previous revision3452 self.initialParent =parseRevision(self.branch)34533454 self.importChanges(changes)34553456if not self.silent:3457print""3458iflen(self.updatedBranches) >0:3459 sys.stdout.write("Updated branches: ")3460for b in self.updatedBranches:3461 sys.stdout.write("%s"% b)3462 sys.stdout.write("\n")34633464ifgitConfigBool("git-p4.importLabels"):3465 self.importLabels =True34663467if self.importLabels:3468 p4Labels =getP4Labels(self.depotPaths)3469 gitTags =getGitTags()34703471 missingP4Labels = p4Labels - gitTags3472 self.importP4Labels(self.gitStream, missingP4Labels)34733474 self.gitStream.close()3475if self.importProcess.wait() !=0:3476die("fast-import failed:%s"% self.gitError.read())3477 self.gitOutput.close()3478 self.gitError.close()34793480# Cleanup temporary branches created during import3481if self.tempBranches != []:3482for branch in self.tempBranches:3483read_pipe("git update-ref -d%s"% branch)3484 os.rmdir(os.path.join(os.environ.get("GIT_DIR",".git"), self.tempBranchLocation))34853486# Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow3487# a convenient shortcut refname "p4".3488if self.importIntoRemotes:3489 head_ref = self.refPrefix +"HEAD"3490if notgitBranchExists(head_ref)andgitBranchExists(self.branch):3491system(["git","symbolic-ref", head_ref, self.branch])34923493return True34943495classP4Rebase(Command):3496def__init__(self):3497 Command.__init__(self)3498 self.options = [3499 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),3500]3501 self.importLabels =False3502 self.description = ("Fetches the latest revision from perforce and "3503+"rebases the current work (branch) against it")35043505defrun(self, args):3506 sync =P4Sync()3507 sync.importLabels = self.importLabels3508 sync.run([])35093510return self.rebase()35113512defrebase(self):3513if os.system("git update-index --refresh") !=0:3514die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");3515iflen(read_pipe("git diff-index HEAD --")) >0:3516die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");35173518[upstream, settings] =findUpstreamBranchPoint()3519iflen(upstream) ==0:3520die("Cannot find upstream branchpoint for rebase")35213522# the branchpoint may be p4/foo~3, so strip off the parent3523 upstream = re.sub("~[0-9]+$","", upstream)35243525print"Rebasing the current branch onto%s"% upstream3526 oldHead =read_pipe("git rev-parse HEAD").strip()3527system("git rebase%s"% upstream)3528system("git diff-tree --stat --summary -M%sHEAD --"% oldHead)3529return True35303531classP4Clone(P4Sync):3532def__init__(self):3533 P4Sync.__init__(self)3534 self.description ="Creates a new git repository and imports from Perforce into it"3535 self.usage ="usage: %prog [options] //depot/path[@revRange]"3536 self.options += [3537 optparse.make_option("--destination", dest="cloneDestination",3538 action='store', default=None,3539help="where to leave result of the clone"),3540 optparse.make_option("--bare", dest="cloneBare",3541 action="store_true", default=False),3542]3543 self.cloneDestination =None3544 self.needsGit =False3545 self.cloneBare =False35463547defdefaultDestination(self, args):3548## TODO: use common prefix of args?3549 depotPath = args[0]3550 depotDir = re.sub("(@[^@]*)$","", depotPath)3551 depotDir = re.sub("(#[^#]*)$","", depotDir)3552 depotDir = re.sub(r"\.\.\.$","", depotDir)3553 depotDir = re.sub(r"/$","", depotDir)3554return os.path.split(depotDir)[1]35553556defrun(self, args):3557iflen(args) <1:3558return False35593560if self.keepRepoPath and not self.cloneDestination:3561 sys.stderr.write("Must specify destination for --keep-path\n")3562 sys.exit(1)35633564 depotPaths = args35653566if not self.cloneDestination andlen(depotPaths) >1:3567 self.cloneDestination = depotPaths[-1]3568 depotPaths = depotPaths[:-1]35693570 self.cloneExclude = ["/"+p for p in self.cloneExclude]3571for p in depotPaths:3572if not p.startswith("//"):3573 sys.stderr.write('Depot paths must start with "//":%s\n'% p)3574return False35753576if not self.cloneDestination:3577 self.cloneDestination = self.defaultDestination(args)35783579print"Importing from%sinto%s"% (', '.join(depotPaths), self.cloneDestination)35803581if not os.path.exists(self.cloneDestination):3582 os.makedirs(self.cloneDestination)3583chdir(self.cloneDestination)35843585 init_cmd = ["git","init"]3586if self.cloneBare:3587 init_cmd.append("--bare")3588 retcode = subprocess.call(init_cmd)3589if retcode:3590raiseCalledProcessError(retcode, init_cmd)35913592if not P4Sync.run(self, depotPaths):3593return False35943595# create a master branch and check out a work tree3596ifgitBranchExists(self.branch):3597system(["git","branch","master", self.branch ])3598if not self.cloneBare:3599system(["git","checkout","-f"])3600else:3601print'Not checking out any branch, use ' \3602'"git checkout -q -b master <branch>"'36033604# auto-set this variable if invoked with --use-client-spec3605if self.useClientSpec_from_options:3606system("git config --bool git-p4.useclientspec true")36073608return True36093610classP4Branches(Command):3611def__init__(self):3612 Command.__init__(self)3613 self.options = [ ]3614 self.description = ("Shows the git branches that hold imports and their "3615+"corresponding perforce depot paths")3616 self.verbose =False36173618defrun(self, args):3619iforiginP4BranchesExist():3620createOrUpdateBranchesFromOrigin()36213622 cmdline ="git rev-parse --symbolic "3623 cmdline +=" --remotes"36243625for line inread_pipe_lines(cmdline):3626 line = line.strip()36273628if not line.startswith('p4/')or line =="p4/HEAD":3629continue3630 branch = line36313632 log =extractLogMessageFromGitCommit("refs/remotes/%s"% branch)3633 settings =extractSettingsGitLog(log)36343635print"%s<=%s(%s)"% (branch,",".join(settings["depot-paths"]), settings["change"])3636return True36373638classHelpFormatter(optparse.IndentedHelpFormatter):3639def__init__(self):3640 optparse.IndentedHelpFormatter.__init__(self)36413642defformat_description(self, description):3643if description:3644return description +"\n"3645else:3646return""36473648defprintUsage(commands):3649print"usage:%s<command> [options]"% sys.argv[0]3650print""3651print"valid commands:%s"%", ".join(commands)3652print""3653print"Try%s<command> --help for command specific help."% sys.argv[0]3654print""36553656commands = {3657"debug": P4Debug,3658"submit": P4Submit,3659"commit": P4Submit,3660"sync": P4Sync,3661"rebase": P4Rebase,3662"clone": P4Clone,3663"rollback": P4RollBack,3664"branches": P4Branches3665}366636673668defmain():3669iflen(sys.argv[1:]) ==0:3670printUsage(commands.keys())3671 sys.exit(2)36723673 cmdName = sys.argv[1]3674try:3675 klass = commands[cmdName]3676 cmd =klass()3677exceptKeyError:3678print"unknown command%s"% cmdName3679print""3680printUsage(commands.keys())3681 sys.exit(2)36823683 options = cmd.options3684 cmd.gitdir = os.environ.get("GIT_DIR",None)36853686 args = sys.argv[2:]36873688 options.append(optparse.make_option("--verbose","-v", dest="verbose", action="store_true"))3689if cmd.needsGit:3690 options.append(optparse.make_option("--git-dir", dest="gitdir"))36913692 parser = optparse.OptionParser(cmd.usage.replace("%prog","%prog "+ cmdName),3693 options,3694 description = cmd.description,3695 formatter =HelpFormatter())36963697(cmd, args) = parser.parse_args(sys.argv[2:], cmd);3698global verbose3699 verbose = cmd.verbose3700if cmd.needsGit:3701if cmd.gitdir ==None:3702 cmd.gitdir = os.path.abspath(".git")3703if notisValidGitDir(cmd.gitdir):3704 cmd.gitdir =read_pipe("git rev-parse --git-dir").strip()3705if os.path.exists(cmd.gitdir):3706 cdup =read_pipe("git rev-parse --show-cdup").strip()3707iflen(cdup) >0:3708chdir(cdup);37093710if notisValidGitDir(cmd.gitdir):3711ifisValidGitDir(cmd.gitdir +"/.git"):3712 cmd.gitdir +="/.git"3713else:3714die("fatal: cannot locate git repository at%s"% cmd.gitdir)37153716 os.environ["GIT_DIR"] = cmd.gitdir37173718if not cmd.run(args):3719 parser.print_help()3720 sys.exit(2)372137223723if __name__ =='__main__':3724main()