contrib / fast-import / git-p4on commit git-p4: recognize all p4 filetypes (9cffb8c)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121#
 122# Canonicalize the p4 type and return a tuple of the
 123# base type, plus any modifiers.  See "p4 help filetypes"
 124# for a list and explanation.
 125#
 126def split_p4_type(p4type):
 127
 128    p4_filetypes_historical = {
 129        "ctempobj": "binary+Sw",
 130        "ctext": "text+C",
 131        "cxtext": "text+Cx",
 132        "ktext": "text+k",
 133        "kxtext": "text+kx",
 134        "ltext": "text+F",
 135        "tempobj": "binary+FSw",
 136        "ubinary": "binary+F",
 137        "uresource": "resource+F",
 138        "uxbinary": "binary+Fx",
 139        "xbinary": "binary+x",
 140        "xltext": "text+Fx",
 141        "xtempobj": "binary+Swx",
 142        "xtext": "text+x",
 143        "xunicode": "unicode+x",
 144        "xutf16": "utf16+x",
 145    }
 146    if p4type in p4_filetypes_historical:
 147        p4type = p4_filetypes_historical[p4type]
 148    mods = ""
 149    s = p4type.split("+")
 150    base = s[0]
 151    mods = ""
 152    if len(s) > 1:
 153        mods = s[1]
 154    return (base, mods)
 155
 156
 157def setP4ExecBit(file, mode):
 158    # Reopens an already open file and changes the execute bit to match
 159    # the execute bit setting in the passed in mode.
 160
 161    p4Type = "+x"
 162
 163    if not isModeExec(mode):
 164        p4Type = getP4OpenedType(file)
 165        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 166        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 167        if p4Type[-1] == "+":
 168            p4Type = p4Type[0:-1]
 169
 170    p4_system("reopen -t %s %s" % (p4Type, file))
 171
 172def getP4OpenedType(file):
 173    # Returns the perforce file type for the given file.
 174
 175    result = p4_read_pipe("opened %s" % file)
 176    match = re.match(".*\((.+)\)\r?$", result)
 177    if match:
 178        return match.group(1)
 179    else:
 180        die("Could not determine file type for %s (result: '%s')" % (file, result))
 181
 182def diffTreePattern():
 183    # This is a simple generator for the diff tree regex pattern. This could be
 184    # a class variable if this and parseDiffTreeEntry were a part of a class.
 185    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 186    while True:
 187        yield pattern
 188
 189def parseDiffTreeEntry(entry):
 190    """Parses a single diff tree entry into its component elements.
 191
 192    See git-diff-tree(1) manpage for details about the format of the diff
 193    output. This method returns a dictionary with the following elements:
 194
 195    src_mode - The mode of the source file
 196    dst_mode - The mode of the destination file
 197    src_sha1 - The sha1 for the source file
 198    dst_sha1 - The sha1 fr the destination file
 199    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 200    status_score - The score for the status (applicable for 'C' and 'R'
 201                   statuses). This is None if there is no score.
 202    src - The path for the source file.
 203    dst - The path for the destination file. This is only present for
 204          copy or renames. If it is not present, this is None.
 205
 206    If the pattern is not matched, None is returned."""
 207
 208    match = diffTreePattern().next().match(entry)
 209    if match:
 210        return {
 211            'src_mode': match.group(1),
 212            'dst_mode': match.group(2),
 213            'src_sha1': match.group(3),
 214            'dst_sha1': match.group(4),
 215            'status': match.group(5),
 216            'status_score': match.group(6),
 217            'src': match.group(7),
 218            'dst': match.group(10)
 219        }
 220    return None
 221
 222def isModeExec(mode):
 223    # Returns True if the given git mode represents an executable file,
 224    # otherwise False.
 225    return mode[-3:] == "755"
 226
 227def isModeExecChanged(src_mode, dst_mode):
 228    return isModeExec(src_mode) != isModeExec(dst_mode)
 229
 230def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 231    cmd = p4_build_cmd("-G %s" % (cmd))
 232    if verbose:
 233        sys.stderr.write("Opening pipe: %s\n" % cmd)
 234
 235    # Use a temporary file to avoid deadlocks without
 236    # subprocess.communicate(), which would put another copy
 237    # of stdout into memory.
 238    stdin_file = None
 239    if stdin is not None:
 240        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 241        stdin_file.write(stdin)
 242        stdin_file.flush()
 243        stdin_file.seek(0)
 244
 245    p4 = subprocess.Popen(cmd, shell=True,
 246                          stdin=stdin_file,
 247                          stdout=subprocess.PIPE)
 248
 249    result = []
 250    try:
 251        while True:
 252            entry = marshal.load(p4.stdout)
 253            if cb is not None:
 254                cb(entry)
 255            else:
 256                result.append(entry)
 257    except EOFError:
 258        pass
 259    exitCode = p4.wait()
 260    if exitCode != 0:
 261        entry = {}
 262        entry["p4ExitCode"] = exitCode
 263        result.append(entry)
 264
 265    return result
 266
 267def p4Cmd(cmd):
 268    list = p4CmdList(cmd)
 269    result = {}
 270    for entry in list:
 271        result.update(entry)
 272    return result;
 273
 274def p4Where(depotPath):
 275    if not depotPath.endswith("/"):
 276        depotPath += "/"
 277    depotPath = depotPath + "..."
 278    outputList = p4CmdList("where %s" % depotPath)
 279    output = None
 280    for entry in outputList:
 281        if "depotFile" in entry:
 282            if entry["depotFile"] == depotPath:
 283                output = entry
 284                break
 285        elif "data" in entry:
 286            data = entry.get("data")
 287            space = data.find(" ")
 288            if data[:space] == depotPath:
 289                output = entry
 290                break
 291    if output == None:
 292        return ""
 293    if output["code"] == "error":
 294        return ""
 295    clientPath = ""
 296    if "path" in output:
 297        clientPath = output.get("path")
 298    elif "data" in output:
 299        data = output.get("data")
 300        lastSpace = data.rfind(" ")
 301        clientPath = data[lastSpace + 1:]
 302
 303    if clientPath.endswith("..."):
 304        clientPath = clientPath[:-3]
 305    return clientPath
 306
 307def currentGitBranch():
 308    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 309
 310def isValidGitDir(path):
 311    if (os.path.exists(path + "/HEAD")
 312        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 313        return True;
 314    return False
 315
 316def parseRevision(ref):
 317    return read_pipe("git rev-parse %s" % ref).strip()
 318
 319def extractLogMessageFromGitCommit(commit):
 320    logMessage = ""
 321
 322    ## fixme: title is first line of commit, not 1st paragraph.
 323    foundTitle = False
 324    for log in read_pipe_lines("git cat-file commit %s" % commit):
 325       if not foundTitle:
 326           if len(log) == 1:
 327               foundTitle = True
 328           continue
 329
 330       logMessage += log
 331    return logMessage
 332
 333def extractSettingsGitLog(log):
 334    values = {}
 335    for line in log.split("\n"):
 336        line = line.strip()
 337        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 338        if not m:
 339            continue
 340
 341        assignments = m.group(1).split (':')
 342        for a in assignments:
 343            vals = a.split ('=')
 344            key = vals[0].strip()
 345            val = ('='.join (vals[1:])).strip()
 346            if val.endswith ('\"') and val.startswith('"'):
 347                val = val[1:-1]
 348
 349            values[key] = val
 350
 351    paths = values.get("depot-paths")
 352    if not paths:
 353        paths = values.get("depot-path")
 354    if paths:
 355        values['depot-paths'] = paths.split(',')
 356    return values
 357
 358def gitBranchExists(branch):
 359    proc = subprocess.Popen(["git", "rev-parse", branch],
 360                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 361    return proc.wait() == 0;
 362
 363_gitConfig = {}
 364def gitConfig(key, args = None): # set args to "--bool", for instance
 365    if not _gitConfig.has_key(key):
 366        argsFilter = ""
 367        if args != None:
 368            argsFilter = "%s " % args
 369        cmd = "git config %s%s" % (argsFilter, key)
 370        _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 371    return _gitConfig[key]
 372
 373def gitConfigList(key):
 374    if not _gitConfig.has_key(key):
 375        _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 376    return _gitConfig[key]
 377
 378def p4BranchesInGit(branchesAreInRemotes = True):
 379    branches = {}
 380
 381    cmdline = "git rev-parse --symbolic "
 382    if branchesAreInRemotes:
 383        cmdline += " --remotes"
 384    else:
 385        cmdline += " --branches"
 386
 387    for line in read_pipe_lines(cmdline):
 388        line = line.strip()
 389
 390        ## only import to p4/
 391        if not line.startswith('p4/') or line == "p4/HEAD":
 392            continue
 393        branch = line
 394
 395        # strip off p4
 396        branch = re.sub ("^p4/", "", line)
 397
 398        branches[branch] = parseRevision(line)
 399    return branches
 400
 401def findUpstreamBranchPoint(head = "HEAD"):
 402    branches = p4BranchesInGit()
 403    # map from depot-path to branch name
 404    branchByDepotPath = {}
 405    for branch in branches.keys():
 406        tip = branches[branch]
 407        log = extractLogMessageFromGitCommit(tip)
 408        settings = extractSettingsGitLog(log)
 409        if settings.has_key("depot-paths"):
 410            paths = ",".join(settings["depot-paths"])
 411            branchByDepotPath[paths] = "remotes/p4/" + branch
 412
 413    settings = None
 414    parent = 0
 415    while parent < 65535:
 416        commit = head + "~%s" % parent
 417        log = extractLogMessageFromGitCommit(commit)
 418        settings = extractSettingsGitLog(log)
 419        if settings.has_key("depot-paths"):
 420            paths = ",".join(settings["depot-paths"])
 421            if branchByDepotPath.has_key(paths):
 422                return [branchByDepotPath[paths], settings]
 423
 424        parent = parent + 1
 425
 426    return ["", settings]
 427
 428def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 429    if not silent:
 430        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 431               % localRefPrefix)
 432
 433    originPrefix = "origin/p4/"
 434
 435    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 436        line = line.strip()
 437        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 438            continue
 439
 440        headName = line[len(originPrefix):]
 441        remoteHead = localRefPrefix + headName
 442        originHead = line
 443
 444        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 445        if (not original.has_key('depot-paths')
 446            or not original.has_key('change')):
 447            continue
 448
 449        update = False
 450        if not gitBranchExists(remoteHead):
 451            if verbose:
 452                print "creating %s" % remoteHead
 453            update = True
 454        else:
 455            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 456            if settings.has_key('change') > 0:
 457                if settings['depot-paths'] == original['depot-paths']:
 458                    originP4Change = int(original['change'])
 459                    p4Change = int(settings['change'])
 460                    if originP4Change > p4Change:
 461                        print ("%s (%s) is newer than %s (%s). "
 462                               "Updating p4 branch from origin."
 463                               % (originHead, originP4Change,
 464                                  remoteHead, p4Change))
 465                        update = True
 466                else:
 467                    print ("Ignoring: %s was imported from %s while "
 468                           "%s was imported from %s"
 469                           % (originHead, ','.join(original['depot-paths']),
 470                              remoteHead, ','.join(settings['depot-paths'])))
 471
 472        if update:
 473            system("git update-ref %s %s" % (remoteHead, originHead))
 474
 475def originP4BranchesExist():
 476        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 477
 478def p4ChangesForPaths(depotPaths, changeRange):
 479    assert depotPaths
 480    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 481                                                        for p in depotPaths]))
 482
 483    changes = {}
 484    for line in output:
 485        changeNum = int(line.split(" ")[1])
 486        changes[changeNum] = True
 487
 488    changelist = changes.keys()
 489    changelist.sort()
 490    return changelist
 491
 492def p4PathStartsWith(path, prefix):
 493    # This method tries to remedy a potential mixed-case issue:
 494    #
 495    # If UserA adds  //depot/DirA/file1
 496    # and UserB adds //depot/dira/file2
 497    #
 498    # we may or may not have a problem. If you have core.ignorecase=true,
 499    # we treat DirA and dira as the same directory
 500    ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 501    if ignorecase:
 502        return path.lower().startswith(prefix.lower())
 503    return path.startswith(prefix)
 504
 505class Command:
 506    def __init__(self):
 507        self.usage = "usage: %prog [options]"
 508        self.needsGit = True
 509
 510class P4UserMap:
 511    def __init__(self):
 512        self.userMapFromPerforceServer = False
 513
 514    def getUserCacheFilename(self):
 515        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 516        return home + "/.gitp4-usercache.txt"
 517
 518    def getUserMapFromPerforceServer(self):
 519        if self.userMapFromPerforceServer:
 520            return
 521        self.users = {}
 522        self.emails = {}
 523
 524        for output in p4CmdList("users"):
 525            if not output.has_key("User"):
 526                continue
 527            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 528            self.emails[output["Email"]] = output["User"]
 529
 530
 531        s = ''
 532        for (key, val) in self.users.items():
 533            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 534
 535        open(self.getUserCacheFilename(), "wb").write(s)
 536        self.userMapFromPerforceServer = True
 537
 538    def loadUserMapFromCache(self):
 539        self.users = {}
 540        self.userMapFromPerforceServer = False
 541        try:
 542            cache = open(self.getUserCacheFilename(), "rb")
 543            lines = cache.readlines()
 544            cache.close()
 545            for line in lines:
 546                entry = line.strip().split("\t")
 547                self.users[entry[0]] = entry[1]
 548        except IOError:
 549            self.getUserMapFromPerforceServer()
 550
 551class P4Debug(Command):
 552    def __init__(self):
 553        Command.__init__(self)
 554        self.options = [
 555            optparse.make_option("--verbose", dest="verbose", action="store_true",
 556                                 default=False),
 557            ]
 558        self.description = "A tool to debug the output of p4 -G."
 559        self.needsGit = False
 560        self.verbose = False
 561
 562    def run(self, args):
 563        j = 0
 564        for output in p4CmdList(" ".join(args)):
 565            print 'Element: %d' % j
 566            j += 1
 567            print output
 568        return True
 569
 570class P4RollBack(Command):
 571    def __init__(self):
 572        Command.__init__(self)
 573        self.options = [
 574            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 575            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 576        ]
 577        self.description = "A tool to debug the multi-branch import. Don't use :)"
 578        self.verbose = False
 579        self.rollbackLocalBranches = False
 580
 581    def run(self, args):
 582        if len(args) != 1:
 583            return False
 584        maxChange = int(args[0])
 585
 586        if "p4ExitCode" in p4Cmd("changes -m 1"):
 587            die("Problems executing p4");
 588
 589        if self.rollbackLocalBranches:
 590            refPrefix = "refs/heads/"
 591            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 592        else:
 593            refPrefix = "refs/remotes/"
 594            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 595
 596        for line in lines:
 597            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 598                line = line.strip()
 599                ref = refPrefix + line
 600                log = extractLogMessageFromGitCommit(ref)
 601                settings = extractSettingsGitLog(log)
 602
 603                depotPaths = settings['depot-paths']
 604                change = settings['change']
 605
 606                changed = False
 607
 608                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 609                                                           for p in depotPaths]))) == 0:
 610                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 611                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 612                    continue
 613
 614                while change and int(change) > maxChange:
 615                    changed = True
 616                    if self.verbose:
 617                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 618                    system("git update-ref %s \"%s^\"" % (ref, ref))
 619                    log = extractLogMessageFromGitCommit(ref)
 620                    settings =  extractSettingsGitLog(log)
 621
 622
 623                    depotPaths = settings['depot-paths']
 624                    change = settings['change']
 625
 626                if changed:
 627                    print "%s rewound to %s" % (ref, change)
 628
 629        return True
 630
 631class P4Submit(Command, P4UserMap):
 632    def __init__(self):
 633        Command.__init__(self)
 634        P4UserMap.__init__(self)
 635        self.options = [
 636                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 637                optparse.make_option("--origin", dest="origin"),
 638                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 639                # preserve the user, requires relevant p4 permissions
 640                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 641        ]
 642        self.description = "Submit changes from git to the perforce depot."
 643        self.usage += " [name of git branch to submit into perforce depot]"
 644        self.interactive = True
 645        self.origin = ""
 646        self.detectRenames = False
 647        self.verbose = False
 648        self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 649        self.isWindows = (platform.system() == "Windows")
 650        self.myP4UserId = None
 651
 652    def check(self):
 653        if len(p4CmdList("opened ...")) > 0:
 654            die("You have files opened with perforce! Close them before starting the sync.")
 655
 656    # replaces everything between 'Description:' and the next P4 submit template field with the
 657    # commit message
 658    def prepareLogMessage(self, template, message):
 659        result = ""
 660
 661        inDescriptionSection = False
 662
 663        for line in template.split("\n"):
 664            if line.startswith("#"):
 665                result += line + "\n"
 666                continue
 667
 668            if inDescriptionSection:
 669                if line.startswith("Files:") or line.startswith("Jobs:"):
 670                    inDescriptionSection = False
 671                else:
 672                    continue
 673            else:
 674                if line.startswith("Description:"):
 675                    inDescriptionSection = True
 676                    line += "\n"
 677                    for messageLine in message.split("\n"):
 678                        line += "\t" + messageLine + "\n"
 679
 680            result += line + "\n"
 681
 682        return result
 683
 684    def p4UserForCommit(self,id):
 685        # Return the tuple (perforce user,git email) for a given git commit id
 686        self.getUserMapFromPerforceServer()
 687        gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 688        gitEmail = gitEmail.strip()
 689        if not self.emails.has_key(gitEmail):
 690            return (None,gitEmail)
 691        else:
 692            return (self.emails[gitEmail],gitEmail)
 693
 694    def checkValidP4Users(self,commits):
 695        # check if any git authors cannot be mapped to p4 users
 696        for id in commits:
 697            (user,email) = self.p4UserForCommit(id)
 698            if not user:
 699                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 700                if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 701                    print "%s" % msg
 702                else:
 703                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 704
 705    def lastP4Changelist(self):
 706        # Get back the last changelist number submitted in this client spec. This
 707        # then gets used to patch up the username in the change. If the same
 708        # client spec is being used by multiple processes then this might go
 709        # wrong.
 710        results = p4CmdList("client -o")        # find the current client
 711        client = None
 712        for r in results:
 713            if r.has_key('Client'):
 714                client = r['Client']
 715                break
 716        if not client:
 717            die("could not get client spec")
 718        results = p4CmdList("changes -c %s -m 1" % client)
 719        for r in results:
 720            if r.has_key('change'):
 721                return r['change']
 722        die("Could not get changelist number for last submit - cannot patch up user details")
 723
 724    def modifyChangelistUser(self, changelist, newUser):
 725        # fixup the user field of a changelist after it has been submitted.
 726        changes = p4CmdList("change -o %s" % changelist)
 727        if len(changes) != 1:
 728            die("Bad output from p4 change modifying %s to user %s" %
 729                (changelist, newUser))
 730
 731        c = changes[0]
 732        if c['User'] == newUser: return   # nothing to do
 733        c['User'] = newUser
 734        input = marshal.dumps(c)
 735
 736        result = p4CmdList("change -f -i", stdin=input)
 737        for r in result:
 738            if r.has_key('code'):
 739                if r['code'] == 'error':
 740                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 741            if r.has_key('data'):
 742                print("Updated user field for changelist %s to %s" % (changelist, newUser))
 743                return
 744        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 745
 746    def canChangeChangelists(self):
 747        # check to see if we have p4 admin or super-user permissions, either of
 748        # which are required to modify changelists.
 749        results = p4CmdList("protects %s" % self.depotPath)
 750        for r in results:
 751            if r.has_key('perm'):
 752                if r['perm'] == 'admin':
 753                    return 1
 754                if r['perm'] == 'super':
 755                    return 1
 756        return 0
 757
 758    def p4UserId(self):
 759        if self.myP4UserId:
 760            return self.myP4UserId
 761
 762        results = p4CmdList("user -o")
 763        for r in results:
 764            if r.has_key('User'):
 765                self.myP4UserId = r['User']
 766                return r['User']
 767        die("Could not find your p4 user id")
 768
 769    def p4UserIsMe(self, p4User):
 770        # return True if the given p4 user is actually me
 771        me = self.p4UserId()
 772        if not p4User or p4User != me:
 773            return False
 774        else:
 775            return True
 776
 777    def prepareSubmitTemplate(self):
 778        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 779        template = ""
 780        inFilesSection = False
 781        for line in p4_read_pipe_lines("change -o"):
 782            if line.endswith("\r\n"):
 783                line = line[:-2] + "\n"
 784            if inFilesSection:
 785                if line.startswith("\t"):
 786                    # path starts and ends with a tab
 787                    path = line[1:]
 788                    lastTab = path.rfind("\t")
 789                    if lastTab != -1:
 790                        path = path[:lastTab]
 791                        if not p4PathStartsWith(path, self.depotPath):
 792                            continue
 793                else:
 794                    inFilesSection = False
 795            else:
 796                if line.startswith("Files:"):
 797                    inFilesSection = True
 798
 799            template += line
 800
 801        return template
 802
 803    def applyCommit(self, id):
 804        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 805
 806        (p4User, gitEmail) = self.p4UserForCommit(id)
 807
 808        if not self.detectRenames:
 809            # If not explicitly set check the config variable
 810            self.detectRenames = gitConfig("git-p4.detectRenames")
 811
 812        if self.detectRenames.lower() == "false" or self.detectRenames == "":
 813            diffOpts = ""
 814        elif self.detectRenames.lower() == "true":
 815            diffOpts = "-M"
 816        else:
 817            diffOpts = "-M%s" % self.detectRenames
 818
 819        detectCopies = gitConfig("git-p4.detectCopies")
 820        if detectCopies.lower() == "true":
 821            diffOpts += " -C"
 822        elif detectCopies != "" and detectCopies.lower() != "false":
 823            diffOpts += " -C%s" % detectCopies
 824
 825        if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 826            diffOpts += " --find-copies-harder"
 827
 828        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 829        filesToAdd = set()
 830        filesToDelete = set()
 831        editedFiles = set()
 832        filesToChangeExecBit = {}
 833        for line in diff:
 834            diff = parseDiffTreeEntry(line)
 835            modifier = diff['status']
 836            path = diff['src']
 837            if modifier == "M":
 838                p4_system("edit \"%s\"" % path)
 839                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 840                    filesToChangeExecBit[path] = diff['dst_mode']
 841                editedFiles.add(path)
 842            elif modifier == "A":
 843                filesToAdd.add(path)
 844                filesToChangeExecBit[path] = diff['dst_mode']
 845                if path in filesToDelete:
 846                    filesToDelete.remove(path)
 847            elif modifier == "D":
 848                filesToDelete.add(path)
 849                if path in filesToAdd:
 850                    filesToAdd.remove(path)
 851            elif modifier == "C":
 852                src, dest = diff['src'], diff['dst']
 853                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 854                if diff['src_sha1'] != diff['dst_sha1']:
 855                    p4_system("edit \"%s\"" % (dest))
 856                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 857                    p4_system("edit \"%s\"" % (dest))
 858                    filesToChangeExecBit[dest] = diff['dst_mode']
 859                os.unlink(dest)
 860                editedFiles.add(dest)
 861            elif modifier == "R":
 862                src, dest = diff['src'], diff['dst']
 863                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 864                if diff['src_sha1'] != diff['dst_sha1']:
 865                    p4_system("edit \"%s\"" % (dest))
 866                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 867                    p4_system("edit \"%s\"" % (dest))
 868                    filesToChangeExecBit[dest] = diff['dst_mode']
 869                os.unlink(dest)
 870                editedFiles.add(dest)
 871                filesToDelete.add(src)
 872            else:
 873                die("unknown modifier %s for %s" % (modifier, path))
 874
 875        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 876        patchcmd = diffcmd + " | git apply "
 877        tryPatchCmd = patchcmd + "--check -"
 878        applyPatchCmd = patchcmd + "--check --apply -"
 879
 880        if os.system(tryPatchCmd) != 0:
 881            print "Unfortunately applying the change failed!"
 882            print "What do you want to do?"
 883            response = "x"
 884            while response != "s" and response != "a" and response != "w":
 885                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 886                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 887            if response == "s":
 888                print "Skipping! Good luck with the next patches..."
 889                for f in editedFiles:
 890                    p4_system("revert \"%s\"" % f);
 891                for f in filesToAdd:
 892                    system("rm %s" %f)
 893                return
 894            elif response == "a":
 895                os.system(applyPatchCmd)
 896                if len(filesToAdd) > 0:
 897                    print "You may also want to call p4 add on the following files:"
 898                    print " ".join(filesToAdd)
 899                if len(filesToDelete):
 900                    print "The following files should be scheduled for deletion with p4 delete:"
 901                    print " ".join(filesToDelete)
 902                die("Please resolve and submit the conflict manually and "
 903                    + "continue afterwards with git-p4 submit --continue")
 904            elif response == "w":
 905                system(diffcmd + " > patch.txt")
 906                print "Patch saved to patch.txt in %s !" % self.clientPath
 907                die("Please resolve and submit the conflict manually and "
 908                    "continue afterwards with git-p4 submit --continue")
 909
 910        system(applyPatchCmd)
 911
 912        for f in filesToAdd:
 913            p4_system("add \"%s\"" % f)
 914        for f in filesToDelete:
 915            p4_system("revert \"%s\"" % f)
 916            p4_system("delete \"%s\"" % f)
 917
 918        # Set/clear executable bits
 919        for f in filesToChangeExecBit.keys():
 920            mode = filesToChangeExecBit[f]
 921            setP4ExecBit(f, mode)
 922
 923        logMessage = extractLogMessageFromGitCommit(id)
 924        logMessage = logMessage.strip()
 925
 926        template = self.prepareSubmitTemplate()
 927
 928        if self.interactive:
 929            submitTemplate = self.prepareLogMessage(template, logMessage)
 930
 931            if self.preserveUser:
 932               submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
 933
 934            if os.environ.has_key("P4DIFF"):
 935                del(os.environ["P4DIFF"])
 936            diff = ""
 937            for editedFile in editedFiles:
 938                diff += p4_read_pipe("diff -du %r" % editedFile)
 939
 940            newdiff = ""
 941            for newFile in filesToAdd:
 942                newdiff += "==== new file ====\n"
 943                newdiff += "--- /dev/null\n"
 944                newdiff += "+++ %s\n" % newFile
 945                f = open(newFile, "r")
 946                for line in f.readlines():
 947                    newdiff += "+" + line
 948                f.close()
 949
 950            if self.checkAuthorship and not self.p4UserIsMe(p4User):
 951                submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
 952                submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
 953                submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
 954
 955            separatorLine = "######## everything below this line is just the diff #######\n"
 956
 957            [handle, fileName] = tempfile.mkstemp()
 958            tmpFile = os.fdopen(handle, "w+")
 959            if self.isWindows:
 960                submitTemplate = submitTemplate.replace("\n", "\r\n")
 961                separatorLine = separatorLine.replace("\n", "\r\n")
 962                newdiff = newdiff.replace("\n", "\r\n")
 963            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 964            tmpFile.close()
 965            mtime = os.stat(fileName).st_mtime
 966            if os.environ.has_key("P4EDITOR"):
 967                editor = os.environ.get("P4EDITOR")
 968            else:
 969                editor = read_pipe("git var GIT_EDITOR").strip()
 970            system(editor + " " + fileName)
 971
 972            if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 973                checkModTime = False
 974            else:
 975                checkModTime = True
 976
 977            response = "y"
 978            if checkModTime and (os.stat(fileName).st_mtime <= mtime):
 979                response = "x"
 980                while response != "y" and response != "n":
 981                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 982
 983            if response == "y":
 984                tmpFile = open(fileName, "rb")
 985                message = tmpFile.read()
 986                tmpFile.close()
 987                submitTemplate = message[:message.index(separatorLine)]
 988                if self.isWindows:
 989                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 990                p4_write_pipe("submit -i", submitTemplate)
 991
 992                if self.preserveUser:
 993                    if p4User:
 994                        # Get last changelist number. Cannot easily get it from
 995                        # the submit command output as the output is unmarshalled.
 996                        changelist = self.lastP4Changelist()
 997                        self.modifyChangelistUser(changelist, p4User)
 998
 999            else:
1000                for f in editedFiles:
1001                    p4_system("revert \"%s\"" % f);
1002                for f in filesToAdd:
1003                    p4_system("revert \"%s\"" % f);
1004                    system("rm %s" %f)
1005
1006            os.remove(fileName)
1007        else:
1008            fileName = "submit.txt"
1009            file = open(fileName, "w+")
1010            file.write(self.prepareLogMessage(template, logMessage))
1011            file.close()
1012            print ("Perforce submit template written as %s. "
1013                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1014                   % (fileName, fileName))
1015
1016    def run(self, args):
1017        if len(args) == 0:
1018            self.master = currentGitBranch()
1019            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1020                die("Detecting current git branch failed!")
1021        elif len(args) == 1:
1022            self.master = args[0]
1023        else:
1024            return False
1025
1026        allowSubmit = gitConfig("git-p4.allowSubmit")
1027        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1028            die("%s is not in git-p4.allowSubmit" % self.master)
1029
1030        [upstream, settings] = findUpstreamBranchPoint()
1031        self.depotPath = settings['depot-paths'][0]
1032        if len(self.origin) == 0:
1033            self.origin = upstream
1034
1035        if self.preserveUser:
1036            if not self.canChangeChangelists():
1037                die("Cannot preserve user names without p4 super-user or admin permissions")
1038
1039        if self.verbose:
1040            print "Origin branch is " + self.origin
1041
1042        if len(self.depotPath) == 0:
1043            print "Internal error: cannot locate perforce depot path from existing branches"
1044            sys.exit(128)
1045
1046        self.clientPath = p4Where(self.depotPath)
1047
1048        if len(self.clientPath) == 0:
1049            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1050            sys.exit(128)
1051
1052        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1053        self.oldWorkingDirectory = os.getcwd()
1054
1055        chdir(self.clientPath)
1056        print "Synchronizing p4 checkout..."
1057        p4_system("sync ...")
1058
1059        self.check()
1060
1061        commits = []
1062        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1063            commits.append(line.strip())
1064        commits.reverse()
1065
1066        if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1067            self.checkAuthorship = False
1068        else:
1069            self.checkAuthorship = True
1070
1071        if self.preserveUser:
1072            self.checkValidP4Users(commits)
1073
1074        while len(commits) > 0:
1075            commit = commits[0]
1076            commits = commits[1:]
1077            self.applyCommit(commit)
1078            if not self.interactive:
1079                break
1080
1081        if len(commits) == 0:
1082            print "All changes applied!"
1083            chdir(self.oldWorkingDirectory)
1084
1085            sync = P4Sync()
1086            sync.run([])
1087
1088            rebase = P4Rebase()
1089            rebase.rebase()
1090
1091        return True
1092
1093class P4Sync(Command, P4UserMap):
1094    delete_actions = ( "delete", "move/delete", "purge" )
1095
1096    def __init__(self):
1097        Command.__init__(self)
1098        P4UserMap.__init__(self)
1099        self.options = [
1100                optparse.make_option("--branch", dest="branch"),
1101                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1102                optparse.make_option("--changesfile", dest="changesFile"),
1103                optparse.make_option("--silent", dest="silent", action="store_true"),
1104                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1105                optparse.make_option("--verbose", dest="verbose", action="store_true"),
1106                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1107                                     help="Import into refs/heads/ , not refs/remotes"),
1108                optparse.make_option("--max-changes", dest="maxChanges"),
1109                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1110                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1111                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1112                                     help="Only sync files that are included in the Perforce Client Spec")
1113        ]
1114        self.description = """Imports from Perforce into a git repository.\n
1115    example:
1116    //depot/my/project/ -- to import the current head
1117    //depot/my/project/@all -- to import everything
1118    //depot/my/project/@1,6 -- to import only from revision 1 to 6
1119
1120    (a ... is not needed in the path p4 specification, it's added implicitly)"""
1121
1122        self.usage += " //depot/path[@revRange]"
1123        self.silent = False
1124        self.createdBranches = set()
1125        self.committedChanges = set()
1126        self.branch = ""
1127        self.detectBranches = False
1128        self.detectLabels = False
1129        self.changesFile = ""
1130        self.syncWithOrigin = True
1131        self.verbose = False
1132        self.importIntoRemotes = True
1133        self.maxChanges = ""
1134        self.isWindows = (platform.system() == "Windows")
1135        self.keepRepoPath = False
1136        self.depotPaths = None
1137        self.p4BranchesInGit = []
1138        self.cloneExclude = []
1139        self.useClientSpec = False
1140        self.clientSpecDirs = []
1141
1142        if gitConfig("git-p4.syncFromOrigin") == "false":
1143            self.syncWithOrigin = False
1144
1145    #
1146    # P4 wildcards are not allowed in filenames.  P4 complains
1147    # if you simply add them, but you can force it with "-f", in
1148    # which case it translates them into %xx encoding internally.
1149    # Search for and fix just these four characters.  Do % last so
1150    # that fixing it does not inadvertently create new %-escapes.
1151    #
1152    def wildcard_decode(self, path):
1153        # Cannot have * in a filename in windows; untested as to
1154        # what p4 would do in such a case.
1155        if not self.isWindows:
1156            path = path.replace("%2A", "*")
1157        path = path.replace("%23", "#") \
1158                   .replace("%40", "@") \
1159                   .replace("%25", "%")
1160        return path
1161
1162    def extractFilesFromCommit(self, commit):
1163        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1164                             for path in self.cloneExclude]
1165        files = []
1166        fnum = 0
1167        while commit.has_key("depotFile%s" % fnum):
1168            path =  commit["depotFile%s" % fnum]
1169
1170            if [p for p in self.cloneExclude
1171                if p4PathStartsWith(path, p)]:
1172                found = False
1173            else:
1174                found = [p for p in self.depotPaths
1175                         if p4PathStartsWith(path, p)]
1176            if not found:
1177                fnum = fnum + 1
1178                continue
1179
1180            file = {}
1181            file["path"] = path
1182            file["rev"] = commit["rev%s" % fnum]
1183            file["action"] = commit["action%s" % fnum]
1184            file["type"] = commit["type%s" % fnum]
1185            files.append(file)
1186            fnum = fnum + 1
1187        return files
1188
1189    def stripRepoPath(self, path, prefixes):
1190        if self.useClientSpec:
1191
1192            # if using the client spec, we use the output directory
1193            # specified in the client.  For example, a view
1194            #   //depot/foo/branch/... //client/branch/foo/...
1195            # will end up putting all foo/branch files into
1196            #  branch/foo/
1197            for val in self.clientSpecDirs:
1198                if path.startswith(val[0]):
1199                    # replace the depot path with the client path
1200                    path = path.replace(val[0], val[1][1])
1201                    # now strip out the client (//client/...)
1202                    path = re.sub("^(//[^/]+/)", '', path)
1203                    # the rest is all path
1204                    return path
1205
1206        if self.keepRepoPath:
1207            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1208
1209        for p in prefixes:
1210            if p4PathStartsWith(path, p):
1211                path = path[len(p):]
1212
1213        return path
1214
1215    def splitFilesIntoBranches(self, commit):
1216        branches = {}
1217        fnum = 0
1218        while commit.has_key("depotFile%s" % fnum):
1219            path =  commit["depotFile%s" % fnum]
1220            found = [p for p in self.depotPaths
1221                     if p4PathStartsWith(path, p)]
1222            if not found:
1223                fnum = fnum + 1
1224                continue
1225
1226            file = {}
1227            file["path"] = path
1228            file["rev"] = commit["rev%s" % fnum]
1229            file["action"] = commit["action%s" % fnum]
1230            file["type"] = commit["type%s" % fnum]
1231            fnum = fnum + 1
1232
1233            relPath = self.stripRepoPath(path, self.depotPaths)
1234
1235            for branch in self.knownBranches.keys():
1236
1237                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1238                if relPath.startswith(branch + "/"):
1239                    if branch not in branches:
1240                        branches[branch] = []
1241                    branches[branch].append(file)
1242                    break
1243
1244        return branches
1245
1246    # output one file from the P4 stream
1247    # - helper for streamP4Files
1248
1249    def streamOneP4File(self, file, contents):
1250        if file["type"] == "apple":
1251            print "\nfile %s is a strange apple file that forks. Ignoring" % \
1252                file['depotFile']
1253            return
1254
1255        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1256        relPath = self.wildcard_decode(relPath)
1257        if verbose:
1258            sys.stderr.write("%s\n" % relPath)
1259
1260        (type_base, type_mods) = split_p4_type(file["type"])
1261
1262        git_mode = "100644"
1263        if "x" in type_mods:
1264            git_mode = "100755"
1265        if type_base == "symlink":
1266            git_mode = "120000"
1267            # p4 print on a symlink contains "target\n"; remove the newline
1268            data = ''.join(contents)
1269            contents = [data[:-1]]
1270
1271        if type_base == "utf16":
1272            # p4 delivers different text in the python output to -G
1273            # than it does when using "print -o", or normal p4 client
1274            # operations.  utf16 is converted to ascii or utf8, perhaps.
1275            # But ascii text saved as -t utf16 is completely mangled.
1276            # Invoke print -o to get the real contents.
1277            text = p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1278            contents = [ text ]
1279
1280        # Perhaps windows wants unicode, utf16 newlines translated too;
1281        # but this is not doing it.
1282        if self.isWindows and type_base == "text":
1283            mangled = []
1284            for data in contents:
1285                data = data.replace("\r\n", "\n")
1286                mangled.append(data)
1287            contents = mangled
1288
1289        # Note that we do not try to de-mangle keywords on utf16 files,
1290        # even though in theory somebody may want that.
1291        if type_base in ("text", "unicode", "binary"):
1292            if "ko" in type_mods:
1293                contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$', r'$\1$', text), contents)
1294            elif "k" in type_mods:
1295                contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$', r'$\1$', text), contents)
1296
1297        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1298
1299        # total length...
1300        length = 0
1301        for d in contents:
1302            length = length + len(d)
1303
1304        self.gitStream.write("data %d\n" % length)
1305        for d in contents:
1306            self.gitStream.write(d)
1307        self.gitStream.write("\n")
1308
1309    def streamOneP4Deletion(self, file):
1310        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1311        if verbose:
1312            sys.stderr.write("delete %s\n" % relPath)
1313        self.gitStream.write("D %s\n" % relPath)
1314
1315    # handle another chunk of streaming data
1316    def streamP4FilesCb(self, marshalled):
1317
1318        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1319            # start of a new file - output the old one first
1320            self.streamOneP4File(self.stream_file, self.stream_contents)
1321            self.stream_file = {}
1322            self.stream_contents = []
1323            self.stream_have_file_info = False
1324
1325        # pick up the new file information... for the
1326        # 'data' field we need to append to our array
1327        for k in marshalled.keys():
1328            if k == 'data':
1329                self.stream_contents.append(marshalled['data'])
1330            else:
1331                self.stream_file[k] = marshalled[k]
1332
1333        self.stream_have_file_info = True
1334
1335    # Stream directly from "p4 files" into "git fast-import"
1336    def streamP4Files(self, files):
1337        filesForCommit = []
1338        filesToRead = []
1339        filesToDelete = []
1340
1341        for f in files:
1342            includeFile = True
1343            for val in self.clientSpecDirs:
1344                if f['path'].startswith(val[0]):
1345                    if val[1][0] <= 0:
1346                        includeFile = False
1347                    break
1348
1349            if includeFile:
1350                filesForCommit.append(f)
1351                if f['action'] in self.delete_actions:
1352                    filesToDelete.append(f)
1353                else:
1354                    filesToRead.append(f)
1355
1356        # deleted files...
1357        for f in filesToDelete:
1358            self.streamOneP4Deletion(f)
1359
1360        if len(filesToRead) > 0:
1361            self.stream_file = {}
1362            self.stream_contents = []
1363            self.stream_have_file_info = False
1364
1365            # curry self argument
1366            def streamP4FilesCbSelf(entry):
1367                self.streamP4FilesCb(entry)
1368
1369            p4CmdList("-x - print",
1370                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1371                                                  for f in filesToRead]),
1372                cb=streamP4FilesCbSelf)
1373
1374            # do the last chunk
1375            if self.stream_file.has_key('depotFile'):
1376                self.streamOneP4File(self.stream_file, self.stream_contents)
1377
1378    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1379        epoch = details["time"]
1380        author = details["user"]
1381        self.branchPrefixes = branchPrefixes
1382
1383        if self.verbose:
1384            print "commit into %s" % branch
1385
1386        # start with reading files; if that fails, we should not
1387        # create a commit.
1388        new_files = []
1389        for f in files:
1390            if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1391                new_files.append (f)
1392            else:
1393                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1394
1395        self.gitStream.write("commit %s\n" % branch)
1396#        gitStream.write("mark :%s\n" % details["change"])
1397        self.committedChanges.add(int(details["change"]))
1398        committer = ""
1399        if author not in self.users:
1400            self.getUserMapFromPerforceServer()
1401        if author in self.users:
1402            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1403        else:
1404            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1405
1406        self.gitStream.write("committer %s\n" % committer)
1407
1408        self.gitStream.write("data <<EOT\n")
1409        self.gitStream.write(details["desc"])
1410        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1411                             % (','.join (branchPrefixes), details["change"]))
1412        if len(details['options']) > 0:
1413            self.gitStream.write(": options = %s" % details['options'])
1414        self.gitStream.write("]\nEOT\n\n")
1415
1416        if len(parent) > 0:
1417            if self.verbose:
1418                print "parent %s" % parent
1419            self.gitStream.write("from %s\n" % parent)
1420
1421        self.streamP4Files(new_files)
1422        self.gitStream.write("\n")
1423
1424        change = int(details["change"])
1425
1426        if self.labels.has_key(change):
1427            label = self.labels[change]
1428            labelDetails = label[0]
1429            labelRevisions = label[1]
1430            if self.verbose:
1431                print "Change %s is labelled %s" % (change, labelDetails)
1432
1433            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1434                                                    for p in branchPrefixes]))
1435
1436            if len(files) == len(labelRevisions):
1437
1438                cleanedFiles = {}
1439                for info in files:
1440                    if info["action"] in self.delete_actions:
1441                        continue
1442                    cleanedFiles[info["depotFile"]] = info["rev"]
1443
1444                if cleanedFiles == labelRevisions:
1445                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1446                    self.gitStream.write("from %s\n" % branch)
1447
1448                    owner = labelDetails["Owner"]
1449                    tagger = ""
1450                    if author in self.users:
1451                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1452                    else:
1453                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1454                    self.gitStream.write("tagger %s\n" % tagger)
1455                    self.gitStream.write("data <<EOT\n")
1456                    self.gitStream.write(labelDetails["Description"])
1457                    self.gitStream.write("EOT\n\n")
1458
1459                else:
1460                    if not self.silent:
1461                        print ("Tag %s does not match with change %s: files do not match."
1462                               % (labelDetails["label"], change))
1463
1464            else:
1465                if not self.silent:
1466                    print ("Tag %s does not match with change %s: file count is different."
1467                           % (labelDetails["label"], change))
1468
1469    def getLabels(self):
1470        self.labels = {}
1471
1472        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1473        if len(l) > 0 and not self.silent:
1474            print "Finding files belonging to labels in %s" % `self.depotPaths`
1475
1476        for output in l:
1477            label = output["label"]
1478            revisions = {}
1479            newestChange = 0
1480            if self.verbose:
1481                print "Querying files for label %s" % label
1482            for file in p4CmdList("files "
1483                                  +  ' '.join (["%s...@%s" % (p, label)
1484                                                for p in self.depotPaths])):
1485                revisions[file["depotFile"]] = file["rev"]
1486                change = int(file["change"])
1487                if change > newestChange:
1488                    newestChange = change
1489
1490            self.labels[newestChange] = [output, revisions]
1491
1492        if self.verbose:
1493            print "Label changes: %s" % self.labels.keys()
1494
1495    def guessProjectName(self):
1496        for p in self.depotPaths:
1497            if p.endswith("/"):
1498                p = p[:-1]
1499            p = p[p.strip().rfind("/") + 1:]
1500            if not p.endswith("/"):
1501               p += "/"
1502            return p
1503
1504    def getBranchMapping(self):
1505        lostAndFoundBranches = set()
1506
1507        user = gitConfig("git-p4.branchUser")
1508        if len(user) > 0:
1509            command = "branches -u %s" % user
1510        else:
1511            command = "branches"
1512
1513        for info in p4CmdList(command):
1514            details = p4Cmd("branch -o %s" % info["branch"])
1515            viewIdx = 0
1516            while details.has_key("View%s" % viewIdx):
1517                paths = details["View%s" % viewIdx].split(" ")
1518                viewIdx = viewIdx + 1
1519                # require standard //depot/foo/... //depot/bar/... mapping
1520                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1521                    continue
1522                source = paths[0]
1523                destination = paths[1]
1524                ## HACK
1525                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1526                    source = source[len(self.depotPaths[0]):-4]
1527                    destination = destination[len(self.depotPaths[0]):-4]
1528
1529                    if destination in self.knownBranches:
1530                        if not self.silent:
1531                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1532                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1533                        continue
1534
1535                    self.knownBranches[destination] = source
1536
1537                    lostAndFoundBranches.discard(destination)
1538
1539                    if source not in self.knownBranches:
1540                        lostAndFoundBranches.add(source)
1541
1542        # Perforce does not strictly require branches to be defined, so we also
1543        # check git config for a branch list.
1544        #
1545        # Example of branch definition in git config file:
1546        # [git-p4]
1547        #   branchList=main:branchA
1548        #   branchList=main:branchB
1549        #   branchList=branchA:branchC
1550        configBranches = gitConfigList("git-p4.branchList")
1551        for branch in configBranches:
1552            if branch:
1553                (source, destination) = branch.split(":")
1554                self.knownBranches[destination] = source
1555
1556                lostAndFoundBranches.discard(destination)
1557
1558                if source not in self.knownBranches:
1559                    lostAndFoundBranches.add(source)
1560
1561
1562        for branch in lostAndFoundBranches:
1563            self.knownBranches[branch] = branch
1564
1565    def getBranchMappingFromGitBranches(self):
1566        branches = p4BranchesInGit(self.importIntoRemotes)
1567        for branch in branches.keys():
1568            if branch == "master":
1569                branch = "main"
1570            else:
1571                branch = branch[len(self.projectName):]
1572            self.knownBranches[branch] = branch
1573
1574    def listExistingP4GitBranches(self):
1575        # branches holds mapping from name to commit
1576        branches = p4BranchesInGit(self.importIntoRemotes)
1577        self.p4BranchesInGit = branches.keys()
1578        for branch in branches.keys():
1579            self.initialParents[self.refPrefix + branch] = branches[branch]
1580
1581    def updateOptionDict(self, d):
1582        option_keys = {}
1583        if self.keepRepoPath:
1584            option_keys['keepRepoPath'] = 1
1585
1586        d["options"] = ' '.join(sorted(option_keys.keys()))
1587
1588    def readOptions(self, d):
1589        self.keepRepoPath = (d.has_key('options')
1590                             and ('keepRepoPath' in d['options']))
1591
1592    def gitRefForBranch(self, branch):
1593        if branch == "main":
1594            return self.refPrefix + "master"
1595
1596        if len(branch) <= 0:
1597            return branch
1598
1599        return self.refPrefix + self.projectName + branch
1600
1601    def gitCommitByP4Change(self, ref, change):
1602        if self.verbose:
1603            print "looking in ref " + ref + " for change %s using bisect..." % change
1604
1605        earliestCommit = ""
1606        latestCommit = parseRevision(ref)
1607
1608        while True:
1609            if self.verbose:
1610                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1611            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1612            if len(next) == 0:
1613                if self.verbose:
1614                    print "argh"
1615                return ""
1616            log = extractLogMessageFromGitCommit(next)
1617            settings = extractSettingsGitLog(log)
1618            currentChange = int(settings['change'])
1619            if self.verbose:
1620                print "current change %s" % currentChange
1621
1622            if currentChange == change:
1623                if self.verbose:
1624                    print "found %s" % next
1625                return next
1626
1627            if currentChange < change:
1628                earliestCommit = "^%s" % next
1629            else:
1630                latestCommit = "%s" % next
1631
1632        return ""
1633
1634    def importNewBranch(self, branch, maxChange):
1635        # make fast-import flush all changes to disk and update the refs using the checkpoint
1636        # command so that we can try to find the branch parent in the git history
1637        self.gitStream.write("checkpoint\n\n");
1638        self.gitStream.flush();
1639        branchPrefix = self.depotPaths[0] + branch + "/"
1640        range = "@1,%s" % maxChange
1641        #print "prefix" + branchPrefix
1642        changes = p4ChangesForPaths([branchPrefix], range)
1643        if len(changes) <= 0:
1644            return False
1645        firstChange = changes[0]
1646        #print "first change in branch: %s" % firstChange
1647        sourceBranch = self.knownBranches[branch]
1648        sourceDepotPath = self.depotPaths[0] + sourceBranch
1649        sourceRef = self.gitRefForBranch(sourceBranch)
1650        #print "source " + sourceBranch
1651
1652        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1653        #print "branch parent: %s" % branchParentChange
1654        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1655        if len(gitParent) > 0:
1656            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1657            #print "parent git commit: %s" % gitParent
1658
1659        self.importChanges(changes)
1660        return True
1661
1662    def importChanges(self, changes):
1663        cnt = 1
1664        for change in changes:
1665            description = p4Cmd("describe %s" % change)
1666            self.updateOptionDict(description)
1667
1668            if not self.silent:
1669                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1670                sys.stdout.flush()
1671            cnt = cnt + 1
1672
1673            try:
1674                if self.detectBranches:
1675                    branches = self.splitFilesIntoBranches(description)
1676                    for branch in branches.keys():
1677                        ## HACK  --hwn
1678                        branchPrefix = self.depotPaths[0] + branch + "/"
1679
1680                        parent = ""
1681
1682                        filesForCommit = branches[branch]
1683
1684                        if self.verbose:
1685                            print "branch is %s" % branch
1686
1687                        self.updatedBranches.add(branch)
1688
1689                        if branch not in self.createdBranches:
1690                            self.createdBranches.add(branch)
1691                            parent = self.knownBranches[branch]
1692                            if parent == branch:
1693                                parent = ""
1694                            else:
1695                                fullBranch = self.projectName + branch
1696                                if fullBranch not in self.p4BranchesInGit:
1697                                    if not self.silent:
1698                                        print("\n    Importing new branch %s" % fullBranch);
1699                                    if self.importNewBranch(branch, change - 1):
1700                                        parent = ""
1701                                        self.p4BranchesInGit.append(fullBranch)
1702                                    if not self.silent:
1703                                        print("\n    Resuming with change %s" % change);
1704
1705                                if self.verbose:
1706                                    print "parent determined through known branches: %s" % parent
1707
1708                        branch = self.gitRefForBranch(branch)
1709                        parent = self.gitRefForBranch(parent)
1710
1711                        if self.verbose:
1712                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1713
1714                        if len(parent) == 0 and branch in self.initialParents:
1715                            parent = self.initialParents[branch]
1716                            del self.initialParents[branch]
1717
1718                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1719                else:
1720                    files = self.extractFilesFromCommit(description)
1721                    self.commit(description, files, self.branch, self.depotPaths,
1722                                self.initialParent)
1723                    self.initialParent = ""
1724            except IOError:
1725                print self.gitError.read()
1726                sys.exit(1)
1727
1728    def importHeadRevision(self, revision):
1729        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1730
1731        details = {}
1732        details["user"] = "git perforce import user"
1733        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1734                           % (' '.join(self.depotPaths), revision))
1735        details["change"] = revision
1736        newestRevision = 0
1737
1738        fileCnt = 0
1739        for info in p4CmdList("files "
1740                              +  ' '.join(["%s...%s"
1741                                           % (p, revision)
1742                                           for p in self.depotPaths])):
1743
1744            if 'code' in info and info['code'] == 'error':
1745                sys.stderr.write("p4 returned an error: %s\n"
1746                                 % info['data'])
1747                if info['data'].find("must refer to client") >= 0:
1748                    sys.stderr.write("This particular p4 error is misleading.\n")
1749                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
1750                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1751                sys.exit(1)
1752            if 'p4ExitCode' in info:
1753                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1754                sys.exit(1)
1755
1756
1757            change = int(info["change"])
1758            if change > newestRevision:
1759                newestRevision = change
1760
1761            if info["action"] in self.delete_actions:
1762                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1763                #fileCnt = fileCnt + 1
1764                continue
1765
1766            for prop in ["depotFile", "rev", "action", "type" ]:
1767                details["%s%s" % (prop, fileCnt)] = info[prop]
1768
1769            fileCnt = fileCnt + 1
1770
1771        details["change"] = newestRevision
1772
1773        # Use time from top-most change so that all git-p4 clones of
1774        # the same p4 repo have the same commit SHA1s.
1775        res = p4CmdList("describe -s %d" % newestRevision)
1776        newestTime = None
1777        for r in res:
1778            if r.has_key('time'):
1779                newestTime = int(r['time'])
1780        if newestTime is None:
1781            die("\"describe -s\" on newest change %d did not give a time")
1782        details["time"] = newestTime
1783
1784        self.updateOptionDict(details)
1785        try:
1786            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1787        except IOError:
1788            print "IO error with git fast-import. Is your git version recent enough?"
1789            print self.gitError.read()
1790
1791
1792    def getClientSpec(self):
1793        specList = p4CmdList( "client -o" )
1794        temp = {}
1795        for entry in specList:
1796            for k,v in entry.iteritems():
1797                if k.startswith("View"):
1798
1799                    # p4 has these %%1 to %%9 arguments in specs to
1800                    # reorder paths; which we can't handle (yet :)
1801                    if re.match('%%\d', v) != None:
1802                        print "Sorry, can't handle %%n arguments in client specs"
1803                        sys.exit(1)
1804
1805                    if v.startswith('"'):
1806                        start = 1
1807                    else:
1808                        start = 0
1809                    index = v.find("...")
1810
1811                    # save the "client view"; i.e the RHS of the view
1812                    # line that tells the client where to put the
1813                    # files for this view.
1814                    cv = v[index+3:].strip() # +3 to remove previous '...'
1815
1816                    # if the client view doesn't end with a
1817                    # ... wildcard, then we're going to mess up the
1818                    # output directory, so fail gracefully.
1819                    if not cv.endswith('...'):
1820                        print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1821                        sys.exit(1)
1822                    cv=cv[:-3]
1823
1824                    # now save the view; +index means included, -index
1825                    # means it should be filtered out.
1826                    v = v[start:index]
1827                    if v.startswith("-"):
1828                        v = v[1:]
1829                        include = -len(v)
1830                    else:
1831                        include = len(v)
1832
1833                    temp[v] = (include, cv)
1834
1835        self.clientSpecDirs = temp.items()
1836        self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1837
1838    def run(self, args):
1839        self.depotPaths = []
1840        self.changeRange = ""
1841        self.initialParent = ""
1842        self.previousDepotPaths = []
1843
1844        # map from branch depot path to parent branch
1845        self.knownBranches = {}
1846        self.initialParents = {}
1847        self.hasOrigin = originP4BranchesExist()
1848        if not self.syncWithOrigin:
1849            self.hasOrigin = False
1850
1851        if self.importIntoRemotes:
1852            self.refPrefix = "refs/remotes/p4/"
1853        else:
1854            self.refPrefix = "refs/heads/p4/"
1855
1856        if self.syncWithOrigin and self.hasOrigin:
1857            if not self.silent:
1858                print "Syncing with origin first by calling git fetch origin"
1859            system("git fetch origin")
1860
1861        if len(self.branch) == 0:
1862            self.branch = self.refPrefix + "master"
1863            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1864                system("git update-ref %s refs/heads/p4" % self.branch)
1865                system("git branch -D p4");
1866            # create it /after/ importing, when master exists
1867            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1868                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1869
1870        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1871            self.getClientSpec()
1872
1873        # TODO: should always look at previous commits,
1874        # merge with previous imports, if possible.
1875        if args == []:
1876            if self.hasOrigin:
1877                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1878            self.listExistingP4GitBranches()
1879
1880            if len(self.p4BranchesInGit) > 1:
1881                if not self.silent:
1882                    print "Importing from/into multiple branches"
1883                self.detectBranches = True
1884
1885            if self.verbose:
1886                print "branches: %s" % self.p4BranchesInGit
1887
1888            p4Change = 0
1889            for branch in self.p4BranchesInGit:
1890                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1891
1892                settings = extractSettingsGitLog(logMsg)
1893
1894                self.readOptions(settings)
1895                if (settings.has_key('depot-paths')
1896                    and settings.has_key ('change')):
1897                    change = int(settings['change']) + 1
1898                    p4Change = max(p4Change, change)
1899
1900                    depotPaths = sorted(settings['depot-paths'])
1901                    if self.previousDepotPaths == []:
1902                        self.previousDepotPaths = depotPaths
1903                    else:
1904                        paths = []
1905                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1906                            prev_list = prev.split("/")
1907                            cur_list = cur.split("/")
1908                            for i in range(0, min(len(cur_list), len(prev_list))):
1909                                if cur_list[i] <> prev_list[i]:
1910                                    i = i - 1
1911                                    break
1912
1913                            paths.append ("/".join(cur_list[:i + 1]))
1914
1915                        self.previousDepotPaths = paths
1916
1917            if p4Change > 0:
1918                self.depotPaths = sorted(self.previousDepotPaths)
1919                self.changeRange = "@%s,#head" % p4Change
1920                if not self.detectBranches:
1921                    self.initialParent = parseRevision(self.branch)
1922                if not self.silent and not self.detectBranches:
1923                    print "Performing incremental import into %s git branch" % self.branch
1924
1925        if not self.branch.startswith("refs/"):
1926            self.branch = "refs/heads/" + self.branch
1927
1928        if len(args) == 0 and self.depotPaths:
1929            if not self.silent:
1930                print "Depot paths: %s" % ' '.join(self.depotPaths)
1931        else:
1932            if self.depotPaths and self.depotPaths != args:
1933                print ("previous import used depot path %s and now %s was specified. "
1934                       "This doesn't work!" % (' '.join (self.depotPaths),
1935                                               ' '.join (args)))
1936                sys.exit(1)
1937
1938            self.depotPaths = sorted(args)
1939
1940        revision = ""
1941        self.users = {}
1942
1943        newPaths = []
1944        for p in self.depotPaths:
1945            if p.find("@") != -1:
1946                atIdx = p.index("@")
1947                self.changeRange = p[atIdx:]
1948                if self.changeRange == "@all":
1949                    self.changeRange = ""
1950                elif ',' not in self.changeRange:
1951                    revision = self.changeRange
1952                    self.changeRange = ""
1953                p = p[:atIdx]
1954            elif p.find("#") != -1:
1955                hashIdx = p.index("#")
1956                revision = p[hashIdx:]
1957                p = p[:hashIdx]
1958            elif self.previousDepotPaths == []:
1959                revision = "#head"
1960
1961            p = re.sub ("\.\.\.$", "", p)
1962            if not p.endswith("/"):
1963                p += "/"
1964
1965            newPaths.append(p)
1966
1967        self.depotPaths = newPaths
1968
1969
1970        self.loadUserMapFromCache()
1971        self.labels = {}
1972        if self.detectLabels:
1973            self.getLabels();
1974
1975        if self.detectBranches:
1976            ## FIXME - what's a P4 projectName ?
1977            self.projectName = self.guessProjectName()
1978
1979            if self.hasOrigin:
1980                self.getBranchMappingFromGitBranches()
1981            else:
1982                self.getBranchMapping()
1983            if self.verbose:
1984                print "p4-git branches: %s" % self.p4BranchesInGit
1985                print "initial parents: %s" % self.initialParents
1986            for b in self.p4BranchesInGit:
1987                if b != "master":
1988
1989                    ## FIXME
1990                    b = b[len(self.projectName):]
1991                self.createdBranches.add(b)
1992
1993        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1994
1995        importProcess = subprocess.Popen(["git", "fast-import"],
1996                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1997                                         stderr=subprocess.PIPE);
1998        self.gitOutput = importProcess.stdout
1999        self.gitStream = importProcess.stdin
2000        self.gitError = importProcess.stderr
2001
2002        if revision:
2003            self.importHeadRevision(revision)
2004        else:
2005            changes = []
2006
2007            if len(self.changesFile) > 0:
2008                output = open(self.changesFile).readlines()
2009                changeSet = set()
2010                for line in output:
2011                    changeSet.add(int(line))
2012
2013                for change in changeSet:
2014                    changes.append(change)
2015
2016                changes.sort()
2017            else:
2018                # catch "git-p4 sync" with no new branches, in a repo that
2019                # does not have any existing git-p4 branches
2020                if len(args) == 0 and not self.p4BranchesInGit:
2021                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2022                if self.verbose:
2023                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2024                                                              self.changeRange)
2025                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2026
2027                if len(self.maxChanges) > 0:
2028                    changes = changes[:min(int(self.maxChanges), len(changes))]
2029
2030            if len(changes) == 0:
2031                if not self.silent:
2032                    print "No changes to import!"
2033                return True
2034
2035            if not self.silent and not self.detectBranches:
2036                print "Import destination: %s" % self.branch
2037
2038            self.updatedBranches = set()
2039
2040            self.importChanges(changes)
2041
2042            if not self.silent:
2043                print ""
2044                if len(self.updatedBranches) > 0:
2045                    sys.stdout.write("Updated branches: ")
2046                    for b in self.updatedBranches:
2047                        sys.stdout.write("%s " % b)
2048                    sys.stdout.write("\n")
2049
2050        self.gitStream.close()
2051        if importProcess.wait() != 0:
2052            die("fast-import failed: %s" % self.gitError.read())
2053        self.gitOutput.close()
2054        self.gitError.close()
2055
2056        return True
2057
2058class P4Rebase(Command):
2059    def __init__(self):
2060        Command.__init__(self)
2061        self.options = [ ]
2062        self.description = ("Fetches the latest revision from perforce and "
2063                            + "rebases the current work (branch) against it")
2064        self.verbose = False
2065
2066    def run(self, args):
2067        sync = P4Sync()
2068        sync.run([])
2069
2070        return self.rebase()
2071
2072    def rebase(self):
2073        if os.system("git update-index --refresh") != 0:
2074            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2075        if len(read_pipe("git diff-index HEAD --")) > 0:
2076            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2077
2078        [upstream, settings] = findUpstreamBranchPoint()
2079        if len(upstream) == 0:
2080            die("Cannot find upstream branchpoint for rebase")
2081
2082        # the branchpoint may be p4/foo~3, so strip off the parent
2083        upstream = re.sub("~[0-9]+$", "", upstream)
2084
2085        print "Rebasing the current branch onto %s" % upstream
2086        oldHead = read_pipe("git rev-parse HEAD").strip()
2087        system("git rebase %s" % upstream)
2088        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2089        return True
2090
2091class P4Clone(P4Sync):
2092    def __init__(self):
2093        P4Sync.__init__(self)
2094        self.description = "Creates a new git repository and imports from Perforce into it"
2095        self.usage = "usage: %prog [options] //depot/path[@revRange]"
2096        self.options += [
2097            optparse.make_option("--destination", dest="cloneDestination",
2098                                 action='store', default=None,
2099                                 help="where to leave result of the clone"),
2100            optparse.make_option("-/", dest="cloneExclude",
2101                                 action="append", type="string",
2102                                 help="exclude depot path"),
2103            optparse.make_option("--bare", dest="cloneBare",
2104                                 action="store_true", default=False),
2105        ]
2106        self.cloneDestination = None
2107        self.needsGit = False
2108        self.cloneBare = False
2109
2110    # This is required for the "append" cloneExclude action
2111    def ensure_value(self, attr, value):
2112        if not hasattr(self, attr) or getattr(self, attr) is None:
2113            setattr(self, attr, value)
2114        return getattr(self, attr)
2115
2116    def defaultDestination(self, args):
2117        ## TODO: use common prefix of args?
2118        depotPath = args[0]
2119        depotDir = re.sub("(@[^@]*)$", "", depotPath)
2120        depotDir = re.sub("(#[^#]*)$", "", depotDir)
2121        depotDir = re.sub(r"\.\.\.$", "", depotDir)
2122        depotDir = re.sub(r"/$", "", depotDir)
2123        return os.path.split(depotDir)[1]
2124
2125    def run(self, args):
2126        if len(args) < 1:
2127            return False
2128
2129        if self.keepRepoPath and not self.cloneDestination:
2130            sys.stderr.write("Must specify destination for --keep-path\n")
2131            sys.exit(1)
2132
2133        depotPaths = args
2134
2135        if not self.cloneDestination and len(depotPaths) > 1:
2136            self.cloneDestination = depotPaths[-1]
2137            depotPaths = depotPaths[:-1]
2138
2139        self.cloneExclude = ["/"+p for p in self.cloneExclude]
2140        for p in depotPaths:
2141            if not p.startswith("//"):
2142                return False
2143
2144        if not self.cloneDestination:
2145            self.cloneDestination = self.defaultDestination(args)
2146
2147        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2148
2149        if not os.path.exists(self.cloneDestination):
2150            os.makedirs(self.cloneDestination)
2151        chdir(self.cloneDestination)
2152
2153        init_cmd = [ "git", "init" ]
2154        if self.cloneBare:
2155            init_cmd.append("--bare")
2156        subprocess.check_call(init_cmd)
2157
2158        if not P4Sync.run(self, depotPaths):
2159            return False
2160        if self.branch != "master":
2161            if self.importIntoRemotes:
2162                masterbranch = "refs/remotes/p4/master"
2163            else:
2164                masterbranch = "refs/heads/p4/master"
2165            if gitBranchExists(masterbranch):
2166                system("git branch master %s" % masterbranch)
2167                if not self.cloneBare:
2168                    system("git checkout -f")
2169            else:
2170                print "Could not detect main branch. No checkout/master branch created."
2171
2172        return True
2173
2174class P4Branches(Command):
2175    def __init__(self):
2176        Command.__init__(self)
2177        self.options = [ ]
2178        self.description = ("Shows the git branches that hold imports and their "
2179                            + "corresponding perforce depot paths")
2180        self.verbose = False
2181
2182    def run(self, args):
2183        if originP4BranchesExist():
2184            createOrUpdateBranchesFromOrigin()
2185
2186        cmdline = "git rev-parse --symbolic "
2187        cmdline += " --remotes"
2188
2189        for line in read_pipe_lines(cmdline):
2190            line = line.strip()
2191
2192            if not line.startswith('p4/') or line == "p4/HEAD":
2193                continue
2194            branch = line
2195
2196            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2197            settings = extractSettingsGitLog(log)
2198
2199            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2200        return True
2201
2202class HelpFormatter(optparse.IndentedHelpFormatter):
2203    def __init__(self):
2204        optparse.IndentedHelpFormatter.__init__(self)
2205
2206    def format_description(self, description):
2207        if description:
2208            return description + "\n"
2209        else:
2210            return ""
2211
2212def printUsage(commands):
2213    print "usage: %s <command> [options]" % sys.argv[0]
2214    print ""
2215    print "valid commands: %s" % ", ".join(commands)
2216    print ""
2217    print "Try %s <command> --help for command specific help." % sys.argv[0]
2218    print ""
2219
2220commands = {
2221    "debug" : P4Debug,
2222    "submit" : P4Submit,
2223    "commit" : P4Submit,
2224    "sync" : P4Sync,
2225    "rebase" : P4Rebase,
2226    "clone" : P4Clone,
2227    "rollback" : P4RollBack,
2228    "branches" : P4Branches
2229}
2230
2231
2232def main():
2233    if len(sys.argv[1:]) == 0:
2234        printUsage(commands.keys())
2235        sys.exit(2)
2236
2237    cmd = ""
2238    cmdName = sys.argv[1]
2239    try:
2240        klass = commands[cmdName]
2241        cmd = klass()
2242    except KeyError:
2243        print "unknown command %s" % cmdName
2244        print ""
2245        printUsage(commands.keys())
2246        sys.exit(2)
2247
2248    options = cmd.options
2249    cmd.gitdir = os.environ.get("GIT_DIR", None)
2250
2251    args = sys.argv[2:]
2252
2253    if len(options) > 0:
2254        options.append(optparse.make_option("--git-dir", dest="gitdir"))
2255
2256        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2257                                       options,
2258                                       description = cmd.description,
2259                                       formatter = HelpFormatter())
2260
2261        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2262    global verbose
2263    verbose = cmd.verbose
2264    if cmd.needsGit:
2265        if cmd.gitdir == None:
2266            cmd.gitdir = os.path.abspath(".git")
2267            if not isValidGitDir(cmd.gitdir):
2268                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2269                if os.path.exists(cmd.gitdir):
2270                    cdup = read_pipe("git rev-parse --show-cdup").strip()
2271                    if len(cdup) > 0:
2272                        chdir(cdup);
2273
2274        if not isValidGitDir(cmd.gitdir):
2275            if isValidGitDir(cmd.gitdir + "/.git"):
2276                cmd.gitdir += "/.git"
2277            else:
2278                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2279
2280        os.environ["GIT_DIR"] = cmd.gitdir
2281
2282    if not cmd.run(args):
2283        parser.print_help()
2284
2285
2286if __name__ == '__main__':
2287    main()