Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'bc/maint-keep-pack' (c9d8563)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    output = p4Cmd("where %s..." % depotPath)
 249    if output["code"] == "error":
 250        return ""
 251    clientPath = ""
 252    if "path" in output:
 253        clientPath = output.get("path")
 254    elif "data" in output:
 255        data = output.get("data")
 256        lastSpace = data.rfind(" ")
 257        clientPath = data[lastSpace + 1:]
 258
 259    if clientPath.endswith("..."):
 260        clientPath = clientPath[:-3]
 261    return clientPath
 262
 263def currentGitBranch():
 264    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266def isValidGitDir(path):
 267    if (os.path.exists(path + "/HEAD")
 268        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269        return True;
 270    return False
 271
 272def parseRevision(ref):
 273    return read_pipe("git rev-parse %s" % ref).strip()
 274
 275def extractLogMessageFromGitCommit(commit):
 276    logMessage = ""
 277
 278    ## fixme: title is first line of commit, not 1st paragraph.
 279    foundTitle = False
 280    for log in read_pipe_lines("git cat-file commit %s" % commit):
 281       if not foundTitle:
 282           if len(log) == 1:
 283               foundTitle = True
 284           continue
 285
 286       logMessage += log
 287    return logMessage
 288
 289def extractSettingsGitLog(log):
 290    values = {}
 291    for line in log.split("\n"):
 292        line = line.strip()
 293        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294        if not m:
 295            continue
 296
 297        assignments = m.group(1).split (':')
 298        for a in assignments:
 299            vals = a.split ('=')
 300            key = vals[0].strip()
 301            val = ('='.join (vals[1:])).strip()
 302            if val.endswith ('\"') and val.startswith('"'):
 303                val = val[1:-1]
 304
 305            values[key] = val
 306
 307    paths = values.get("depot-paths")
 308    if not paths:
 309        paths = values.get("depot-path")
 310    if paths:
 311        values['depot-paths'] = paths.split(',')
 312    return values
 313
 314def gitBranchExists(branch):
 315    proc = subprocess.Popen(["git", "rev-parse", branch],
 316                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317    return proc.wait() == 0;
 318
 319_gitConfig = {}
 320def gitConfig(key):
 321    if not _gitConfig.has_key(key):
 322        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 323    return _gitConfig[key]
 324
 325def p4BranchesInGit(branchesAreInRemotes = True):
 326    branches = {}
 327
 328    cmdline = "git rev-parse --symbolic "
 329    if branchesAreInRemotes:
 330        cmdline += " --remotes"
 331    else:
 332        cmdline += " --branches"
 333
 334    for line in read_pipe_lines(cmdline):
 335        line = line.strip()
 336
 337        ## only import to p4/
 338        if not line.startswith('p4/') or line == "p4/HEAD":
 339            continue
 340        branch = line
 341
 342        # strip off p4
 343        branch = re.sub ("^p4/", "", line)
 344
 345        branches[branch] = parseRevision(line)
 346    return branches
 347
 348def findUpstreamBranchPoint(head = "HEAD"):
 349    branches = p4BranchesInGit()
 350    # map from depot-path to branch name
 351    branchByDepotPath = {}
 352    for branch in branches.keys():
 353        tip = branches[branch]
 354        log = extractLogMessageFromGitCommit(tip)
 355        settings = extractSettingsGitLog(log)
 356        if settings.has_key("depot-paths"):
 357            paths = ",".join(settings["depot-paths"])
 358            branchByDepotPath[paths] = "remotes/p4/" + branch
 359
 360    settings = None
 361    parent = 0
 362    while parent < 65535:
 363        commit = head + "~%s" % parent
 364        log = extractLogMessageFromGitCommit(commit)
 365        settings = extractSettingsGitLog(log)
 366        if settings.has_key("depot-paths"):
 367            paths = ",".join(settings["depot-paths"])
 368            if branchByDepotPath.has_key(paths):
 369                return [branchByDepotPath[paths], settings]
 370
 371        parent = parent + 1
 372
 373    return ["", settings]
 374
 375def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 376    if not silent:
 377        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 378               % localRefPrefix)
 379
 380    originPrefix = "origin/p4/"
 381
 382    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 383        line = line.strip()
 384        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 385            continue
 386
 387        headName = line[len(originPrefix):]
 388        remoteHead = localRefPrefix + headName
 389        originHead = line
 390
 391        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 392        if (not original.has_key('depot-paths')
 393            or not original.has_key('change')):
 394            continue
 395
 396        update = False
 397        if not gitBranchExists(remoteHead):
 398            if verbose:
 399                print "creating %s" % remoteHead
 400            update = True
 401        else:
 402            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 403            if settings.has_key('change') > 0:
 404                if settings['depot-paths'] == original['depot-paths']:
 405                    originP4Change = int(original['change'])
 406                    p4Change = int(settings['change'])
 407                    if originP4Change > p4Change:
 408                        print ("%s (%s) is newer than %s (%s). "
 409                               "Updating p4 branch from origin."
 410                               % (originHead, originP4Change,
 411                                  remoteHead, p4Change))
 412                        update = True
 413                else:
 414                    print ("Ignoring: %s was imported from %s while "
 415                           "%s was imported from %s"
 416                           % (originHead, ','.join(original['depot-paths']),
 417                              remoteHead, ','.join(settings['depot-paths'])))
 418
 419        if update:
 420            system("git update-ref %s %s" % (remoteHead, originHead))
 421
 422def originP4BranchesExist():
 423        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 424
 425def p4ChangesForPaths(depotPaths, changeRange):
 426    assert depotPaths
 427    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 428                                                        for p in depotPaths]))
 429
 430    changes = []
 431    for line in output:
 432        changeNum = line.split(" ")[1]
 433        changes.append(int(changeNum))
 434
 435    changes.sort()
 436    return changes
 437
 438class Command:
 439    def __init__(self):
 440        self.usage = "usage: %prog [options]"
 441        self.needsGit = True
 442
 443class P4Debug(Command):
 444    def __init__(self):
 445        Command.__init__(self)
 446        self.options = [
 447            optparse.make_option("--verbose", dest="verbose", action="store_true",
 448                                 default=False),
 449            ]
 450        self.description = "A tool to debug the output of p4 -G."
 451        self.needsGit = False
 452        self.verbose = False
 453
 454    def run(self, args):
 455        j = 0
 456        for output in p4CmdList(" ".join(args)):
 457            print 'Element: %d' % j
 458            j += 1
 459            print output
 460        return True
 461
 462class P4RollBack(Command):
 463    def __init__(self):
 464        Command.__init__(self)
 465        self.options = [
 466            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 467            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 468        ]
 469        self.description = "A tool to debug the multi-branch import. Don't use :)"
 470        self.verbose = False
 471        self.rollbackLocalBranches = False
 472
 473    def run(self, args):
 474        if len(args) != 1:
 475            return False
 476        maxChange = int(args[0])
 477
 478        if "p4ExitCode" in p4Cmd("changes -m 1"):
 479            die("Problems executing p4");
 480
 481        if self.rollbackLocalBranches:
 482            refPrefix = "refs/heads/"
 483            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 484        else:
 485            refPrefix = "refs/remotes/"
 486            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 487
 488        for line in lines:
 489            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 490                line = line.strip()
 491                ref = refPrefix + line
 492                log = extractLogMessageFromGitCommit(ref)
 493                settings = extractSettingsGitLog(log)
 494
 495                depotPaths = settings['depot-paths']
 496                change = settings['change']
 497
 498                changed = False
 499
 500                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 501                                                           for p in depotPaths]))) == 0:
 502                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 503                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 504                    continue
 505
 506                while change and int(change) > maxChange:
 507                    changed = True
 508                    if self.verbose:
 509                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 510                    system("git update-ref %s \"%s^\"" % (ref, ref))
 511                    log = extractLogMessageFromGitCommit(ref)
 512                    settings =  extractSettingsGitLog(log)
 513
 514
 515                    depotPaths = settings['depot-paths']
 516                    change = settings['change']
 517
 518                if changed:
 519                    print "%s rewound to %s" % (ref, change)
 520
 521        return True
 522
 523class P4Submit(Command):
 524    def __init__(self):
 525        Command.__init__(self)
 526        self.options = [
 527                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 528                optparse.make_option("--origin", dest="origin"),
 529                optparse.make_option("-M", dest="detectRename", action="store_true"),
 530        ]
 531        self.description = "Submit changes from git to the perforce depot."
 532        self.usage += " [name of git branch to submit into perforce depot]"
 533        self.interactive = True
 534        self.origin = ""
 535        self.detectRename = False
 536        self.verbose = False
 537        self.isWindows = (platform.system() == "Windows")
 538
 539    def check(self):
 540        if len(p4CmdList("opened ...")) > 0:
 541            die("You have files opened with perforce! Close them before starting the sync.")
 542
 543    # replaces everything between 'Description:' and the next P4 submit template field with the
 544    # commit message
 545    def prepareLogMessage(self, template, message):
 546        result = ""
 547
 548        inDescriptionSection = False
 549
 550        for line in template.split("\n"):
 551            if line.startswith("#"):
 552                result += line + "\n"
 553                continue
 554
 555            if inDescriptionSection:
 556                if line.startswith("Files:"):
 557                    inDescriptionSection = False
 558                else:
 559                    continue
 560            else:
 561                if line.startswith("Description:"):
 562                    inDescriptionSection = True
 563                    line += "\n"
 564                    for messageLine in message.split("\n"):
 565                        line += "\t" + messageLine + "\n"
 566
 567            result += line + "\n"
 568
 569        return result
 570
 571    def prepareSubmitTemplate(self):
 572        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 573        template = ""
 574        inFilesSection = False
 575        for line in p4_read_pipe_lines("change -o"):
 576            if line.endswith("\r\n"):
 577                line = line[:-2] + "\n"
 578            if inFilesSection:
 579                if line.startswith("\t"):
 580                    # path starts and ends with a tab
 581                    path = line[1:]
 582                    lastTab = path.rfind("\t")
 583                    if lastTab != -1:
 584                        path = path[:lastTab]
 585                        if not path.startswith(self.depotPath):
 586                            continue
 587                else:
 588                    inFilesSection = False
 589            else:
 590                if line.startswith("Files:"):
 591                    inFilesSection = True
 592
 593            template += line
 594
 595        return template
 596
 597    def applyCommit(self, id):
 598        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 599        diffOpts = ("", "-M")[self.detectRename]
 600        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 601        filesToAdd = set()
 602        filesToDelete = set()
 603        editedFiles = set()
 604        filesToChangeExecBit = {}
 605        for line in diff:
 606            diff = parseDiffTreeEntry(line)
 607            modifier = diff['status']
 608            path = diff['src']
 609            if modifier == "M":
 610                p4_system("edit \"%s\"" % path)
 611                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 612                    filesToChangeExecBit[path] = diff['dst_mode']
 613                editedFiles.add(path)
 614            elif modifier == "A":
 615                filesToAdd.add(path)
 616                filesToChangeExecBit[path] = diff['dst_mode']
 617                if path in filesToDelete:
 618                    filesToDelete.remove(path)
 619            elif modifier == "D":
 620                filesToDelete.add(path)
 621                if path in filesToAdd:
 622                    filesToAdd.remove(path)
 623            elif modifier == "R":
 624                src, dest = diff['src'], diff['dst']
 625                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 626                p4_system("edit \"%s\"" % (dest))
 627                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 628                    filesToChangeExecBit[dest] = diff['dst_mode']
 629                os.unlink(dest)
 630                editedFiles.add(dest)
 631                filesToDelete.add(src)
 632            else:
 633                die("unknown modifier %s for %s" % (modifier, path))
 634
 635        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 636        patchcmd = diffcmd + " | git apply "
 637        tryPatchCmd = patchcmd + "--check -"
 638        applyPatchCmd = patchcmd + "--check --apply -"
 639
 640        if os.system(tryPatchCmd) != 0:
 641            print "Unfortunately applying the change failed!"
 642            print "What do you want to do?"
 643            response = "x"
 644            while response != "s" and response != "a" and response != "w":
 645                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 646                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 647            if response == "s":
 648                print "Skipping! Good luck with the next patches..."
 649                for f in editedFiles:
 650                    p4_system("revert \"%s\"" % f);
 651                for f in filesToAdd:
 652                    system("rm %s" %f)
 653                return
 654            elif response == "a":
 655                os.system(applyPatchCmd)
 656                if len(filesToAdd) > 0:
 657                    print "You may also want to call p4 add on the following files:"
 658                    print " ".join(filesToAdd)
 659                if len(filesToDelete):
 660                    print "The following files should be scheduled for deletion with p4 delete:"
 661                    print " ".join(filesToDelete)
 662                die("Please resolve and submit the conflict manually and "
 663                    + "continue afterwards with git-p4 submit --continue")
 664            elif response == "w":
 665                system(diffcmd + " > patch.txt")
 666                print "Patch saved to patch.txt in %s !" % self.clientPath
 667                die("Please resolve and submit the conflict manually and "
 668                    "continue afterwards with git-p4 submit --continue")
 669
 670        system(applyPatchCmd)
 671
 672        for f in filesToAdd:
 673            p4_system("add \"%s\"" % f)
 674        for f in filesToDelete:
 675            p4_system("revert \"%s\"" % f)
 676            p4_system("delete \"%s\"" % f)
 677
 678        # Set/clear executable bits
 679        for f in filesToChangeExecBit.keys():
 680            mode = filesToChangeExecBit[f]
 681            setP4ExecBit(f, mode)
 682
 683        logMessage = extractLogMessageFromGitCommit(id)
 684        logMessage = logMessage.strip()
 685
 686        template = self.prepareSubmitTemplate()
 687
 688        if self.interactive:
 689            submitTemplate = self.prepareLogMessage(template, logMessage)
 690            if os.environ.has_key("P4DIFF"):
 691                del(os.environ["P4DIFF"])
 692            diff = p4_read_pipe("diff -du ...")
 693
 694            newdiff = ""
 695            for newFile in filesToAdd:
 696                newdiff += "==== new file ====\n"
 697                newdiff += "--- /dev/null\n"
 698                newdiff += "+++ %s\n" % newFile
 699                f = open(newFile, "r")
 700                for line in f.readlines():
 701                    newdiff += "+" + line
 702                f.close()
 703
 704            separatorLine = "######## everything below this line is just the diff #######\n"
 705
 706            [handle, fileName] = tempfile.mkstemp()
 707            tmpFile = os.fdopen(handle, "w+")
 708            if self.isWindows:
 709                submitTemplate = submitTemplate.replace("\n", "\r\n")
 710                separatorLine = separatorLine.replace("\n", "\r\n")
 711                newdiff = newdiff.replace("\n", "\r\n")
 712            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 713            tmpFile.close()
 714            mtime = os.stat(fileName).st_mtime
 715            defaultEditor = "vi"
 716            if platform.system() == "Windows":
 717                defaultEditor = "notepad"
 718            if os.environ.has_key("P4EDITOR"):
 719                editor = os.environ.get("P4EDITOR")
 720            else:
 721                editor = os.environ.get("EDITOR", defaultEditor);
 722            system(editor + " " + fileName)
 723
 724            response = "y"
 725            if os.stat(fileName).st_mtime <= mtime:
 726                response = "x"
 727                while response != "y" and response != "n":
 728                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 729
 730            if response == "y":
 731                tmpFile = open(fileName, "rb")
 732                message = tmpFile.read()
 733                tmpFile.close()
 734                submitTemplate = message[:message.index(separatorLine)]
 735                if self.isWindows:
 736                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 737                p4_write_pipe("submit -i", submitTemplate)
 738            else:
 739                for f in editedFiles:
 740                    p4_system("revert \"%s\"" % f);
 741                for f in filesToAdd:
 742                    p4_system("revert \"%s\"" % f);
 743                    system("rm %s" %f)
 744
 745            os.remove(fileName)
 746        else:
 747            fileName = "submit.txt"
 748            file = open(fileName, "w+")
 749            file.write(self.prepareLogMessage(template, logMessage))
 750            file.close()
 751            print ("Perforce submit template written as %s. "
 752                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 753                   % (fileName, fileName))
 754
 755    def run(self, args):
 756        if len(args) == 0:
 757            self.master = currentGitBranch()
 758            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 759                die("Detecting current git branch failed!")
 760        elif len(args) == 1:
 761            self.master = args[0]
 762        else:
 763            return False
 764
 765        allowSubmit = gitConfig("git-p4.allowSubmit")
 766        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 767            die("%s is not in git-p4.allowSubmit" % self.master)
 768
 769        [upstream, settings] = findUpstreamBranchPoint()
 770        self.depotPath = settings['depot-paths'][0]
 771        if len(self.origin) == 0:
 772            self.origin = upstream
 773
 774        if self.verbose:
 775            print "Origin branch is " + self.origin
 776
 777        if len(self.depotPath) == 0:
 778            print "Internal error: cannot locate perforce depot path from existing branches"
 779            sys.exit(128)
 780
 781        self.clientPath = p4Where(self.depotPath)
 782
 783        if len(self.clientPath) == 0:
 784            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 785            sys.exit(128)
 786
 787        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 788        self.oldWorkingDirectory = os.getcwd()
 789
 790        chdir(self.clientPath)
 791        print "Syncronizing p4 checkout..."
 792        p4_system("sync ...")
 793
 794        self.check()
 795
 796        commits = []
 797        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 798            commits.append(line.strip())
 799        commits.reverse()
 800
 801        while len(commits) > 0:
 802            commit = commits[0]
 803            commits = commits[1:]
 804            self.applyCommit(commit)
 805            if not self.interactive:
 806                break
 807
 808        if len(commits) == 0:
 809            print "All changes applied!"
 810            chdir(self.oldWorkingDirectory)
 811
 812            sync = P4Sync()
 813            sync.run([])
 814
 815            rebase = P4Rebase()
 816            rebase.rebase()
 817
 818        return True
 819
 820class P4Sync(Command):
 821    def __init__(self):
 822        Command.__init__(self)
 823        self.options = [
 824                optparse.make_option("--branch", dest="branch"),
 825                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 826                optparse.make_option("--changesfile", dest="changesFile"),
 827                optparse.make_option("--silent", dest="silent", action="store_true"),
 828                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 829                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 830                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 831                                     help="Import into refs/heads/ , not refs/remotes"),
 832                optparse.make_option("--max-changes", dest="maxChanges"),
 833                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 834                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 835                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 836                                     help="Only sync files that are included in the Perforce Client Spec")
 837        ]
 838        self.description = """Imports from Perforce into a git repository.\n
 839    example:
 840    //depot/my/project/ -- to import the current head
 841    //depot/my/project/@all -- to import everything
 842    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 843
 844    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 845
 846        self.usage += " //depot/path[@revRange]"
 847        self.silent = False
 848        self.createdBranches = Set()
 849        self.committedChanges = Set()
 850        self.branch = ""
 851        self.detectBranches = False
 852        self.detectLabels = False
 853        self.changesFile = ""
 854        self.syncWithOrigin = True
 855        self.verbose = False
 856        self.importIntoRemotes = True
 857        self.maxChanges = ""
 858        self.isWindows = (platform.system() == "Windows")
 859        self.keepRepoPath = False
 860        self.depotPaths = None
 861        self.p4BranchesInGit = []
 862        self.cloneExclude = []
 863        self.useClientSpec = False
 864        self.clientSpecDirs = []
 865
 866        if gitConfig("git-p4.syncFromOrigin") == "false":
 867            self.syncWithOrigin = False
 868
 869    def extractFilesFromCommit(self, commit):
 870        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 871                             for path in self.cloneExclude]
 872        files = []
 873        fnum = 0
 874        while commit.has_key("depotFile%s" % fnum):
 875            path =  commit["depotFile%s" % fnum]
 876
 877            if [p for p in self.cloneExclude
 878                if path.startswith (p)]:
 879                found = False
 880            else:
 881                found = [p for p in self.depotPaths
 882                         if path.startswith (p)]
 883            if not found:
 884                fnum = fnum + 1
 885                continue
 886
 887            file = {}
 888            file["path"] = path
 889            file["rev"] = commit["rev%s" % fnum]
 890            file["action"] = commit["action%s" % fnum]
 891            file["type"] = commit["type%s" % fnum]
 892            files.append(file)
 893            fnum = fnum + 1
 894        return files
 895
 896    def stripRepoPath(self, path, prefixes):
 897        if self.keepRepoPath:
 898            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 899
 900        for p in prefixes:
 901            if path.startswith(p):
 902                path = path[len(p):]
 903
 904        return path
 905
 906    def splitFilesIntoBranches(self, commit):
 907        branches = {}
 908        fnum = 0
 909        while commit.has_key("depotFile%s" % fnum):
 910            path =  commit["depotFile%s" % fnum]
 911            found = [p for p in self.depotPaths
 912                     if path.startswith (p)]
 913            if not found:
 914                fnum = fnum + 1
 915                continue
 916
 917            file = {}
 918            file["path"] = path
 919            file["rev"] = commit["rev%s" % fnum]
 920            file["action"] = commit["action%s" % fnum]
 921            file["type"] = commit["type%s" % fnum]
 922            fnum = fnum + 1
 923
 924            relPath = self.stripRepoPath(path, self.depotPaths)
 925
 926            for branch in self.knownBranches.keys():
 927
 928                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 929                if relPath.startswith(branch + "/"):
 930                    if branch not in branches:
 931                        branches[branch] = []
 932                    branches[branch].append(file)
 933                    break
 934
 935        return branches
 936
 937    ## Should move this out, doesn't use SELF.
 938    def readP4Files(self, files):
 939        filesForCommit = []
 940        filesToRead = []
 941
 942        for f in files:
 943            includeFile = True
 944            for val in self.clientSpecDirs:
 945                if f['path'].startswith(val[0]):
 946                    if val[1] <= 0:
 947                        includeFile = False
 948                    break
 949
 950            if includeFile:
 951                filesForCommit.append(f)
 952                if f['action'] not in ('delete', 'purge'):
 953                    filesToRead.append(f)
 954
 955        filedata = []
 956        if len(filesToRead) > 0:
 957            filedata = p4CmdList('-x - print',
 958                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 959                                                  for f in filesToRead]),
 960                                 stdin_mode='w+')
 961
 962            if "p4ExitCode" in filedata[0]:
 963                die("Problems executing p4. Error: [%d]."
 964                    % (filedata[0]['p4ExitCode']));
 965
 966        j = 0;
 967        contents = {}
 968        while j < len(filedata):
 969            stat = filedata[j]
 970            j += 1
 971            text = ''
 972            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 973                text += filedata[j]['data']
 974                del filedata[j]['data']
 975                j += 1
 976
 977            if not stat.has_key('depotFile'):
 978                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 979                continue
 980
 981            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 982                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 983            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 984                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 985
 986            contents[stat['depotFile']] = text
 987
 988        for f in filesForCommit:
 989            path = f['path']
 990            if contents.has_key(path):
 991                f['data'] = contents[path]
 992
 993        return filesForCommit
 994
 995    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 996        epoch = details["time"]
 997        author = details["user"]
 998
 999        if self.verbose:
1000            print "commit into %s" % branch
1001
1002        # start with reading files; if that fails, we should not
1003        # create a commit.
1004        new_files = []
1005        for f in files:
1006            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1007                new_files.append (f)
1008            else:
1009                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1010        files = self.readP4Files(new_files)
1011
1012        self.gitStream.write("commit %s\n" % branch)
1013#        gitStream.write("mark :%s\n" % details["change"])
1014        self.committedChanges.add(int(details["change"]))
1015        committer = ""
1016        if author not in self.users:
1017            self.getUserMapFromPerforceServer()
1018        if author in self.users:
1019            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1020        else:
1021            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1022
1023        self.gitStream.write("committer %s\n" % committer)
1024
1025        self.gitStream.write("data <<EOT\n")
1026        self.gitStream.write(details["desc"])
1027        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1028                             % (','.join (branchPrefixes), details["change"]))
1029        if len(details['options']) > 0:
1030            self.gitStream.write(": options = %s" % details['options'])
1031        self.gitStream.write("]\nEOT\n\n")
1032
1033        if len(parent) > 0:
1034            if self.verbose:
1035                print "parent %s" % parent
1036            self.gitStream.write("from %s\n" % parent)
1037
1038        for file in files:
1039            if file["type"] == "apple":
1040                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1041                continue
1042
1043            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1044            if file["action"] in ("delete", "purge"):
1045                self.gitStream.write("D %s\n" % relPath)
1046            else:
1047                data = file['data']
1048
1049                mode = "644"
1050                if isP4Exec(file["type"]):
1051                    mode = "755"
1052                elif file["type"] == "symlink":
1053                    mode = "120000"
1054                    # p4 print on a symlink contains "target\n", so strip it off
1055                    data = data[:-1]
1056
1057                if self.isWindows and file["type"].endswith("text"):
1058                    data = data.replace("\r\n", "\n")
1059
1060                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1061                self.gitStream.write("data %s\n" % len(data))
1062                self.gitStream.write(data)
1063                self.gitStream.write("\n")
1064
1065        self.gitStream.write("\n")
1066
1067        change = int(details["change"])
1068
1069        if self.labels.has_key(change):
1070            label = self.labels[change]
1071            labelDetails = label[0]
1072            labelRevisions = label[1]
1073            if self.verbose:
1074                print "Change %s is labelled %s" % (change, labelDetails)
1075
1076            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1077                                                    for p in branchPrefixes]))
1078
1079            if len(files) == len(labelRevisions):
1080
1081                cleanedFiles = {}
1082                for info in files:
1083                    if info["action"] in ("delete", "purge"):
1084                        continue
1085                    cleanedFiles[info["depotFile"]] = info["rev"]
1086
1087                if cleanedFiles == labelRevisions:
1088                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1089                    self.gitStream.write("from %s\n" % branch)
1090
1091                    owner = labelDetails["Owner"]
1092                    tagger = ""
1093                    if author in self.users:
1094                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1095                    else:
1096                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1097                    self.gitStream.write("tagger %s\n" % tagger)
1098                    self.gitStream.write("data <<EOT\n")
1099                    self.gitStream.write(labelDetails["Description"])
1100                    self.gitStream.write("EOT\n\n")
1101
1102                else:
1103                    if not self.silent:
1104                        print ("Tag %s does not match with change %s: files do not match."
1105                               % (labelDetails["label"], change))
1106
1107            else:
1108                if not self.silent:
1109                    print ("Tag %s does not match with change %s: file count is different."
1110                           % (labelDetails["label"], change))
1111
1112    def getUserCacheFilename(self):
1113        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1114        return home + "/.gitp4-usercache.txt"
1115
1116    def getUserMapFromPerforceServer(self):
1117        if self.userMapFromPerforceServer:
1118            return
1119        self.users = {}
1120
1121        for output in p4CmdList("users"):
1122            if not output.has_key("User"):
1123                continue
1124            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1125
1126
1127        s = ''
1128        for (key, val) in self.users.items():
1129            s += "%s\t%s\n" % (key, val)
1130
1131        open(self.getUserCacheFilename(), "wb").write(s)
1132        self.userMapFromPerforceServer = True
1133
1134    def loadUserMapFromCache(self):
1135        self.users = {}
1136        self.userMapFromPerforceServer = False
1137        try:
1138            cache = open(self.getUserCacheFilename(), "rb")
1139            lines = cache.readlines()
1140            cache.close()
1141            for line in lines:
1142                entry = line.strip().split("\t")
1143                self.users[entry[0]] = entry[1]
1144        except IOError:
1145            self.getUserMapFromPerforceServer()
1146
1147    def getLabels(self):
1148        self.labels = {}
1149
1150        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1151        if len(l) > 0 and not self.silent:
1152            print "Finding files belonging to labels in %s" % `self.depotPaths`
1153
1154        for output in l:
1155            label = output["label"]
1156            revisions = {}
1157            newestChange = 0
1158            if self.verbose:
1159                print "Querying files for label %s" % label
1160            for file in p4CmdList("files "
1161                                  +  ' '.join (["%s...@%s" % (p, label)
1162                                                for p in self.depotPaths])):
1163                revisions[file["depotFile"]] = file["rev"]
1164                change = int(file["change"])
1165                if change > newestChange:
1166                    newestChange = change
1167
1168            self.labels[newestChange] = [output, revisions]
1169
1170        if self.verbose:
1171            print "Label changes: %s" % self.labels.keys()
1172
1173    def guessProjectName(self):
1174        for p in self.depotPaths:
1175            if p.endswith("/"):
1176                p = p[:-1]
1177            p = p[p.strip().rfind("/") + 1:]
1178            if not p.endswith("/"):
1179               p += "/"
1180            return p
1181
1182    def getBranchMapping(self):
1183        lostAndFoundBranches = set()
1184
1185        for info in p4CmdList("branches"):
1186            details = p4Cmd("branch -o %s" % info["branch"])
1187            viewIdx = 0
1188            while details.has_key("View%s" % viewIdx):
1189                paths = details["View%s" % viewIdx].split(" ")
1190                viewIdx = viewIdx + 1
1191                # require standard //depot/foo/... //depot/bar/... mapping
1192                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1193                    continue
1194                source = paths[0]
1195                destination = paths[1]
1196                ## HACK
1197                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1198                    source = source[len(self.depotPaths[0]):-4]
1199                    destination = destination[len(self.depotPaths[0]):-4]
1200
1201                    if destination in self.knownBranches:
1202                        if not self.silent:
1203                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1204                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1205                        continue
1206
1207                    self.knownBranches[destination] = source
1208
1209                    lostAndFoundBranches.discard(destination)
1210
1211                    if source not in self.knownBranches:
1212                        lostAndFoundBranches.add(source)
1213
1214
1215        for branch in lostAndFoundBranches:
1216            self.knownBranches[branch] = branch
1217
1218    def getBranchMappingFromGitBranches(self):
1219        branches = p4BranchesInGit(self.importIntoRemotes)
1220        for branch in branches.keys():
1221            if branch == "master":
1222                branch = "main"
1223            else:
1224                branch = branch[len(self.projectName):]
1225            self.knownBranches[branch] = branch
1226
1227    def listExistingP4GitBranches(self):
1228        # branches holds mapping from name to commit
1229        branches = p4BranchesInGit(self.importIntoRemotes)
1230        self.p4BranchesInGit = branches.keys()
1231        for branch in branches.keys():
1232            self.initialParents[self.refPrefix + branch] = branches[branch]
1233
1234    def updateOptionDict(self, d):
1235        option_keys = {}
1236        if self.keepRepoPath:
1237            option_keys['keepRepoPath'] = 1
1238
1239        d["options"] = ' '.join(sorted(option_keys.keys()))
1240
1241    def readOptions(self, d):
1242        self.keepRepoPath = (d.has_key('options')
1243                             and ('keepRepoPath' in d['options']))
1244
1245    def gitRefForBranch(self, branch):
1246        if branch == "main":
1247            return self.refPrefix + "master"
1248
1249        if len(branch) <= 0:
1250            return branch
1251
1252        return self.refPrefix + self.projectName + branch
1253
1254    def gitCommitByP4Change(self, ref, change):
1255        if self.verbose:
1256            print "looking in ref " + ref + " for change %s using bisect..." % change
1257
1258        earliestCommit = ""
1259        latestCommit = parseRevision(ref)
1260
1261        while True:
1262            if self.verbose:
1263                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1264            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1265            if len(next) == 0:
1266                if self.verbose:
1267                    print "argh"
1268                return ""
1269            log = extractLogMessageFromGitCommit(next)
1270            settings = extractSettingsGitLog(log)
1271            currentChange = int(settings['change'])
1272            if self.verbose:
1273                print "current change %s" % currentChange
1274
1275            if currentChange == change:
1276                if self.verbose:
1277                    print "found %s" % next
1278                return next
1279
1280            if currentChange < change:
1281                earliestCommit = "^%s" % next
1282            else:
1283                latestCommit = "%s" % next
1284
1285        return ""
1286
1287    def importNewBranch(self, branch, maxChange):
1288        # make fast-import flush all changes to disk and update the refs using the checkpoint
1289        # command so that we can try to find the branch parent in the git history
1290        self.gitStream.write("checkpoint\n\n");
1291        self.gitStream.flush();
1292        branchPrefix = self.depotPaths[0] + branch + "/"
1293        range = "@1,%s" % maxChange
1294        #print "prefix" + branchPrefix
1295        changes = p4ChangesForPaths([branchPrefix], range)
1296        if len(changes) <= 0:
1297            return False
1298        firstChange = changes[0]
1299        #print "first change in branch: %s" % firstChange
1300        sourceBranch = self.knownBranches[branch]
1301        sourceDepotPath = self.depotPaths[0] + sourceBranch
1302        sourceRef = self.gitRefForBranch(sourceBranch)
1303        #print "source " + sourceBranch
1304
1305        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1306        #print "branch parent: %s" % branchParentChange
1307        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1308        if len(gitParent) > 0:
1309            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1310            #print "parent git commit: %s" % gitParent
1311
1312        self.importChanges(changes)
1313        return True
1314
1315    def importChanges(self, changes):
1316        cnt = 1
1317        for change in changes:
1318            description = p4Cmd("describe %s" % change)
1319            self.updateOptionDict(description)
1320
1321            if not self.silent:
1322                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1323                sys.stdout.flush()
1324            cnt = cnt + 1
1325
1326            try:
1327                if self.detectBranches:
1328                    branches = self.splitFilesIntoBranches(description)
1329                    for branch in branches.keys():
1330                        ## HACK  --hwn
1331                        branchPrefix = self.depotPaths[0] + branch + "/"
1332
1333                        parent = ""
1334
1335                        filesForCommit = branches[branch]
1336
1337                        if self.verbose:
1338                            print "branch is %s" % branch
1339
1340                        self.updatedBranches.add(branch)
1341
1342                        if branch not in self.createdBranches:
1343                            self.createdBranches.add(branch)
1344                            parent = self.knownBranches[branch]
1345                            if parent == branch:
1346                                parent = ""
1347                            else:
1348                                fullBranch = self.projectName + branch
1349                                if fullBranch not in self.p4BranchesInGit:
1350                                    if not self.silent:
1351                                        print("\n    Importing new branch %s" % fullBranch);
1352                                    if self.importNewBranch(branch, change - 1):
1353                                        parent = ""
1354                                        self.p4BranchesInGit.append(fullBranch)
1355                                    if not self.silent:
1356                                        print("\n    Resuming with change %s" % change);
1357
1358                                if self.verbose:
1359                                    print "parent determined through known branches: %s" % parent
1360
1361                        branch = self.gitRefForBranch(branch)
1362                        parent = self.gitRefForBranch(parent)
1363
1364                        if self.verbose:
1365                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1366
1367                        if len(parent) == 0 and branch in self.initialParents:
1368                            parent = self.initialParents[branch]
1369                            del self.initialParents[branch]
1370
1371                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1372                else:
1373                    files = self.extractFilesFromCommit(description)
1374                    self.commit(description, files, self.branch, self.depotPaths,
1375                                self.initialParent)
1376                    self.initialParent = ""
1377            except IOError:
1378                print self.gitError.read()
1379                sys.exit(1)
1380
1381    def importHeadRevision(self, revision):
1382        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1383
1384        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1385        details["desc"] = ("Initial import of %s from the state at revision %s"
1386                           % (' '.join(self.depotPaths), revision))
1387        details["change"] = revision
1388        newestRevision = 0
1389
1390        fileCnt = 0
1391        for info in p4CmdList("files "
1392                              +  ' '.join(["%s...%s"
1393                                           % (p, revision)
1394                                           for p in self.depotPaths])):
1395
1396            if info['code'] == 'error':
1397                sys.stderr.write("p4 returned an error: %s\n"
1398                                 % info['data'])
1399                sys.exit(1)
1400
1401
1402            change = int(info["change"])
1403            if change > newestRevision:
1404                newestRevision = change
1405
1406            if info["action"] in ("delete", "purge"):
1407                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1408                #fileCnt = fileCnt + 1
1409                continue
1410
1411            for prop in ["depotFile", "rev", "action", "type" ]:
1412                details["%s%s" % (prop, fileCnt)] = info[prop]
1413
1414            fileCnt = fileCnt + 1
1415
1416        details["change"] = newestRevision
1417        self.updateOptionDict(details)
1418        try:
1419            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1420        except IOError:
1421            print "IO error with git fast-import. Is your git version recent enough?"
1422            print self.gitError.read()
1423
1424
1425    def getClientSpec(self):
1426        specList = p4CmdList( "client -o" )
1427        temp = {}
1428        for entry in specList:
1429            for k,v in entry.iteritems():
1430                if k.startswith("View"):
1431                    if v.startswith('"'):
1432                        start = 1
1433                    else:
1434                        start = 0
1435                    index = v.find("...")
1436                    v = v[start:index]
1437                    if v.startswith("-"):
1438                        v = v[1:]
1439                        temp[v] = -len(v)
1440                    else:
1441                        temp[v] = len(v)
1442        self.clientSpecDirs = temp.items()
1443        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1444
1445    def run(self, args):
1446        self.depotPaths = []
1447        self.changeRange = ""
1448        self.initialParent = ""
1449        self.previousDepotPaths = []
1450
1451        # map from branch depot path to parent branch
1452        self.knownBranches = {}
1453        self.initialParents = {}
1454        self.hasOrigin = originP4BranchesExist()
1455        if not self.syncWithOrigin:
1456            self.hasOrigin = False
1457
1458        if self.importIntoRemotes:
1459            self.refPrefix = "refs/remotes/p4/"
1460        else:
1461            self.refPrefix = "refs/heads/p4/"
1462
1463        if self.syncWithOrigin and self.hasOrigin:
1464            if not self.silent:
1465                print "Syncing with origin first by calling git fetch origin"
1466            system("git fetch origin")
1467
1468        if len(self.branch) == 0:
1469            self.branch = self.refPrefix + "master"
1470            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1471                system("git update-ref %s refs/heads/p4" % self.branch)
1472                system("git branch -D p4");
1473            # create it /after/ importing, when master exists
1474            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1475                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1476
1477        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1478            self.getClientSpec()
1479
1480        # TODO: should always look at previous commits,
1481        # merge with previous imports, if possible.
1482        if args == []:
1483            if self.hasOrigin:
1484                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1485            self.listExistingP4GitBranches()
1486
1487            if len(self.p4BranchesInGit) > 1:
1488                if not self.silent:
1489                    print "Importing from/into multiple branches"
1490                self.detectBranches = True
1491
1492            if self.verbose:
1493                print "branches: %s" % self.p4BranchesInGit
1494
1495            p4Change = 0
1496            for branch in self.p4BranchesInGit:
1497                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1498
1499                settings = extractSettingsGitLog(logMsg)
1500
1501                self.readOptions(settings)
1502                if (settings.has_key('depot-paths')
1503                    and settings.has_key ('change')):
1504                    change = int(settings['change']) + 1
1505                    p4Change = max(p4Change, change)
1506
1507                    depotPaths = sorted(settings['depot-paths'])
1508                    if self.previousDepotPaths == []:
1509                        self.previousDepotPaths = depotPaths
1510                    else:
1511                        paths = []
1512                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1513                            for i in range(0, min(len(cur), len(prev))):
1514                                if cur[i] <> prev[i]:
1515                                    i = i - 1
1516                                    break
1517
1518                            paths.append (cur[:i + 1])
1519
1520                        self.previousDepotPaths = paths
1521
1522            if p4Change > 0:
1523                self.depotPaths = sorted(self.previousDepotPaths)
1524                self.changeRange = "@%s,#head" % p4Change
1525                if not self.detectBranches:
1526                    self.initialParent = parseRevision(self.branch)
1527                if not self.silent and not self.detectBranches:
1528                    print "Performing incremental import into %s git branch" % self.branch
1529
1530        if not self.branch.startswith("refs/"):
1531            self.branch = "refs/heads/" + self.branch
1532
1533        if len(args) == 0 and self.depotPaths:
1534            if not self.silent:
1535                print "Depot paths: %s" % ' '.join(self.depotPaths)
1536        else:
1537            if self.depotPaths and self.depotPaths != args:
1538                print ("previous import used depot path %s and now %s was specified. "
1539                       "This doesn't work!" % (' '.join (self.depotPaths),
1540                                               ' '.join (args)))
1541                sys.exit(1)
1542
1543            self.depotPaths = sorted(args)
1544
1545        revision = ""
1546        self.users = {}
1547
1548        newPaths = []
1549        for p in self.depotPaths:
1550            if p.find("@") != -1:
1551                atIdx = p.index("@")
1552                self.changeRange = p[atIdx:]
1553                if self.changeRange == "@all":
1554                    self.changeRange = ""
1555                elif ',' not in self.changeRange:
1556                    revision = self.changeRange
1557                    self.changeRange = ""
1558                p = p[:atIdx]
1559            elif p.find("#") != -1:
1560                hashIdx = p.index("#")
1561                revision = p[hashIdx:]
1562                p = p[:hashIdx]
1563            elif self.previousDepotPaths == []:
1564                revision = "#head"
1565
1566            p = re.sub ("\.\.\.$", "", p)
1567            if not p.endswith("/"):
1568                p += "/"
1569
1570            newPaths.append(p)
1571
1572        self.depotPaths = newPaths
1573
1574
1575        self.loadUserMapFromCache()
1576        self.labels = {}
1577        if self.detectLabels:
1578            self.getLabels();
1579
1580        if self.detectBranches:
1581            ## FIXME - what's a P4 projectName ?
1582            self.projectName = self.guessProjectName()
1583
1584            if self.hasOrigin:
1585                self.getBranchMappingFromGitBranches()
1586            else:
1587                self.getBranchMapping()
1588            if self.verbose:
1589                print "p4-git branches: %s" % self.p4BranchesInGit
1590                print "initial parents: %s" % self.initialParents
1591            for b in self.p4BranchesInGit:
1592                if b != "master":
1593
1594                    ## FIXME
1595                    b = b[len(self.projectName):]
1596                self.createdBranches.add(b)
1597
1598        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1599
1600        importProcess = subprocess.Popen(["git", "fast-import"],
1601                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1602                                         stderr=subprocess.PIPE);
1603        self.gitOutput = importProcess.stdout
1604        self.gitStream = importProcess.stdin
1605        self.gitError = importProcess.stderr
1606
1607        if revision:
1608            self.importHeadRevision(revision)
1609        else:
1610            changes = []
1611
1612            if len(self.changesFile) > 0:
1613                output = open(self.changesFile).readlines()
1614                changeSet = Set()
1615                for line in output:
1616                    changeSet.add(int(line))
1617
1618                for change in changeSet:
1619                    changes.append(change)
1620
1621                changes.sort()
1622            else:
1623                if self.verbose:
1624                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1625                                                              self.changeRange)
1626                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1627
1628                if len(self.maxChanges) > 0:
1629                    changes = changes[:min(int(self.maxChanges), len(changes))]
1630
1631            if len(changes) == 0:
1632                if not self.silent:
1633                    print "No changes to import!"
1634                return True
1635
1636            if not self.silent and not self.detectBranches:
1637                print "Import destination: %s" % self.branch
1638
1639            self.updatedBranches = set()
1640
1641            self.importChanges(changes)
1642
1643            if not self.silent:
1644                print ""
1645                if len(self.updatedBranches) > 0:
1646                    sys.stdout.write("Updated branches: ")
1647                    for b in self.updatedBranches:
1648                        sys.stdout.write("%s " % b)
1649                    sys.stdout.write("\n")
1650
1651        self.gitStream.close()
1652        if importProcess.wait() != 0:
1653            die("fast-import failed: %s" % self.gitError.read())
1654        self.gitOutput.close()
1655        self.gitError.close()
1656
1657        return True
1658
1659class P4Rebase(Command):
1660    def __init__(self):
1661        Command.__init__(self)
1662        self.options = [ ]
1663        self.description = ("Fetches the latest revision from perforce and "
1664                            + "rebases the current work (branch) against it")
1665        self.verbose = False
1666
1667    def run(self, args):
1668        sync = P4Sync()
1669        sync.run([])
1670
1671        return self.rebase()
1672
1673    def rebase(self):
1674        if os.system("git update-index --refresh") != 0:
1675            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1676        if len(read_pipe("git diff-index HEAD --")) > 0:
1677            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1678
1679        [upstream, settings] = findUpstreamBranchPoint()
1680        if len(upstream) == 0:
1681            die("Cannot find upstream branchpoint for rebase")
1682
1683        # the branchpoint may be p4/foo~3, so strip off the parent
1684        upstream = re.sub("~[0-9]+$", "", upstream)
1685
1686        print "Rebasing the current branch onto %s" % upstream
1687        oldHead = read_pipe("git rev-parse HEAD").strip()
1688        system("git rebase %s" % upstream)
1689        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1690        return True
1691
1692class P4Clone(P4Sync):
1693    def __init__(self):
1694        P4Sync.__init__(self)
1695        self.description = "Creates a new git repository and imports from Perforce into it"
1696        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1697        self.options += [
1698            optparse.make_option("--destination", dest="cloneDestination",
1699                                 action='store', default=None,
1700                                 help="where to leave result of the clone"),
1701            optparse.make_option("-/", dest="cloneExclude",
1702                                 action="append", type="string",
1703                                 help="exclude depot path")
1704        ]
1705        self.cloneDestination = None
1706        self.needsGit = False
1707
1708    # This is required for the "append" cloneExclude action
1709    def ensure_value(self, attr, value):
1710        if not hasattr(self, attr) or getattr(self, attr) is None:
1711            setattr(self, attr, value)
1712        return getattr(self, attr)
1713
1714    def defaultDestination(self, args):
1715        ## TODO: use common prefix of args?
1716        depotPath = args[0]
1717        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1718        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1719        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1720        depotDir = re.sub(r"/$", "", depotDir)
1721        return os.path.split(depotDir)[1]
1722
1723    def run(self, args):
1724        if len(args) < 1:
1725            return False
1726
1727        if self.keepRepoPath and not self.cloneDestination:
1728            sys.stderr.write("Must specify destination for --keep-path\n")
1729            sys.exit(1)
1730
1731        depotPaths = args
1732
1733        if not self.cloneDestination and len(depotPaths) > 1:
1734            self.cloneDestination = depotPaths[-1]
1735            depotPaths = depotPaths[:-1]
1736
1737        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1738        for p in depotPaths:
1739            if not p.startswith("//"):
1740                return False
1741
1742        if not self.cloneDestination:
1743            self.cloneDestination = self.defaultDestination(args)
1744
1745        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1746        if not os.path.exists(self.cloneDestination):
1747            os.makedirs(self.cloneDestination)
1748        chdir(self.cloneDestination)
1749        system("git init")
1750        self.gitdir = os.getcwd() + "/.git"
1751        if not P4Sync.run(self, depotPaths):
1752            return False
1753        if self.branch != "master":
1754            if self.importIntoRemotes:
1755                masterbranch = "refs/remotes/p4/master"
1756            else:
1757                masterbranch = "refs/heads/p4/master"
1758            if gitBranchExists(masterbranch):
1759                system("git branch master %s" % masterbranch)
1760                system("git checkout -f")
1761            else:
1762                print "Could not detect main branch. No checkout/master branch created."
1763
1764        return True
1765
1766class P4Branches(Command):
1767    def __init__(self):
1768        Command.__init__(self)
1769        self.options = [ ]
1770        self.description = ("Shows the git branches that hold imports and their "
1771                            + "corresponding perforce depot paths")
1772        self.verbose = False
1773
1774    def run(self, args):
1775        if originP4BranchesExist():
1776            createOrUpdateBranchesFromOrigin()
1777
1778        cmdline = "git rev-parse --symbolic "
1779        cmdline += " --remotes"
1780
1781        for line in read_pipe_lines(cmdline):
1782            line = line.strip()
1783
1784            if not line.startswith('p4/') or line == "p4/HEAD":
1785                continue
1786            branch = line
1787
1788            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1789            settings = extractSettingsGitLog(log)
1790
1791            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1792        return True
1793
1794class HelpFormatter(optparse.IndentedHelpFormatter):
1795    def __init__(self):
1796        optparse.IndentedHelpFormatter.__init__(self)
1797
1798    def format_description(self, description):
1799        if description:
1800            return description + "\n"
1801        else:
1802            return ""
1803
1804def printUsage(commands):
1805    print "usage: %s <command> [options]" % sys.argv[0]
1806    print ""
1807    print "valid commands: %s" % ", ".join(commands)
1808    print ""
1809    print "Try %s <command> --help for command specific help." % sys.argv[0]
1810    print ""
1811
1812commands = {
1813    "debug" : P4Debug,
1814    "submit" : P4Submit,
1815    "commit" : P4Submit,
1816    "sync" : P4Sync,
1817    "rebase" : P4Rebase,
1818    "clone" : P4Clone,
1819    "rollback" : P4RollBack,
1820    "branches" : P4Branches
1821}
1822
1823
1824def main():
1825    if len(sys.argv[1:]) == 0:
1826        printUsage(commands.keys())
1827        sys.exit(2)
1828
1829    cmd = ""
1830    cmdName = sys.argv[1]
1831    try:
1832        klass = commands[cmdName]
1833        cmd = klass()
1834    except KeyError:
1835        print "unknown command %s" % cmdName
1836        print ""
1837        printUsage(commands.keys())
1838        sys.exit(2)
1839
1840    options = cmd.options
1841    cmd.gitdir = os.environ.get("GIT_DIR", None)
1842
1843    args = sys.argv[2:]
1844
1845    if len(options) > 0:
1846        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1847
1848        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1849                                       options,
1850                                       description = cmd.description,
1851                                       formatter = HelpFormatter())
1852
1853        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1854    global verbose
1855    verbose = cmd.verbose
1856    if cmd.needsGit:
1857        if cmd.gitdir == None:
1858            cmd.gitdir = os.path.abspath(".git")
1859            if not isValidGitDir(cmd.gitdir):
1860                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1861                if os.path.exists(cmd.gitdir):
1862                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1863                    if len(cdup) > 0:
1864                        chdir(cdup);
1865
1866        if not isValidGitDir(cmd.gitdir):
1867            if isValidGitDir(cmd.gitdir + "/.git"):
1868                cmd.gitdir += "/.git"
1869            else:
1870                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1871
1872        os.environ["GIT_DIR"] = cmd.gitdir
1873
1874    if not cmd.run(args):
1875        parser.print_help()
1876
1877
1878if __name__ == '__main__':
1879    main()