Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Utilise our new p4_read_pipe and p4_write_pipe wrappers (a7d3ef9)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def die(msg):
  55    if verbose:
  56        raise Exception(msg)
  57    else:
  58        sys.stderr.write(msg + "\n")
  59        sys.exit(1)
  60
  61def write_pipe(c, str):
  62    if verbose:
  63        sys.stderr.write('Writing pipe: %s\n' % c)
  64
  65    pipe = os.popen(c, 'w')
  66    val = pipe.write(str)
  67    if pipe.close():
  68        die('Command failed: %s' % c)
  69
  70    return val
  71
  72def p4_write_pipe(c, str):
  73    real_cmd = p4_build_cmd(c)
  74    return write_pipe(c, str)
  75
  76def read_pipe(c, ignore_error=False):
  77    if verbose:
  78        sys.stderr.write('Reading pipe: %s\n' % c)
  79
  80    pipe = os.popen(c, 'rb')
  81    val = pipe.read()
  82    if pipe.close() and not ignore_error:
  83        die('Command failed: %s' % c)
  84
  85    return val
  86
  87def p4_read_pipe(c, ignore_error=False):
  88    real_cmd = p4_build_cmd(c)
  89    return read_pipe(real_cmd, ignore_error)
  90
  91def read_pipe_lines(c):
  92    if verbose:
  93        sys.stderr.write('Reading pipe: %s\n' % c)
  94    ## todo: check return status
  95    pipe = os.popen(c, 'rb')
  96    val = pipe.readlines()
  97    if pipe.close():
  98        die('Command failed: %s' % c)
  99
 100    return val
 101
 102def p4_read_pipe_lines(c):
 103    """Specifically invoke p4 on the command supplied. """
 104    real_cmd = p4_build_cmd(c)
 105    return read_pipe_lines(real_cmd)
 106
 107def system(cmd):
 108    if verbose:
 109        sys.stderr.write("executing %s\n" % cmd)
 110    if os.system(cmd) != 0:
 111        die("command failed: %s" % cmd)
 112
 113def p4_system(cmd):
 114    """Specifically invoke p4 as the system command. """
 115    real_cmd = p4_build_cmd(cmd)
 116    return system(real_cmd)
 117
 118def isP4Exec(kind):
 119    """Determine if a Perforce 'kind' should have execute permission
 120
 121    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 122    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 123    a plus sign, it is also executable"""
 124    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 125
 126def setP4ExecBit(file, mode):
 127    # Reopens an already open file and changes the execute bit to match
 128    # the execute bit setting in the passed in mode.
 129
 130    p4Type = "+x"
 131
 132    if not isModeExec(mode):
 133        p4Type = getP4OpenedType(file)
 134        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 135        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 136        if p4Type[-1] == "+":
 137            p4Type = p4Type[0:-1]
 138
 139    p4_system("reopen -t %s %s" % (p4Type, file))
 140
 141def getP4OpenedType(file):
 142    # Returns the perforce file type for the given file.
 143
 144    result = p4_read_pipe("opened %s" % file)
 145    match = re.match(".*\((.+)\)\r?$", result)
 146    if match:
 147        return match.group(1)
 148    else:
 149        die("Could not determine file type for %s (result: '%s')" % (file, result))
 150
 151def diffTreePattern():
 152    # This is a simple generator for the diff tree regex pattern. This could be
 153    # a class variable if this and parseDiffTreeEntry were a part of a class.
 154    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 155    while True:
 156        yield pattern
 157
 158def parseDiffTreeEntry(entry):
 159    """Parses a single diff tree entry into its component elements.
 160
 161    See git-diff-tree(1) manpage for details about the format of the diff
 162    output. This method returns a dictionary with the following elements:
 163
 164    src_mode - The mode of the source file
 165    dst_mode - The mode of the destination file
 166    src_sha1 - The sha1 for the source file
 167    dst_sha1 - The sha1 fr the destination file
 168    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 169    status_score - The score for the status (applicable for 'C' and 'R'
 170                   statuses). This is None if there is no score.
 171    src - The path for the source file.
 172    dst - The path for the destination file. This is only present for
 173          copy or renames. If it is not present, this is None.
 174
 175    If the pattern is not matched, None is returned."""
 176
 177    match = diffTreePattern().next().match(entry)
 178    if match:
 179        return {
 180            'src_mode': match.group(1),
 181            'dst_mode': match.group(2),
 182            'src_sha1': match.group(3),
 183            'dst_sha1': match.group(4),
 184            'status': match.group(5),
 185            'status_score': match.group(6),
 186            'src': match.group(7),
 187            'dst': match.group(10)
 188        }
 189    return None
 190
 191def isModeExec(mode):
 192    # Returns True if the given git mode represents an executable file,
 193    # otherwise False.
 194    return mode[-3:] == "755"
 195
 196def isModeExecChanged(src_mode, dst_mode):
 197    return isModeExec(src_mode) != isModeExec(dst_mode)
 198
 199def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 200    cmd = p4_build_cmd("-G %s" % (cmd))
 201    if verbose:
 202        sys.stderr.write("Opening pipe: %s\n" % cmd)
 203
 204    # Use a temporary file to avoid deadlocks without
 205    # subprocess.communicate(), which would put another copy
 206    # of stdout into memory.
 207    stdin_file = None
 208    if stdin is not None:
 209        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 210        stdin_file.write(stdin)
 211        stdin_file.flush()
 212        stdin_file.seek(0)
 213
 214    p4 = subprocess.Popen(cmd, shell=True,
 215                          stdin=stdin_file,
 216                          stdout=subprocess.PIPE)
 217
 218    result = []
 219    try:
 220        while True:
 221            entry = marshal.load(p4.stdout)
 222            result.append(entry)
 223    except EOFError:
 224        pass
 225    exitCode = p4.wait()
 226    if exitCode != 0:
 227        entry = {}
 228        entry["p4ExitCode"] = exitCode
 229        result.append(entry)
 230
 231    return result
 232
 233def p4Cmd(cmd):
 234    list = p4CmdList(cmd)
 235    result = {}
 236    for entry in list:
 237        result.update(entry)
 238    return result;
 239
 240def p4Where(depotPath):
 241    if not depotPath.endswith("/"):
 242        depotPath += "/"
 243    output = p4Cmd("where %s..." % depotPath)
 244    if output["code"] == "error":
 245        return ""
 246    clientPath = ""
 247    if "path" in output:
 248        clientPath = output.get("path")
 249    elif "data" in output:
 250        data = output.get("data")
 251        lastSpace = data.rfind(" ")
 252        clientPath = data[lastSpace + 1:]
 253
 254    if clientPath.endswith("..."):
 255        clientPath = clientPath[:-3]
 256    return clientPath
 257
 258def currentGitBranch():
 259    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 260
 261def isValidGitDir(path):
 262    if (os.path.exists(path + "/HEAD")
 263        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 264        return True;
 265    return False
 266
 267def parseRevision(ref):
 268    return read_pipe("git rev-parse %s" % ref).strip()
 269
 270def extractLogMessageFromGitCommit(commit):
 271    logMessage = ""
 272
 273    ## fixme: title is first line of commit, not 1st paragraph.
 274    foundTitle = False
 275    for log in read_pipe_lines("git cat-file commit %s" % commit):
 276       if not foundTitle:
 277           if len(log) == 1:
 278               foundTitle = True
 279           continue
 280
 281       logMessage += log
 282    return logMessage
 283
 284def extractSettingsGitLog(log):
 285    values = {}
 286    for line in log.split("\n"):
 287        line = line.strip()
 288        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 289        if not m:
 290            continue
 291
 292        assignments = m.group(1).split (':')
 293        for a in assignments:
 294            vals = a.split ('=')
 295            key = vals[0].strip()
 296            val = ('='.join (vals[1:])).strip()
 297            if val.endswith ('\"') and val.startswith('"'):
 298                val = val[1:-1]
 299
 300            values[key] = val
 301
 302    paths = values.get("depot-paths")
 303    if not paths:
 304        paths = values.get("depot-path")
 305    if paths:
 306        values['depot-paths'] = paths.split(',')
 307    return values
 308
 309def gitBranchExists(branch):
 310    proc = subprocess.Popen(["git", "rev-parse", branch],
 311                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 312    return proc.wait() == 0;
 313
 314def gitConfig(key):
 315    return read_pipe("git config %s" % key, ignore_error=True).strip()
 316
 317def p4BranchesInGit(branchesAreInRemotes = True):
 318    branches = {}
 319
 320    cmdline = "git rev-parse --symbolic "
 321    if branchesAreInRemotes:
 322        cmdline += " --remotes"
 323    else:
 324        cmdline += " --branches"
 325
 326    for line in read_pipe_lines(cmdline):
 327        line = line.strip()
 328
 329        ## only import to p4/
 330        if not line.startswith('p4/') or line == "p4/HEAD":
 331            continue
 332        branch = line
 333
 334        # strip off p4
 335        branch = re.sub ("^p4/", "", line)
 336
 337        branches[branch] = parseRevision(line)
 338    return branches
 339
 340def findUpstreamBranchPoint(head = "HEAD"):
 341    branches = p4BranchesInGit()
 342    # map from depot-path to branch name
 343    branchByDepotPath = {}
 344    for branch in branches.keys():
 345        tip = branches[branch]
 346        log = extractLogMessageFromGitCommit(tip)
 347        settings = extractSettingsGitLog(log)
 348        if settings.has_key("depot-paths"):
 349            paths = ",".join(settings["depot-paths"])
 350            branchByDepotPath[paths] = "remotes/p4/" + branch
 351
 352    settings = None
 353    parent = 0
 354    while parent < 65535:
 355        commit = head + "~%s" % parent
 356        log = extractLogMessageFromGitCommit(commit)
 357        settings = extractSettingsGitLog(log)
 358        if settings.has_key("depot-paths"):
 359            paths = ",".join(settings["depot-paths"])
 360            if branchByDepotPath.has_key(paths):
 361                return [branchByDepotPath[paths], settings]
 362
 363        parent = parent + 1
 364
 365    return ["", settings]
 366
 367def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 368    if not silent:
 369        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 370               % localRefPrefix)
 371
 372    originPrefix = "origin/p4/"
 373
 374    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 375        line = line.strip()
 376        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 377            continue
 378
 379        headName = line[len(originPrefix):]
 380        remoteHead = localRefPrefix + headName
 381        originHead = line
 382
 383        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 384        if (not original.has_key('depot-paths')
 385            or not original.has_key('change')):
 386            continue
 387
 388        update = False
 389        if not gitBranchExists(remoteHead):
 390            if verbose:
 391                print "creating %s" % remoteHead
 392            update = True
 393        else:
 394            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 395            if settings.has_key('change') > 0:
 396                if settings['depot-paths'] == original['depot-paths']:
 397                    originP4Change = int(original['change'])
 398                    p4Change = int(settings['change'])
 399                    if originP4Change > p4Change:
 400                        print ("%s (%s) is newer than %s (%s). "
 401                               "Updating p4 branch from origin."
 402                               % (originHead, originP4Change,
 403                                  remoteHead, p4Change))
 404                        update = True
 405                else:
 406                    print ("Ignoring: %s was imported from %s while "
 407                           "%s was imported from %s"
 408                           % (originHead, ','.join(original['depot-paths']),
 409                              remoteHead, ','.join(settings['depot-paths'])))
 410
 411        if update:
 412            system("git update-ref %s %s" % (remoteHead, originHead))
 413
 414def originP4BranchesExist():
 415        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 416
 417def p4ChangesForPaths(depotPaths, changeRange):
 418    assert depotPaths
 419    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 420                                                        for p in depotPaths]))
 421
 422    changes = []
 423    for line in output:
 424        changeNum = line.split(" ")[1]
 425        changes.append(int(changeNum))
 426
 427    changes.sort()
 428    return changes
 429
 430class Command:
 431    def __init__(self):
 432        self.usage = "usage: %prog [options]"
 433        self.needsGit = True
 434
 435class P4Debug(Command):
 436    def __init__(self):
 437        Command.__init__(self)
 438        self.options = [
 439            optparse.make_option("--verbose", dest="verbose", action="store_true",
 440                                 default=False),
 441            ]
 442        self.description = "A tool to debug the output of p4 -G."
 443        self.needsGit = False
 444        self.verbose = False
 445
 446    def run(self, args):
 447        j = 0
 448        for output in p4CmdList(" ".join(args)):
 449            print 'Element: %d' % j
 450            j += 1
 451            print output
 452        return True
 453
 454class P4RollBack(Command):
 455    def __init__(self):
 456        Command.__init__(self)
 457        self.options = [
 458            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 459            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 460        ]
 461        self.description = "A tool to debug the multi-branch import. Don't use :)"
 462        self.verbose = False
 463        self.rollbackLocalBranches = False
 464
 465    def run(self, args):
 466        if len(args) != 1:
 467            return False
 468        maxChange = int(args[0])
 469
 470        if "p4ExitCode" in p4Cmd("changes -m 1"):
 471            die("Problems executing p4");
 472
 473        if self.rollbackLocalBranches:
 474            refPrefix = "refs/heads/"
 475            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 476        else:
 477            refPrefix = "refs/remotes/"
 478            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 479
 480        for line in lines:
 481            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 482                line = line.strip()
 483                ref = refPrefix + line
 484                log = extractLogMessageFromGitCommit(ref)
 485                settings = extractSettingsGitLog(log)
 486
 487                depotPaths = settings['depot-paths']
 488                change = settings['change']
 489
 490                changed = False
 491
 492                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 493                                                           for p in depotPaths]))) == 0:
 494                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 495                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 496                    continue
 497
 498                while change and int(change) > maxChange:
 499                    changed = True
 500                    if self.verbose:
 501                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 502                    system("git update-ref %s \"%s^\"" % (ref, ref))
 503                    log = extractLogMessageFromGitCommit(ref)
 504                    settings =  extractSettingsGitLog(log)
 505
 506
 507                    depotPaths = settings['depot-paths']
 508                    change = settings['change']
 509
 510                if changed:
 511                    print "%s rewound to %s" % (ref, change)
 512
 513        return True
 514
 515class P4Submit(Command):
 516    def __init__(self):
 517        Command.__init__(self)
 518        self.options = [
 519                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 520                optparse.make_option("--origin", dest="origin"),
 521                optparse.make_option("-M", dest="detectRename", action="store_true"),
 522        ]
 523        self.description = "Submit changes from git to the perforce depot."
 524        self.usage += " [name of git branch to submit into perforce depot]"
 525        self.interactive = True
 526        self.origin = ""
 527        self.detectRename = False
 528        self.verbose = False
 529        self.isWindows = (platform.system() == "Windows")
 530
 531    def check(self):
 532        if len(p4CmdList("opened ...")) > 0:
 533            die("You have files opened with perforce! Close them before starting the sync.")
 534
 535    # replaces everything between 'Description:' and the next P4 submit template field with the
 536    # commit message
 537    def prepareLogMessage(self, template, message):
 538        result = ""
 539
 540        inDescriptionSection = False
 541
 542        for line in template.split("\n"):
 543            if line.startswith("#"):
 544                result += line + "\n"
 545                continue
 546
 547            if inDescriptionSection:
 548                if line.startswith("Files:"):
 549                    inDescriptionSection = False
 550                else:
 551                    continue
 552            else:
 553                if line.startswith("Description:"):
 554                    inDescriptionSection = True
 555                    line += "\n"
 556                    for messageLine in message.split("\n"):
 557                        line += "\t" + messageLine + "\n"
 558
 559            result += line + "\n"
 560
 561        return result
 562
 563    def prepareSubmitTemplate(self):
 564        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 565        template = ""
 566        inFilesSection = False
 567        for line in p4_read_pipe_lines("change -o"):
 568            if line.endswith("\r\n"):
 569                line = line[:-2] + "\n"
 570            if inFilesSection:
 571                if line.startswith("\t"):
 572                    # path starts and ends with a tab
 573                    path = line[1:]
 574                    lastTab = path.rfind("\t")
 575                    if lastTab != -1:
 576                        path = path[:lastTab]
 577                        if not path.startswith(self.depotPath):
 578                            continue
 579                else:
 580                    inFilesSection = False
 581            else:
 582                if line.startswith("Files:"):
 583                    inFilesSection = True
 584
 585            template += line
 586
 587        return template
 588
 589    def applyCommit(self, id):
 590        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 591        diffOpts = ("", "-M")[self.detectRename]
 592        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 593        filesToAdd = set()
 594        filesToDelete = set()
 595        editedFiles = set()
 596        filesToChangeExecBit = {}
 597        for line in diff:
 598            diff = parseDiffTreeEntry(line)
 599            modifier = diff['status']
 600            path = diff['src']
 601            if modifier == "M":
 602                p4_system("edit \"%s\"" % path)
 603                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 604                    filesToChangeExecBit[path] = diff['dst_mode']
 605                editedFiles.add(path)
 606            elif modifier == "A":
 607                filesToAdd.add(path)
 608                filesToChangeExecBit[path] = diff['dst_mode']
 609                if path in filesToDelete:
 610                    filesToDelete.remove(path)
 611            elif modifier == "D":
 612                filesToDelete.add(path)
 613                if path in filesToAdd:
 614                    filesToAdd.remove(path)
 615            elif modifier == "R":
 616                src, dest = diff['src'], diff['dst']
 617                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 618                p4_system("edit \"%s\"" % (dest))
 619                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 620                    filesToChangeExecBit[dest] = diff['dst_mode']
 621                os.unlink(dest)
 622                editedFiles.add(dest)
 623                filesToDelete.add(src)
 624            else:
 625                die("unknown modifier %s for %s" % (modifier, path))
 626
 627        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 628        patchcmd = diffcmd + " | git apply "
 629        tryPatchCmd = patchcmd + "--check -"
 630        applyPatchCmd = patchcmd + "--check --apply -"
 631
 632        if os.system(tryPatchCmd) != 0:
 633            print "Unfortunately applying the change failed!"
 634            print "What do you want to do?"
 635            response = "x"
 636            while response != "s" and response != "a" and response != "w":
 637                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 638                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 639            if response == "s":
 640                print "Skipping! Good luck with the next patches..."
 641                for f in editedFiles:
 642                    p4_system("revert \"%s\"" % f);
 643                for f in filesToAdd:
 644                    system("rm %s" %f)
 645                return
 646            elif response == "a":
 647                os.system(applyPatchCmd)
 648                if len(filesToAdd) > 0:
 649                    print "You may also want to call p4 add on the following files:"
 650                    print " ".join(filesToAdd)
 651                if len(filesToDelete):
 652                    print "The following files should be scheduled for deletion with p4 delete:"
 653                    print " ".join(filesToDelete)
 654                die("Please resolve and submit the conflict manually and "
 655                    + "continue afterwards with git-p4 submit --continue")
 656            elif response == "w":
 657                system(diffcmd + " > patch.txt")
 658                print "Patch saved to patch.txt in %s !" % self.clientPath
 659                die("Please resolve and submit the conflict manually and "
 660                    "continue afterwards with git-p4 submit --continue")
 661
 662        system(applyPatchCmd)
 663
 664        for f in filesToAdd:
 665            p4_system("add \"%s\"" % f)
 666        for f in filesToDelete:
 667            p4_system("revert \"%s\"" % f)
 668            p4_system("delete \"%s\"" % f)
 669
 670        # Set/clear executable bits
 671        for f in filesToChangeExecBit.keys():
 672            mode = filesToChangeExecBit[f]
 673            setP4ExecBit(f, mode)
 674
 675        logMessage = extractLogMessageFromGitCommit(id)
 676        logMessage = logMessage.strip()
 677
 678        template = self.prepareSubmitTemplate()
 679
 680        if self.interactive:
 681            submitTemplate = self.prepareLogMessage(template, logMessage)
 682            if os.environ.has_key("P4DIFF"):
 683                del(os.environ["P4DIFF"])
 684            diff = p4_read_pipe("diff -du ...")
 685
 686            newdiff = ""
 687            for newFile in filesToAdd:
 688                newdiff += "==== new file ====\n"
 689                newdiff += "--- /dev/null\n"
 690                newdiff += "+++ %s\n" % newFile
 691                f = open(newFile, "r")
 692                for line in f.readlines():
 693                    newdiff += "+" + line
 694                f.close()
 695
 696            separatorLine = "######## everything below this line is just the diff #######\n"
 697
 698            [handle, fileName] = tempfile.mkstemp()
 699            tmpFile = os.fdopen(handle, "w+")
 700            if self.isWindows:
 701                submitTemplate = submitTemplate.replace("\n", "\r\n")
 702                separatorLine = separatorLine.replace("\n", "\r\n")
 703                newdiff = newdiff.replace("\n", "\r\n")
 704            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 705            tmpFile.close()
 706            defaultEditor = "vi"
 707            if platform.system() == "Windows":
 708                defaultEditor = "notepad"
 709            if os.environ.has_key("P4EDITOR"):
 710                editor = os.environ.get("P4EDITOR")
 711            else:
 712                editor = os.environ.get("EDITOR", defaultEditor);
 713            system(editor + " " + fileName)
 714            tmpFile = open(fileName, "rb")
 715            message = tmpFile.read()
 716            tmpFile.close()
 717            os.remove(fileName)
 718            submitTemplate = message[:message.index(separatorLine)]
 719            if self.isWindows:
 720                submitTemplate = submitTemplate.replace("\r\n", "\n")
 721
 722            p4_write_pipe("submit -i", submitTemplate)
 723        else:
 724            fileName = "submit.txt"
 725            file = open(fileName, "w+")
 726            file.write(self.prepareLogMessage(template, logMessage))
 727            file.close()
 728            print ("Perforce submit template written as %s. "
 729                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 730                   % (fileName, fileName))
 731
 732    def run(self, args):
 733        if len(args) == 0:
 734            self.master = currentGitBranch()
 735            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 736                die("Detecting current git branch failed!")
 737        elif len(args) == 1:
 738            self.master = args[0]
 739        else:
 740            return False
 741
 742        allowSubmit = gitConfig("git-p4.allowSubmit")
 743        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 744            die("%s is not in git-p4.allowSubmit" % self.master)
 745
 746        [upstream, settings] = findUpstreamBranchPoint()
 747        self.depotPath = settings['depot-paths'][0]
 748        if len(self.origin) == 0:
 749            self.origin = upstream
 750
 751        if self.verbose:
 752            print "Origin branch is " + self.origin
 753
 754        if len(self.depotPath) == 0:
 755            print "Internal error: cannot locate perforce depot path from existing branches"
 756            sys.exit(128)
 757
 758        self.clientPath = p4Where(self.depotPath)
 759
 760        if len(self.clientPath) == 0:
 761            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 762            sys.exit(128)
 763
 764        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 765        self.oldWorkingDirectory = os.getcwd()
 766
 767        os.chdir(self.clientPath)
 768        print "Syncronizing p4 checkout..."
 769        p4_system("sync ...")
 770
 771        self.check()
 772
 773        commits = []
 774        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 775            commits.append(line.strip())
 776        commits.reverse()
 777
 778        while len(commits) > 0:
 779            commit = commits[0]
 780            commits = commits[1:]
 781            self.applyCommit(commit)
 782            if not self.interactive:
 783                break
 784
 785        if len(commits) == 0:
 786            print "All changes applied!"
 787            os.chdir(self.oldWorkingDirectory)
 788
 789            sync = P4Sync()
 790            sync.run([])
 791
 792            rebase = P4Rebase()
 793            rebase.rebase()
 794
 795        return True
 796
 797class P4Sync(Command):
 798    def __init__(self):
 799        Command.__init__(self)
 800        self.options = [
 801                optparse.make_option("--branch", dest="branch"),
 802                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 803                optparse.make_option("--changesfile", dest="changesFile"),
 804                optparse.make_option("--silent", dest="silent", action="store_true"),
 805                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 806                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 807                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 808                                     help="Import into refs/heads/ , not refs/remotes"),
 809                optparse.make_option("--max-changes", dest="maxChanges"),
 810                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 811                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 812                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 813                                     help="Only sync files that are included in the Perforce Client Spec")
 814        ]
 815        self.description = """Imports from Perforce into a git repository.\n
 816    example:
 817    //depot/my/project/ -- to import the current head
 818    //depot/my/project/@all -- to import everything
 819    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 820
 821    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 822
 823        self.usage += " //depot/path[@revRange]"
 824        self.silent = False
 825        self.createdBranches = Set()
 826        self.committedChanges = Set()
 827        self.branch = ""
 828        self.detectBranches = False
 829        self.detectLabels = False
 830        self.changesFile = ""
 831        self.syncWithOrigin = True
 832        self.verbose = False
 833        self.importIntoRemotes = True
 834        self.maxChanges = ""
 835        self.isWindows = (platform.system() == "Windows")
 836        self.keepRepoPath = False
 837        self.depotPaths = None
 838        self.p4BranchesInGit = []
 839        self.cloneExclude = []
 840        self.useClientSpec = False
 841        self.clientSpecDirs = []
 842
 843        if gitConfig("git-p4.syncFromOrigin") == "false":
 844            self.syncWithOrigin = False
 845
 846    def extractFilesFromCommit(self, commit):
 847        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 848                             for path in self.cloneExclude]
 849        files = []
 850        fnum = 0
 851        while commit.has_key("depotFile%s" % fnum):
 852            path =  commit["depotFile%s" % fnum]
 853
 854            if [p for p in self.cloneExclude
 855                if path.startswith (p)]:
 856                found = False
 857            else:
 858                found = [p for p in self.depotPaths
 859                         if path.startswith (p)]
 860            if not found:
 861                fnum = fnum + 1
 862                continue
 863
 864            file = {}
 865            file["path"] = path
 866            file["rev"] = commit["rev%s" % fnum]
 867            file["action"] = commit["action%s" % fnum]
 868            file["type"] = commit["type%s" % fnum]
 869            files.append(file)
 870            fnum = fnum + 1
 871        return files
 872
 873    def stripRepoPath(self, path, prefixes):
 874        if self.keepRepoPath:
 875            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 876
 877        for p in prefixes:
 878            if path.startswith(p):
 879                path = path[len(p):]
 880
 881        return path
 882
 883    def splitFilesIntoBranches(self, commit):
 884        branches = {}
 885        fnum = 0
 886        while commit.has_key("depotFile%s" % fnum):
 887            path =  commit["depotFile%s" % fnum]
 888            found = [p for p in self.depotPaths
 889                     if path.startswith (p)]
 890            if not found:
 891                fnum = fnum + 1
 892                continue
 893
 894            file = {}
 895            file["path"] = path
 896            file["rev"] = commit["rev%s" % fnum]
 897            file["action"] = commit["action%s" % fnum]
 898            file["type"] = commit["type%s" % fnum]
 899            fnum = fnum + 1
 900
 901            relPath = self.stripRepoPath(path, self.depotPaths)
 902
 903            for branch in self.knownBranches.keys():
 904
 905                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 906                if relPath.startswith(branch + "/"):
 907                    if branch not in branches:
 908                        branches[branch] = []
 909                    branches[branch].append(file)
 910                    break
 911
 912        return branches
 913
 914    ## Should move this out, doesn't use SELF.
 915    def readP4Files(self, files):
 916        filesForCommit = []
 917        filesToRead = []
 918
 919        for f in files:
 920            includeFile = True
 921            for val in self.clientSpecDirs:
 922                if f['path'].startswith(val[0]):
 923                    if val[1] <= 0:
 924                        includeFile = False
 925                    break
 926
 927            if includeFile:
 928                filesForCommit.append(f)
 929                if f['action'] != 'delete':
 930                    filesToRead.append(f)
 931
 932        filedata = []
 933        if len(filesToRead) > 0:
 934            filedata = p4CmdList('-x - print',
 935                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 936                                                  for f in filesToRead]),
 937                                 stdin_mode='w+')
 938
 939            if "p4ExitCode" in filedata[0]:
 940                die("Problems executing p4. Error: [%d]."
 941                    % (filedata[0]['p4ExitCode']));
 942
 943        j = 0;
 944        contents = {}
 945        while j < len(filedata):
 946            stat = filedata[j]
 947            j += 1
 948            text = [];
 949            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 950                text.append(filedata[j]['data'])
 951                j += 1
 952            text = ''.join(text)
 953
 954            if not stat.has_key('depotFile'):
 955                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 956                continue
 957
 958            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 959                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 960            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 961                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 962
 963            contents[stat['depotFile']] = text
 964
 965        for f in filesForCommit:
 966            path = f['path']
 967            if contents.has_key(path):
 968                f['data'] = contents[path]
 969
 970        return filesForCommit
 971
 972    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 973        epoch = details["time"]
 974        author = details["user"]
 975
 976        if self.verbose:
 977            print "commit into %s" % branch
 978
 979        # start with reading files; if that fails, we should not
 980        # create a commit.
 981        new_files = []
 982        for f in files:
 983            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 984                new_files.append (f)
 985            else:
 986                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 987        files = self.readP4Files(new_files)
 988
 989        self.gitStream.write("commit %s\n" % branch)
 990#        gitStream.write("mark :%s\n" % details["change"])
 991        self.committedChanges.add(int(details["change"]))
 992        committer = ""
 993        if author not in self.users:
 994            self.getUserMapFromPerforceServer()
 995        if author in self.users:
 996            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 997        else:
 998            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 999
1000        self.gitStream.write("committer %s\n" % committer)
1001
1002        self.gitStream.write("data <<EOT\n")
1003        self.gitStream.write(details["desc"])
1004        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1005                             % (','.join (branchPrefixes), details["change"]))
1006        if len(details['options']) > 0:
1007            self.gitStream.write(": options = %s" % details['options'])
1008        self.gitStream.write("]\nEOT\n\n")
1009
1010        if len(parent) > 0:
1011            if self.verbose:
1012                print "parent %s" % parent
1013            self.gitStream.write("from %s\n" % parent)
1014
1015        for file in files:
1016            if file["type"] == "apple":
1017                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1018                continue
1019
1020            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1021            if file["action"] == "delete":
1022                self.gitStream.write("D %s\n" % relPath)
1023            else:
1024                data = file['data']
1025
1026                mode = "644"
1027                if isP4Exec(file["type"]):
1028                    mode = "755"
1029                elif file["type"] == "symlink":
1030                    mode = "120000"
1031                    # p4 print on a symlink contains "target\n", so strip it off
1032                    data = data[:-1]
1033
1034                if self.isWindows and file["type"].endswith("text"):
1035                    data = data.replace("\r\n", "\n")
1036
1037                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1038                self.gitStream.write("data %s\n" % len(data))
1039                self.gitStream.write(data)
1040                self.gitStream.write("\n")
1041
1042        self.gitStream.write("\n")
1043
1044        change = int(details["change"])
1045
1046        if self.labels.has_key(change):
1047            label = self.labels[change]
1048            labelDetails = label[0]
1049            labelRevisions = label[1]
1050            if self.verbose:
1051                print "Change %s is labelled %s" % (change, labelDetails)
1052
1053            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1054                                                    for p in branchPrefixes]))
1055
1056            if len(files) == len(labelRevisions):
1057
1058                cleanedFiles = {}
1059                for info in files:
1060                    if info["action"] == "delete":
1061                        continue
1062                    cleanedFiles[info["depotFile"]] = info["rev"]
1063
1064                if cleanedFiles == labelRevisions:
1065                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1066                    self.gitStream.write("from %s\n" % branch)
1067
1068                    owner = labelDetails["Owner"]
1069                    tagger = ""
1070                    if author in self.users:
1071                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1072                    else:
1073                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1074                    self.gitStream.write("tagger %s\n" % tagger)
1075                    self.gitStream.write("data <<EOT\n")
1076                    self.gitStream.write(labelDetails["Description"])
1077                    self.gitStream.write("EOT\n\n")
1078
1079                else:
1080                    if not self.silent:
1081                        print ("Tag %s does not match with change %s: files do not match."
1082                               % (labelDetails["label"], change))
1083
1084            else:
1085                if not self.silent:
1086                    print ("Tag %s does not match with change %s: file count is different."
1087                           % (labelDetails["label"], change))
1088
1089    def getUserCacheFilename(self):
1090        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1091        return home + "/.gitp4-usercache.txt"
1092
1093    def getUserMapFromPerforceServer(self):
1094        if self.userMapFromPerforceServer:
1095            return
1096        self.users = {}
1097
1098        for output in p4CmdList("users"):
1099            if not output.has_key("User"):
1100                continue
1101            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1102
1103
1104        s = ''
1105        for (key, val) in self.users.items():
1106            s += "%s\t%s\n" % (key, val)
1107
1108        open(self.getUserCacheFilename(), "wb").write(s)
1109        self.userMapFromPerforceServer = True
1110
1111    def loadUserMapFromCache(self):
1112        self.users = {}
1113        self.userMapFromPerforceServer = False
1114        try:
1115            cache = open(self.getUserCacheFilename(), "rb")
1116            lines = cache.readlines()
1117            cache.close()
1118            for line in lines:
1119                entry = line.strip().split("\t")
1120                self.users[entry[0]] = entry[1]
1121        except IOError:
1122            self.getUserMapFromPerforceServer()
1123
1124    def getLabels(self):
1125        self.labels = {}
1126
1127        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1128        if len(l) > 0 and not self.silent:
1129            print "Finding files belonging to labels in %s" % `self.depotPaths`
1130
1131        for output in l:
1132            label = output["label"]
1133            revisions = {}
1134            newestChange = 0
1135            if self.verbose:
1136                print "Querying files for label %s" % label
1137            for file in p4CmdList("files "
1138                                  +  ' '.join (["%s...@%s" % (p, label)
1139                                                for p in self.depotPaths])):
1140                revisions[file["depotFile"]] = file["rev"]
1141                change = int(file["change"])
1142                if change > newestChange:
1143                    newestChange = change
1144
1145            self.labels[newestChange] = [output, revisions]
1146
1147        if self.verbose:
1148            print "Label changes: %s" % self.labels.keys()
1149
1150    def guessProjectName(self):
1151        for p in self.depotPaths:
1152            if p.endswith("/"):
1153                p = p[:-1]
1154            p = p[p.strip().rfind("/") + 1:]
1155            if not p.endswith("/"):
1156               p += "/"
1157            return p
1158
1159    def getBranchMapping(self):
1160        lostAndFoundBranches = set()
1161
1162        for info in p4CmdList("branches"):
1163            details = p4Cmd("branch -o %s" % info["branch"])
1164            viewIdx = 0
1165            while details.has_key("View%s" % viewIdx):
1166                paths = details["View%s" % viewIdx].split(" ")
1167                viewIdx = viewIdx + 1
1168                # require standard //depot/foo/... //depot/bar/... mapping
1169                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1170                    continue
1171                source = paths[0]
1172                destination = paths[1]
1173                ## HACK
1174                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1175                    source = source[len(self.depotPaths[0]):-4]
1176                    destination = destination[len(self.depotPaths[0]):-4]
1177
1178                    if destination in self.knownBranches:
1179                        if not self.silent:
1180                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1181                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1182                        continue
1183
1184                    self.knownBranches[destination] = source
1185
1186                    lostAndFoundBranches.discard(destination)
1187
1188                    if source not in self.knownBranches:
1189                        lostAndFoundBranches.add(source)
1190
1191
1192        for branch in lostAndFoundBranches:
1193            self.knownBranches[branch] = branch
1194
1195    def getBranchMappingFromGitBranches(self):
1196        branches = p4BranchesInGit(self.importIntoRemotes)
1197        for branch in branches.keys():
1198            if branch == "master":
1199                branch = "main"
1200            else:
1201                branch = branch[len(self.projectName):]
1202            self.knownBranches[branch] = branch
1203
1204    def listExistingP4GitBranches(self):
1205        # branches holds mapping from name to commit
1206        branches = p4BranchesInGit(self.importIntoRemotes)
1207        self.p4BranchesInGit = branches.keys()
1208        for branch in branches.keys():
1209            self.initialParents[self.refPrefix + branch] = branches[branch]
1210
1211    def updateOptionDict(self, d):
1212        option_keys = {}
1213        if self.keepRepoPath:
1214            option_keys['keepRepoPath'] = 1
1215
1216        d["options"] = ' '.join(sorted(option_keys.keys()))
1217
1218    def readOptions(self, d):
1219        self.keepRepoPath = (d.has_key('options')
1220                             and ('keepRepoPath' in d['options']))
1221
1222    def gitRefForBranch(self, branch):
1223        if branch == "main":
1224            return self.refPrefix + "master"
1225
1226        if len(branch) <= 0:
1227            return branch
1228
1229        return self.refPrefix + self.projectName + branch
1230
1231    def gitCommitByP4Change(self, ref, change):
1232        if self.verbose:
1233            print "looking in ref " + ref + " for change %s using bisect..." % change
1234
1235        earliestCommit = ""
1236        latestCommit = parseRevision(ref)
1237
1238        while True:
1239            if self.verbose:
1240                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1241            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1242            if len(next) == 0:
1243                if self.verbose:
1244                    print "argh"
1245                return ""
1246            log = extractLogMessageFromGitCommit(next)
1247            settings = extractSettingsGitLog(log)
1248            currentChange = int(settings['change'])
1249            if self.verbose:
1250                print "current change %s" % currentChange
1251
1252            if currentChange == change:
1253                if self.verbose:
1254                    print "found %s" % next
1255                return next
1256
1257            if currentChange < change:
1258                earliestCommit = "^%s" % next
1259            else:
1260                latestCommit = "%s" % next
1261
1262        return ""
1263
1264    def importNewBranch(self, branch, maxChange):
1265        # make fast-import flush all changes to disk and update the refs using the checkpoint
1266        # command so that we can try to find the branch parent in the git history
1267        self.gitStream.write("checkpoint\n\n");
1268        self.gitStream.flush();
1269        branchPrefix = self.depotPaths[0] + branch + "/"
1270        range = "@1,%s" % maxChange
1271        #print "prefix" + branchPrefix
1272        changes = p4ChangesForPaths([branchPrefix], range)
1273        if len(changes) <= 0:
1274            return False
1275        firstChange = changes[0]
1276        #print "first change in branch: %s" % firstChange
1277        sourceBranch = self.knownBranches[branch]
1278        sourceDepotPath = self.depotPaths[0] + sourceBranch
1279        sourceRef = self.gitRefForBranch(sourceBranch)
1280        #print "source " + sourceBranch
1281
1282        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1283        #print "branch parent: %s" % branchParentChange
1284        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1285        if len(gitParent) > 0:
1286            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1287            #print "parent git commit: %s" % gitParent
1288
1289        self.importChanges(changes)
1290        return True
1291
1292    def importChanges(self, changes):
1293        cnt = 1
1294        for change in changes:
1295            description = p4Cmd("describe %s" % change)
1296            self.updateOptionDict(description)
1297
1298            if not self.silent:
1299                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1300                sys.stdout.flush()
1301            cnt = cnt + 1
1302
1303            try:
1304                if self.detectBranches:
1305                    branches = self.splitFilesIntoBranches(description)
1306                    for branch in branches.keys():
1307                        ## HACK  --hwn
1308                        branchPrefix = self.depotPaths[0] + branch + "/"
1309
1310                        parent = ""
1311
1312                        filesForCommit = branches[branch]
1313
1314                        if self.verbose:
1315                            print "branch is %s" % branch
1316
1317                        self.updatedBranches.add(branch)
1318
1319                        if branch not in self.createdBranches:
1320                            self.createdBranches.add(branch)
1321                            parent = self.knownBranches[branch]
1322                            if parent == branch:
1323                                parent = ""
1324                            else:
1325                                fullBranch = self.projectName + branch
1326                                if fullBranch not in self.p4BranchesInGit:
1327                                    if not self.silent:
1328                                        print("\n    Importing new branch %s" % fullBranch);
1329                                    if self.importNewBranch(branch, change - 1):
1330                                        parent = ""
1331                                        self.p4BranchesInGit.append(fullBranch)
1332                                    if not self.silent:
1333                                        print("\n    Resuming with change %s" % change);
1334
1335                                if self.verbose:
1336                                    print "parent determined through known branches: %s" % parent
1337
1338                        branch = self.gitRefForBranch(branch)
1339                        parent = self.gitRefForBranch(parent)
1340
1341                        if self.verbose:
1342                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1343
1344                        if len(parent) == 0 and branch in self.initialParents:
1345                            parent = self.initialParents[branch]
1346                            del self.initialParents[branch]
1347
1348                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1349                else:
1350                    files = self.extractFilesFromCommit(description)
1351                    self.commit(description, files, self.branch, self.depotPaths,
1352                                self.initialParent)
1353                    self.initialParent = ""
1354            except IOError:
1355                print self.gitError.read()
1356                sys.exit(1)
1357
1358    def importHeadRevision(self, revision):
1359        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1360
1361        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1362        details["desc"] = ("Initial import of %s from the state at revision %s"
1363                           % (' '.join(self.depotPaths), revision))
1364        details["change"] = revision
1365        newestRevision = 0
1366
1367        fileCnt = 0
1368        for info in p4CmdList("files "
1369                              +  ' '.join(["%s...%s"
1370                                           % (p, revision)
1371                                           for p in self.depotPaths])):
1372
1373            if info['code'] == 'error':
1374                sys.stderr.write("p4 returned an error: %s\n"
1375                                 % info['data'])
1376                sys.exit(1)
1377
1378
1379            change = int(info["change"])
1380            if change > newestRevision:
1381                newestRevision = change
1382
1383            if info["action"] == "delete":
1384                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1385                #fileCnt = fileCnt + 1
1386                continue
1387
1388            for prop in ["depotFile", "rev", "action", "type" ]:
1389                details["%s%s" % (prop, fileCnt)] = info[prop]
1390
1391            fileCnt = fileCnt + 1
1392
1393        details["change"] = newestRevision
1394        self.updateOptionDict(details)
1395        try:
1396            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1397        except IOError:
1398            print "IO error with git fast-import. Is your git version recent enough?"
1399            print self.gitError.read()
1400
1401
1402    def getClientSpec(self):
1403        specList = p4CmdList( "client -o" )
1404        temp = {}
1405        for entry in specList:
1406            for k,v in entry.iteritems():
1407                if k.startswith("View"):
1408                    if v.startswith('"'):
1409                        start = 1
1410                    else:
1411                        start = 0
1412                    index = v.find("...")
1413                    v = v[start:index]
1414                    if v.startswith("-"):
1415                        v = v[1:]
1416                        temp[v] = -len(v)
1417                    else:
1418                        temp[v] = len(v)
1419        self.clientSpecDirs = temp.items()
1420        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1421
1422    def run(self, args):
1423        self.depotPaths = []
1424        self.changeRange = ""
1425        self.initialParent = ""
1426        self.previousDepotPaths = []
1427
1428        # map from branch depot path to parent branch
1429        self.knownBranches = {}
1430        self.initialParents = {}
1431        self.hasOrigin = originP4BranchesExist()
1432        if not self.syncWithOrigin:
1433            self.hasOrigin = False
1434
1435        if self.importIntoRemotes:
1436            self.refPrefix = "refs/remotes/p4/"
1437        else:
1438            self.refPrefix = "refs/heads/p4/"
1439
1440        if self.syncWithOrigin and self.hasOrigin:
1441            if not self.silent:
1442                print "Syncing with origin first by calling git fetch origin"
1443            system("git fetch origin")
1444
1445        if len(self.branch) == 0:
1446            self.branch = self.refPrefix + "master"
1447            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1448                system("git update-ref %s refs/heads/p4" % self.branch)
1449                system("git branch -D p4");
1450            # create it /after/ importing, when master exists
1451            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1452                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1453
1454        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1455            self.getClientSpec()
1456
1457        # TODO: should always look at previous commits,
1458        # merge with previous imports, if possible.
1459        if args == []:
1460            if self.hasOrigin:
1461                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1462            self.listExistingP4GitBranches()
1463
1464            if len(self.p4BranchesInGit) > 1:
1465                if not self.silent:
1466                    print "Importing from/into multiple branches"
1467                self.detectBranches = True
1468
1469            if self.verbose:
1470                print "branches: %s" % self.p4BranchesInGit
1471
1472            p4Change = 0
1473            for branch in self.p4BranchesInGit:
1474                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1475
1476                settings = extractSettingsGitLog(logMsg)
1477
1478                self.readOptions(settings)
1479                if (settings.has_key('depot-paths')
1480                    and settings.has_key ('change')):
1481                    change = int(settings['change']) + 1
1482                    p4Change = max(p4Change, change)
1483
1484                    depotPaths = sorted(settings['depot-paths'])
1485                    if self.previousDepotPaths == []:
1486                        self.previousDepotPaths = depotPaths
1487                    else:
1488                        paths = []
1489                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1490                            for i in range(0, min(len(cur), len(prev))):
1491                                if cur[i] <> prev[i]:
1492                                    i = i - 1
1493                                    break
1494
1495                            paths.append (cur[:i + 1])
1496
1497                        self.previousDepotPaths = paths
1498
1499            if p4Change > 0:
1500                self.depotPaths = sorted(self.previousDepotPaths)
1501                self.changeRange = "@%s,#head" % p4Change
1502                if not self.detectBranches:
1503                    self.initialParent = parseRevision(self.branch)
1504                if not self.silent and not self.detectBranches:
1505                    print "Performing incremental import into %s git branch" % self.branch
1506
1507        if not self.branch.startswith("refs/"):
1508            self.branch = "refs/heads/" + self.branch
1509
1510        if len(args) == 0 and self.depotPaths:
1511            if not self.silent:
1512                print "Depot paths: %s" % ' '.join(self.depotPaths)
1513        else:
1514            if self.depotPaths and self.depotPaths != args:
1515                print ("previous import used depot path %s and now %s was specified. "
1516                       "This doesn't work!" % (' '.join (self.depotPaths),
1517                                               ' '.join (args)))
1518                sys.exit(1)
1519
1520            self.depotPaths = sorted(args)
1521
1522        revision = ""
1523        self.users = {}
1524
1525        newPaths = []
1526        for p in self.depotPaths:
1527            if p.find("@") != -1:
1528                atIdx = p.index("@")
1529                self.changeRange = p[atIdx:]
1530                if self.changeRange == "@all":
1531                    self.changeRange = ""
1532                elif ',' not in self.changeRange:
1533                    revision = self.changeRange
1534                    self.changeRange = ""
1535                p = p[:atIdx]
1536            elif p.find("#") != -1:
1537                hashIdx = p.index("#")
1538                revision = p[hashIdx:]
1539                p = p[:hashIdx]
1540            elif self.previousDepotPaths == []:
1541                revision = "#head"
1542
1543            p = re.sub ("\.\.\.$", "", p)
1544            if not p.endswith("/"):
1545                p += "/"
1546
1547            newPaths.append(p)
1548
1549        self.depotPaths = newPaths
1550
1551
1552        self.loadUserMapFromCache()
1553        self.labels = {}
1554        if self.detectLabels:
1555            self.getLabels();
1556
1557        if self.detectBranches:
1558            ## FIXME - what's a P4 projectName ?
1559            self.projectName = self.guessProjectName()
1560
1561            if self.hasOrigin:
1562                self.getBranchMappingFromGitBranches()
1563            else:
1564                self.getBranchMapping()
1565            if self.verbose:
1566                print "p4-git branches: %s" % self.p4BranchesInGit
1567                print "initial parents: %s" % self.initialParents
1568            for b in self.p4BranchesInGit:
1569                if b != "master":
1570
1571                    ## FIXME
1572                    b = b[len(self.projectName):]
1573                self.createdBranches.add(b)
1574
1575        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1576
1577        importProcess = subprocess.Popen(["git", "fast-import"],
1578                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1579                                         stderr=subprocess.PIPE);
1580        self.gitOutput = importProcess.stdout
1581        self.gitStream = importProcess.stdin
1582        self.gitError = importProcess.stderr
1583
1584        if revision:
1585            self.importHeadRevision(revision)
1586        else:
1587            changes = []
1588
1589            if len(self.changesFile) > 0:
1590                output = open(self.changesFile).readlines()
1591                changeSet = Set()
1592                for line in output:
1593                    changeSet.add(int(line))
1594
1595                for change in changeSet:
1596                    changes.append(change)
1597
1598                changes.sort()
1599            else:
1600                if self.verbose:
1601                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1602                                                              self.changeRange)
1603                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1604
1605                if len(self.maxChanges) > 0:
1606                    changes = changes[:min(int(self.maxChanges), len(changes))]
1607
1608            if len(changes) == 0:
1609                if not self.silent:
1610                    print "No changes to import!"
1611                return True
1612
1613            if not self.silent and not self.detectBranches:
1614                print "Import destination: %s" % self.branch
1615
1616            self.updatedBranches = set()
1617
1618            self.importChanges(changes)
1619
1620            if not self.silent:
1621                print ""
1622                if len(self.updatedBranches) > 0:
1623                    sys.stdout.write("Updated branches: ")
1624                    for b in self.updatedBranches:
1625                        sys.stdout.write("%s " % b)
1626                    sys.stdout.write("\n")
1627
1628        self.gitStream.close()
1629        if importProcess.wait() != 0:
1630            die("fast-import failed: %s" % self.gitError.read())
1631        self.gitOutput.close()
1632        self.gitError.close()
1633
1634        return True
1635
1636class P4Rebase(Command):
1637    def __init__(self):
1638        Command.__init__(self)
1639        self.options = [ ]
1640        self.description = ("Fetches the latest revision from perforce and "
1641                            + "rebases the current work (branch) against it")
1642        self.verbose = False
1643
1644    def run(self, args):
1645        sync = P4Sync()
1646        sync.run([])
1647
1648        return self.rebase()
1649
1650    def rebase(self):
1651        if os.system("git update-index --refresh") != 0:
1652            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1653        if len(read_pipe("git diff-index HEAD --")) > 0:
1654            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1655
1656        [upstream, settings] = findUpstreamBranchPoint()
1657        if len(upstream) == 0:
1658            die("Cannot find upstream branchpoint for rebase")
1659
1660        # the branchpoint may be p4/foo~3, so strip off the parent
1661        upstream = re.sub("~[0-9]+$", "", upstream)
1662
1663        print "Rebasing the current branch onto %s" % upstream
1664        oldHead = read_pipe("git rev-parse HEAD").strip()
1665        system("git rebase %s" % upstream)
1666        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1667        return True
1668
1669class P4Clone(P4Sync):
1670    def __init__(self):
1671        P4Sync.__init__(self)
1672        self.description = "Creates a new git repository and imports from Perforce into it"
1673        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1674        self.options += [
1675            optparse.make_option("--destination", dest="cloneDestination",
1676                                 action='store', default=None,
1677                                 help="where to leave result of the clone"),
1678            optparse.make_option("-/", dest="cloneExclude",
1679                                 action="append", type="string",
1680                                 help="exclude depot path")
1681        ]
1682        self.cloneDestination = None
1683        self.needsGit = False
1684
1685    # This is required for the "append" cloneExclude action
1686    def ensure_value(self, attr, value):
1687        if not hasattr(self, attr) or getattr(self, attr) is None:
1688            setattr(self, attr, value)
1689        return getattr(self, attr)
1690
1691    def defaultDestination(self, args):
1692        ## TODO: use common prefix of args?
1693        depotPath = args[0]
1694        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1695        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1696        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1697        depotDir = re.sub(r"/$", "", depotDir)
1698        return os.path.split(depotDir)[1]
1699
1700    def run(self, args):
1701        if len(args) < 1:
1702            return False
1703
1704        if self.keepRepoPath and not self.cloneDestination:
1705            sys.stderr.write("Must specify destination for --keep-path\n")
1706            sys.exit(1)
1707
1708        depotPaths = args
1709
1710        if not self.cloneDestination and len(depotPaths) > 1:
1711            self.cloneDestination = depotPaths[-1]
1712            depotPaths = depotPaths[:-1]
1713
1714        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1715        for p in depotPaths:
1716            if not p.startswith("//"):
1717                return False
1718
1719        if not self.cloneDestination:
1720            self.cloneDestination = self.defaultDestination(args)
1721
1722        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1723        if not os.path.exists(self.cloneDestination):
1724            os.makedirs(self.cloneDestination)
1725        os.chdir(self.cloneDestination)
1726        system("git init")
1727        self.gitdir = os.getcwd() + "/.git"
1728        if not P4Sync.run(self, depotPaths):
1729            return False
1730        if self.branch != "master":
1731            if gitBranchExists("refs/remotes/p4/master"):
1732                system("git branch master refs/remotes/p4/master")
1733                system("git checkout -f")
1734            else:
1735                print "Could not detect main branch. No checkout/master branch created."
1736
1737        return True
1738
1739class P4Branches(Command):
1740    def __init__(self):
1741        Command.__init__(self)
1742        self.options = [ ]
1743        self.description = ("Shows the git branches that hold imports and their "
1744                            + "corresponding perforce depot paths")
1745        self.verbose = False
1746
1747    def run(self, args):
1748        if originP4BranchesExist():
1749            createOrUpdateBranchesFromOrigin()
1750
1751        cmdline = "git rev-parse --symbolic "
1752        cmdline += " --remotes"
1753
1754        for line in read_pipe_lines(cmdline):
1755            line = line.strip()
1756
1757            if not line.startswith('p4/') or line == "p4/HEAD":
1758                continue
1759            branch = line
1760
1761            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1762            settings = extractSettingsGitLog(log)
1763
1764            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1765        return True
1766
1767class HelpFormatter(optparse.IndentedHelpFormatter):
1768    def __init__(self):
1769        optparse.IndentedHelpFormatter.__init__(self)
1770
1771    def format_description(self, description):
1772        if description:
1773            return description + "\n"
1774        else:
1775            return ""
1776
1777def printUsage(commands):
1778    print "usage: %s <command> [options]" % sys.argv[0]
1779    print ""
1780    print "valid commands: %s" % ", ".join(commands)
1781    print ""
1782    print "Try %s <command> --help for command specific help." % sys.argv[0]
1783    print ""
1784
1785commands = {
1786    "debug" : P4Debug,
1787    "submit" : P4Submit,
1788    "commit" : P4Submit,
1789    "sync" : P4Sync,
1790    "rebase" : P4Rebase,
1791    "clone" : P4Clone,
1792    "rollback" : P4RollBack,
1793    "branches" : P4Branches
1794}
1795
1796
1797def main():
1798    if len(sys.argv[1:]) == 0:
1799        printUsage(commands.keys())
1800        sys.exit(2)
1801
1802    cmd = ""
1803    cmdName = sys.argv[1]
1804    try:
1805        klass = commands[cmdName]
1806        cmd = klass()
1807    except KeyError:
1808        print "unknown command %s" % cmdName
1809        print ""
1810        printUsage(commands.keys())
1811        sys.exit(2)
1812
1813    options = cmd.options
1814    cmd.gitdir = os.environ.get("GIT_DIR", None)
1815
1816    args = sys.argv[2:]
1817
1818    if len(options) > 0:
1819        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1820
1821        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1822                                       options,
1823                                       description = cmd.description,
1824                                       formatter = HelpFormatter())
1825
1826        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1827    global verbose
1828    verbose = cmd.verbose
1829    if cmd.needsGit:
1830        if cmd.gitdir == None:
1831            cmd.gitdir = os.path.abspath(".git")
1832            if not isValidGitDir(cmd.gitdir):
1833                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1834                if os.path.exists(cmd.gitdir):
1835                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1836                    if len(cdup) > 0:
1837                        os.chdir(cdup);
1838
1839        if not isValidGitDir(cmd.gitdir):
1840            if isValidGitDir(cmd.gitdir + "/.git"):
1841                cmd.gitdir += "/.git"
1842            else:
1843                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1844
1845        os.environ["GIT_DIR"] = cmd.gitdir
1846
1847    if not cmd.run(args):
1848        parser.print_help()
1849
1850
1851if __name__ == '__main__':
1852    main()