Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Add a single command that will be used to construct the 'p4' command (21a5075)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s %s" % ("p4", cmd)
  28    return real_cmd
  29
  30def die(msg):
  31    if verbose:
  32        raise Exception(msg)
  33    else:
  34        sys.stderr.write(msg + "\n")
  35        sys.exit(1)
  36
  37def write_pipe(c, str):
  38    if verbose:
  39        sys.stderr.write('Writing pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'w')
  42    val = pipe.write(str)
  43    if pipe.close():
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48def read_pipe(c, ignore_error=False):
  49    if verbose:
  50        sys.stderr.write('Reading pipe: %s\n' % c)
  51
  52    pipe = os.popen(c, 'rb')
  53    val = pipe.read()
  54    if pipe.close() and not ignore_error:
  55        die('Command failed: %s' % c)
  56
  57    return val
  58
  59
  60def read_pipe_lines(c):
  61    if verbose:
  62        sys.stderr.write('Reading pipe: %s\n' % c)
  63    ## todo: check return status
  64    pipe = os.popen(c, 'rb')
  65    val = pipe.readlines()
  66    if pipe.close():
  67        die('Command failed: %s' % c)
  68
  69    return val
  70
  71def p4_read_pipe_lines(c):
  72    """Specifically invoke p4 on the command supplied. """
  73    real_cmd = "%s %s" % ("p4", c)
  74    if verbose:
  75        print real_cmd
  76    return read_pipe_lines(real_cmd)
  77
  78def system(cmd):
  79    if verbose:
  80        sys.stderr.write("executing %s\n" % cmd)
  81    if os.system(cmd) != 0:
  82        die("command failed: %s" % cmd)
  83
  84def p4_system(cmd):
  85    """Specifically invoke p4 as the system command. """
  86    real_cmd = "%s %s" % ("p4", cmd)
  87    if verbose:
  88        print real_cmd
  89    return system(real_cmd)
  90
  91def isP4Exec(kind):
  92    """Determine if a Perforce 'kind' should have execute permission
  93
  94    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  95    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  96    a plus sign, it is also executable"""
  97    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  98
  99def setP4ExecBit(file, mode):
 100    # Reopens an already open file and changes the execute bit to match
 101    # the execute bit setting in the passed in mode.
 102
 103    p4Type = "+x"
 104
 105    if not isModeExec(mode):
 106        p4Type = getP4OpenedType(file)
 107        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 108        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 109        if p4Type[-1] == "+":
 110            p4Type = p4Type[0:-1]
 111
 112    p4_system("reopen -t %s %s" % (p4Type, file))
 113
 114def getP4OpenedType(file):
 115    # Returns the perforce file type for the given file.
 116
 117    result = read_pipe("p4 opened %s" % file)
 118    match = re.match(".*\((.+)\)\r?$", result)
 119    if match:
 120        return match.group(1)
 121    else:
 122        die("Could not determine file type for %s (result: '%s')" % (file, result))
 123
 124def diffTreePattern():
 125    # This is a simple generator for the diff tree regex pattern. This could be
 126    # a class variable if this and parseDiffTreeEntry were a part of a class.
 127    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 128    while True:
 129        yield pattern
 130
 131def parseDiffTreeEntry(entry):
 132    """Parses a single diff tree entry into its component elements.
 133
 134    See git-diff-tree(1) manpage for details about the format of the diff
 135    output. This method returns a dictionary with the following elements:
 136
 137    src_mode - The mode of the source file
 138    dst_mode - The mode of the destination file
 139    src_sha1 - The sha1 for the source file
 140    dst_sha1 - The sha1 fr the destination file
 141    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 142    status_score - The score for the status (applicable for 'C' and 'R'
 143                   statuses). This is None if there is no score.
 144    src - The path for the source file.
 145    dst - The path for the destination file. This is only present for
 146          copy or renames. If it is not present, this is None.
 147
 148    If the pattern is not matched, None is returned."""
 149
 150    match = diffTreePattern().next().match(entry)
 151    if match:
 152        return {
 153            'src_mode': match.group(1),
 154            'dst_mode': match.group(2),
 155            'src_sha1': match.group(3),
 156            'dst_sha1': match.group(4),
 157            'status': match.group(5),
 158            'status_score': match.group(6),
 159            'src': match.group(7),
 160            'dst': match.group(10)
 161        }
 162    return None
 163
 164def isModeExec(mode):
 165    # Returns True if the given git mode represents an executable file,
 166    # otherwise False.
 167    return mode[-3:] == "755"
 168
 169def isModeExecChanged(src_mode, dst_mode):
 170    return isModeExec(src_mode) != isModeExec(dst_mode)
 171
 172def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 173    cmd = "p4 -G %s" % cmd
 174    if verbose:
 175        sys.stderr.write("Opening pipe: %s\n" % cmd)
 176
 177    # Use a temporary file to avoid deadlocks without
 178    # subprocess.communicate(), which would put another copy
 179    # of stdout into memory.
 180    stdin_file = None
 181    if stdin is not None:
 182        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 183        stdin_file.write(stdin)
 184        stdin_file.flush()
 185        stdin_file.seek(0)
 186
 187    p4 = subprocess.Popen(cmd, shell=True,
 188                          stdin=stdin_file,
 189                          stdout=subprocess.PIPE)
 190
 191    result = []
 192    try:
 193        while True:
 194            entry = marshal.load(p4.stdout)
 195            result.append(entry)
 196    except EOFError:
 197        pass
 198    exitCode = p4.wait()
 199    if exitCode != 0:
 200        entry = {}
 201        entry["p4ExitCode"] = exitCode
 202        result.append(entry)
 203
 204    return result
 205
 206def p4Cmd(cmd):
 207    list = p4CmdList(cmd)
 208    result = {}
 209    for entry in list:
 210        result.update(entry)
 211    return result;
 212
 213def p4Where(depotPath):
 214    if not depotPath.endswith("/"):
 215        depotPath += "/"
 216    output = p4Cmd("where %s..." % depotPath)
 217    if output["code"] == "error":
 218        return ""
 219    clientPath = ""
 220    if "path" in output:
 221        clientPath = output.get("path")
 222    elif "data" in output:
 223        data = output.get("data")
 224        lastSpace = data.rfind(" ")
 225        clientPath = data[lastSpace + 1:]
 226
 227    if clientPath.endswith("..."):
 228        clientPath = clientPath[:-3]
 229    return clientPath
 230
 231def currentGitBranch():
 232    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 233
 234def isValidGitDir(path):
 235    if (os.path.exists(path + "/HEAD")
 236        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 237        return True;
 238    return False
 239
 240def parseRevision(ref):
 241    return read_pipe("git rev-parse %s" % ref).strip()
 242
 243def extractLogMessageFromGitCommit(commit):
 244    logMessage = ""
 245
 246    ## fixme: title is first line of commit, not 1st paragraph.
 247    foundTitle = False
 248    for log in read_pipe_lines("git cat-file commit %s" % commit):
 249       if not foundTitle:
 250           if len(log) == 1:
 251               foundTitle = True
 252           continue
 253
 254       logMessage += log
 255    return logMessage
 256
 257def extractSettingsGitLog(log):
 258    values = {}
 259    for line in log.split("\n"):
 260        line = line.strip()
 261        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 262        if not m:
 263            continue
 264
 265        assignments = m.group(1).split (':')
 266        for a in assignments:
 267            vals = a.split ('=')
 268            key = vals[0].strip()
 269            val = ('='.join (vals[1:])).strip()
 270            if val.endswith ('\"') and val.startswith('"'):
 271                val = val[1:-1]
 272
 273            values[key] = val
 274
 275    paths = values.get("depot-paths")
 276    if not paths:
 277        paths = values.get("depot-path")
 278    if paths:
 279        values['depot-paths'] = paths.split(',')
 280    return values
 281
 282def gitBranchExists(branch):
 283    proc = subprocess.Popen(["git", "rev-parse", branch],
 284                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 285    return proc.wait() == 0;
 286
 287def gitConfig(key):
 288    return read_pipe("git config %s" % key, ignore_error=True).strip()
 289
 290def p4BranchesInGit(branchesAreInRemotes = True):
 291    branches = {}
 292
 293    cmdline = "git rev-parse --symbolic "
 294    if branchesAreInRemotes:
 295        cmdline += " --remotes"
 296    else:
 297        cmdline += " --branches"
 298
 299    for line in read_pipe_lines(cmdline):
 300        line = line.strip()
 301
 302        ## only import to p4/
 303        if not line.startswith('p4/') or line == "p4/HEAD":
 304            continue
 305        branch = line
 306
 307        # strip off p4
 308        branch = re.sub ("^p4/", "", line)
 309
 310        branches[branch] = parseRevision(line)
 311    return branches
 312
 313def findUpstreamBranchPoint(head = "HEAD"):
 314    branches = p4BranchesInGit()
 315    # map from depot-path to branch name
 316    branchByDepotPath = {}
 317    for branch in branches.keys():
 318        tip = branches[branch]
 319        log = extractLogMessageFromGitCommit(tip)
 320        settings = extractSettingsGitLog(log)
 321        if settings.has_key("depot-paths"):
 322            paths = ",".join(settings["depot-paths"])
 323            branchByDepotPath[paths] = "remotes/p4/" + branch
 324
 325    settings = None
 326    parent = 0
 327    while parent < 65535:
 328        commit = head + "~%s" % parent
 329        log = extractLogMessageFromGitCommit(commit)
 330        settings = extractSettingsGitLog(log)
 331        if settings.has_key("depot-paths"):
 332            paths = ",".join(settings["depot-paths"])
 333            if branchByDepotPath.has_key(paths):
 334                return [branchByDepotPath[paths], settings]
 335
 336        parent = parent + 1
 337
 338    return ["", settings]
 339
 340def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 341    if not silent:
 342        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 343               % localRefPrefix)
 344
 345    originPrefix = "origin/p4/"
 346
 347    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 348        line = line.strip()
 349        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 350            continue
 351
 352        headName = line[len(originPrefix):]
 353        remoteHead = localRefPrefix + headName
 354        originHead = line
 355
 356        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 357        if (not original.has_key('depot-paths')
 358            or not original.has_key('change')):
 359            continue
 360
 361        update = False
 362        if not gitBranchExists(remoteHead):
 363            if verbose:
 364                print "creating %s" % remoteHead
 365            update = True
 366        else:
 367            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 368            if settings.has_key('change') > 0:
 369                if settings['depot-paths'] == original['depot-paths']:
 370                    originP4Change = int(original['change'])
 371                    p4Change = int(settings['change'])
 372                    if originP4Change > p4Change:
 373                        print ("%s (%s) is newer than %s (%s). "
 374                               "Updating p4 branch from origin."
 375                               % (originHead, originP4Change,
 376                                  remoteHead, p4Change))
 377                        update = True
 378                else:
 379                    print ("Ignoring: %s was imported from %s while "
 380                           "%s was imported from %s"
 381                           % (originHead, ','.join(original['depot-paths']),
 382                              remoteHead, ','.join(settings['depot-paths'])))
 383
 384        if update:
 385            system("git update-ref %s %s" % (remoteHead, originHead))
 386
 387def originP4BranchesExist():
 388        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 389
 390def p4ChangesForPaths(depotPaths, changeRange):
 391    assert depotPaths
 392    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 393                                                        for p in depotPaths]))
 394
 395    changes = []
 396    for line in output:
 397        changeNum = line.split(" ")[1]
 398        changes.append(int(changeNum))
 399
 400    changes.sort()
 401    return changes
 402
 403class Command:
 404    def __init__(self):
 405        self.usage = "usage: %prog [options]"
 406        self.needsGit = True
 407
 408class P4Debug(Command):
 409    def __init__(self):
 410        Command.__init__(self)
 411        self.options = [
 412            optparse.make_option("--verbose", dest="verbose", action="store_true",
 413                                 default=False),
 414            ]
 415        self.description = "A tool to debug the output of p4 -G."
 416        self.needsGit = False
 417        self.verbose = False
 418
 419    def run(self, args):
 420        j = 0
 421        for output in p4CmdList(" ".join(args)):
 422            print 'Element: %d' % j
 423            j += 1
 424            print output
 425        return True
 426
 427class P4RollBack(Command):
 428    def __init__(self):
 429        Command.__init__(self)
 430        self.options = [
 431            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 432            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 433        ]
 434        self.description = "A tool to debug the multi-branch import. Don't use :)"
 435        self.verbose = False
 436        self.rollbackLocalBranches = False
 437
 438    def run(self, args):
 439        if len(args) != 1:
 440            return False
 441        maxChange = int(args[0])
 442
 443        if "p4ExitCode" in p4Cmd("changes -m 1"):
 444            die("Problems executing p4");
 445
 446        if self.rollbackLocalBranches:
 447            refPrefix = "refs/heads/"
 448            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 449        else:
 450            refPrefix = "refs/remotes/"
 451            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 452
 453        for line in lines:
 454            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 455                line = line.strip()
 456                ref = refPrefix + line
 457                log = extractLogMessageFromGitCommit(ref)
 458                settings = extractSettingsGitLog(log)
 459
 460                depotPaths = settings['depot-paths']
 461                change = settings['change']
 462
 463                changed = False
 464
 465                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 466                                                           for p in depotPaths]))) == 0:
 467                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 468                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 469                    continue
 470
 471                while change and int(change) > maxChange:
 472                    changed = True
 473                    if self.verbose:
 474                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 475                    system("git update-ref %s \"%s^\"" % (ref, ref))
 476                    log = extractLogMessageFromGitCommit(ref)
 477                    settings =  extractSettingsGitLog(log)
 478
 479
 480                    depotPaths = settings['depot-paths']
 481                    change = settings['change']
 482
 483                if changed:
 484                    print "%s rewound to %s" % (ref, change)
 485
 486        return True
 487
 488class P4Submit(Command):
 489    def __init__(self):
 490        Command.__init__(self)
 491        self.options = [
 492                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 493                optparse.make_option("--origin", dest="origin"),
 494                optparse.make_option("-M", dest="detectRename", action="store_true"),
 495        ]
 496        self.description = "Submit changes from git to the perforce depot."
 497        self.usage += " [name of git branch to submit into perforce depot]"
 498        self.interactive = True
 499        self.origin = ""
 500        self.detectRename = False
 501        self.verbose = False
 502        self.isWindows = (platform.system() == "Windows")
 503
 504    def check(self):
 505        if len(p4CmdList("opened ...")) > 0:
 506            die("You have files opened with perforce! Close them before starting the sync.")
 507
 508    # replaces everything between 'Description:' and the next P4 submit template field with the
 509    # commit message
 510    def prepareLogMessage(self, template, message):
 511        result = ""
 512
 513        inDescriptionSection = False
 514
 515        for line in template.split("\n"):
 516            if line.startswith("#"):
 517                result += line + "\n"
 518                continue
 519
 520            if inDescriptionSection:
 521                if line.startswith("Files:"):
 522                    inDescriptionSection = False
 523                else:
 524                    continue
 525            else:
 526                if line.startswith("Description:"):
 527                    inDescriptionSection = True
 528                    line += "\n"
 529                    for messageLine in message.split("\n"):
 530                        line += "\t" + messageLine + "\n"
 531
 532            result += line + "\n"
 533
 534        return result
 535
 536    def prepareSubmitTemplate(self):
 537        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 538        template = ""
 539        inFilesSection = False
 540        for line in p4_read_pipe_lines("change -o"):
 541            if line.endswith("\r\n"):
 542                line = line[:-2] + "\n"
 543            if inFilesSection:
 544                if line.startswith("\t"):
 545                    # path starts and ends with a tab
 546                    path = line[1:]
 547                    lastTab = path.rfind("\t")
 548                    if lastTab != -1:
 549                        path = path[:lastTab]
 550                        if not path.startswith(self.depotPath):
 551                            continue
 552                else:
 553                    inFilesSection = False
 554            else:
 555                if line.startswith("Files:"):
 556                    inFilesSection = True
 557
 558            template += line
 559
 560        return template
 561
 562    def applyCommit(self, id):
 563        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 564        diffOpts = ("", "-M")[self.detectRename]
 565        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 566        filesToAdd = set()
 567        filesToDelete = set()
 568        editedFiles = set()
 569        filesToChangeExecBit = {}
 570        for line in diff:
 571            diff = parseDiffTreeEntry(line)
 572            modifier = diff['status']
 573            path = diff['src']
 574            if modifier == "M":
 575                p4_system("edit \"%s\"" % path)
 576                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 577                    filesToChangeExecBit[path] = diff['dst_mode']
 578                editedFiles.add(path)
 579            elif modifier == "A":
 580                filesToAdd.add(path)
 581                filesToChangeExecBit[path] = diff['dst_mode']
 582                if path in filesToDelete:
 583                    filesToDelete.remove(path)
 584            elif modifier == "D":
 585                filesToDelete.add(path)
 586                if path in filesToAdd:
 587                    filesToAdd.remove(path)
 588            elif modifier == "R":
 589                src, dest = diff['src'], diff['dst']
 590                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 591                p4_system("edit \"%s\"" % (dest))
 592                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 593                    filesToChangeExecBit[dest] = diff['dst_mode']
 594                os.unlink(dest)
 595                editedFiles.add(dest)
 596                filesToDelete.add(src)
 597            else:
 598                die("unknown modifier %s for %s" % (modifier, path))
 599
 600        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 601        patchcmd = diffcmd + " | git apply "
 602        tryPatchCmd = patchcmd + "--check -"
 603        applyPatchCmd = patchcmd + "--check --apply -"
 604
 605        if os.system(tryPatchCmd) != 0:
 606            print "Unfortunately applying the change failed!"
 607            print "What do you want to do?"
 608            response = "x"
 609            while response != "s" and response != "a" and response != "w":
 610                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 611                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 612            if response == "s":
 613                print "Skipping! Good luck with the next patches..."
 614                for f in editedFiles:
 615                    p4_system("revert \"%s\"" % f);
 616                for f in filesToAdd:
 617                    system("rm %s" %f)
 618                return
 619            elif response == "a":
 620                os.system(applyPatchCmd)
 621                if len(filesToAdd) > 0:
 622                    print "You may also want to call p4 add on the following files:"
 623                    print " ".join(filesToAdd)
 624                if len(filesToDelete):
 625                    print "The following files should be scheduled for deletion with p4 delete:"
 626                    print " ".join(filesToDelete)
 627                die("Please resolve and submit the conflict manually and "
 628                    + "continue afterwards with git-p4 submit --continue")
 629            elif response == "w":
 630                system(diffcmd + " > patch.txt")
 631                print "Patch saved to patch.txt in %s !" % self.clientPath
 632                die("Please resolve and submit the conflict manually and "
 633                    "continue afterwards with git-p4 submit --continue")
 634
 635        system(applyPatchCmd)
 636
 637        for f in filesToAdd:
 638            p4_system("add \"%s\"" % f)
 639        for f in filesToDelete:
 640            p4_system("revert \"%s\"" % f)
 641            p4_system("delete \"%s\"" % f)
 642
 643        # Set/clear executable bits
 644        for f in filesToChangeExecBit.keys():
 645            mode = filesToChangeExecBit[f]
 646            setP4ExecBit(f, mode)
 647
 648        logMessage = extractLogMessageFromGitCommit(id)
 649        logMessage = logMessage.strip()
 650
 651        template = self.prepareSubmitTemplate()
 652
 653        if self.interactive:
 654            submitTemplate = self.prepareLogMessage(template, logMessage)
 655            if os.environ.has_key("P4DIFF"):
 656                del(os.environ["P4DIFF"])
 657            diff = read_pipe("p4 diff -du ...")
 658
 659            newdiff = ""
 660            for newFile in filesToAdd:
 661                newdiff += "==== new file ====\n"
 662                newdiff += "--- /dev/null\n"
 663                newdiff += "+++ %s\n" % newFile
 664                f = open(newFile, "r")
 665                for line in f.readlines():
 666                    newdiff += "+" + line
 667                f.close()
 668
 669            separatorLine = "######## everything below this line is just the diff #######\n"
 670
 671            [handle, fileName] = tempfile.mkstemp()
 672            tmpFile = os.fdopen(handle, "w+")
 673            if self.isWindows:
 674                submitTemplate = submitTemplate.replace("\n", "\r\n")
 675                separatorLine = separatorLine.replace("\n", "\r\n")
 676                newdiff = newdiff.replace("\n", "\r\n")
 677            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 678            tmpFile.close()
 679            defaultEditor = "vi"
 680            if platform.system() == "Windows":
 681                defaultEditor = "notepad"
 682            if os.environ.has_key("P4EDITOR"):
 683                editor = os.environ.get("P4EDITOR")
 684            else:
 685                editor = os.environ.get("EDITOR", defaultEditor);
 686            system(editor + " " + fileName)
 687            tmpFile = open(fileName, "rb")
 688            message = tmpFile.read()
 689            tmpFile.close()
 690            os.remove(fileName)
 691            submitTemplate = message[:message.index(separatorLine)]
 692            if self.isWindows:
 693                submitTemplate = submitTemplate.replace("\r\n", "\n")
 694
 695            write_pipe("p4 submit -i", submitTemplate)
 696        else:
 697            fileName = "submit.txt"
 698            file = open(fileName, "w+")
 699            file.write(self.prepareLogMessage(template, logMessage))
 700            file.close()
 701            print ("Perforce submit template written as %s. "
 702                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 703                   % (fileName, fileName))
 704
 705    def run(self, args):
 706        if len(args) == 0:
 707            self.master = currentGitBranch()
 708            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 709                die("Detecting current git branch failed!")
 710        elif len(args) == 1:
 711            self.master = args[0]
 712        else:
 713            return False
 714
 715        allowSubmit = gitConfig("git-p4.allowSubmit")
 716        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 717            die("%s is not in git-p4.allowSubmit" % self.master)
 718
 719        [upstream, settings] = findUpstreamBranchPoint()
 720        self.depotPath = settings['depot-paths'][0]
 721        if len(self.origin) == 0:
 722            self.origin = upstream
 723
 724        if self.verbose:
 725            print "Origin branch is " + self.origin
 726
 727        if len(self.depotPath) == 0:
 728            print "Internal error: cannot locate perforce depot path from existing branches"
 729            sys.exit(128)
 730
 731        self.clientPath = p4Where(self.depotPath)
 732
 733        if len(self.clientPath) == 0:
 734            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 735            sys.exit(128)
 736
 737        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 738        self.oldWorkingDirectory = os.getcwd()
 739
 740        os.chdir(self.clientPath)
 741        print "Syncronizing p4 checkout..."
 742        p4_system("sync ...")
 743
 744        self.check()
 745
 746        commits = []
 747        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 748            commits.append(line.strip())
 749        commits.reverse()
 750
 751        while len(commits) > 0:
 752            commit = commits[0]
 753            commits = commits[1:]
 754            self.applyCommit(commit)
 755            if not self.interactive:
 756                break
 757
 758        if len(commits) == 0:
 759            print "All changes applied!"
 760            os.chdir(self.oldWorkingDirectory)
 761
 762            sync = P4Sync()
 763            sync.run([])
 764
 765            rebase = P4Rebase()
 766            rebase.rebase()
 767
 768        return True
 769
 770class P4Sync(Command):
 771    def __init__(self):
 772        Command.__init__(self)
 773        self.options = [
 774                optparse.make_option("--branch", dest="branch"),
 775                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 776                optparse.make_option("--changesfile", dest="changesFile"),
 777                optparse.make_option("--silent", dest="silent", action="store_true"),
 778                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 779                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 780                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 781                                     help="Import into refs/heads/ , not refs/remotes"),
 782                optparse.make_option("--max-changes", dest="maxChanges"),
 783                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 784                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 785                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 786                                     help="Only sync files that are included in the Perforce Client Spec")
 787        ]
 788        self.description = """Imports from Perforce into a git repository.\n
 789    example:
 790    //depot/my/project/ -- to import the current head
 791    //depot/my/project/@all -- to import everything
 792    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 793
 794    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 795
 796        self.usage += " //depot/path[@revRange]"
 797        self.silent = False
 798        self.createdBranches = Set()
 799        self.committedChanges = Set()
 800        self.branch = ""
 801        self.detectBranches = False
 802        self.detectLabels = False
 803        self.changesFile = ""
 804        self.syncWithOrigin = True
 805        self.verbose = False
 806        self.importIntoRemotes = True
 807        self.maxChanges = ""
 808        self.isWindows = (platform.system() == "Windows")
 809        self.keepRepoPath = False
 810        self.depotPaths = None
 811        self.p4BranchesInGit = []
 812        self.cloneExclude = []
 813        self.useClientSpec = False
 814        self.clientSpecDirs = []
 815
 816        if gitConfig("git-p4.syncFromOrigin") == "false":
 817            self.syncWithOrigin = False
 818
 819    def extractFilesFromCommit(self, commit):
 820        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 821                             for path in self.cloneExclude]
 822        files = []
 823        fnum = 0
 824        while commit.has_key("depotFile%s" % fnum):
 825            path =  commit["depotFile%s" % fnum]
 826
 827            if [p for p in self.cloneExclude
 828                if path.startswith (p)]:
 829                found = False
 830            else:
 831                found = [p for p in self.depotPaths
 832                         if path.startswith (p)]
 833            if not found:
 834                fnum = fnum + 1
 835                continue
 836
 837            file = {}
 838            file["path"] = path
 839            file["rev"] = commit["rev%s" % fnum]
 840            file["action"] = commit["action%s" % fnum]
 841            file["type"] = commit["type%s" % fnum]
 842            files.append(file)
 843            fnum = fnum + 1
 844        return files
 845
 846    def stripRepoPath(self, path, prefixes):
 847        if self.keepRepoPath:
 848            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 849
 850        for p in prefixes:
 851            if path.startswith(p):
 852                path = path[len(p):]
 853
 854        return path
 855
 856    def splitFilesIntoBranches(self, commit):
 857        branches = {}
 858        fnum = 0
 859        while commit.has_key("depotFile%s" % fnum):
 860            path =  commit["depotFile%s" % fnum]
 861            found = [p for p in self.depotPaths
 862                     if path.startswith (p)]
 863            if not found:
 864                fnum = fnum + 1
 865                continue
 866
 867            file = {}
 868            file["path"] = path
 869            file["rev"] = commit["rev%s" % fnum]
 870            file["action"] = commit["action%s" % fnum]
 871            file["type"] = commit["type%s" % fnum]
 872            fnum = fnum + 1
 873
 874            relPath = self.stripRepoPath(path, self.depotPaths)
 875
 876            for branch in self.knownBranches.keys():
 877
 878                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 879                if relPath.startswith(branch + "/"):
 880                    if branch not in branches:
 881                        branches[branch] = []
 882                    branches[branch].append(file)
 883                    break
 884
 885        return branches
 886
 887    ## Should move this out, doesn't use SELF.
 888    def readP4Files(self, files):
 889        filesForCommit = []
 890        filesToRead = []
 891
 892        for f in files:
 893            includeFile = True
 894            for val in self.clientSpecDirs:
 895                if f['path'].startswith(val[0]):
 896                    if val[1] <= 0:
 897                        includeFile = False
 898                    break
 899
 900            if includeFile:
 901                filesForCommit.append(f)
 902                if f['action'] != 'delete':
 903                    filesToRead.append(f)
 904
 905        filedata = []
 906        if len(filesToRead) > 0:
 907            filedata = p4CmdList('-x - print',
 908                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 909                                                  for f in filesToRead]),
 910                                 stdin_mode='w+')
 911
 912            if "p4ExitCode" in filedata[0]:
 913                die("Problems executing p4. Error: [%d]."
 914                    % (filedata[0]['p4ExitCode']));
 915
 916        j = 0;
 917        contents = {}
 918        while j < len(filedata):
 919            stat = filedata[j]
 920            j += 1
 921            text = [];
 922            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 923                text.append(filedata[j]['data'])
 924                j += 1
 925            text = ''.join(text)
 926
 927            if not stat.has_key('depotFile'):
 928                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 929                continue
 930
 931            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 932                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 933            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 934                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 935
 936            contents[stat['depotFile']] = text
 937
 938        for f in filesForCommit:
 939            path = f['path']
 940            if contents.has_key(path):
 941                f['data'] = contents[path]
 942
 943        return filesForCommit
 944
 945    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 946        epoch = details["time"]
 947        author = details["user"]
 948
 949        if self.verbose:
 950            print "commit into %s" % branch
 951
 952        # start with reading files; if that fails, we should not
 953        # create a commit.
 954        new_files = []
 955        for f in files:
 956            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 957                new_files.append (f)
 958            else:
 959                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 960        files = self.readP4Files(new_files)
 961
 962        self.gitStream.write("commit %s\n" % branch)
 963#        gitStream.write("mark :%s\n" % details["change"])
 964        self.committedChanges.add(int(details["change"]))
 965        committer = ""
 966        if author not in self.users:
 967            self.getUserMapFromPerforceServer()
 968        if author in self.users:
 969            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 970        else:
 971            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 972
 973        self.gitStream.write("committer %s\n" % committer)
 974
 975        self.gitStream.write("data <<EOT\n")
 976        self.gitStream.write(details["desc"])
 977        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 978                             % (','.join (branchPrefixes), details["change"]))
 979        if len(details['options']) > 0:
 980            self.gitStream.write(": options = %s" % details['options'])
 981        self.gitStream.write("]\nEOT\n\n")
 982
 983        if len(parent) > 0:
 984            if self.verbose:
 985                print "parent %s" % parent
 986            self.gitStream.write("from %s\n" % parent)
 987
 988        for file in files:
 989            if file["type"] == "apple":
 990                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 991                continue
 992
 993            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 994            if file["action"] == "delete":
 995                self.gitStream.write("D %s\n" % relPath)
 996            else:
 997                data = file['data']
 998
 999                mode = "644"
1000                if isP4Exec(file["type"]):
1001                    mode = "755"
1002                elif file["type"] == "symlink":
1003                    mode = "120000"
1004                    # p4 print on a symlink contains "target\n", so strip it off
1005                    data = data[:-1]
1006
1007                if self.isWindows and file["type"].endswith("text"):
1008                    data = data.replace("\r\n", "\n")
1009
1010                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1011                self.gitStream.write("data %s\n" % len(data))
1012                self.gitStream.write(data)
1013                self.gitStream.write("\n")
1014
1015        self.gitStream.write("\n")
1016
1017        change = int(details["change"])
1018
1019        if self.labels.has_key(change):
1020            label = self.labels[change]
1021            labelDetails = label[0]
1022            labelRevisions = label[1]
1023            if self.verbose:
1024                print "Change %s is labelled %s" % (change, labelDetails)
1025
1026            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1027                                                    for p in branchPrefixes]))
1028
1029            if len(files) == len(labelRevisions):
1030
1031                cleanedFiles = {}
1032                for info in files:
1033                    if info["action"] == "delete":
1034                        continue
1035                    cleanedFiles[info["depotFile"]] = info["rev"]
1036
1037                if cleanedFiles == labelRevisions:
1038                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1039                    self.gitStream.write("from %s\n" % branch)
1040
1041                    owner = labelDetails["Owner"]
1042                    tagger = ""
1043                    if author in self.users:
1044                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1045                    else:
1046                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1047                    self.gitStream.write("tagger %s\n" % tagger)
1048                    self.gitStream.write("data <<EOT\n")
1049                    self.gitStream.write(labelDetails["Description"])
1050                    self.gitStream.write("EOT\n\n")
1051
1052                else:
1053                    if not self.silent:
1054                        print ("Tag %s does not match with change %s: files do not match."
1055                               % (labelDetails["label"], change))
1056
1057            else:
1058                if not self.silent:
1059                    print ("Tag %s does not match with change %s: file count is different."
1060                           % (labelDetails["label"], change))
1061
1062    def getUserCacheFilename(self):
1063        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1064        return home + "/.gitp4-usercache.txt"
1065
1066    def getUserMapFromPerforceServer(self):
1067        if self.userMapFromPerforceServer:
1068            return
1069        self.users = {}
1070
1071        for output in p4CmdList("users"):
1072            if not output.has_key("User"):
1073                continue
1074            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1075
1076
1077        s = ''
1078        for (key, val) in self.users.items():
1079            s += "%s\t%s\n" % (key, val)
1080
1081        open(self.getUserCacheFilename(), "wb").write(s)
1082        self.userMapFromPerforceServer = True
1083
1084    def loadUserMapFromCache(self):
1085        self.users = {}
1086        self.userMapFromPerforceServer = False
1087        try:
1088            cache = open(self.getUserCacheFilename(), "rb")
1089            lines = cache.readlines()
1090            cache.close()
1091            for line in lines:
1092                entry = line.strip().split("\t")
1093                self.users[entry[0]] = entry[1]
1094        except IOError:
1095            self.getUserMapFromPerforceServer()
1096
1097    def getLabels(self):
1098        self.labels = {}
1099
1100        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1101        if len(l) > 0 and not self.silent:
1102            print "Finding files belonging to labels in %s" % `self.depotPaths`
1103
1104        for output in l:
1105            label = output["label"]
1106            revisions = {}
1107            newestChange = 0
1108            if self.verbose:
1109                print "Querying files for label %s" % label
1110            for file in p4CmdList("files "
1111                                  +  ' '.join (["%s...@%s" % (p, label)
1112                                                for p in self.depotPaths])):
1113                revisions[file["depotFile"]] = file["rev"]
1114                change = int(file["change"])
1115                if change > newestChange:
1116                    newestChange = change
1117
1118            self.labels[newestChange] = [output, revisions]
1119
1120        if self.verbose:
1121            print "Label changes: %s" % self.labels.keys()
1122
1123    def guessProjectName(self):
1124        for p in self.depotPaths:
1125            if p.endswith("/"):
1126                p = p[:-1]
1127            p = p[p.strip().rfind("/") + 1:]
1128            if not p.endswith("/"):
1129               p += "/"
1130            return p
1131
1132    def getBranchMapping(self):
1133        lostAndFoundBranches = set()
1134
1135        for info in p4CmdList("branches"):
1136            details = p4Cmd("branch -o %s" % info["branch"])
1137            viewIdx = 0
1138            while details.has_key("View%s" % viewIdx):
1139                paths = details["View%s" % viewIdx].split(" ")
1140                viewIdx = viewIdx + 1
1141                # require standard //depot/foo/... //depot/bar/... mapping
1142                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1143                    continue
1144                source = paths[0]
1145                destination = paths[1]
1146                ## HACK
1147                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1148                    source = source[len(self.depotPaths[0]):-4]
1149                    destination = destination[len(self.depotPaths[0]):-4]
1150
1151                    if destination in self.knownBranches:
1152                        if not self.silent:
1153                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1154                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1155                        continue
1156
1157                    self.knownBranches[destination] = source
1158
1159                    lostAndFoundBranches.discard(destination)
1160
1161                    if source not in self.knownBranches:
1162                        lostAndFoundBranches.add(source)
1163
1164
1165        for branch in lostAndFoundBranches:
1166            self.knownBranches[branch] = branch
1167
1168    def getBranchMappingFromGitBranches(self):
1169        branches = p4BranchesInGit(self.importIntoRemotes)
1170        for branch in branches.keys():
1171            if branch == "master":
1172                branch = "main"
1173            else:
1174                branch = branch[len(self.projectName):]
1175            self.knownBranches[branch] = branch
1176
1177    def listExistingP4GitBranches(self):
1178        # branches holds mapping from name to commit
1179        branches = p4BranchesInGit(self.importIntoRemotes)
1180        self.p4BranchesInGit = branches.keys()
1181        for branch in branches.keys():
1182            self.initialParents[self.refPrefix + branch] = branches[branch]
1183
1184    def updateOptionDict(self, d):
1185        option_keys = {}
1186        if self.keepRepoPath:
1187            option_keys['keepRepoPath'] = 1
1188
1189        d["options"] = ' '.join(sorted(option_keys.keys()))
1190
1191    def readOptions(self, d):
1192        self.keepRepoPath = (d.has_key('options')
1193                             and ('keepRepoPath' in d['options']))
1194
1195    def gitRefForBranch(self, branch):
1196        if branch == "main":
1197            return self.refPrefix + "master"
1198
1199        if len(branch) <= 0:
1200            return branch
1201
1202        return self.refPrefix + self.projectName + branch
1203
1204    def gitCommitByP4Change(self, ref, change):
1205        if self.verbose:
1206            print "looking in ref " + ref + " for change %s using bisect..." % change
1207
1208        earliestCommit = ""
1209        latestCommit = parseRevision(ref)
1210
1211        while True:
1212            if self.verbose:
1213                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1214            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1215            if len(next) == 0:
1216                if self.verbose:
1217                    print "argh"
1218                return ""
1219            log = extractLogMessageFromGitCommit(next)
1220            settings = extractSettingsGitLog(log)
1221            currentChange = int(settings['change'])
1222            if self.verbose:
1223                print "current change %s" % currentChange
1224
1225            if currentChange == change:
1226                if self.verbose:
1227                    print "found %s" % next
1228                return next
1229
1230            if currentChange < change:
1231                earliestCommit = "^%s" % next
1232            else:
1233                latestCommit = "%s" % next
1234
1235        return ""
1236
1237    def importNewBranch(self, branch, maxChange):
1238        # make fast-import flush all changes to disk and update the refs using the checkpoint
1239        # command so that we can try to find the branch parent in the git history
1240        self.gitStream.write("checkpoint\n\n");
1241        self.gitStream.flush();
1242        branchPrefix = self.depotPaths[0] + branch + "/"
1243        range = "@1,%s" % maxChange
1244        #print "prefix" + branchPrefix
1245        changes = p4ChangesForPaths([branchPrefix], range)
1246        if len(changes) <= 0:
1247            return False
1248        firstChange = changes[0]
1249        #print "first change in branch: %s" % firstChange
1250        sourceBranch = self.knownBranches[branch]
1251        sourceDepotPath = self.depotPaths[0] + sourceBranch
1252        sourceRef = self.gitRefForBranch(sourceBranch)
1253        #print "source " + sourceBranch
1254
1255        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1256        #print "branch parent: %s" % branchParentChange
1257        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1258        if len(gitParent) > 0:
1259            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1260            #print "parent git commit: %s" % gitParent
1261
1262        self.importChanges(changes)
1263        return True
1264
1265    def importChanges(self, changes):
1266        cnt = 1
1267        for change in changes:
1268            description = p4Cmd("describe %s" % change)
1269            self.updateOptionDict(description)
1270
1271            if not self.silent:
1272                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1273                sys.stdout.flush()
1274            cnt = cnt + 1
1275
1276            try:
1277                if self.detectBranches:
1278                    branches = self.splitFilesIntoBranches(description)
1279                    for branch in branches.keys():
1280                        ## HACK  --hwn
1281                        branchPrefix = self.depotPaths[0] + branch + "/"
1282
1283                        parent = ""
1284
1285                        filesForCommit = branches[branch]
1286
1287                        if self.verbose:
1288                            print "branch is %s" % branch
1289
1290                        self.updatedBranches.add(branch)
1291
1292                        if branch not in self.createdBranches:
1293                            self.createdBranches.add(branch)
1294                            parent = self.knownBranches[branch]
1295                            if parent == branch:
1296                                parent = ""
1297                            else:
1298                                fullBranch = self.projectName + branch
1299                                if fullBranch not in self.p4BranchesInGit:
1300                                    if not self.silent:
1301                                        print("\n    Importing new branch %s" % fullBranch);
1302                                    if self.importNewBranch(branch, change - 1):
1303                                        parent = ""
1304                                        self.p4BranchesInGit.append(fullBranch)
1305                                    if not self.silent:
1306                                        print("\n    Resuming with change %s" % change);
1307
1308                                if self.verbose:
1309                                    print "parent determined through known branches: %s" % parent
1310
1311                        branch = self.gitRefForBranch(branch)
1312                        parent = self.gitRefForBranch(parent)
1313
1314                        if self.verbose:
1315                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1316
1317                        if len(parent) == 0 and branch in self.initialParents:
1318                            parent = self.initialParents[branch]
1319                            del self.initialParents[branch]
1320
1321                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1322                else:
1323                    files = self.extractFilesFromCommit(description)
1324                    self.commit(description, files, self.branch, self.depotPaths,
1325                                self.initialParent)
1326                    self.initialParent = ""
1327            except IOError:
1328                print self.gitError.read()
1329                sys.exit(1)
1330
1331    def importHeadRevision(self, revision):
1332        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1333
1334        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1335        details["desc"] = ("Initial import of %s from the state at revision %s"
1336                           % (' '.join(self.depotPaths), revision))
1337        details["change"] = revision
1338        newestRevision = 0
1339
1340        fileCnt = 0
1341        for info in p4CmdList("files "
1342                              +  ' '.join(["%s...%s"
1343                                           % (p, revision)
1344                                           for p in self.depotPaths])):
1345
1346            if info['code'] == 'error':
1347                sys.stderr.write("p4 returned an error: %s\n"
1348                                 % info['data'])
1349                sys.exit(1)
1350
1351
1352            change = int(info["change"])
1353            if change > newestRevision:
1354                newestRevision = change
1355
1356            if info["action"] == "delete":
1357                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1358                #fileCnt = fileCnt + 1
1359                continue
1360
1361            for prop in ["depotFile", "rev", "action", "type" ]:
1362                details["%s%s" % (prop, fileCnt)] = info[prop]
1363
1364            fileCnt = fileCnt + 1
1365
1366        details["change"] = newestRevision
1367        self.updateOptionDict(details)
1368        try:
1369            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1370        except IOError:
1371            print "IO error with git fast-import. Is your git version recent enough?"
1372            print self.gitError.read()
1373
1374
1375    def getClientSpec(self):
1376        specList = p4CmdList( "client -o" )
1377        temp = {}
1378        for entry in specList:
1379            for k,v in entry.iteritems():
1380                if k.startswith("View"):
1381                    if v.startswith('"'):
1382                        start = 1
1383                    else:
1384                        start = 0
1385                    index = v.find("...")
1386                    v = v[start:index]
1387                    if v.startswith("-"):
1388                        v = v[1:]
1389                        temp[v] = -len(v)
1390                    else:
1391                        temp[v] = len(v)
1392        self.clientSpecDirs = temp.items()
1393        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1394
1395    def run(self, args):
1396        self.depotPaths = []
1397        self.changeRange = ""
1398        self.initialParent = ""
1399        self.previousDepotPaths = []
1400
1401        # map from branch depot path to parent branch
1402        self.knownBranches = {}
1403        self.initialParents = {}
1404        self.hasOrigin = originP4BranchesExist()
1405        if not self.syncWithOrigin:
1406            self.hasOrigin = False
1407
1408        if self.importIntoRemotes:
1409            self.refPrefix = "refs/remotes/p4/"
1410        else:
1411            self.refPrefix = "refs/heads/p4/"
1412
1413        if self.syncWithOrigin and self.hasOrigin:
1414            if not self.silent:
1415                print "Syncing with origin first by calling git fetch origin"
1416            system("git fetch origin")
1417
1418        if len(self.branch) == 0:
1419            self.branch = self.refPrefix + "master"
1420            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1421                system("git update-ref %s refs/heads/p4" % self.branch)
1422                system("git branch -D p4");
1423            # create it /after/ importing, when master exists
1424            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1425                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1426
1427        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1428            self.getClientSpec()
1429
1430        # TODO: should always look at previous commits,
1431        # merge with previous imports, if possible.
1432        if args == []:
1433            if self.hasOrigin:
1434                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1435            self.listExistingP4GitBranches()
1436
1437            if len(self.p4BranchesInGit) > 1:
1438                if not self.silent:
1439                    print "Importing from/into multiple branches"
1440                self.detectBranches = True
1441
1442            if self.verbose:
1443                print "branches: %s" % self.p4BranchesInGit
1444
1445            p4Change = 0
1446            for branch in self.p4BranchesInGit:
1447                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1448
1449                settings = extractSettingsGitLog(logMsg)
1450
1451                self.readOptions(settings)
1452                if (settings.has_key('depot-paths')
1453                    and settings.has_key ('change')):
1454                    change = int(settings['change']) + 1
1455                    p4Change = max(p4Change, change)
1456
1457                    depotPaths = sorted(settings['depot-paths'])
1458                    if self.previousDepotPaths == []:
1459                        self.previousDepotPaths = depotPaths
1460                    else:
1461                        paths = []
1462                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1463                            for i in range(0, min(len(cur), len(prev))):
1464                                if cur[i] <> prev[i]:
1465                                    i = i - 1
1466                                    break
1467
1468                            paths.append (cur[:i + 1])
1469
1470                        self.previousDepotPaths = paths
1471
1472            if p4Change > 0:
1473                self.depotPaths = sorted(self.previousDepotPaths)
1474                self.changeRange = "@%s,#head" % p4Change
1475                if not self.detectBranches:
1476                    self.initialParent = parseRevision(self.branch)
1477                if not self.silent and not self.detectBranches:
1478                    print "Performing incremental import into %s git branch" % self.branch
1479
1480        if not self.branch.startswith("refs/"):
1481            self.branch = "refs/heads/" + self.branch
1482
1483        if len(args) == 0 and self.depotPaths:
1484            if not self.silent:
1485                print "Depot paths: %s" % ' '.join(self.depotPaths)
1486        else:
1487            if self.depotPaths and self.depotPaths != args:
1488                print ("previous import used depot path %s and now %s was specified. "
1489                       "This doesn't work!" % (' '.join (self.depotPaths),
1490                                               ' '.join (args)))
1491                sys.exit(1)
1492
1493            self.depotPaths = sorted(args)
1494
1495        revision = ""
1496        self.users = {}
1497
1498        newPaths = []
1499        for p in self.depotPaths:
1500            if p.find("@") != -1:
1501                atIdx = p.index("@")
1502                self.changeRange = p[atIdx:]
1503                if self.changeRange == "@all":
1504                    self.changeRange = ""
1505                elif ',' not in self.changeRange:
1506                    revision = self.changeRange
1507                    self.changeRange = ""
1508                p = p[:atIdx]
1509            elif p.find("#") != -1:
1510                hashIdx = p.index("#")
1511                revision = p[hashIdx:]
1512                p = p[:hashIdx]
1513            elif self.previousDepotPaths == []:
1514                revision = "#head"
1515
1516            p = re.sub ("\.\.\.$", "", p)
1517            if not p.endswith("/"):
1518                p += "/"
1519
1520            newPaths.append(p)
1521
1522        self.depotPaths = newPaths
1523
1524
1525        self.loadUserMapFromCache()
1526        self.labels = {}
1527        if self.detectLabels:
1528            self.getLabels();
1529
1530        if self.detectBranches:
1531            ## FIXME - what's a P4 projectName ?
1532            self.projectName = self.guessProjectName()
1533
1534            if self.hasOrigin:
1535                self.getBranchMappingFromGitBranches()
1536            else:
1537                self.getBranchMapping()
1538            if self.verbose:
1539                print "p4-git branches: %s" % self.p4BranchesInGit
1540                print "initial parents: %s" % self.initialParents
1541            for b in self.p4BranchesInGit:
1542                if b != "master":
1543
1544                    ## FIXME
1545                    b = b[len(self.projectName):]
1546                self.createdBranches.add(b)
1547
1548        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1549
1550        importProcess = subprocess.Popen(["git", "fast-import"],
1551                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1552                                         stderr=subprocess.PIPE);
1553        self.gitOutput = importProcess.stdout
1554        self.gitStream = importProcess.stdin
1555        self.gitError = importProcess.stderr
1556
1557        if revision:
1558            self.importHeadRevision(revision)
1559        else:
1560            changes = []
1561
1562            if len(self.changesFile) > 0:
1563                output = open(self.changesFile).readlines()
1564                changeSet = Set()
1565                for line in output:
1566                    changeSet.add(int(line))
1567
1568                for change in changeSet:
1569                    changes.append(change)
1570
1571                changes.sort()
1572            else:
1573                if self.verbose:
1574                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1575                                                              self.changeRange)
1576                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1577
1578                if len(self.maxChanges) > 0:
1579                    changes = changes[:min(int(self.maxChanges), len(changes))]
1580
1581            if len(changes) == 0:
1582                if not self.silent:
1583                    print "No changes to import!"
1584                return True
1585
1586            if not self.silent and not self.detectBranches:
1587                print "Import destination: %s" % self.branch
1588
1589            self.updatedBranches = set()
1590
1591            self.importChanges(changes)
1592
1593            if not self.silent:
1594                print ""
1595                if len(self.updatedBranches) > 0:
1596                    sys.stdout.write("Updated branches: ")
1597                    for b in self.updatedBranches:
1598                        sys.stdout.write("%s " % b)
1599                    sys.stdout.write("\n")
1600
1601        self.gitStream.close()
1602        if importProcess.wait() != 0:
1603            die("fast-import failed: %s" % self.gitError.read())
1604        self.gitOutput.close()
1605        self.gitError.close()
1606
1607        return True
1608
1609class P4Rebase(Command):
1610    def __init__(self):
1611        Command.__init__(self)
1612        self.options = [ ]
1613        self.description = ("Fetches the latest revision from perforce and "
1614                            + "rebases the current work (branch) against it")
1615        self.verbose = False
1616
1617    def run(self, args):
1618        sync = P4Sync()
1619        sync.run([])
1620
1621        return self.rebase()
1622
1623    def rebase(self):
1624        if os.system("git update-index --refresh") != 0:
1625            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1626        if len(read_pipe("git diff-index HEAD --")) > 0:
1627            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1628
1629        [upstream, settings] = findUpstreamBranchPoint()
1630        if len(upstream) == 0:
1631            die("Cannot find upstream branchpoint for rebase")
1632
1633        # the branchpoint may be p4/foo~3, so strip off the parent
1634        upstream = re.sub("~[0-9]+$", "", upstream)
1635
1636        print "Rebasing the current branch onto %s" % upstream
1637        oldHead = read_pipe("git rev-parse HEAD").strip()
1638        system("git rebase %s" % upstream)
1639        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1640        return True
1641
1642class P4Clone(P4Sync):
1643    def __init__(self):
1644        P4Sync.__init__(self)
1645        self.description = "Creates a new git repository and imports from Perforce into it"
1646        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1647        self.options += [
1648            optparse.make_option("--destination", dest="cloneDestination",
1649                                 action='store', default=None,
1650                                 help="where to leave result of the clone"),
1651            optparse.make_option("-/", dest="cloneExclude",
1652                                 action="append", type="string",
1653                                 help="exclude depot path")
1654        ]
1655        self.cloneDestination = None
1656        self.needsGit = False
1657
1658    # This is required for the "append" cloneExclude action
1659    def ensure_value(self, attr, value):
1660        if not hasattr(self, attr) or getattr(self, attr) is None:
1661            setattr(self, attr, value)
1662        return getattr(self, attr)
1663
1664    def defaultDestination(self, args):
1665        ## TODO: use common prefix of args?
1666        depotPath = args[0]
1667        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1668        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1669        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1670        depotDir = re.sub(r"/$", "", depotDir)
1671        return os.path.split(depotDir)[1]
1672
1673    def run(self, args):
1674        if len(args) < 1:
1675            return False
1676
1677        if self.keepRepoPath and not self.cloneDestination:
1678            sys.stderr.write("Must specify destination for --keep-path\n")
1679            sys.exit(1)
1680
1681        depotPaths = args
1682
1683        if not self.cloneDestination and len(depotPaths) > 1:
1684            self.cloneDestination = depotPaths[-1]
1685            depotPaths = depotPaths[:-1]
1686
1687        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1688        for p in depotPaths:
1689            if not p.startswith("//"):
1690                return False
1691
1692        if not self.cloneDestination:
1693            self.cloneDestination = self.defaultDestination(args)
1694
1695        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1696        if not os.path.exists(self.cloneDestination):
1697            os.makedirs(self.cloneDestination)
1698        os.chdir(self.cloneDestination)
1699        system("git init")
1700        self.gitdir = os.getcwd() + "/.git"
1701        if not P4Sync.run(self, depotPaths):
1702            return False
1703        if self.branch != "master":
1704            if gitBranchExists("refs/remotes/p4/master"):
1705                system("git branch master refs/remotes/p4/master")
1706                system("git checkout -f")
1707            else:
1708                print "Could not detect main branch. No checkout/master branch created."
1709
1710        return True
1711
1712class P4Branches(Command):
1713    def __init__(self):
1714        Command.__init__(self)
1715        self.options = [ ]
1716        self.description = ("Shows the git branches that hold imports and their "
1717                            + "corresponding perforce depot paths")
1718        self.verbose = False
1719
1720    def run(self, args):
1721        if originP4BranchesExist():
1722            createOrUpdateBranchesFromOrigin()
1723
1724        cmdline = "git rev-parse --symbolic "
1725        cmdline += " --remotes"
1726
1727        for line in read_pipe_lines(cmdline):
1728            line = line.strip()
1729
1730            if not line.startswith('p4/') or line == "p4/HEAD":
1731                continue
1732            branch = line
1733
1734            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1735            settings = extractSettingsGitLog(log)
1736
1737            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1738        return True
1739
1740class HelpFormatter(optparse.IndentedHelpFormatter):
1741    def __init__(self):
1742        optparse.IndentedHelpFormatter.__init__(self)
1743
1744    def format_description(self, description):
1745        if description:
1746            return description + "\n"
1747        else:
1748            return ""
1749
1750def printUsage(commands):
1751    print "usage: %s <command> [options]" % sys.argv[0]
1752    print ""
1753    print "valid commands: %s" % ", ".join(commands)
1754    print ""
1755    print "Try %s <command> --help for command specific help." % sys.argv[0]
1756    print ""
1757
1758commands = {
1759    "debug" : P4Debug,
1760    "submit" : P4Submit,
1761    "commit" : P4Submit,
1762    "sync" : P4Sync,
1763    "rebase" : P4Rebase,
1764    "clone" : P4Clone,
1765    "rollback" : P4RollBack,
1766    "branches" : P4Branches
1767}
1768
1769
1770def main():
1771    if len(sys.argv[1:]) == 0:
1772        printUsage(commands.keys())
1773        sys.exit(2)
1774
1775    cmd = ""
1776    cmdName = sys.argv[1]
1777    try:
1778        klass = commands[cmdName]
1779        cmd = klass()
1780    except KeyError:
1781        print "unknown command %s" % cmdName
1782        print ""
1783        printUsage(commands.keys())
1784        sys.exit(2)
1785
1786    options = cmd.options
1787    cmd.gitdir = os.environ.get("GIT_DIR", None)
1788
1789    args = sys.argv[2:]
1790
1791    if len(options) > 0:
1792        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1793
1794        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1795                                       options,
1796                                       description = cmd.description,
1797                                       formatter = HelpFormatter())
1798
1799        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1800    global verbose
1801    verbose = cmd.verbose
1802    if cmd.needsGit:
1803        if cmd.gitdir == None:
1804            cmd.gitdir = os.path.abspath(".git")
1805            if not isValidGitDir(cmd.gitdir):
1806                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1807                if os.path.exists(cmd.gitdir):
1808                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1809                    if len(cdup) > 0:
1810                        os.chdir(cdup);
1811
1812        if not isValidGitDir(cmd.gitdir):
1813            if isValidGitDir(cmd.gitdir + "/.git"):
1814                cmd.gitdir += "/.git"
1815            else:
1816                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1817
1818        os.environ["GIT_DIR"] = cmd.gitdir
1819
1820    if not cmd.run(args):
1821        parser.print_help()
1822
1823
1824if __name__ == '__main__':
1825    main()