Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Utilise the new 'p4_read_pipe_lines' command (b340fa4)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def p4_read_pipe_lines(c):
  61    """Specifically invoke p4 on the command supplied. """
  62    real_cmd = "%s %s" % ("p4", c)
  63    if verbose:
  64        print real_cmd
  65    return read_pipe_lines(real_cmd)
  66
  67def system(cmd):
  68    if verbose:
  69        sys.stderr.write("executing %s\n" % cmd)
  70    if os.system(cmd) != 0:
  71        die("command failed: %s" % cmd)
  72
  73def isP4Exec(kind):
  74    """Determine if a Perforce 'kind' should have execute permission
  75
  76    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  77    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  78    a plus sign, it is also executable"""
  79    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  80
  81def setP4ExecBit(file, mode):
  82    # Reopens an already open file and changes the execute bit to match
  83    # the execute bit setting in the passed in mode.
  84
  85    p4Type = "+x"
  86
  87    if not isModeExec(mode):
  88        p4Type = getP4OpenedType(file)
  89        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  90        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  91        if p4Type[-1] == "+":
  92            p4Type = p4Type[0:-1]
  93
  94    system("p4 reopen -t %s %s" % (p4Type, file))
  95
  96def getP4OpenedType(file):
  97    # Returns the perforce file type for the given file.
  98
  99    result = read_pipe("p4 opened %s" % file)
 100    match = re.match(".*\((.+)\)\r?$", result)
 101    if match:
 102        return match.group(1)
 103    else:
 104        die("Could not determine file type for %s (result: '%s')" % (file, result))
 105
 106def diffTreePattern():
 107    # This is a simple generator for the diff tree regex pattern. This could be
 108    # a class variable if this and parseDiffTreeEntry were a part of a class.
 109    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 110    while True:
 111        yield pattern
 112
 113def parseDiffTreeEntry(entry):
 114    """Parses a single diff tree entry into its component elements.
 115
 116    See git-diff-tree(1) manpage for details about the format of the diff
 117    output. This method returns a dictionary with the following elements:
 118
 119    src_mode - The mode of the source file
 120    dst_mode - The mode of the destination file
 121    src_sha1 - The sha1 for the source file
 122    dst_sha1 - The sha1 fr the destination file
 123    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 124    status_score - The score for the status (applicable for 'C' and 'R'
 125                   statuses). This is None if there is no score.
 126    src - The path for the source file.
 127    dst - The path for the destination file. This is only present for
 128          copy or renames. If it is not present, this is None.
 129
 130    If the pattern is not matched, None is returned."""
 131
 132    match = diffTreePattern().next().match(entry)
 133    if match:
 134        return {
 135            'src_mode': match.group(1),
 136            'dst_mode': match.group(2),
 137            'src_sha1': match.group(3),
 138            'dst_sha1': match.group(4),
 139            'status': match.group(5),
 140            'status_score': match.group(6),
 141            'src': match.group(7),
 142            'dst': match.group(10)
 143        }
 144    return None
 145
 146def isModeExec(mode):
 147    # Returns True if the given git mode represents an executable file,
 148    # otherwise False.
 149    return mode[-3:] == "755"
 150
 151def isModeExecChanged(src_mode, dst_mode):
 152    return isModeExec(src_mode) != isModeExec(dst_mode)
 153
 154def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 155    cmd = "p4 -G %s" % cmd
 156    if verbose:
 157        sys.stderr.write("Opening pipe: %s\n" % cmd)
 158
 159    # Use a temporary file to avoid deadlocks without
 160    # subprocess.communicate(), which would put another copy
 161    # of stdout into memory.
 162    stdin_file = None
 163    if stdin is not None:
 164        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 165        stdin_file.write(stdin)
 166        stdin_file.flush()
 167        stdin_file.seek(0)
 168
 169    p4 = subprocess.Popen(cmd, shell=True,
 170                          stdin=stdin_file,
 171                          stdout=subprocess.PIPE)
 172
 173    result = []
 174    try:
 175        while True:
 176            entry = marshal.load(p4.stdout)
 177            result.append(entry)
 178    except EOFError:
 179        pass
 180    exitCode = p4.wait()
 181    if exitCode != 0:
 182        entry = {}
 183        entry["p4ExitCode"] = exitCode
 184        result.append(entry)
 185
 186    return result
 187
 188def p4Cmd(cmd):
 189    list = p4CmdList(cmd)
 190    result = {}
 191    for entry in list:
 192        result.update(entry)
 193    return result;
 194
 195def p4Where(depotPath):
 196    if not depotPath.endswith("/"):
 197        depotPath += "/"
 198    output = p4Cmd("where %s..." % depotPath)
 199    if output["code"] == "error":
 200        return ""
 201    clientPath = ""
 202    if "path" in output:
 203        clientPath = output.get("path")
 204    elif "data" in output:
 205        data = output.get("data")
 206        lastSpace = data.rfind(" ")
 207        clientPath = data[lastSpace + 1:]
 208
 209    if clientPath.endswith("..."):
 210        clientPath = clientPath[:-3]
 211    return clientPath
 212
 213def currentGitBranch():
 214    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 215
 216def isValidGitDir(path):
 217    if (os.path.exists(path + "/HEAD")
 218        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 219        return True;
 220    return False
 221
 222def parseRevision(ref):
 223    return read_pipe("git rev-parse %s" % ref).strip()
 224
 225def extractLogMessageFromGitCommit(commit):
 226    logMessage = ""
 227
 228    ## fixme: title is first line of commit, not 1st paragraph.
 229    foundTitle = False
 230    for log in read_pipe_lines("git cat-file commit %s" % commit):
 231       if not foundTitle:
 232           if len(log) == 1:
 233               foundTitle = True
 234           continue
 235
 236       logMessage += log
 237    return logMessage
 238
 239def extractSettingsGitLog(log):
 240    values = {}
 241    for line in log.split("\n"):
 242        line = line.strip()
 243        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 244        if not m:
 245            continue
 246
 247        assignments = m.group(1).split (':')
 248        for a in assignments:
 249            vals = a.split ('=')
 250            key = vals[0].strip()
 251            val = ('='.join (vals[1:])).strip()
 252            if val.endswith ('\"') and val.startswith('"'):
 253                val = val[1:-1]
 254
 255            values[key] = val
 256
 257    paths = values.get("depot-paths")
 258    if not paths:
 259        paths = values.get("depot-path")
 260    if paths:
 261        values['depot-paths'] = paths.split(',')
 262    return values
 263
 264def gitBranchExists(branch):
 265    proc = subprocess.Popen(["git", "rev-parse", branch],
 266                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 267    return proc.wait() == 0;
 268
 269def gitConfig(key):
 270    return read_pipe("git config %s" % key, ignore_error=True).strip()
 271
 272def p4BranchesInGit(branchesAreInRemotes = True):
 273    branches = {}
 274
 275    cmdline = "git rev-parse --symbolic "
 276    if branchesAreInRemotes:
 277        cmdline += " --remotes"
 278    else:
 279        cmdline += " --branches"
 280
 281    for line in read_pipe_lines(cmdline):
 282        line = line.strip()
 283
 284        ## only import to p4/
 285        if not line.startswith('p4/') or line == "p4/HEAD":
 286            continue
 287        branch = line
 288
 289        # strip off p4
 290        branch = re.sub ("^p4/", "", line)
 291
 292        branches[branch] = parseRevision(line)
 293    return branches
 294
 295def findUpstreamBranchPoint(head = "HEAD"):
 296    branches = p4BranchesInGit()
 297    # map from depot-path to branch name
 298    branchByDepotPath = {}
 299    for branch in branches.keys():
 300        tip = branches[branch]
 301        log = extractLogMessageFromGitCommit(tip)
 302        settings = extractSettingsGitLog(log)
 303        if settings.has_key("depot-paths"):
 304            paths = ",".join(settings["depot-paths"])
 305            branchByDepotPath[paths] = "remotes/p4/" + branch
 306
 307    settings = None
 308    parent = 0
 309    while parent < 65535:
 310        commit = head + "~%s" % parent
 311        log = extractLogMessageFromGitCommit(commit)
 312        settings = extractSettingsGitLog(log)
 313        if settings.has_key("depot-paths"):
 314            paths = ",".join(settings["depot-paths"])
 315            if branchByDepotPath.has_key(paths):
 316                return [branchByDepotPath[paths], settings]
 317
 318        parent = parent + 1
 319
 320    return ["", settings]
 321
 322def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 323    if not silent:
 324        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 325               % localRefPrefix)
 326
 327    originPrefix = "origin/p4/"
 328
 329    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 330        line = line.strip()
 331        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 332            continue
 333
 334        headName = line[len(originPrefix):]
 335        remoteHead = localRefPrefix + headName
 336        originHead = line
 337
 338        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 339        if (not original.has_key('depot-paths')
 340            or not original.has_key('change')):
 341            continue
 342
 343        update = False
 344        if not gitBranchExists(remoteHead):
 345            if verbose:
 346                print "creating %s" % remoteHead
 347            update = True
 348        else:
 349            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 350            if settings.has_key('change') > 0:
 351                if settings['depot-paths'] == original['depot-paths']:
 352                    originP4Change = int(original['change'])
 353                    p4Change = int(settings['change'])
 354                    if originP4Change > p4Change:
 355                        print ("%s (%s) is newer than %s (%s). "
 356                               "Updating p4 branch from origin."
 357                               % (originHead, originP4Change,
 358                                  remoteHead, p4Change))
 359                        update = True
 360                else:
 361                    print ("Ignoring: %s was imported from %s while "
 362                           "%s was imported from %s"
 363                           % (originHead, ','.join(original['depot-paths']),
 364                              remoteHead, ','.join(settings['depot-paths'])))
 365
 366        if update:
 367            system("git update-ref %s %s" % (remoteHead, originHead))
 368
 369def originP4BranchesExist():
 370        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 371
 372def p4ChangesForPaths(depotPaths, changeRange):
 373    assert depotPaths
 374    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 375                                                        for p in depotPaths]))
 376
 377    changes = []
 378    for line in output:
 379        changeNum = line.split(" ")[1]
 380        changes.append(int(changeNum))
 381
 382    changes.sort()
 383    return changes
 384
 385class Command:
 386    def __init__(self):
 387        self.usage = "usage: %prog [options]"
 388        self.needsGit = True
 389
 390class P4Debug(Command):
 391    def __init__(self):
 392        Command.__init__(self)
 393        self.options = [
 394            optparse.make_option("--verbose", dest="verbose", action="store_true",
 395                                 default=False),
 396            ]
 397        self.description = "A tool to debug the output of p4 -G."
 398        self.needsGit = False
 399        self.verbose = False
 400
 401    def run(self, args):
 402        j = 0
 403        for output in p4CmdList(" ".join(args)):
 404            print 'Element: %d' % j
 405            j += 1
 406            print output
 407        return True
 408
 409class P4RollBack(Command):
 410    def __init__(self):
 411        Command.__init__(self)
 412        self.options = [
 413            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 414            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 415        ]
 416        self.description = "A tool to debug the multi-branch import. Don't use :)"
 417        self.verbose = False
 418        self.rollbackLocalBranches = False
 419
 420    def run(self, args):
 421        if len(args) != 1:
 422            return False
 423        maxChange = int(args[0])
 424
 425        if "p4ExitCode" in p4Cmd("changes -m 1"):
 426            die("Problems executing p4");
 427
 428        if self.rollbackLocalBranches:
 429            refPrefix = "refs/heads/"
 430            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 431        else:
 432            refPrefix = "refs/remotes/"
 433            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 434
 435        for line in lines:
 436            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 437                line = line.strip()
 438                ref = refPrefix + line
 439                log = extractLogMessageFromGitCommit(ref)
 440                settings = extractSettingsGitLog(log)
 441
 442                depotPaths = settings['depot-paths']
 443                change = settings['change']
 444
 445                changed = False
 446
 447                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 448                                                           for p in depotPaths]))) == 0:
 449                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 450                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 451                    continue
 452
 453                while change and int(change) > maxChange:
 454                    changed = True
 455                    if self.verbose:
 456                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 457                    system("git update-ref %s \"%s^\"" % (ref, ref))
 458                    log = extractLogMessageFromGitCommit(ref)
 459                    settings =  extractSettingsGitLog(log)
 460
 461
 462                    depotPaths = settings['depot-paths']
 463                    change = settings['change']
 464
 465                if changed:
 466                    print "%s rewound to %s" % (ref, change)
 467
 468        return True
 469
 470class P4Submit(Command):
 471    def __init__(self):
 472        Command.__init__(self)
 473        self.options = [
 474                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 475                optparse.make_option("--origin", dest="origin"),
 476                optparse.make_option("-M", dest="detectRename", action="store_true"),
 477        ]
 478        self.description = "Submit changes from git to the perforce depot."
 479        self.usage += " [name of git branch to submit into perforce depot]"
 480        self.interactive = True
 481        self.origin = ""
 482        self.detectRename = False
 483        self.verbose = False
 484        self.isWindows = (platform.system() == "Windows")
 485
 486    def check(self):
 487        if len(p4CmdList("opened ...")) > 0:
 488            die("You have files opened with perforce! Close them before starting the sync.")
 489
 490    # replaces everything between 'Description:' and the next P4 submit template field with the
 491    # commit message
 492    def prepareLogMessage(self, template, message):
 493        result = ""
 494
 495        inDescriptionSection = False
 496
 497        for line in template.split("\n"):
 498            if line.startswith("#"):
 499                result += line + "\n"
 500                continue
 501
 502            if inDescriptionSection:
 503                if line.startswith("Files:"):
 504                    inDescriptionSection = False
 505                else:
 506                    continue
 507            else:
 508                if line.startswith("Description:"):
 509                    inDescriptionSection = True
 510                    line += "\n"
 511                    for messageLine in message.split("\n"):
 512                        line += "\t" + messageLine + "\n"
 513
 514            result += line + "\n"
 515
 516        return result
 517
 518    def prepareSubmitTemplate(self):
 519        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 520        template = ""
 521        inFilesSection = False
 522        for line in p4_read_pipe_lines("change -o"):
 523            if line.endswith("\r\n"):
 524                line = line[:-2] + "\n"
 525            if inFilesSection:
 526                if line.startswith("\t"):
 527                    # path starts and ends with a tab
 528                    path = line[1:]
 529                    lastTab = path.rfind("\t")
 530                    if lastTab != -1:
 531                        path = path[:lastTab]
 532                        if not path.startswith(self.depotPath):
 533                            continue
 534                else:
 535                    inFilesSection = False
 536            else:
 537                if line.startswith("Files:"):
 538                    inFilesSection = True
 539
 540            template += line
 541
 542        return template
 543
 544    def applyCommit(self, id):
 545        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 546        diffOpts = ("", "-M")[self.detectRename]
 547        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 548        filesToAdd = set()
 549        filesToDelete = set()
 550        editedFiles = set()
 551        filesToChangeExecBit = {}
 552        for line in diff:
 553            diff = parseDiffTreeEntry(line)
 554            modifier = diff['status']
 555            path = diff['src']
 556            if modifier == "M":
 557                system("p4 edit \"%s\"" % path)
 558                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 559                    filesToChangeExecBit[path] = diff['dst_mode']
 560                editedFiles.add(path)
 561            elif modifier == "A":
 562                filesToAdd.add(path)
 563                filesToChangeExecBit[path] = diff['dst_mode']
 564                if path in filesToDelete:
 565                    filesToDelete.remove(path)
 566            elif modifier == "D":
 567                filesToDelete.add(path)
 568                if path in filesToAdd:
 569                    filesToAdd.remove(path)
 570            elif modifier == "R":
 571                src, dest = diff['src'], diff['dst']
 572                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 573                system("p4 edit \"%s\"" % (dest))
 574                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 575                    filesToChangeExecBit[dest] = diff['dst_mode']
 576                os.unlink(dest)
 577                editedFiles.add(dest)
 578                filesToDelete.add(src)
 579            else:
 580                die("unknown modifier %s for %s" % (modifier, path))
 581
 582        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 583        patchcmd = diffcmd + " | git apply "
 584        tryPatchCmd = patchcmd + "--check -"
 585        applyPatchCmd = patchcmd + "--check --apply -"
 586
 587        if os.system(tryPatchCmd) != 0:
 588            print "Unfortunately applying the change failed!"
 589            print "What do you want to do?"
 590            response = "x"
 591            while response != "s" and response != "a" and response != "w":
 592                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 593                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 594            if response == "s":
 595                print "Skipping! Good luck with the next patches..."
 596                for f in editedFiles:
 597                    system("p4 revert \"%s\"" % f);
 598                for f in filesToAdd:
 599                    system("rm %s" %f)
 600                return
 601            elif response == "a":
 602                os.system(applyPatchCmd)
 603                if len(filesToAdd) > 0:
 604                    print "You may also want to call p4 add on the following files:"
 605                    print " ".join(filesToAdd)
 606                if len(filesToDelete):
 607                    print "The following files should be scheduled for deletion with p4 delete:"
 608                    print " ".join(filesToDelete)
 609                die("Please resolve and submit the conflict manually and "
 610                    + "continue afterwards with git-p4 submit --continue")
 611            elif response == "w":
 612                system(diffcmd + " > patch.txt")
 613                print "Patch saved to patch.txt in %s !" % self.clientPath
 614                die("Please resolve and submit the conflict manually and "
 615                    "continue afterwards with git-p4 submit --continue")
 616
 617        system(applyPatchCmd)
 618
 619        for f in filesToAdd:
 620            system("p4 add \"%s\"" % f)
 621        for f in filesToDelete:
 622            system("p4 revert \"%s\"" % f)
 623            system("p4 delete \"%s\"" % f)
 624
 625        # Set/clear executable bits
 626        for f in filesToChangeExecBit.keys():
 627            mode = filesToChangeExecBit[f]
 628            setP4ExecBit(f, mode)
 629
 630        logMessage = extractLogMessageFromGitCommit(id)
 631        logMessage = logMessage.strip()
 632
 633        template = self.prepareSubmitTemplate()
 634
 635        if self.interactive:
 636            submitTemplate = self.prepareLogMessage(template, logMessage)
 637            if os.environ.has_key("P4DIFF"):
 638                del(os.environ["P4DIFF"])
 639            diff = read_pipe("p4 diff -du ...")
 640
 641            newdiff = ""
 642            for newFile in filesToAdd:
 643                newdiff += "==== new file ====\n"
 644                newdiff += "--- /dev/null\n"
 645                newdiff += "+++ %s\n" % newFile
 646                f = open(newFile, "r")
 647                for line in f.readlines():
 648                    newdiff += "+" + line
 649                f.close()
 650
 651            separatorLine = "######## everything below this line is just the diff #######\n"
 652
 653            [handle, fileName] = tempfile.mkstemp()
 654            tmpFile = os.fdopen(handle, "w+")
 655            if self.isWindows:
 656                submitTemplate = submitTemplate.replace("\n", "\r\n")
 657                separatorLine = separatorLine.replace("\n", "\r\n")
 658                newdiff = newdiff.replace("\n", "\r\n")
 659            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 660            tmpFile.close()
 661            defaultEditor = "vi"
 662            if platform.system() == "Windows":
 663                defaultEditor = "notepad"
 664            if os.environ.has_key("P4EDITOR"):
 665                editor = os.environ.get("P4EDITOR")
 666            else:
 667                editor = os.environ.get("EDITOR", defaultEditor);
 668            system(editor + " " + fileName)
 669            tmpFile = open(fileName, "rb")
 670            message = tmpFile.read()
 671            tmpFile.close()
 672            os.remove(fileName)
 673            submitTemplate = message[:message.index(separatorLine)]
 674            if self.isWindows:
 675                submitTemplate = submitTemplate.replace("\r\n", "\n")
 676
 677            write_pipe("p4 submit -i", submitTemplate)
 678        else:
 679            fileName = "submit.txt"
 680            file = open(fileName, "w+")
 681            file.write(self.prepareLogMessage(template, logMessage))
 682            file.close()
 683            print ("Perforce submit template written as %s. "
 684                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 685                   % (fileName, fileName))
 686
 687    def run(self, args):
 688        if len(args) == 0:
 689            self.master = currentGitBranch()
 690            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 691                die("Detecting current git branch failed!")
 692        elif len(args) == 1:
 693            self.master = args[0]
 694        else:
 695            return False
 696
 697        allowSubmit = gitConfig("git-p4.allowSubmit")
 698        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 699            die("%s is not in git-p4.allowSubmit" % self.master)
 700
 701        [upstream, settings] = findUpstreamBranchPoint()
 702        self.depotPath = settings['depot-paths'][0]
 703        if len(self.origin) == 0:
 704            self.origin = upstream
 705
 706        if self.verbose:
 707            print "Origin branch is " + self.origin
 708
 709        if len(self.depotPath) == 0:
 710            print "Internal error: cannot locate perforce depot path from existing branches"
 711            sys.exit(128)
 712
 713        self.clientPath = p4Where(self.depotPath)
 714
 715        if len(self.clientPath) == 0:
 716            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 717            sys.exit(128)
 718
 719        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 720        self.oldWorkingDirectory = os.getcwd()
 721
 722        os.chdir(self.clientPath)
 723        print "Syncronizing p4 checkout..."
 724        system("p4 sync ...")
 725
 726        self.check()
 727
 728        commits = []
 729        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 730            commits.append(line.strip())
 731        commits.reverse()
 732
 733        while len(commits) > 0:
 734            commit = commits[0]
 735            commits = commits[1:]
 736            self.applyCommit(commit)
 737            if not self.interactive:
 738                break
 739
 740        if len(commits) == 0:
 741            print "All changes applied!"
 742            os.chdir(self.oldWorkingDirectory)
 743
 744            sync = P4Sync()
 745            sync.run([])
 746
 747            rebase = P4Rebase()
 748            rebase.rebase()
 749
 750        return True
 751
 752class P4Sync(Command):
 753    def __init__(self):
 754        Command.__init__(self)
 755        self.options = [
 756                optparse.make_option("--branch", dest="branch"),
 757                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 758                optparse.make_option("--changesfile", dest="changesFile"),
 759                optparse.make_option("--silent", dest="silent", action="store_true"),
 760                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 761                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 762                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 763                                     help="Import into refs/heads/ , not refs/remotes"),
 764                optparse.make_option("--max-changes", dest="maxChanges"),
 765                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 766                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 767                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 768                                     help="Only sync files that are included in the Perforce Client Spec")
 769        ]
 770        self.description = """Imports from Perforce into a git repository.\n
 771    example:
 772    //depot/my/project/ -- to import the current head
 773    //depot/my/project/@all -- to import everything
 774    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 775
 776    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 777
 778        self.usage += " //depot/path[@revRange]"
 779        self.silent = False
 780        self.createdBranches = Set()
 781        self.committedChanges = Set()
 782        self.branch = ""
 783        self.detectBranches = False
 784        self.detectLabels = False
 785        self.changesFile = ""
 786        self.syncWithOrigin = True
 787        self.verbose = False
 788        self.importIntoRemotes = True
 789        self.maxChanges = ""
 790        self.isWindows = (platform.system() == "Windows")
 791        self.keepRepoPath = False
 792        self.depotPaths = None
 793        self.p4BranchesInGit = []
 794        self.cloneExclude = []
 795        self.useClientSpec = False
 796        self.clientSpecDirs = []
 797
 798        if gitConfig("git-p4.syncFromOrigin") == "false":
 799            self.syncWithOrigin = False
 800
 801    def extractFilesFromCommit(self, commit):
 802        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 803                             for path in self.cloneExclude]
 804        files = []
 805        fnum = 0
 806        while commit.has_key("depotFile%s" % fnum):
 807            path =  commit["depotFile%s" % fnum]
 808
 809            if [p for p in self.cloneExclude
 810                if path.startswith (p)]:
 811                found = False
 812            else:
 813                found = [p for p in self.depotPaths
 814                         if path.startswith (p)]
 815            if not found:
 816                fnum = fnum + 1
 817                continue
 818
 819            file = {}
 820            file["path"] = path
 821            file["rev"] = commit["rev%s" % fnum]
 822            file["action"] = commit["action%s" % fnum]
 823            file["type"] = commit["type%s" % fnum]
 824            files.append(file)
 825            fnum = fnum + 1
 826        return files
 827
 828    def stripRepoPath(self, path, prefixes):
 829        if self.keepRepoPath:
 830            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 831
 832        for p in prefixes:
 833            if path.startswith(p):
 834                path = path[len(p):]
 835
 836        return path
 837
 838    def splitFilesIntoBranches(self, commit):
 839        branches = {}
 840        fnum = 0
 841        while commit.has_key("depotFile%s" % fnum):
 842            path =  commit["depotFile%s" % fnum]
 843            found = [p for p in self.depotPaths
 844                     if path.startswith (p)]
 845            if not found:
 846                fnum = fnum + 1
 847                continue
 848
 849            file = {}
 850            file["path"] = path
 851            file["rev"] = commit["rev%s" % fnum]
 852            file["action"] = commit["action%s" % fnum]
 853            file["type"] = commit["type%s" % fnum]
 854            fnum = fnum + 1
 855
 856            relPath = self.stripRepoPath(path, self.depotPaths)
 857
 858            for branch in self.knownBranches.keys():
 859
 860                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 861                if relPath.startswith(branch + "/"):
 862                    if branch not in branches:
 863                        branches[branch] = []
 864                    branches[branch].append(file)
 865                    break
 866
 867        return branches
 868
 869    ## Should move this out, doesn't use SELF.
 870    def readP4Files(self, files):
 871        filesForCommit = []
 872        filesToRead = []
 873
 874        for f in files:
 875            includeFile = True
 876            for val in self.clientSpecDirs:
 877                if f['path'].startswith(val[0]):
 878                    if val[1] <= 0:
 879                        includeFile = False
 880                    break
 881
 882            if includeFile:
 883                filesForCommit.append(f)
 884                if f['action'] != 'delete':
 885                    filesToRead.append(f)
 886
 887        filedata = []
 888        if len(filesToRead) > 0:
 889            filedata = p4CmdList('-x - print',
 890                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 891                                                  for f in filesToRead]),
 892                                 stdin_mode='w+')
 893
 894            if "p4ExitCode" in filedata[0]:
 895                die("Problems executing p4. Error: [%d]."
 896                    % (filedata[0]['p4ExitCode']));
 897
 898        j = 0;
 899        contents = {}
 900        while j < len(filedata):
 901            stat = filedata[j]
 902            j += 1
 903            text = [];
 904            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 905                text.append(filedata[j]['data'])
 906                j += 1
 907            text = ''.join(text)
 908
 909            if not stat.has_key('depotFile'):
 910                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 911                continue
 912
 913            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 914                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 915            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 916                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 917
 918            contents[stat['depotFile']] = text
 919
 920        for f in filesForCommit:
 921            path = f['path']
 922            if contents.has_key(path):
 923                f['data'] = contents[path]
 924
 925        return filesForCommit
 926
 927    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 928        epoch = details["time"]
 929        author = details["user"]
 930
 931        if self.verbose:
 932            print "commit into %s" % branch
 933
 934        # start with reading files; if that fails, we should not
 935        # create a commit.
 936        new_files = []
 937        for f in files:
 938            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 939                new_files.append (f)
 940            else:
 941                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 942        files = self.readP4Files(new_files)
 943
 944        self.gitStream.write("commit %s\n" % branch)
 945#        gitStream.write("mark :%s\n" % details["change"])
 946        self.committedChanges.add(int(details["change"]))
 947        committer = ""
 948        if author not in self.users:
 949            self.getUserMapFromPerforceServer()
 950        if author in self.users:
 951            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 952        else:
 953            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 954
 955        self.gitStream.write("committer %s\n" % committer)
 956
 957        self.gitStream.write("data <<EOT\n")
 958        self.gitStream.write(details["desc"])
 959        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 960                             % (','.join (branchPrefixes), details["change"]))
 961        if len(details['options']) > 0:
 962            self.gitStream.write(": options = %s" % details['options'])
 963        self.gitStream.write("]\nEOT\n\n")
 964
 965        if len(parent) > 0:
 966            if self.verbose:
 967                print "parent %s" % parent
 968            self.gitStream.write("from %s\n" % parent)
 969
 970        for file in files:
 971            if file["type"] == "apple":
 972                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 973                continue
 974
 975            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 976            if file["action"] == "delete":
 977                self.gitStream.write("D %s\n" % relPath)
 978            else:
 979                data = file['data']
 980
 981                mode = "644"
 982                if isP4Exec(file["type"]):
 983                    mode = "755"
 984                elif file["type"] == "symlink":
 985                    mode = "120000"
 986                    # p4 print on a symlink contains "target\n", so strip it off
 987                    data = data[:-1]
 988
 989                if self.isWindows and file["type"].endswith("text"):
 990                    data = data.replace("\r\n", "\n")
 991
 992                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 993                self.gitStream.write("data %s\n" % len(data))
 994                self.gitStream.write(data)
 995                self.gitStream.write("\n")
 996
 997        self.gitStream.write("\n")
 998
 999        change = int(details["change"])
1000
1001        if self.labels.has_key(change):
1002            label = self.labels[change]
1003            labelDetails = label[0]
1004            labelRevisions = label[1]
1005            if self.verbose:
1006                print "Change %s is labelled %s" % (change, labelDetails)
1007
1008            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1009                                                    for p in branchPrefixes]))
1010
1011            if len(files) == len(labelRevisions):
1012
1013                cleanedFiles = {}
1014                for info in files:
1015                    if info["action"] == "delete":
1016                        continue
1017                    cleanedFiles[info["depotFile"]] = info["rev"]
1018
1019                if cleanedFiles == labelRevisions:
1020                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1021                    self.gitStream.write("from %s\n" % branch)
1022
1023                    owner = labelDetails["Owner"]
1024                    tagger = ""
1025                    if author in self.users:
1026                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1027                    else:
1028                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1029                    self.gitStream.write("tagger %s\n" % tagger)
1030                    self.gitStream.write("data <<EOT\n")
1031                    self.gitStream.write(labelDetails["Description"])
1032                    self.gitStream.write("EOT\n\n")
1033
1034                else:
1035                    if not self.silent:
1036                        print ("Tag %s does not match with change %s: files do not match."
1037                               % (labelDetails["label"], change))
1038
1039            else:
1040                if not self.silent:
1041                    print ("Tag %s does not match with change %s: file count is different."
1042                           % (labelDetails["label"], change))
1043
1044    def getUserCacheFilename(self):
1045        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1046        return home + "/.gitp4-usercache.txt"
1047
1048    def getUserMapFromPerforceServer(self):
1049        if self.userMapFromPerforceServer:
1050            return
1051        self.users = {}
1052
1053        for output in p4CmdList("users"):
1054            if not output.has_key("User"):
1055                continue
1056            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1057
1058
1059        s = ''
1060        for (key, val) in self.users.items():
1061            s += "%s\t%s\n" % (key, val)
1062
1063        open(self.getUserCacheFilename(), "wb").write(s)
1064        self.userMapFromPerforceServer = True
1065
1066    def loadUserMapFromCache(self):
1067        self.users = {}
1068        self.userMapFromPerforceServer = False
1069        try:
1070            cache = open(self.getUserCacheFilename(), "rb")
1071            lines = cache.readlines()
1072            cache.close()
1073            for line in lines:
1074                entry = line.strip().split("\t")
1075                self.users[entry[0]] = entry[1]
1076        except IOError:
1077            self.getUserMapFromPerforceServer()
1078
1079    def getLabels(self):
1080        self.labels = {}
1081
1082        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1083        if len(l) > 0 and not self.silent:
1084            print "Finding files belonging to labels in %s" % `self.depotPaths`
1085
1086        for output in l:
1087            label = output["label"]
1088            revisions = {}
1089            newestChange = 0
1090            if self.verbose:
1091                print "Querying files for label %s" % label
1092            for file in p4CmdList("files "
1093                                  +  ' '.join (["%s...@%s" % (p, label)
1094                                                for p in self.depotPaths])):
1095                revisions[file["depotFile"]] = file["rev"]
1096                change = int(file["change"])
1097                if change > newestChange:
1098                    newestChange = change
1099
1100            self.labels[newestChange] = [output, revisions]
1101
1102        if self.verbose:
1103            print "Label changes: %s" % self.labels.keys()
1104
1105    def guessProjectName(self):
1106        for p in self.depotPaths:
1107            if p.endswith("/"):
1108                p = p[:-1]
1109            p = p[p.strip().rfind("/") + 1:]
1110            if not p.endswith("/"):
1111               p += "/"
1112            return p
1113
1114    def getBranchMapping(self):
1115        lostAndFoundBranches = set()
1116
1117        for info in p4CmdList("branches"):
1118            details = p4Cmd("branch -o %s" % info["branch"])
1119            viewIdx = 0
1120            while details.has_key("View%s" % viewIdx):
1121                paths = details["View%s" % viewIdx].split(" ")
1122                viewIdx = viewIdx + 1
1123                # require standard //depot/foo/... //depot/bar/... mapping
1124                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1125                    continue
1126                source = paths[0]
1127                destination = paths[1]
1128                ## HACK
1129                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1130                    source = source[len(self.depotPaths[0]):-4]
1131                    destination = destination[len(self.depotPaths[0]):-4]
1132
1133                    if destination in self.knownBranches:
1134                        if not self.silent:
1135                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1136                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1137                        continue
1138
1139                    self.knownBranches[destination] = source
1140
1141                    lostAndFoundBranches.discard(destination)
1142
1143                    if source not in self.knownBranches:
1144                        lostAndFoundBranches.add(source)
1145
1146
1147        for branch in lostAndFoundBranches:
1148            self.knownBranches[branch] = branch
1149
1150    def getBranchMappingFromGitBranches(self):
1151        branches = p4BranchesInGit(self.importIntoRemotes)
1152        for branch in branches.keys():
1153            if branch == "master":
1154                branch = "main"
1155            else:
1156                branch = branch[len(self.projectName):]
1157            self.knownBranches[branch] = branch
1158
1159    def listExistingP4GitBranches(self):
1160        # branches holds mapping from name to commit
1161        branches = p4BranchesInGit(self.importIntoRemotes)
1162        self.p4BranchesInGit = branches.keys()
1163        for branch in branches.keys():
1164            self.initialParents[self.refPrefix + branch] = branches[branch]
1165
1166    def updateOptionDict(self, d):
1167        option_keys = {}
1168        if self.keepRepoPath:
1169            option_keys['keepRepoPath'] = 1
1170
1171        d["options"] = ' '.join(sorted(option_keys.keys()))
1172
1173    def readOptions(self, d):
1174        self.keepRepoPath = (d.has_key('options')
1175                             and ('keepRepoPath' in d['options']))
1176
1177    def gitRefForBranch(self, branch):
1178        if branch == "main":
1179            return self.refPrefix + "master"
1180
1181        if len(branch) <= 0:
1182            return branch
1183
1184        return self.refPrefix + self.projectName + branch
1185
1186    def gitCommitByP4Change(self, ref, change):
1187        if self.verbose:
1188            print "looking in ref " + ref + " for change %s using bisect..." % change
1189
1190        earliestCommit = ""
1191        latestCommit = parseRevision(ref)
1192
1193        while True:
1194            if self.verbose:
1195                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1196            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1197            if len(next) == 0:
1198                if self.verbose:
1199                    print "argh"
1200                return ""
1201            log = extractLogMessageFromGitCommit(next)
1202            settings = extractSettingsGitLog(log)
1203            currentChange = int(settings['change'])
1204            if self.verbose:
1205                print "current change %s" % currentChange
1206
1207            if currentChange == change:
1208                if self.verbose:
1209                    print "found %s" % next
1210                return next
1211
1212            if currentChange < change:
1213                earliestCommit = "^%s" % next
1214            else:
1215                latestCommit = "%s" % next
1216
1217        return ""
1218
1219    def importNewBranch(self, branch, maxChange):
1220        # make fast-import flush all changes to disk and update the refs using the checkpoint
1221        # command so that we can try to find the branch parent in the git history
1222        self.gitStream.write("checkpoint\n\n");
1223        self.gitStream.flush();
1224        branchPrefix = self.depotPaths[0] + branch + "/"
1225        range = "@1,%s" % maxChange
1226        #print "prefix" + branchPrefix
1227        changes = p4ChangesForPaths([branchPrefix], range)
1228        if len(changes) <= 0:
1229            return False
1230        firstChange = changes[0]
1231        #print "first change in branch: %s" % firstChange
1232        sourceBranch = self.knownBranches[branch]
1233        sourceDepotPath = self.depotPaths[0] + sourceBranch
1234        sourceRef = self.gitRefForBranch(sourceBranch)
1235        #print "source " + sourceBranch
1236
1237        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1238        #print "branch parent: %s" % branchParentChange
1239        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1240        if len(gitParent) > 0:
1241            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1242            #print "parent git commit: %s" % gitParent
1243
1244        self.importChanges(changes)
1245        return True
1246
1247    def importChanges(self, changes):
1248        cnt = 1
1249        for change in changes:
1250            description = p4Cmd("describe %s" % change)
1251            self.updateOptionDict(description)
1252
1253            if not self.silent:
1254                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1255                sys.stdout.flush()
1256            cnt = cnt + 1
1257
1258            try:
1259                if self.detectBranches:
1260                    branches = self.splitFilesIntoBranches(description)
1261                    for branch in branches.keys():
1262                        ## HACK  --hwn
1263                        branchPrefix = self.depotPaths[0] + branch + "/"
1264
1265                        parent = ""
1266
1267                        filesForCommit = branches[branch]
1268
1269                        if self.verbose:
1270                            print "branch is %s" % branch
1271
1272                        self.updatedBranches.add(branch)
1273
1274                        if branch not in self.createdBranches:
1275                            self.createdBranches.add(branch)
1276                            parent = self.knownBranches[branch]
1277                            if parent == branch:
1278                                parent = ""
1279                            else:
1280                                fullBranch = self.projectName + branch
1281                                if fullBranch not in self.p4BranchesInGit:
1282                                    if not self.silent:
1283                                        print("\n    Importing new branch %s" % fullBranch);
1284                                    if self.importNewBranch(branch, change - 1):
1285                                        parent = ""
1286                                        self.p4BranchesInGit.append(fullBranch)
1287                                    if not self.silent:
1288                                        print("\n    Resuming with change %s" % change);
1289
1290                                if self.verbose:
1291                                    print "parent determined through known branches: %s" % parent
1292
1293                        branch = self.gitRefForBranch(branch)
1294                        parent = self.gitRefForBranch(parent)
1295
1296                        if self.verbose:
1297                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1298
1299                        if len(parent) == 0 and branch in self.initialParents:
1300                            parent = self.initialParents[branch]
1301                            del self.initialParents[branch]
1302
1303                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1304                else:
1305                    files = self.extractFilesFromCommit(description)
1306                    self.commit(description, files, self.branch, self.depotPaths,
1307                                self.initialParent)
1308                    self.initialParent = ""
1309            except IOError:
1310                print self.gitError.read()
1311                sys.exit(1)
1312
1313    def importHeadRevision(self, revision):
1314        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1315
1316        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1317        details["desc"] = ("Initial import of %s from the state at revision %s"
1318                           % (' '.join(self.depotPaths), revision))
1319        details["change"] = revision
1320        newestRevision = 0
1321
1322        fileCnt = 0
1323        for info in p4CmdList("files "
1324                              +  ' '.join(["%s...%s"
1325                                           % (p, revision)
1326                                           for p in self.depotPaths])):
1327
1328            if info['code'] == 'error':
1329                sys.stderr.write("p4 returned an error: %s\n"
1330                                 % info['data'])
1331                sys.exit(1)
1332
1333
1334            change = int(info["change"])
1335            if change > newestRevision:
1336                newestRevision = change
1337
1338            if info["action"] == "delete":
1339                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1340                #fileCnt = fileCnt + 1
1341                continue
1342
1343            for prop in ["depotFile", "rev", "action", "type" ]:
1344                details["%s%s" % (prop, fileCnt)] = info[prop]
1345
1346            fileCnt = fileCnt + 1
1347
1348        details["change"] = newestRevision
1349        self.updateOptionDict(details)
1350        try:
1351            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1352        except IOError:
1353            print "IO error with git fast-import. Is your git version recent enough?"
1354            print self.gitError.read()
1355
1356
1357    def getClientSpec(self):
1358        specList = p4CmdList( "client -o" )
1359        temp = {}
1360        for entry in specList:
1361            for k,v in entry.iteritems():
1362                if k.startswith("View"):
1363                    if v.startswith('"'):
1364                        start = 1
1365                    else:
1366                        start = 0
1367                    index = v.find("...")
1368                    v = v[start:index]
1369                    if v.startswith("-"):
1370                        v = v[1:]
1371                        temp[v] = -len(v)
1372                    else:
1373                        temp[v] = len(v)
1374        self.clientSpecDirs = temp.items()
1375        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1376
1377    def run(self, args):
1378        self.depotPaths = []
1379        self.changeRange = ""
1380        self.initialParent = ""
1381        self.previousDepotPaths = []
1382
1383        # map from branch depot path to parent branch
1384        self.knownBranches = {}
1385        self.initialParents = {}
1386        self.hasOrigin = originP4BranchesExist()
1387        if not self.syncWithOrigin:
1388            self.hasOrigin = False
1389
1390        if self.importIntoRemotes:
1391            self.refPrefix = "refs/remotes/p4/"
1392        else:
1393            self.refPrefix = "refs/heads/p4/"
1394
1395        if self.syncWithOrigin and self.hasOrigin:
1396            if not self.silent:
1397                print "Syncing with origin first by calling git fetch origin"
1398            system("git fetch origin")
1399
1400        if len(self.branch) == 0:
1401            self.branch = self.refPrefix + "master"
1402            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1403                system("git update-ref %s refs/heads/p4" % self.branch)
1404                system("git branch -D p4");
1405            # create it /after/ importing, when master exists
1406            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1407                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1408
1409        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1410            self.getClientSpec()
1411
1412        # TODO: should always look at previous commits,
1413        # merge with previous imports, if possible.
1414        if args == []:
1415            if self.hasOrigin:
1416                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1417            self.listExistingP4GitBranches()
1418
1419            if len(self.p4BranchesInGit) > 1:
1420                if not self.silent:
1421                    print "Importing from/into multiple branches"
1422                self.detectBranches = True
1423
1424            if self.verbose:
1425                print "branches: %s" % self.p4BranchesInGit
1426
1427            p4Change = 0
1428            for branch in self.p4BranchesInGit:
1429                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1430
1431                settings = extractSettingsGitLog(logMsg)
1432
1433                self.readOptions(settings)
1434                if (settings.has_key('depot-paths')
1435                    and settings.has_key ('change')):
1436                    change = int(settings['change']) + 1
1437                    p4Change = max(p4Change, change)
1438
1439                    depotPaths = sorted(settings['depot-paths'])
1440                    if self.previousDepotPaths == []:
1441                        self.previousDepotPaths = depotPaths
1442                    else:
1443                        paths = []
1444                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1445                            for i in range(0, min(len(cur), len(prev))):
1446                                if cur[i] <> prev[i]:
1447                                    i = i - 1
1448                                    break
1449
1450                            paths.append (cur[:i + 1])
1451
1452                        self.previousDepotPaths = paths
1453
1454            if p4Change > 0:
1455                self.depotPaths = sorted(self.previousDepotPaths)
1456                self.changeRange = "@%s,#head" % p4Change
1457                if not self.detectBranches:
1458                    self.initialParent = parseRevision(self.branch)
1459                if not self.silent and not self.detectBranches:
1460                    print "Performing incremental import into %s git branch" % self.branch
1461
1462        if not self.branch.startswith("refs/"):
1463            self.branch = "refs/heads/" + self.branch
1464
1465        if len(args) == 0 and self.depotPaths:
1466            if not self.silent:
1467                print "Depot paths: %s" % ' '.join(self.depotPaths)
1468        else:
1469            if self.depotPaths and self.depotPaths != args:
1470                print ("previous import used depot path %s and now %s was specified. "
1471                       "This doesn't work!" % (' '.join (self.depotPaths),
1472                                               ' '.join (args)))
1473                sys.exit(1)
1474
1475            self.depotPaths = sorted(args)
1476
1477        revision = ""
1478        self.users = {}
1479
1480        newPaths = []
1481        for p in self.depotPaths:
1482            if p.find("@") != -1:
1483                atIdx = p.index("@")
1484                self.changeRange = p[atIdx:]
1485                if self.changeRange == "@all":
1486                    self.changeRange = ""
1487                elif ',' not in self.changeRange:
1488                    revision = self.changeRange
1489                    self.changeRange = ""
1490                p = p[:atIdx]
1491            elif p.find("#") != -1:
1492                hashIdx = p.index("#")
1493                revision = p[hashIdx:]
1494                p = p[:hashIdx]
1495            elif self.previousDepotPaths == []:
1496                revision = "#head"
1497
1498            p = re.sub ("\.\.\.$", "", p)
1499            if not p.endswith("/"):
1500                p += "/"
1501
1502            newPaths.append(p)
1503
1504        self.depotPaths = newPaths
1505
1506
1507        self.loadUserMapFromCache()
1508        self.labels = {}
1509        if self.detectLabels:
1510            self.getLabels();
1511
1512        if self.detectBranches:
1513            ## FIXME - what's a P4 projectName ?
1514            self.projectName = self.guessProjectName()
1515
1516            if self.hasOrigin:
1517                self.getBranchMappingFromGitBranches()
1518            else:
1519                self.getBranchMapping()
1520            if self.verbose:
1521                print "p4-git branches: %s" % self.p4BranchesInGit
1522                print "initial parents: %s" % self.initialParents
1523            for b in self.p4BranchesInGit:
1524                if b != "master":
1525
1526                    ## FIXME
1527                    b = b[len(self.projectName):]
1528                self.createdBranches.add(b)
1529
1530        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1531
1532        importProcess = subprocess.Popen(["git", "fast-import"],
1533                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1534                                         stderr=subprocess.PIPE);
1535        self.gitOutput = importProcess.stdout
1536        self.gitStream = importProcess.stdin
1537        self.gitError = importProcess.stderr
1538
1539        if revision:
1540            self.importHeadRevision(revision)
1541        else:
1542            changes = []
1543
1544            if len(self.changesFile) > 0:
1545                output = open(self.changesFile).readlines()
1546                changeSet = Set()
1547                for line in output:
1548                    changeSet.add(int(line))
1549
1550                for change in changeSet:
1551                    changes.append(change)
1552
1553                changes.sort()
1554            else:
1555                if self.verbose:
1556                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1557                                                              self.changeRange)
1558                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1559
1560                if len(self.maxChanges) > 0:
1561                    changes = changes[:min(int(self.maxChanges), len(changes))]
1562
1563            if len(changes) == 0:
1564                if not self.silent:
1565                    print "No changes to import!"
1566                return True
1567
1568            if not self.silent and not self.detectBranches:
1569                print "Import destination: %s" % self.branch
1570
1571            self.updatedBranches = set()
1572
1573            self.importChanges(changes)
1574
1575            if not self.silent:
1576                print ""
1577                if len(self.updatedBranches) > 0:
1578                    sys.stdout.write("Updated branches: ")
1579                    for b in self.updatedBranches:
1580                        sys.stdout.write("%s " % b)
1581                    sys.stdout.write("\n")
1582
1583        self.gitStream.close()
1584        if importProcess.wait() != 0:
1585            die("fast-import failed: %s" % self.gitError.read())
1586        self.gitOutput.close()
1587        self.gitError.close()
1588
1589        return True
1590
1591class P4Rebase(Command):
1592    def __init__(self):
1593        Command.__init__(self)
1594        self.options = [ ]
1595        self.description = ("Fetches the latest revision from perforce and "
1596                            + "rebases the current work (branch) against it")
1597        self.verbose = False
1598
1599    def run(self, args):
1600        sync = P4Sync()
1601        sync.run([])
1602
1603        return self.rebase()
1604
1605    def rebase(self):
1606        if os.system("git update-index --refresh") != 0:
1607            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1608        if len(read_pipe("git diff-index HEAD --")) > 0:
1609            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1610
1611        [upstream, settings] = findUpstreamBranchPoint()
1612        if len(upstream) == 0:
1613            die("Cannot find upstream branchpoint for rebase")
1614
1615        # the branchpoint may be p4/foo~3, so strip off the parent
1616        upstream = re.sub("~[0-9]+$", "", upstream)
1617
1618        print "Rebasing the current branch onto %s" % upstream
1619        oldHead = read_pipe("git rev-parse HEAD").strip()
1620        system("git rebase %s" % upstream)
1621        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1622        return True
1623
1624class P4Clone(P4Sync):
1625    def __init__(self):
1626        P4Sync.__init__(self)
1627        self.description = "Creates a new git repository and imports from Perforce into it"
1628        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1629        self.options += [
1630            optparse.make_option("--destination", dest="cloneDestination",
1631                                 action='store', default=None,
1632                                 help="where to leave result of the clone"),
1633            optparse.make_option("-/", dest="cloneExclude",
1634                                 action="append", type="string",
1635                                 help="exclude depot path")
1636        ]
1637        self.cloneDestination = None
1638        self.needsGit = False
1639
1640    # This is required for the "append" cloneExclude action
1641    def ensure_value(self, attr, value):
1642        if not hasattr(self, attr) or getattr(self, attr) is None:
1643            setattr(self, attr, value)
1644        return getattr(self, attr)
1645
1646    def defaultDestination(self, args):
1647        ## TODO: use common prefix of args?
1648        depotPath = args[0]
1649        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1650        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1651        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1652        depotDir = re.sub(r"/$", "", depotDir)
1653        return os.path.split(depotDir)[1]
1654
1655    def run(self, args):
1656        if len(args) < 1:
1657            return False
1658
1659        if self.keepRepoPath and not self.cloneDestination:
1660            sys.stderr.write("Must specify destination for --keep-path\n")
1661            sys.exit(1)
1662
1663        depotPaths = args
1664
1665        if not self.cloneDestination and len(depotPaths) > 1:
1666            self.cloneDestination = depotPaths[-1]
1667            depotPaths = depotPaths[:-1]
1668
1669        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1670        for p in depotPaths:
1671            if not p.startswith("//"):
1672                return False
1673
1674        if not self.cloneDestination:
1675            self.cloneDestination = self.defaultDestination(args)
1676
1677        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1678        if not os.path.exists(self.cloneDestination):
1679            os.makedirs(self.cloneDestination)
1680        os.chdir(self.cloneDestination)
1681        system("git init")
1682        self.gitdir = os.getcwd() + "/.git"
1683        if not P4Sync.run(self, depotPaths):
1684            return False
1685        if self.branch != "master":
1686            if gitBranchExists("refs/remotes/p4/master"):
1687                system("git branch master refs/remotes/p4/master")
1688                system("git checkout -f")
1689            else:
1690                print "Could not detect main branch. No checkout/master branch created."
1691
1692        return True
1693
1694class P4Branches(Command):
1695    def __init__(self):
1696        Command.__init__(self)
1697        self.options = [ ]
1698        self.description = ("Shows the git branches that hold imports and their "
1699                            + "corresponding perforce depot paths")
1700        self.verbose = False
1701
1702    def run(self, args):
1703        if originP4BranchesExist():
1704            createOrUpdateBranchesFromOrigin()
1705
1706        cmdline = "git rev-parse --symbolic "
1707        cmdline += " --remotes"
1708
1709        for line in read_pipe_lines(cmdline):
1710            line = line.strip()
1711
1712            if not line.startswith('p4/') or line == "p4/HEAD":
1713                continue
1714            branch = line
1715
1716            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1717            settings = extractSettingsGitLog(log)
1718
1719            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1720        return True
1721
1722class HelpFormatter(optparse.IndentedHelpFormatter):
1723    def __init__(self):
1724        optparse.IndentedHelpFormatter.__init__(self)
1725
1726    def format_description(self, description):
1727        if description:
1728            return description + "\n"
1729        else:
1730            return ""
1731
1732def printUsage(commands):
1733    print "usage: %s <command> [options]" % sys.argv[0]
1734    print ""
1735    print "valid commands: %s" % ", ".join(commands)
1736    print ""
1737    print "Try %s <command> --help for command specific help." % sys.argv[0]
1738    print ""
1739
1740commands = {
1741    "debug" : P4Debug,
1742    "submit" : P4Submit,
1743    "commit" : P4Submit,
1744    "sync" : P4Sync,
1745    "rebase" : P4Rebase,
1746    "clone" : P4Clone,
1747    "rollback" : P4RollBack,
1748    "branches" : P4Branches
1749}
1750
1751
1752def main():
1753    if len(sys.argv[1:]) == 0:
1754        printUsage(commands.keys())
1755        sys.exit(2)
1756
1757    cmd = ""
1758    cmdName = sys.argv[1]
1759    try:
1760        klass = commands[cmdName]
1761        cmd = klass()
1762    except KeyError:
1763        print "unknown command %s" % cmdName
1764        print ""
1765        printUsage(commands.keys())
1766        sys.exit(2)
1767
1768    options = cmd.options
1769    cmd.gitdir = os.environ.get("GIT_DIR", None)
1770
1771    args = sys.argv[2:]
1772
1773    if len(options) > 0:
1774        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1775
1776        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1777                                       options,
1778                                       description = cmd.description,
1779                                       formatter = HelpFormatter())
1780
1781        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1782    global verbose
1783    verbose = cmd.verbose
1784    if cmd.needsGit:
1785        if cmd.gitdir == None:
1786            cmd.gitdir = os.path.abspath(".git")
1787            if not isValidGitDir(cmd.gitdir):
1788                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1789                if os.path.exists(cmd.gitdir):
1790                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1791                    if len(cdup) > 0:
1792                        os.chdir(cdup);
1793
1794        if not isValidGitDir(cmd.gitdir):
1795            if isValidGitDir(cmd.gitdir + "/.git"):
1796                cmd.gitdir += "/.git"
1797            else:
1798                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1799
1800        os.environ["GIT_DIR"] = cmd.gitdir
1801
1802    if not cmd.run(args):
1803        parser.print_help()
1804
1805
1806if __name__ == '__main__':
1807    main()