Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Have a command that specifically invokes 'p4' (via system) (bf9320f)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def p4_read_pipe_lines(c):
  61    """Specifically invoke p4 on the command supplied. """
  62    real_cmd = "%s %s" % ("p4", c)
  63    if verbose:
  64        print real_cmd
  65    return read_pipe_lines(real_cmd)
  66
  67def system(cmd):
  68    if verbose:
  69        sys.stderr.write("executing %s\n" % cmd)
  70    if os.system(cmd) != 0:
  71        die("command failed: %s" % cmd)
  72
  73def p4_system(cmd):
  74    """Specifically invoke p4 as the system command. """
  75    real_cmd = "%s %s" % ("p4", cmd)
  76    if verbose:
  77        print real_cmd
  78    return system(real_cmd)
  79
  80def isP4Exec(kind):
  81    """Determine if a Perforce 'kind' should have execute permission
  82
  83    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  84    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  85    a plus sign, it is also executable"""
  86    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  87
  88def setP4ExecBit(file, mode):
  89    # Reopens an already open file and changes the execute bit to match
  90    # the execute bit setting in the passed in mode.
  91
  92    p4Type = "+x"
  93
  94    if not isModeExec(mode):
  95        p4Type = getP4OpenedType(file)
  96        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  97        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  98        if p4Type[-1] == "+":
  99            p4Type = p4Type[0:-1]
 100
 101    system("p4 reopen -t %s %s" % (p4Type, file))
 102
 103def getP4OpenedType(file):
 104    # Returns the perforce file type for the given file.
 105
 106    result = read_pipe("p4 opened %s" % file)
 107    match = re.match(".*\((.+)\)\r?$", result)
 108    if match:
 109        return match.group(1)
 110    else:
 111        die("Could not determine file type for %s (result: '%s')" % (file, result))
 112
 113def diffTreePattern():
 114    # This is a simple generator for the diff tree regex pattern. This could be
 115    # a class variable if this and parseDiffTreeEntry were a part of a class.
 116    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 117    while True:
 118        yield pattern
 119
 120def parseDiffTreeEntry(entry):
 121    """Parses a single diff tree entry into its component elements.
 122
 123    See git-diff-tree(1) manpage for details about the format of the diff
 124    output. This method returns a dictionary with the following elements:
 125
 126    src_mode - The mode of the source file
 127    dst_mode - The mode of the destination file
 128    src_sha1 - The sha1 for the source file
 129    dst_sha1 - The sha1 fr the destination file
 130    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 131    status_score - The score for the status (applicable for 'C' and 'R'
 132                   statuses). This is None if there is no score.
 133    src - The path for the source file.
 134    dst - The path for the destination file. This is only present for
 135          copy or renames. If it is not present, this is None.
 136
 137    If the pattern is not matched, None is returned."""
 138
 139    match = diffTreePattern().next().match(entry)
 140    if match:
 141        return {
 142            'src_mode': match.group(1),
 143            'dst_mode': match.group(2),
 144            'src_sha1': match.group(3),
 145            'dst_sha1': match.group(4),
 146            'status': match.group(5),
 147            'status_score': match.group(6),
 148            'src': match.group(7),
 149            'dst': match.group(10)
 150        }
 151    return None
 152
 153def isModeExec(mode):
 154    # Returns True if the given git mode represents an executable file,
 155    # otherwise False.
 156    return mode[-3:] == "755"
 157
 158def isModeExecChanged(src_mode, dst_mode):
 159    return isModeExec(src_mode) != isModeExec(dst_mode)
 160
 161def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 162    cmd = "p4 -G %s" % cmd
 163    if verbose:
 164        sys.stderr.write("Opening pipe: %s\n" % cmd)
 165
 166    # Use a temporary file to avoid deadlocks without
 167    # subprocess.communicate(), which would put another copy
 168    # of stdout into memory.
 169    stdin_file = None
 170    if stdin is not None:
 171        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 172        stdin_file.write(stdin)
 173        stdin_file.flush()
 174        stdin_file.seek(0)
 175
 176    p4 = subprocess.Popen(cmd, shell=True,
 177                          stdin=stdin_file,
 178                          stdout=subprocess.PIPE)
 179
 180    result = []
 181    try:
 182        while True:
 183            entry = marshal.load(p4.stdout)
 184            result.append(entry)
 185    except EOFError:
 186        pass
 187    exitCode = p4.wait()
 188    if exitCode != 0:
 189        entry = {}
 190        entry["p4ExitCode"] = exitCode
 191        result.append(entry)
 192
 193    return result
 194
 195def p4Cmd(cmd):
 196    list = p4CmdList(cmd)
 197    result = {}
 198    for entry in list:
 199        result.update(entry)
 200    return result;
 201
 202def p4Where(depotPath):
 203    if not depotPath.endswith("/"):
 204        depotPath += "/"
 205    output = p4Cmd("where %s..." % depotPath)
 206    if output["code"] == "error":
 207        return ""
 208    clientPath = ""
 209    if "path" in output:
 210        clientPath = output.get("path")
 211    elif "data" in output:
 212        data = output.get("data")
 213        lastSpace = data.rfind(" ")
 214        clientPath = data[lastSpace + 1:]
 215
 216    if clientPath.endswith("..."):
 217        clientPath = clientPath[:-3]
 218    return clientPath
 219
 220def currentGitBranch():
 221    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 222
 223def isValidGitDir(path):
 224    if (os.path.exists(path + "/HEAD")
 225        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 226        return True;
 227    return False
 228
 229def parseRevision(ref):
 230    return read_pipe("git rev-parse %s" % ref).strip()
 231
 232def extractLogMessageFromGitCommit(commit):
 233    logMessage = ""
 234
 235    ## fixme: title is first line of commit, not 1st paragraph.
 236    foundTitle = False
 237    for log in read_pipe_lines("git cat-file commit %s" % commit):
 238       if not foundTitle:
 239           if len(log) == 1:
 240               foundTitle = True
 241           continue
 242
 243       logMessage += log
 244    return logMessage
 245
 246def extractSettingsGitLog(log):
 247    values = {}
 248    for line in log.split("\n"):
 249        line = line.strip()
 250        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 251        if not m:
 252            continue
 253
 254        assignments = m.group(1).split (':')
 255        for a in assignments:
 256            vals = a.split ('=')
 257            key = vals[0].strip()
 258            val = ('='.join (vals[1:])).strip()
 259            if val.endswith ('\"') and val.startswith('"'):
 260                val = val[1:-1]
 261
 262            values[key] = val
 263
 264    paths = values.get("depot-paths")
 265    if not paths:
 266        paths = values.get("depot-path")
 267    if paths:
 268        values['depot-paths'] = paths.split(',')
 269    return values
 270
 271def gitBranchExists(branch):
 272    proc = subprocess.Popen(["git", "rev-parse", branch],
 273                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 274    return proc.wait() == 0;
 275
 276def gitConfig(key):
 277    return read_pipe("git config %s" % key, ignore_error=True).strip()
 278
 279def p4BranchesInGit(branchesAreInRemotes = True):
 280    branches = {}
 281
 282    cmdline = "git rev-parse --symbolic "
 283    if branchesAreInRemotes:
 284        cmdline += " --remotes"
 285    else:
 286        cmdline += " --branches"
 287
 288    for line in read_pipe_lines(cmdline):
 289        line = line.strip()
 290
 291        ## only import to p4/
 292        if not line.startswith('p4/') or line == "p4/HEAD":
 293            continue
 294        branch = line
 295
 296        # strip off p4
 297        branch = re.sub ("^p4/", "", line)
 298
 299        branches[branch] = parseRevision(line)
 300    return branches
 301
 302def findUpstreamBranchPoint(head = "HEAD"):
 303    branches = p4BranchesInGit()
 304    # map from depot-path to branch name
 305    branchByDepotPath = {}
 306    for branch in branches.keys():
 307        tip = branches[branch]
 308        log = extractLogMessageFromGitCommit(tip)
 309        settings = extractSettingsGitLog(log)
 310        if settings.has_key("depot-paths"):
 311            paths = ",".join(settings["depot-paths"])
 312            branchByDepotPath[paths] = "remotes/p4/" + branch
 313
 314    settings = None
 315    parent = 0
 316    while parent < 65535:
 317        commit = head + "~%s" % parent
 318        log = extractLogMessageFromGitCommit(commit)
 319        settings = extractSettingsGitLog(log)
 320        if settings.has_key("depot-paths"):
 321            paths = ",".join(settings["depot-paths"])
 322            if branchByDepotPath.has_key(paths):
 323                return [branchByDepotPath[paths], settings]
 324
 325        parent = parent + 1
 326
 327    return ["", settings]
 328
 329def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 330    if not silent:
 331        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 332               % localRefPrefix)
 333
 334    originPrefix = "origin/p4/"
 335
 336    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 337        line = line.strip()
 338        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 339            continue
 340
 341        headName = line[len(originPrefix):]
 342        remoteHead = localRefPrefix + headName
 343        originHead = line
 344
 345        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 346        if (not original.has_key('depot-paths')
 347            or not original.has_key('change')):
 348            continue
 349
 350        update = False
 351        if not gitBranchExists(remoteHead):
 352            if verbose:
 353                print "creating %s" % remoteHead
 354            update = True
 355        else:
 356            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 357            if settings.has_key('change') > 0:
 358                if settings['depot-paths'] == original['depot-paths']:
 359                    originP4Change = int(original['change'])
 360                    p4Change = int(settings['change'])
 361                    if originP4Change > p4Change:
 362                        print ("%s (%s) is newer than %s (%s). "
 363                               "Updating p4 branch from origin."
 364                               % (originHead, originP4Change,
 365                                  remoteHead, p4Change))
 366                        update = True
 367                else:
 368                    print ("Ignoring: %s was imported from %s while "
 369                           "%s was imported from %s"
 370                           % (originHead, ','.join(original['depot-paths']),
 371                              remoteHead, ','.join(settings['depot-paths'])))
 372
 373        if update:
 374            system("git update-ref %s %s" % (remoteHead, originHead))
 375
 376def originP4BranchesExist():
 377        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 378
 379def p4ChangesForPaths(depotPaths, changeRange):
 380    assert depotPaths
 381    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 382                                                        for p in depotPaths]))
 383
 384    changes = []
 385    for line in output:
 386        changeNum = line.split(" ")[1]
 387        changes.append(int(changeNum))
 388
 389    changes.sort()
 390    return changes
 391
 392class Command:
 393    def __init__(self):
 394        self.usage = "usage: %prog [options]"
 395        self.needsGit = True
 396
 397class P4Debug(Command):
 398    def __init__(self):
 399        Command.__init__(self)
 400        self.options = [
 401            optparse.make_option("--verbose", dest="verbose", action="store_true",
 402                                 default=False),
 403            ]
 404        self.description = "A tool to debug the output of p4 -G."
 405        self.needsGit = False
 406        self.verbose = False
 407
 408    def run(self, args):
 409        j = 0
 410        for output in p4CmdList(" ".join(args)):
 411            print 'Element: %d' % j
 412            j += 1
 413            print output
 414        return True
 415
 416class P4RollBack(Command):
 417    def __init__(self):
 418        Command.__init__(self)
 419        self.options = [
 420            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 421            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 422        ]
 423        self.description = "A tool to debug the multi-branch import. Don't use :)"
 424        self.verbose = False
 425        self.rollbackLocalBranches = False
 426
 427    def run(self, args):
 428        if len(args) != 1:
 429            return False
 430        maxChange = int(args[0])
 431
 432        if "p4ExitCode" in p4Cmd("changes -m 1"):
 433            die("Problems executing p4");
 434
 435        if self.rollbackLocalBranches:
 436            refPrefix = "refs/heads/"
 437            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 438        else:
 439            refPrefix = "refs/remotes/"
 440            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 441
 442        for line in lines:
 443            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 444                line = line.strip()
 445                ref = refPrefix + line
 446                log = extractLogMessageFromGitCommit(ref)
 447                settings = extractSettingsGitLog(log)
 448
 449                depotPaths = settings['depot-paths']
 450                change = settings['change']
 451
 452                changed = False
 453
 454                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 455                                                           for p in depotPaths]))) == 0:
 456                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 457                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 458                    continue
 459
 460                while change and int(change) > maxChange:
 461                    changed = True
 462                    if self.verbose:
 463                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 464                    system("git update-ref %s \"%s^\"" % (ref, ref))
 465                    log = extractLogMessageFromGitCommit(ref)
 466                    settings =  extractSettingsGitLog(log)
 467
 468
 469                    depotPaths = settings['depot-paths']
 470                    change = settings['change']
 471
 472                if changed:
 473                    print "%s rewound to %s" % (ref, change)
 474
 475        return True
 476
 477class P4Submit(Command):
 478    def __init__(self):
 479        Command.__init__(self)
 480        self.options = [
 481                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 482                optparse.make_option("--origin", dest="origin"),
 483                optparse.make_option("-M", dest="detectRename", action="store_true"),
 484        ]
 485        self.description = "Submit changes from git to the perforce depot."
 486        self.usage += " [name of git branch to submit into perforce depot]"
 487        self.interactive = True
 488        self.origin = ""
 489        self.detectRename = False
 490        self.verbose = False
 491        self.isWindows = (platform.system() == "Windows")
 492
 493    def check(self):
 494        if len(p4CmdList("opened ...")) > 0:
 495            die("You have files opened with perforce! Close them before starting the sync.")
 496
 497    # replaces everything between 'Description:' and the next P4 submit template field with the
 498    # commit message
 499    def prepareLogMessage(self, template, message):
 500        result = ""
 501
 502        inDescriptionSection = False
 503
 504        for line in template.split("\n"):
 505            if line.startswith("#"):
 506                result += line + "\n"
 507                continue
 508
 509            if inDescriptionSection:
 510                if line.startswith("Files:"):
 511                    inDescriptionSection = False
 512                else:
 513                    continue
 514            else:
 515                if line.startswith("Description:"):
 516                    inDescriptionSection = True
 517                    line += "\n"
 518                    for messageLine in message.split("\n"):
 519                        line += "\t" + messageLine + "\n"
 520
 521            result += line + "\n"
 522
 523        return result
 524
 525    def prepareSubmitTemplate(self):
 526        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 527        template = ""
 528        inFilesSection = False
 529        for line in p4_read_pipe_lines("change -o"):
 530            if line.endswith("\r\n"):
 531                line = line[:-2] + "\n"
 532            if inFilesSection:
 533                if line.startswith("\t"):
 534                    # path starts and ends with a tab
 535                    path = line[1:]
 536                    lastTab = path.rfind("\t")
 537                    if lastTab != -1:
 538                        path = path[:lastTab]
 539                        if not path.startswith(self.depotPath):
 540                            continue
 541                else:
 542                    inFilesSection = False
 543            else:
 544                if line.startswith("Files:"):
 545                    inFilesSection = True
 546
 547            template += line
 548
 549        return template
 550
 551    def applyCommit(self, id):
 552        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 553        diffOpts = ("", "-M")[self.detectRename]
 554        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 555        filesToAdd = set()
 556        filesToDelete = set()
 557        editedFiles = set()
 558        filesToChangeExecBit = {}
 559        for line in diff:
 560            diff = parseDiffTreeEntry(line)
 561            modifier = diff['status']
 562            path = diff['src']
 563            if modifier == "M":
 564                system("p4 edit \"%s\"" % path)
 565                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                    filesToChangeExecBit[path] = diff['dst_mode']
 567                editedFiles.add(path)
 568            elif modifier == "A":
 569                filesToAdd.add(path)
 570                filesToChangeExecBit[path] = diff['dst_mode']
 571                if path in filesToDelete:
 572                    filesToDelete.remove(path)
 573            elif modifier == "D":
 574                filesToDelete.add(path)
 575                if path in filesToAdd:
 576                    filesToAdd.remove(path)
 577            elif modifier == "R":
 578                src, dest = diff['src'], diff['dst']
 579                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 580                system("p4 edit \"%s\"" % (dest))
 581                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                    filesToChangeExecBit[dest] = diff['dst_mode']
 583                os.unlink(dest)
 584                editedFiles.add(dest)
 585                filesToDelete.add(src)
 586            else:
 587                die("unknown modifier %s for %s" % (modifier, path))
 588
 589        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 590        patchcmd = diffcmd + " | git apply "
 591        tryPatchCmd = patchcmd + "--check -"
 592        applyPatchCmd = patchcmd + "--check --apply -"
 593
 594        if os.system(tryPatchCmd) != 0:
 595            print "Unfortunately applying the change failed!"
 596            print "What do you want to do?"
 597            response = "x"
 598            while response != "s" and response != "a" and response != "w":
 599                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 600                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 601            if response == "s":
 602                print "Skipping! Good luck with the next patches..."
 603                for f in editedFiles:
 604                    system("p4 revert \"%s\"" % f);
 605                for f in filesToAdd:
 606                    system("rm %s" %f)
 607                return
 608            elif response == "a":
 609                os.system(applyPatchCmd)
 610                if len(filesToAdd) > 0:
 611                    print "You may also want to call p4 add on the following files:"
 612                    print " ".join(filesToAdd)
 613                if len(filesToDelete):
 614                    print "The following files should be scheduled for deletion with p4 delete:"
 615                    print " ".join(filesToDelete)
 616                die("Please resolve and submit the conflict manually and "
 617                    + "continue afterwards with git-p4 submit --continue")
 618            elif response == "w":
 619                system(diffcmd + " > patch.txt")
 620                print "Patch saved to patch.txt in %s !" % self.clientPath
 621                die("Please resolve and submit the conflict manually and "
 622                    "continue afterwards with git-p4 submit --continue")
 623
 624        system(applyPatchCmd)
 625
 626        for f in filesToAdd:
 627            system("p4 add \"%s\"" % f)
 628        for f in filesToDelete:
 629            system("p4 revert \"%s\"" % f)
 630            system("p4 delete \"%s\"" % f)
 631
 632        # Set/clear executable bits
 633        for f in filesToChangeExecBit.keys():
 634            mode = filesToChangeExecBit[f]
 635            setP4ExecBit(f, mode)
 636
 637        logMessage = extractLogMessageFromGitCommit(id)
 638        logMessage = logMessage.strip()
 639
 640        template = self.prepareSubmitTemplate()
 641
 642        if self.interactive:
 643            submitTemplate = self.prepareLogMessage(template, logMessage)
 644            if os.environ.has_key("P4DIFF"):
 645                del(os.environ["P4DIFF"])
 646            diff = read_pipe("p4 diff -du ...")
 647
 648            newdiff = ""
 649            for newFile in filesToAdd:
 650                newdiff += "==== new file ====\n"
 651                newdiff += "--- /dev/null\n"
 652                newdiff += "+++ %s\n" % newFile
 653                f = open(newFile, "r")
 654                for line in f.readlines():
 655                    newdiff += "+" + line
 656                f.close()
 657
 658            separatorLine = "######## everything below this line is just the diff #######\n"
 659
 660            [handle, fileName] = tempfile.mkstemp()
 661            tmpFile = os.fdopen(handle, "w+")
 662            if self.isWindows:
 663                submitTemplate = submitTemplate.replace("\n", "\r\n")
 664                separatorLine = separatorLine.replace("\n", "\r\n")
 665                newdiff = newdiff.replace("\n", "\r\n")
 666            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 667            tmpFile.close()
 668            defaultEditor = "vi"
 669            if platform.system() == "Windows":
 670                defaultEditor = "notepad"
 671            if os.environ.has_key("P4EDITOR"):
 672                editor = os.environ.get("P4EDITOR")
 673            else:
 674                editor = os.environ.get("EDITOR", defaultEditor);
 675            system(editor + " " + fileName)
 676            tmpFile = open(fileName, "rb")
 677            message = tmpFile.read()
 678            tmpFile.close()
 679            os.remove(fileName)
 680            submitTemplate = message[:message.index(separatorLine)]
 681            if self.isWindows:
 682                submitTemplate = submitTemplate.replace("\r\n", "\n")
 683
 684            write_pipe("p4 submit -i", submitTemplate)
 685        else:
 686            fileName = "submit.txt"
 687            file = open(fileName, "w+")
 688            file.write(self.prepareLogMessage(template, logMessage))
 689            file.close()
 690            print ("Perforce submit template written as %s. "
 691                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 692                   % (fileName, fileName))
 693
 694    def run(self, args):
 695        if len(args) == 0:
 696            self.master = currentGitBranch()
 697            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 698                die("Detecting current git branch failed!")
 699        elif len(args) == 1:
 700            self.master = args[0]
 701        else:
 702            return False
 703
 704        allowSubmit = gitConfig("git-p4.allowSubmit")
 705        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 706            die("%s is not in git-p4.allowSubmit" % self.master)
 707
 708        [upstream, settings] = findUpstreamBranchPoint()
 709        self.depotPath = settings['depot-paths'][0]
 710        if len(self.origin) == 0:
 711            self.origin = upstream
 712
 713        if self.verbose:
 714            print "Origin branch is " + self.origin
 715
 716        if len(self.depotPath) == 0:
 717            print "Internal error: cannot locate perforce depot path from existing branches"
 718            sys.exit(128)
 719
 720        self.clientPath = p4Where(self.depotPath)
 721
 722        if len(self.clientPath) == 0:
 723            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 724            sys.exit(128)
 725
 726        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 727        self.oldWorkingDirectory = os.getcwd()
 728
 729        os.chdir(self.clientPath)
 730        print "Syncronizing p4 checkout..."
 731        system("p4 sync ...")
 732
 733        self.check()
 734
 735        commits = []
 736        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 737            commits.append(line.strip())
 738        commits.reverse()
 739
 740        while len(commits) > 0:
 741            commit = commits[0]
 742            commits = commits[1:]
 743            self.applyCommit(commit)
 744            if not self.interactive:
 745                break
 746
 747        if len(commits) == 0:
 748            print "All changes applied!"
 749            os.chdir(self.oldWorkingDirectory)
 750
 751            sync = P4Sync()
 752            sync.run([])
 753
 754            rebase = P4Rebase()
 755            rebase.rebase()
 756
 757        return True
 758
 759class P4Sync(Command):
 760    def __init__(self):
 761        Command.__init__(self)
 762        self.options = [
 763                optparse.make_option("--branch", dest="branch"),
 764                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 765                optparse.make_option("--changesfile", dest="changesFile"),
 766                optparse.make_option("--silent", dest="silent", action="store_true"),
 767                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 768                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 769                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 770                                     help="Import into refs/heads/ , not refs/remotes"),
 771                optparse.make_option("--max-changes", dest="maxChanges"),
 772                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 773                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 774                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 775                                     help="Only sync files that are included in the Perforce Client Spec")
 776        ]
 777        self.description = """Imports from Perforce into a git repository.\n
 778    example:
 779    //depot/my/project/ -- to import the current head
 780    //depot/my/project/@all -- to import everything
 781    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 782
 783    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 784
 785        self.usage += " //depot/path[@revRange]"
 786        self.silent = False
 787        self.createdBranches = Set()
 788        self.committedChanges = Set()
 789        self.branch = ""
 790        self.detectBranches = False
 791        self.detectLabels = False
 792        self.changesFile = ""
 793        self.syncWithOrigin = True
 794        self.verbose = False
 795        self.importIntoRemotes = True
 796        self.maxChanges = ""
 797        self.isWindows = (platform.system() == "Windows")
 798        self.keepRepoPath = False
 799        self.depotPaths = None
 800        self.p4BranchesInGit = []
 801        self.cloneExclude = []
 802        self.useClientSpec = False
 803        self.clientSpecDirs = []
 804
 805        if gitConfig("git-p4.syncFromOrigin") == "false":
 806            self.syncWithOrigin = False
 807
 808    def extractFilesFromCommit(self, commit):
 809        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 810                             for path in self.cloneExclude]
 811        files = []
 812        fnum = 0
 813        while commit.has_key("depotFile%s" % fnum):
 814            path =  commit["depotFile%s" % fnum]
 815
 816            if [p for p in self.cloneExclude
 817                if path.startswith (p)]:
 818                found = False
 819            else:
 820                found = [p for p in self.depotPaths
 821                         if path.startswith (p)]
 822            if not found:
 823                fnum = fnum + 1
 824                continue
 825
 826            file = {}
 827            file["path"] = path
 828            file["rev"] = commit["rev%s" % fnum]
 829            file["action"] = commit["action%s" % fnum]
 830            file["type"] = commit["type%s" % fnum]
 831            files.append(file)
 832            fnum = fnum + 1
 833        return files
 834
 835    def stripRepoPath(self, path, prefixes):
 836        if self.keepRepoPath:
 837            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 838
 839        for p in prefixes:
 840            if path.startswith(p):
 841                path = path[len(p):]
 842
 843        return path
 844
 845    def splitFilesIntoBranches(self, commit):
 846        branches = {}
 847        fnum = 0
 848        while commit.has_key("depotFile%s" % fnum):
 849            path =  commit["depotFile%s" % fnum]
 850            found = [p for p in self.depotPaths
 851                     if path.startswith (p)]
 852            if not found:
 853                fnum = fnum + 1
 854                continue
 855
 856            file = {}
 857            file["path"] = path
 858            file["rev"] = commit["rev%s" % fnum]
 859            file["action"] = commit["action%s" % fnum]
 860            file["type"] = commit["type%s" % fnum]
 861            fnum = fnum + 1
 862
 863            relPath = self.stripRepoPath(path, self.depotPaths)
 864
 865            for branch in self.knownBranches.keys():
 866
 867                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 868                if relPath.startswith(branch + "/"):
 869                    if branch not in branches:
 870                        branches[branch] = []
 871                    branches[branch].append(file)
 872                    break
 873
 874        return branches
 875
 876    ## Should move this out, doesn't use SELF.
 877    def readP4Files(self, files):
 878        filesForCommit = []
 879        filesToRead = []
 880
 881        for f in files:
 882            includeFile = True
 883            for val in self.clientSpecDirs:
 884                if f['path'].startswith(val[0]):
 885                    if val[1] <= 0:
 886                        includeFile = False
 887                    break
 888
 889            if includeFile:
 890                filesForCommit.append(f)
 891                if f['action'] != 'delete':
 892                    filesToRead.append(f)
 893
 894        filedata = []
 895        if len(filesToRead) > 0:
 896            filedata = p4CmdList('-x - print',
 897                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 898                                                  for f in filesToRead]),
 899                                 stdin_mode='w+')
 900
 901            if "p4ExitCode" in filedata[0]:
 902                die("Problems executing p4. Error: [%d]."
 903                    % (filedata[0]['p4ExitCode']));
 904
 905        j = 0;
 906        contents = {}
 907        while j < len(filedata):
 908            stat = filedata[j]
 909            j += 1
 910            text = [];
 911            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 912                text.append(filedata[j]['data'])
 913                j += 1
 914            text = ''.join(text)
 915
 916            if not stat.has_key('depotFile'):
 917                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 918                continue
 919
 920            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 921                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 922            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 923                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 924
 925            contents[stat['depotFile']] = text
 926
 927        for f in filesForCommit:
 928            path = f['path']
 929            if contents.has_key(path):
 930                f['data'] = contents[path]
 931
 932        return filesForCommit
 933
 934    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 935        epoch = details["time"]
 936        author = details["user"]
 937
 938        if self.verbose:
 939            print "commit into %s" % branch
 940
 941        # start with reading files; if that fails, we should not
 942        # create a commit.
 943        new_files = []
 944        for f in files:
 945            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 946                new_files.append (f)
 947            else:
 948                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 949        files = self.readP4Files(new_files)
 950
 951        self.gitStream.write("commit %s\n" % branch)
 952#        gitStream.write("mark :%s\n" % details["change"])
 953        self.committedChanges.add(int(details["change"]))
 954        committer = ""
 955        if author not in self.users:
 956            self.getUserMapFromPerforceServer()
 957        if author in self.users:
 958            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 959        else:
 960            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 961
 962        self.gitStream.write("committer %s\n" % committer)
 963
 964        self.gitStream.write("data <<EOT\n")
 965        self.gitStream.write(details["desc"])
 966        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 967                             % (','.join (branchPrefixes), details["change"]))
 968        if len(details['options']) > 0:
 969            self.gitStream.write(": options = %s" % details['options'])
 970        self.gitStream.write("]\nEOT\n\n")
 971
 972        if len(parent) > 0:
 973            if self.verbose:
 974                print "parent %s" % parent
 975            self.gitStream.write("from %s\n" % parent)
 976
 977        for file in files:
 978            if file["type"] == "apple":
 979                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 980                continue
 981
 982            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 983            if file["action"] == "delete":
 984                self.gitStream.write("D %s\n" % relPath)
 985            else:
 986                data = file['data']
 987
 988                mode = "644"
 989                if isP4Exec(file["type"]):
 990                    mode = "755"
 991                elif file["type"] == "symlink":
 992                    mode = "120000"
 993                    # p4 print on a symlink contains "target\n", so strip it off
 994                    data = data[:-1]
 995
 996                if self.isWindows and file["type"].endswith("text"):
 997                    data = data.replace("\r\n", "\n")
 998
 999                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1000                self.gitStream.write("data %s\n" % len(data))
1001                self.gitStream.write(data)
1002                self.gitStream.write("\n")
1003
1004        self.gitStream.write("\n")
1005
1006        change = int(details["change"])
1007
1008        if self.labels.has_key(change):
1009            label = self.labels[change]
1010            labelDetails = label[0]
1011            labelRevisions = label[1]
1012            if self.verbose:
1013                print "Change %s is labelled %s" % (change, labelDetails)
1014
1015            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1016                                                    for p in branchPrefixes]))
1017
1018            if len(files) == len(labelRevisions):
1019
1020                cleanedFiles = {}
1021                for info in files:
1022                    if info["action"] == "delete":
1023                        continue
1024                    cleanedFiles[info["depotFile"]] = info["rev"]
1025
1026                if cleanedFiles == labelRevisions:
1027                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1028                    self.gitStream.write("from %s\n" % branch)
1029
1030                    owner = labelDetails["Owner"]
1031                    tagger = ""
1032                    if author in self.users:
1033                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1034                    else:
1035                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1036                    self.gitStream.write("tagger %s\n" % tagger)
1037                    self.gitStream.write("data <<EOT\n")
1038                    self.gitStream.write(labelDetails["Description"])
1039                    self.gitStream.write("EOT\n\n")
1040
1041                else:
1042                    if not self.silent:
1043                        print ("Tag %s does not match with change %s: files do not match."
1044                               % (labelDetails["label"], change))
1045
1046            else:
1047                if not self.silent:
1048                    print ("Tag %s does not match with change %s: file count is different."
1049                           % (labelDetails["label"], change))
1050
1051    def getUserCacheFilename(self):
1052        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1053        return home + "/.gitp4-usercache.txt"
1054
1055    def getUserMapFromPerforceServer(self):
1056        if self.userMapFromPerforceServer:
1057            return
1058        self.users = {}
1059
1060        for output in p4CmdList("users"):
1061            if not output.has_key("User"):
1062                continue
1063            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1064
1065
1066        s = ''
1067        for (key, val) in self.users.items():
1068            s += "%s\t%s\n" % (key, val)
1069
1070        open(self.getUserCacheFilename(), "wb").write(s)
1071        self.userMapFromPerforceServer = True
1072
1073    def loadUserMapFromCache(self):
1074        self.users = {}
1075        self.userMapFromPerforceServer = False
1076        try:
1077            cache = open(self.getUserCacheFilename(), "rb")
1078            lines = cache.readlines()
1079            cache.close()
1080            for line in lines:
1081                entry = line.strip().split("\t")
1082                self.users[entry[0]] = entry[1]
1083        except IOError:
1084            self.getUserMapFromPerforceServer()
1085
1086    def getLabels(self):
1087        self.labels = {}
1088
1089        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1090        if len(l) > 0 and not self.silent:
1091            print "Finding files belonging to labels in %s" % `self.depotPaths`
1092
1093        for output in l:
1094            label = output["label"]
1095            revisions = {}
1096            newestChange = 0
1097            if self.verbose:
1098                print "Querying files for label %s" % label
1099            for file in p4CmdList("files "
1100                                  +  ' '.join (["%s...@%s" % (p, label)
1101                                                for p in self.depotPaths])):
1102                revisions[file["depotFile"]] = file["rev"]
1103                change = int(file["change"])
1104                if change > newestChange:
1105                    newestChange = change
1106
1107            self.labels[newestChange] = [output, revisions]
1108
1109        if self.verbose:
1110            print "Label changes: %s" % self.labels.keys()
1111
1112    def guessProjectName(self):
1113        for p in self.depotPaths:
1114            if p.endswith("/"):
1115                p = p[:-1]
1116            p = p[p.strip().rfind("/") + 1:]
1117            if not p.endswith("/"):
1118               p += "/"
1119            return p
1120
1121    def getBranchMapping(self):
1122        lostAndFoundBranches = set()
1123
1124        for info in p4CmdList("branches"):
1125            details = p4Cmd("branch -o %s" % info["branch"])
1126            viewIdx = 0
1127            while details.has_key("View%s" % viewIdx):
1128                paths = details["View%s" % viewIdx].split(" ")
1129                viewIdx = viewIdx + 1
1130                # require standard //depot/foo/... //depot/bar/... mapping
1131                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1132                    continue
1133                source = paths[0]
1134                destination = paths[1]
1135                ## HACK
1136                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1137                    source = source[len(self.depotPaths[0]):-4]
1138                    destination = destination[len(self.depotPaths[0]):-4]
1139
1140                    if destination in self.knownBranches:
1141                        if not self.silent:
1142                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1143                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1144                        continue
1145
1146                    self.knownBranches[destination] = source
1147
1148                    lostAndFoundBranches.discard(destination)
1149
1150                    if source not in self.knownBranches:
1151                        lostAndFoundBranches.add(source)
1152
1153
1154        for branch in lostAndFoundBranches:
1155            self.knownBranches[branch] = branch
1156
1157    def getBranchMappingFromGitBranches(self):
1158        branches = p4BranchesInGit(self.importIntoRemotes)
1159        for branch in branches.keys():
1160            if branch == "master":
1161                branch = "main"
1162            else:
1163                branch = branch[len(self.projectName):]
1164            self.knownBranches[branch] = branch
1165
1166    def listExistingP4GitBranches(self):
1167        # branches holds mapping from name to commit
1168        branches = p4BranchesInGit(self.importIntoRemotes)
1169        self.p4BranchesInGit = branches.keys()
1170        for branch in branches.keys():
1171            self.initialParents[self.refPrefix + branch] = branches[branch]
1172
1173    def updateOptionDict(self, d):
1174        option_keys = {}
1175        if self.keepRepoPath:
1176            option_keys['keepRepoPath'] = 1
1177
1178        d["options"] = ' '.join(sorted(option_keys.keys()))
1179
1180    def readOptions(self, d):
1181        self.keepRepoPath = (d.has_key('options')
1182                             and ('keepRepoPath' in d['options']))
1183
1184    def gitRefForBranch(self, branch):
1185        if branch == "main":
1186            return self.refPrefix + "master"
1187
1188        if len(branch) <= 0:
1189            return branch
1190
1191        return self.refPrefix + self.projectName + branch
1192
1193    def gitCommitByP4Change(self, ref, change):
1194        if self.verbose:
1195            print "looking in ref " + ref + " for change %s using bisect..." % change
1196
1197        earliestCommit = ""
1198        latestCommit = parseRevision(ref)
1199
1200        while True:
1201            if self.verbose:
1202                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1203            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1204            if len(next) == 0:
1205                if self.verbose:
1206                    print "argh"
1207                return ""
1208            log = extractLogMessageFromGitCommit(next)
1209            settings = extractSettingsGitLog(log)
1210            currentChange = int(settings['change'])
1211            if self.verbose:
1212                print "current change %s" % currentChange
1213
1214            if currentChange == change:
1215                if self.verbose:
1216                    print "found %s" % next
1217                return next
1218
1219            if currentChange < change:
1220                earliestCommit = "^%s" % next
1221            else:
1222                latestCommit = "%s" % next
1223
1224        return ""
1225
1226    def importNewBranch(self, branch, maxChange):
1227        # make fast-import flush all changes to disk and update the refs using the checkpoint
1228        # command so that we can try to find the branch parent in the git history
1229        self.gitStream.write("checkpoint\n\n");
1230        self.gitStream.flush();
1231        branchPrefix = self.depotPaths[0] + branch + "/"
1232        range = "@1,%s" % maxChange
1233        #print "prefix" + branchPrefix
1234        changes = p4ChangesForPaths([branchPrefix], range)
1235        if len(changes) <= 0:
1236            return False
1237        firstChange = changes[0]
1238        #print "first change in branch: %s" % firstChange
1239        sourceBranch = self.knownBranches[branch]
1240        sourceDepotPath = self.depotPaths[0] + sourceBranch
1241        sourceRef = self.gitRefForBranch(sourceBranch)
1242        #print "source " + sourceBranch
1243
1244        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1245        #print "branch parent: %s" % branchParentChange
1246        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1247        if len(gitParent) > 0:
1248            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1249            #print "parent git commit: %s" % gitParent
1250
1251        self.importChanges(changes)
1252        return True
1253
1254    def importChanges(self, changes):
1255        cnt = 1
1256        for change in changes:
1257            description = p4Cmd("describe %s" % change)
1258            self.updateOptionDict(description)
1259
1260            if not self.silent:
1261                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1262                sys.stdout.flush()
1263            cnt = cnt + 1
1264
1265            try:
1266                if self.detectBranches:
1267                    branches = self.splitFilesIntoBranches(description)
1268                    for branch in branches.keys():
1269                        ## HACK  --hwn
1270                        branchPrefix = self.depotPaths[0] + branch + "/"
1271
1272                        parent = ""
1273
1274                        filesForCommit = branches[branch]
1275
1276                        if self.verbose:
1277                            print "branch is %s" % branch
1278
1279                        self.updatedBranches.add(branch)
1280
1281                        if branch not in self.createdBranches:
1282                            self.createdBranches.add(branch)
1283                            parent = self.knownBranches[branch]
1284                            if parent == branch:
1285                                parent = ""
1286                            else:
1287                                fullBranch = self.projectName + branch
1288                                if fullBranch not in self.p4BranchesInGit:
1289                                    if not self.silent:
1290                                        print("\n    Importing new branch %s" % fullBranch);
1291                                    if self.importNewBranch(branch, change - 1):
1292                                        parent = ""
1293                                        self.p4BranchesInGit.append(fullBranch)
1294                                    if not self.silent:
1295                                        print("\n    Resuming with change %s" % change);
1296
1297                                if self.verbose:
1298                                    print "parent determined through known branches: %s" % parent
1299
1300                        branch = self.gitRefForBranch(branch)
1301                        parent = self.gitRefForBranch(parent)
1302
1303                        if self.verbose:
1304                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1305
1306                        if len(parent) == 0 and branch in self.initialParents:
1307                            parent = self.initialParents[branch]
1308                            del self.initialParents[branch]
1309
1310                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1311                else:
1312                    files = self.extractFilesFromCommit(description)
1313                    self.commit(description, files, self.branch, self.depotPaths,
1314                                self.initialParent)
1315                    self.initialParent = ""
1316            except IOError:
1317                print self.gitError.read()
1318                sys.exit(1)
1319
1320    def importHeadRevision(self, revision):
1321        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1322
1323        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1324        details["desc"] = ("Initial import of %s from the state at revision %s"
1325                           % (' '.join(self.depotPaths), revision))
1326        details["change"] = revision
1327        newestRevision = 0
1328
1329        fileCnt = 0
1330        for info in p4CmdList("files "
1331                              +  ' '.join(["%s...%s"
1332                                           % (p, revision)
1333                                           for p in self.depotPaths])):
1334
1335            if info['code'] == 'error':
1336                sys.stderr.write("p4 returned an error: %s\n"
1337                                 % info['data'])
1338                sys.exit(1)
1339
1340
1341            change = int(info["change"])
1342            if change > newestRevision:
1343                newestRevision = change
1344
1345            if info["action"] == "delete":
1346                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1347                #fileCnt = fileCnt + 1
1348                continue
1349
1350            for prop in ["depotFile", "rev", "action", "type" ]:
1351                details["%s%s" % (prop, fileCnt)] = info[prop]
1352
1353            fileCnt = fileCnt + 1
1354
1355        details["change"] = newestRevision
1356        self.updateOptionDict(details)
1357        try:
1358            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1359        except IOError:
1360            print "IO error with git fast-import. Is your git version recent enough?"
1361            print self.gitError.read()
1362
1363
1364    def getClientSpec(self):
1365        specList = p4CmdList( "client -o" )
1366        temp = {}
1367        for entry in specList:
1368            for k,v in entry.iteritems():
1369                if k.startswith("View"):
1370                    if v.startswith('"'):
1371                        start = 1
1372                    else:
1373                        start = 0
1374                    index = v.find("...")
1375                    v = v[start:index]
1376                    if v.startswith("-"):
1377                        v = v[1:]
1378                        temp[v] = -len(v)
1379                    else:
1380                        temp[v] = len(v)
1381        self.clientSpecDirs = temp.items()
1382        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1383
1384    def run(self, args):
1385        self.depotPaths = []
1386        self.changeRange = ""
1387        self.initialParent = ""
1388        self.previousDepotPaths = []
1389
1390        # map from branch depot path to parent branch
1391        self.knownBranches = {}
1392        self.initialParents = {}
1393        self.hasOrigin = originP4BranchesExist()
1394        if not self.syncWithOrigin:
1395            self.hasOrigin = False
1396
1397        if self.importIntoRemotes:
1398            self.refPrefix = "refs/remotes/p4/"
1399        else:
1400            self.refPrefix = "refs/heads/p4/"
1401
1402        if self.syncWithOrigin and self.hasOrigin:
1403            if not self.silent:
1404                print "Syncing with origin first by calling git fetch origin"
1405            system("git fetch origin")
1406
1407        if len(self.branch) == 0:
1408            self.branch = self.refPrefix + "master"
1409            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1410                system("git update-ref %s refs/heads/p4" % self.branch)
1411                system("git branch -D p4");
1412            # create it /after/ importing, when master exists
1413            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1414                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1415
1416        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1417            self.getClientSpec()
1418
1419        # TODO: should always look at previous commits,
1420        # merge with previous imports, if possible.
1421        if args == []:
1422            if self.hasOrigin:
1423                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1424            self.listExistingP4GitBranches()
1425
1426            if len(self.p4BranchesInGit) > 1:
1427                if not self.silent:
1428                    print "Importing from/into multiple branches"
1429                self.detectBranches = True
1430
1431            if self.verbose:
1432                print "branches: %s" % self.p4BranchesInGit
1433
1434            p4Change = 0
1435            for branch in self.p4BranchesInGit:
1436                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1437
1438                settings = extractSettingsGitLog(logMsg)
1439
1440                self.readOptions(settings)
1441                if (settings.has_key('depot-paths')
1442                    and settings.has_key ('change')):
1443                    change = int(settings['change']) + 1
1444                    p4Change = max(p4Change, change)
1445
1446                    depotPaths = sorted(settings['depot-paths'])
1447                    if self.previousDepotPaths == []:
1448                        self.previousDepotPaths = depotPaths
1449                    else:
1450                        paths = []
1451                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1452                            for i in range(0, min(len(cur), len(prev))):
1453                                if cur[i] <> prev[i]:
1454                                    i = i - 1
1455                                    break
1456
1457                            paths.append (cur[:i + 1])
1458
1459                        self.previousDepotPaths = paths
1460
1461            if p4Change > 0:
1462                self.depotPaths = sorted(self.previousDepotPaths)
1463                self.changeRange = "@%s,#head" % p4Change
1464                if not self.detectBranches:
1465                    self.initialParent = parseRevision(self.branch)
1466                if not self.silent and not self.detectBranches:
1467                    print "Performing incremental import into %s git branch" % self.branch
1468
1469        if not self.branch.startswith("refs/"):
1470            self.branch = "refs/heads/" + self.branch
1471
1472        if len(args) == 0 and self.depotPaths:
1473            if not self.silent:
1474                print "Depot paths: %s" % ' '.join(self.depotPaths)
1475        else:
1476            if self.depotPaths and self.depotPaths != args:
1477                print ("previous import used depot path %s and now %s was specified. "
1478                       "This doesn't work!" % (' '.join (self.depotPaths),
1479                                               ' '.join (args)))
1480                sys.exit(1)
1481
1482            self.depotPaths = sorted(args)
1483
1484        revision = ""
1485        self.users = {}
1486
1487        newPaths = []
1488        for p in self.depotPaths:
1489            if p.find("@") != -1:
1490                atIdx = p.index("@")
1491                self.changeRange = p[atIdx:]
1492                if self.changeRange == "@all":
1493                    self.changeRange = ""
1494                elif ',' not in self.changeRange:
1495                    revision = self.changeRange
1496                    self.changeRange = ""
1497                p = p[:atIdx]
1498            elif p.find("#") != -1:
1499                hashIdx = p.index("#")
1500                revision = p[hashIdx:]
1501                p = p[:hashIdx]
1502            elif self.previousDepotPaths == []:
1503                revision = "#head"
1504
1505            p = re.sub ("\.\.\.$", "", p)
1506            if not p.endswith("/"):
1507                p += "/"
1508
1509            newPaths.append(p)
1510
1511        self.depotPaths = newPaths
1512
1513
1514        self.loadUserMapFromCache()
1515        self.labels = {}
1516        if self.detectLabels:
1517            self.getLabels();
1518
1519        if self.detectBranches:
1520            ## FIXME - what's a P4 projectName ?
1521            self.projectName = self.guessProjectName()
1522
1523            if self.hasOrigin:
1524                self.getBranchMappingFromGitBranches()
1525            else:
1526                self.getBranchMapping()
1527            if self.verbose:
1528                print "p4-git branches: %s" % self.p4BranchesInGit
1529                print "initial parents: %s" % self.initialParents
1530            for b in self.p4BranchesInGit:
1531                if b != "master":
1532
1533                    ## FIXME
1534                    b = b[len(self.projectName):]
1535                self.createdBranches.add(b)
1536
1537        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1538
1539        importProcess = subprocess.Popen(["git", "fast-import"],
1540                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1541                                         stderr=subprocess.PIPE);
1542        self.gitOutput = importProcess.stdout
1543        self.gitStream = importProcess.stdin
1544        self.gitError = importProcess.stderr
1545
1546        if revision:
1547            self.importHeadRevision(revision)
1548        else:
1549            changes = []
1550
1551            if len(self.changesFile) > 0:
1552                output = open(self.changesFile).readlines()
1553                changeSet = Set()
1554                for line in output:
1555                    changeSet.add(int(line))
1556
1557                for change in changeSet:
1558                    changes.append(change)
1559
1560                changes.sort()
1561            else:
1562                if self.verbose:
1563                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1564                                                              self.changeRange)
1565                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1566
1567                if len(self.maxChanges) > 0:
1568                    changes = changes[:min(int(self.maxChanges), len(changes))]
1569
1570            if len(changes) == 0:
1571                if not self.silent:
1572                    print "No changes to import!"
1573                return True
1574
1575            if not self.silent and not self.detectBranches:
1576                print "Import destination: %s" % self.branch
1577
1578            self.updatedBranches = set()
1579
1580            self.importChanges(changes)
1581
1582            if not self.silent:
1583                print ""
1584                if len(self.updatedBranches) > 0:
1585                    sys.stdout.write("Updated branches: ")
1586                    for b in self.updatedBranches:
1587                        sys.stdout.write("%s " % b)
1588                    sys.stdout.write("\n")
1589
1590        self.gitStream.close()
1591        if importProcess.wait() != 0:
1592            die("fast-import failed: %s" % self.gitError.read())
1593        self.gitOutput.close()
1594        self.gitError.close()
1595
1596        return True
1597
1598class P4Rebase(Command):
1599    def __init__(self):
1600        Command.__init__(self)
1601        self.options = [ ]
1602        self.description = ("Fetches the latest revision from perforce and "
1603                            + "rebases the current work (branch) against it")
1604        self.verbose = False
1605
1606    def run(self, args):
1607        sync = P4Sync()
1608        sync.run([])
1609
1610        return self.rebase()
1611
1612    def rebase(self):
1613        if os.system("git update-index --refresh") != 0:
1614            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1615        if len(read_pipe("git diff-index HEAD --")) > 0:
1616            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1617
1618        [upstream, settings] = findUpstreamBranchPoint()
1619        if len(upstream) == 0:
1620            die("Cannot find upstream branchpoint for rebase")
1621
1622        # the branchpoint may be p4/foo~3, so strip off the parent
1623        upstream = re.sub("~[0-9]+$", "", upstream)
1624
1625        print "Rebasing the current branch onto %s" % upstream
1626        oldHead = read_pipe("git rev-parse HEAD").strip()
1627        system("git rebase %s" % upstream)
1628        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1629        return True
1630
1631class P4Clone(P4Sync):
1632    def __init__(self):
1633        P4Sync.__init__(self)
1634        self.description = "Creates a new git repository and imports from Perforce into it"
1635        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1636        self.options += [
1637            optparse.make_option("--destination", dest="cloneDestination",
1638                                 action='store', default=None,
1639                                 help="where to leave result of the clone"),
1640            optparse.make_option("-/", dest="cloneExclude",
1641                                 action="append", type="string",
1642                                 help="exclude depot path")
1643        ]
1644        self.cloneDestination = None
1645        self.needsGit = False
1646
1647    # This is required for the "append" cloneExclude action
1648    def ensure_value(self, attr, value):
1649        if not hasattr(self, attr) or getattr(self, attr) is None:
1650            setattr(self, attr, value)
1651        return getattr(self, attr)
1652
1653    def defaultDestination(self, args):
1654        ## TODO: use common prefix of args?
1655        depotPath = args[0]
1656        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1657        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1658        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1659        depotDir = re.sub(r"/$", "", depotDir)
1660        return os.path.split(depotDir)[1]
1661
1662    def run(self, args):
1663        if len(args) < 1:
1664            return False
1665
1666        if self.keepRepoPath and not self.cloneDestination:
1667            sys.stderr.write("Must specify destination for --keep-path\n")
1668            sys.exit(1)
1669
1670        depotPaths = args
1671
1672        if not self.cloneDestination and len(depotPaths) > 1:
1673            self.cloneDestination = depotPaths[-1]
1674            depotPaths = depotPaths[:-1]
1675
1676        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1677        for p in depotPaths:
1678            if not p.startswith("//"):
1679                return False
1680
1681        if not self.cloneDestination:
1682            self.cloneDestination = self.defaultDestination(args)
1683
1684        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1685        if not os.path.exists(self.cloneDestination):
1686            os.makedirs(self.cloneDestination)
1687        os.chdir(self.cloneDestination)
1688        system("git init")
1689        self.gitdir = os.getcwd() + "/.git"
1690        if not P4Sync.run(self, depotPaths):
1691            return False
1692        if self.branch != "master":
1693            if gitBranchExists("refs/remotes/p4/master"):
1694                system("git branch master refs/remotes/p4/master")
1695                system("git checkout -f")
1696            else:
1697                print "Could not detect main branch. No checkout/master branch created."
1698
1699        return True
1700
1701class P4Branches(Command):
1702    def __init__(self):
1703        Command.__init__(self)
1704        self.options = [ ]
1705        self.description = ("Shows the git branches that hold imports and their "
1706                            + "corresponding perforce depot paths")
1707        self.verbose = False
1708
1709    def run(self, args):
1710        if originP4BranchesExist():
1711            createOrUpdateBranchesFromOrigin()
1712
1713        cmdline = "git rev-parse --symbolic "
1714        cmdline += " --remotes"
1715
1716        for line in read_pipe_lines(cmdline):
1717            line = line.strip()
1718
1719            if not line.startswith('p4/') or line == "p4/HEAD":
1720                continue
1721            branch = line
1722
1723            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1724            settings = extractSettingsGitLog(log)
1725
1726            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1727        return True
1728
1729class HelpFormatter(optparse.IndentedHelpFormatter):
1730    def __init__(self):
1731        optparse.IndentedHelpFormatter.__init__(self)
1732
1733    def format_description(self, description):
1734        if description:
1735            return description + "\n"
1736        else:
1737            return ""
1738
1739def printUsage(commands):
1740    print "usage: %s <command> [options]" % sys.argv[0]
1741    print ""
1742    print "valid commands: %s" % ", ".join(commands)
1743    print ""
1744    print "Try %s <command> --help for command specific help." % sys.argv[0]
1745    print ""
1746
1747commands = {
1748    "debug" : P4Debug,
1749    "submit" : P4Submit,
1750    "commit" : P4Submit,
1751    "sync" : P4Sync,
1752    "rebase" : P4Rebase,
1753    "clone" : P4Clone,
1754    "rollback" : P4RollBack,
1755    "branches" : P4Branches
1756}
1757
1758
1759def main():
1760    if len(sys.argv[1:]) == 0:
1761        printUsage(commands.keys())
1762        sys.exit(2)
1763
1764    cmd = ""
1765    cmdName = sys.argv[1]
1766    try:
1767        klass = commands[cmdName]
1768        cmd = klass()
1769    except KeyError:
1770        print "unknown command %s" % cmdName
1771        print ""
1772        printUsage(commands.keys())
1773        sys.exit(2)
1774
1775    options = cmd.options
1776    cmd.gitdir = os.environ.get("GIT_DIR", None)
1777
1778    args = sys.argv[2:]
1779
1780    if len(options) > 0:
1781        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1782
1783        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1784                                       options,
1785                                       description = cmd.description,
1786                                       formatter = HelpFormatter())
1787
1788        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1789    global verbose
1790    verbose = cmd.verbose
1791    if cmd.needsGit:
1792        if cmd.gitdir == None:
1793            cmd.gitdir = os.path.abspath(".git")
1794            if not isValidGitDir(cmd.gitdir):
1795                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1796                if os.path.exists(cmd.gitdir):
1797                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1798                    if len(cdup) > 0:
1799                        os.chdir(cdup);
1800
1801        if not isValidGitDir(cmd.gitdir):
1802            if isValidGitDir(cmd.gitdir + "/.git"):
1803                cmd.gitdir += "/.git"
1804            else:
1805                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1806
1807        os.environ["GIT_DIR"] = cmd.gitdir
1808
1809    if not cmd.run(args):
1810        parser.print_help()
1811
1812
1813if __name__ == '__main__':
1814    main()