Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Consistently use 'git-p4' for the configuration entries (3cafb7d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def die(msg):
  55    if verbose:
  56        raise Exception(msg)
  57    else:
  58        sys.stderr.write(msg + "\n")
  59        sys.exit(1)
  60
  61def write_pipe(c, str):
  62    if verbose:
  63        sys.stderr.write('Writing pipe: %s\n' % c)
  64
  65    pipe = os.popen(c, 'w')
  66    val = pipe.write(str)
  67    if pipe.close():
  68        die('Command failed: %s' % c)
  69
  70    return val
  71
  72def read_pipe(c, ignore_error=False):
  73    if verbose:
  74        sys.stderr.write('Reading pipe: %s\n' % c)
  75
  76    pipe = os.popen(c, 'rb')
  77    val = pipe.read()
  78    if pipe.close() and not ignore_error:
  79        die('Command failed: %s' % c)
  80
  81    return val
  82
  83
  84def read_pipe_lines(c):
  85    if verbose:
  86        sys.stderr.write('Reading pipe: %s\n' % c)
  87    ## todo: check return status
  88    pipe = os.popen(c, 'rb')
  89    val = pipe.readlines()
  90    if pipe.close():
  91        die('Command failed: %s' % c)
  92
  93    return val
  94
  95def p4_read_pipe_lines(c):
  96    """Specifically invoke p4 on the command supplied. """
  97    real_cmd = p4_build_cmd(c)
  98    return read_pipe_lines(real_cmd)
  99
 100def system(cmd):
 101    if verbose:
 102        sys.stderr.write("executing %s\n" % cmd)
 103    if os.system(cmd) != 0:
 104        die("command failed: %s" % cmd)
 105
 106def p4_system(cmd):
 107    """Specifically invoke p4 as the system command. """
 108    real_cmd = p4_build_cmd(cmd)
 109    return system(real_cmd)
 110
 111def isP4Exec(kind):
 112    """Determine if a Perforce 'kind' should have execute permission
 113
 114    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 115    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 116    a plus sign, it is also executable"""
 117    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 118
 119def setP4ExecBit(file, mode):
 120    # Reopens an already open file and changes the execute bit to match
 121    # the execute bit setting in the passed in mode.
 122
 123    p4Type = "+x"
 124
 125    if not isModeExec(mode):
 126        p4Type = getP4OpenedType(file)
 127        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 128        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 129        if p4Type[-1] == "+":
 130            p4Type = p4Type[0:-1]
 131
 132    p4_system("reopen -t %s %s" % (p4Type, file))
 133
 134def getP4OpenedType(file):
 135    # Returns the perforce file type for the given file.
 136
 137    result = read_pipe("p4 opened %s" % file)
 138    match = re.match(".*\((.+)\)\r?$", result)
 139    if match:
 140        return match.group(1)
 141    else:
 142        die("Could not determine file type for %s (result: '%s')" % (file, result))
 143
 144def diffTreePattern():
 145    # This is a simple generator for the diff tree regex pattern. This could be
 146    # a class variable if this and parseDiffTreeEntry were a part of a class.
 147    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 148    while True:
 149        yield pattern
 150
 151def parseDiffTreeEntry(entry):
 152    """Parses a single diff tree entry into its component elements.
 153
 154    See git-diff-tree(1) manpage for details about the format of the diff
 155    output. This method returns a dictionary with the following elements:
 156
 157    src_mode - The mode of the source file
 158    dst_mode - The mode of the destination file
 159    src_sha1 - The sha1 for the source file
 160    dst_sha1 - The sha1 fr the destination file
 161    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 162    status_score - The score for the status (applicable for 'C' and 'R'
 163                   statuses). This is None if there is no score.
 164    src - The path for the source file.
 165    dst - The path for the destination file. This is only present for
 166          copy or renames. If it is not present, this is None.
 167
 168    If the pattern is not matched, None is returned."""
 169
 170    match = diffTreePattern().next().match(entry)
 171    if match:
 172        return {
 173            'src_mode': match.group(1),
 174            'dst_mode': match.group(2),
 175            'src_sha1': match.group(3),
 176            'dst_sha1': match.group(4),
 177            'status': match.group(5),
 178            'status_score': match.group(6),
 179            'src': match.group(7),
 180            'dst': match.group(10)
 181        }
 182    return None
 183
 184def isModeExec(mode):
 185    # Returns True if the given git mode represents an executable file,
 186    # otherwise False.
 187    return mode[-3:] == "755"
 188
 189def isModeExecChanged(src_mode, dst_mode):
 190    return isModeExec(src_mode) != isModeExec(dst_mode)
 191
 192def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 193    cmd = p4_build_cmd("-G %s" % (cmd))
 194    if verbose:
 195        sys.stderr.write("Opening pipe: %s\n" % cmd)
 196
 197    # Use a temporary file to avoid deadlocks without
 198    # subprocess.communicate(), which would put another copy
 199    # of stdout into memory.
 200    stdin_file = None
 201    if stdin is not None:
 202        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 203        stdin_file.write(stdin)
 204        stdin_file.flush()
 205        stdin_file.seek(0)
 206
 207    p4 = subprocess.Popen(cmd, shell=True,
 208                          stdin=stdin_file,
 209                          stdout=subprocess.PIPE)
 210
 211    result = []
 212    try:
 213        while True:
 214            entry = marshal.load(p4.stdout)
 215            result.append(entry)
 216    except EOFError:
 217        pass
 218    exitCode = p4.wait()
 219    if exitCode != 0:
 220        entry = {}
 221        entry["p4ExitCode"] = exitCode
 222        result.append(entry)
 223
 224    return result
 225
 226def p4Cmd(cmd):
 227    list = p4CmdList(cmd)
 228    result = {}
 229    for entry in list:
 230        result.update(entry)
 231    return result;
 232
 233def p4Where(depotPath):
 234    if not depotPath.endswith("/"):
 235        depotPath += "/"
 236    output = p4Cmd("where %s..." % depotPath)
 237    if output["code"] == "error":
 238        return ""
 239    clientPath = ""
 240    if "path" in output:
 241        clientPath = output.get("path")
 242    elif "data" in output:
 243        data = output.get("data")
 244        lastSpace = data.rfind(" ")
 245        clientPath = data[lastSpace + 1:]
 246
 247    if clientPath.endswith("..."):
 248        clientPath = clientPath[:-3]
 249    return clientPath
 250
 251def currentGitBranch():
 252    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 253
 254def isValidGitDir(path):
 255    if (os.path.exists(path + "/HEAD")
 256        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 257        return True;
 258    return False
 259
 260def parseRevision(ref):
 261    return read_pipe("git rev-parse %s" % ref).strip()
 262
 263def extractLogMessageFromGitCommit(commit):
 264    logMessage = ""
 265
 266    ## fixme: title is first line of commit, not 1st paragraph.
 267    foundTitle = False
 268    for log in read_pipe_lines("git cat-file commit %s" % commit):
 269       if not foundTitle:
 270           if len(log) == 1:
 271               foundTitle = True
 272           continue
 273
 274       logMessage += log
 275    return logMessage
 276
 277def extractSettingsGitLog(log):
 278    values = {}
 279    for line in log.split("\n"):
 280        line = line.strip()
 281        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 282        if not m:
 283            continue
 284
 285        assignments = m.group(1).split (':')
 286        for a in assignments:
 287            vals = a.split ('=')
 288            key = vals[0].strip()
 289            val = ('='.join (vals[1:])).strip()
 290            if val.endswith ('\"') and val.startswith('"'):
 291                val = val[1:-1]
 292
 293            values[key] = val
 294
 295    paths = values.get("depot-paths")
 296    if not paths:
 297        paths = values.get("depot-path")
 298    if paths:
 299        values['depot-paths'] = paths.split(',')
 300    return values
 301
 302def gitBranchExists(branch):
 303    proc = subprocess.Popen(["git", "rev-parse", branch],
 304                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 305    return proc.wait() == 0;
 306
 307def gitConfig(key):
 308    return read_pipe("git config %s" % key, ignore_error=True).strip()
 309
 310def p4BranchesInGit(branchesAreInRemotes = True):
 311    branches = {}
 312
 313    cmdline = "git rev-parse --symbolic "
 314    if branchesAreInRemotes:
 315        cmdline += " --remotes"
 316    else:
 317        cmdline += " --branches"
 318
 319    for line in read_pipe_lines(cmdline):
 320        line = line.strip()
 321
 322        ## only import to p4/
 323        if not line.startswith('p4/') or line == "p4/HEAD":
 324            continue
 325        branch = line
 326
 327        # strip off p4
 328        branch = re.sub ("^p4/", "", line)
 329
 330        branches[branch] = parseRevision(line)
 331    return branches
 332
 333def findUpstreamBranchPoint(head = "HEAD"):
 334    branches = p4BranchesInGit()
 335    # map from depot-path to branch name
 336    branchByDepotPath = {}
 337    for branch in branches.keys():
 338        tip = branches[branch]
 339        log = extractLogMessageFromGitCommit(tip)
 340        settings = extractSettingsGitLog(log)
 341        if settings.has_key("depot-paths"):
 342            paths = ",".join(settings["depot-paths"])
 343            branchByDepotPath[paths] = "remotes/p4/" + branch
 344
 345    settings = None
 346    parent = 0
 347    while parent < 65535:
 348        commit = head + "~%s" % parent
 349        log = extractLogMessageFromGitCommit(commit)
 350        settings = extractSettingsGitLog(log)
 351        if settings.has_key("depot-paths"):
 352            paths = ",".join(settings["depot-paths"])
 353            if branchByDepotPath.has_key(paths):
 354                return [branchByDepotPath[paths], settings]
 355
 356        parent = parent + 1
 357
 358    return ["", settings]
 359
 360def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 361    if not silent:
 362        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 363               % localRefPrefix)
 364
 365    originPrefix = "origin/p4/"
 366
 367    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 368        line = line.strip()
 369        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 370            continue
 371
 372        headName = line[len(originPrefix):]
 373        remoteHead = localRefPrefix + headName
 374        originHead = line
 375
 376        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 377        if (not original.has_key('depot-paths')
 378            or not original.has_key('change')):
 379            continue
 380
 381        update = False
 382        if not gitBranchExists(remoteHead):
 383            if verbose:
 384                print "creating %s" % remoteHead
 385            update = True
 386        else:
 387            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 388            if settings.has_key('change') > 0:
 389                if settings['depot-paths'] == original['depot-paths']:
 390                    originP4Change = int(original['change'])
 391                    p4Change = int(settings['change'])
 392                    if originP4Change > p4Change:
 393                        print ("%s (%s) is newer than %s (%s). "
 394                               "Updating p4 branch from origin."
 395                               % (originHead, originP4Change,
 396                                  remoteHead, p4Change))
 397                        update = True
 398                else:
 399                    print ("Ignoring: %s was imported from %s while "
 400                           "%s was imported from %s"
 401                           % (originHead, ','.join(original['depot-paths']),
 402                              remoteHead, ','.join(settings['depot-paths'])))
 403
 404        if update:
 405            system("git update-ref %s %s" % (remoteHead, originHead))
 406
 407def originP4BranchesExist():
 408        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 409
 410def p4ChangesForPaths(depotPaths, changeRange):
 411    assert depotPaths
 412    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 413                                                        for p in depotPaths]))
 414
 415    changes = []
 416    for line in output:
 417        changeNum = line.split(" ")[1]
 418        changes.append(int(changeNum))
 419
 420    changes.sort()
 421    return changes
 422
 423class Command:
 424    def __init__(self):
 425        self.usage = "usage: %prog [options]"
 426        self.needsGit = True
 427
 428class P4Debug(Command):
 429    def __init__(self):
 430        Command.__init__(self)
 431        self.options = [
 432            optparse.make_option("--verbose", dest="verbose", action="store_true",
 433                                 default=False),
 434            ]
 435        self.description = "A tool to debug the output of p4 -G."
 436        self.needsGit = False
 437        self.verbose = False
 438
 439    def run(self, args):
 440        j = 0
 441        for output in p4CmdList(" ".join(args)):
 442            print 'Element: %d' % j
 443            j += 1
 444            print output
 445        return True
 446
 447class P4RollBack(Command):
 448    def __init__(self):
 449        Command.__init__(self)
 450        self.options = [
 451            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 452            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 453        ]
 454        self.description = "A tool to debug the multi-branch import. Don't use :)"
 455        self.verbose = False
 456        self.rollbackLocalBranches = False
 457
 458    def run(self, args):
 459        if len(args) != 1:
 460            return False
 461        maxChange = int(args[0])
 462
 463        if "p4ExitCode" in p4Cmd("changes -m 1"):
 464            die("Problems executing p4");
 465
 466        if self.rollbackLocalBranches:
 467            refPrefix = "refs/heads/"
 468            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 469        else:
 470            refPrefix = "refs/remotes/"
 471            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 472
 473        for line in lines:
 474            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 475                line = line.strip()
 476                ref = refPrefix + line
 477                log = extractLogMessageFromGitCommit(ref)
 478                settings = extractSettingsGitLog(log)
 479
 480                depotPaths = settings['depot-paths']
 481                change = settings['change']
 482
 483                changed = False
 484
 485                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 486                                                           for p in depotPaths]))) == 0:
 487                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 488                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 489                    continue
 490
 491                while change and int(change) > maxChange:
 492                    changed = True
 493                    if self.verbose:
 494                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 495                    system("git update-ref %s \"%s^\"" % (ref, ref))
 496                    log = extractLogMessageFromGitCommit(ref)
 497                    settings =  extractSettingsGitLog(log)
 498
 499
 500                    depotPaths = settings['depot-paths']
 501                    change = settings['change']
 502
 503                if changed:
 504                    print "%s rewound to %s" % (ref, change)
 505
 506        return True
 507
 508class P4Submit(Command):
 509    def __init__(self):
 510        Command.__init__(self)
 511        self.options = [
 512                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 513                optparse.make_option("--origin", dest="origin"),
 514                optparse.make_option("-M", dest="detectRename", action="store_true"),
 515        ]
 516        self.description = "Submit changes from git to the perforce depot."
 517        self.usage += " [name of git branch to submit into perforce depot]"
 518        self.interactive = True
 519        self.origin = ""
 520        self.detectRename = False
 521        self.verbose = False
 522        self.isWindows = (platform.system() == "Windows")
 523
 524    def check(self):
 525        if len(p4CmdList("opened ...")) > 0:
 526            die("You have files opened with perforce! Close them before starting the sync.")
 527
 528    # replaces everything between 'Description:' and the next P4 submit template field with the
 529    # commit message
 530    def prepareLogMessage(self, template, message):
 531        result = ""
 532
 533        inDescriptionSection = False
 534
 535        for line in template.split("\n"):
 536            if line.startswith("#"):
 537                result += line + "\n"
 538                continue
 539
 540            if inDescriptionSection:
 541                if line.startswith("Files:"):
 542                    inDescriptionSection = False
 543                else:
 544                    continue
 545            else:
 546                if line.startswith("Description:"):
 547                    inDescriptionSection = True
 548                    line += "\n"
 549                    for messageLine in message.split("\n"):
 550                        line += "\t" + messageLine + "\n"
 551
 552            result += line + "\n"
 553
 554        return result
 555
 556    def prepareSubmitTemplate(self):
 557        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 558        template = ""
 559        inFilesSection = False
 560        for line in p4_read_pipe_lines("change -o"):
 561            if line.endswith("\r\n"):
 562                line = line[:-2] + "\n"
 563            if inFilesSection:
 564                if line.startswith("\t"):
 565                    # path starts and ends with a tab
 566                    path = line[1:]
 567                    lastTab = path.rfind("\t")
 568                    if lastTab != -1:
 569                        path = path[:lastTab]
 570                        if not path.startswith(self.depotPath):
 571                            continue
 572                else:
 573                    inFilesSection = False
 574            else:
 575                if line.startswith("Files:"):
 576                    inFilesSection = True
 577
 578            template += line
 579
 580        return template
 581
 582    def applyCommit(self, id):
 583        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 584        diffOpts = ("", "-M")[self.detectRename]
 585        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 586        filesToAdd = set()
 587        filesToDelete = set()
 588        editedFiles = set()
 589        filesToChangeExecBit = {}
 590        for line in diff:
 591            diff = parseDiffTreeEntry(line)
 592            modifier = diff['status']
 593            path = diff['src']
 594            if modifier == "M":
 595                p4_system("edit \"%s\"" % path)
 596                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 597                    filesToChangeExecBit[path] = diff['dst_mode']
 598                editedFiles.add(path)
 599            elif modifier == "A":
 600                filesToAdd.add(path)
 601                filesToChangeExecBit[path] = diff['dst_mode']
 602                if path in filesToDelete:
 603                    filesToDelete.remove(path)
 604            elif modifier == "D":
 605                filesToDelete.add(path)
 606                if path in filesToAdd:
 607                    filesToAdd.remove(path)
 608            elif modifier == "R":
 609                src, dest = diff['src'], diff['dst']
 610                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 611                p4_system("edit \"%s\"" % (dest))
 612                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 613                    filesToChangeExecBit[dest] = diff['dst_mode']
 614                os.unlink(dest)
 615                editedFiles.add(dest)
 616                filesToDelete.add(src)
 617            else:
 618                die("unknown modifier %s for %s" % (modifier, path))
 619
 620        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 621        patchcmd = diffcmd + " | git apply "
 622        tryPatchCmd = patchcmd + "--check -"
 623        applyPatchCmd = patchcmd + "--check --apply -"
 624
 625        if os.system(tryPatchCmd) != 0:
 626            print "Unfortunately applying the change failed!"
 627            print "What do you want to do?"
 628            response = "x"
 629            while response != "s" and response != "a" and response != "w":
 630                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 631                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 632            if response == "s":
 633                print "Skipping! Good luck with the next patches..."
 634                for f in editedFiles:
 635                    p4_system("revert \"%s\"" % f);
 636                for f in filesToAdd:
 637                    system("rm %s" %f)
 638                return
 639            elif response == "a":
 640                os.system(applyPatchCmd)
 641                if len(filesToAdd) > 0:
 642                    print "You may also want to call p4 add on the following files:"
 643                    print " ".join(filesToAdd)
 644                if len(filesToDelete):
 645                    print "The following files should be scheduled for deletion with p4 delete:"
 646                    print " ".join(filesToDelete)
 647                die("Please resolve and submit the conflict manually and "
 648                    + "continue afterwards with git-p4 submit --continue")
 649            elif response == "w":
 650                system(diffcmd + " > patch.txt")
 651                print "Patch saved to patch.txt in %s !" % self.clientPath
 652                die("Please resolve and submit the conflict manually and "
 653                    "continue afterwards with git-p4 submit --continue")
 654
 655        system(applyPatchCmd)
 656
 657        for f in filesToAdd:
 658            p4_system("add \"%s\"" % f)
 659        for f in filesToDelete:
 660            p4_system("revert \"%s\"" % f)
 661            p4_system("delete \"%s\"" % f)
 662
 663        # Set/clear executable bits
 664        for f in filesToChangeExecBit.keys():
 665            mode = filesToChangeExecBit[f]
 666            setP4ExecBit(f, mode)
 667
 668        logMessage = extractLogMessageFromGitCommit(id)
 669        logMessage = logMessage.strip()
 670
 671        template = self.prepareSubmitTemplate()
 672
 673        if self.interactive:
 674            submitTemplate = self.prepareLogMessage(template, logMessage)
 675            if os.environ.has_key("P4DIFF"):
 676                del(os.environ["P4DIFF"])
 677            diff = read_pipe("p4 diff -du ...")
 678
 679            newdiff = ""
 680            for newFile in filesToAdd:
 681                newdiff += "==== new file ====\n"
 682                newdiff += "--- /dev/null\n"
 683                newdiff += "+++ %s\n" % newFile
 684                f = open(newFile, "r")
 685                for line in f.readlines():
 686                    newdiff += "+" + line
 687                f.close()
 688
 689            separatorLine = "######## everything below this line is just the diff #######\n"
 690
 691            [handle, fileName] = tempfile.mkstemp()
 692            tmpFile = os.fdopen(handle, "w+")
 693            if self.isWindows:
 694                submitTemplate = submitTemplate.replace("\n", "\r\n")
 695                separatorLine = separatorLine.replace("\n", "\r\n")
 696                newdiff = newdiff.replace("\n", "\r\n")
 697            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 698            tmpFile.close()
 699            defaultEditor = "vi"
 700            if platform.system() == "Windows":
 701                defaultEditor = "notepad"
 702            if os.environ.has_key("P4EDITOR"):
 703                editor = os.environ.get("P4EDITOR")
 704            else:
 705                editor = os.environ.get("EDITOR", defaultEditor);
 706            system(editor + " " + fileName)
 707            tmpFile = open(fileName, "rb")
 708            message = tmpFile.read()
 709            tmpFile.close()
 710            os.remove(fileName)
 711            submitTemplate = message[:message.index(separatorLine)]
 712            if self.isWindows:
 713                submitTemplate = submitTemplate.replace("\r\n", "\n")
 714
 715            write_pipe("p4 submit -i", submitTemplate)
 716        else:
 717            fileName = "submit.txt"
 718            file = open(fileName, "w+")
 719            file.write(self.prepareLogMessage(template, logMessage))
 720            file.close()
 721            print ("Perforce submit template written as %s. "
 722                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 723                   % (fileName, fileName))
 724
 725    def run(self, args):
 726        if len(args) == 0:
 727            self.master = currentGitBranch()
 728            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 729                die("Detecting current git branch failed!")
 730        elif len(args) == 1:
 731            self.master = args[0]
 732        else:
 733            return False
 734
 735        allowSubmit = gitConfig("git-p4.allowSubmit")
 736        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 737            die("%s is not in git-p4.allowSubmit" % self.master)
 738
 739        [upstream, settings] = findUpstreamBranchPoint()
 740        self.depotPath = settings['depot-paths'][0]
 741        if len(self.origin) == 0:
 742            self.origin = upstream
 743
 744        if self.verbose:
 745            print "Origin branch is " + self.origin
 746
 747        if len(self.depotPath) == 0:
 748            print "Internal error: cannot locate perforce depot path from existing branches"
 749            sys.exit(128)
 750
 751        self.clientPath = p4Where(self.depotPath)
 752
 753        if len(self.clientPath) == 0:
 754            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 755            sys.exit(128)
 756
 757        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 758        self.oldWorkingDirectory = os.getcwd()
 759
 760        os.chdir(self.clientPath)
 761        print "Syncronizing p4 checkout..."
 762        p4_system("sync ...")
 763
 764        self.check()
 765
 766        commits = []
 767        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 768            commits.append(line.strip())
 769        commits.reverse()
 770
 771        while len(commits) > 0:
 772            commit = commits[0]
 773            commits = commits[1:]
 774            self.applyCommit(commit)
 775            if not self.interactive:
 776                break
 777
 778        if len(commits) == 0:
 779            print "All changes applied!"
 780            os.chdir(self.oldWorkingDirectory)
 781
 782            sync = P4Sync()
 783            sync.run([])
 784
 785            rebase = P4Rebase()
 786            rebase.rebase()
 787
 788        return True
 789
 790class P4Sync(Command):
 791    def __init__(self):
 792        Command.__init__(self)
 793        self.options = [
 794                optparse.make_option("--branch", dest="branch"),
 795                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 796                optparse.make_option("--changesfile", dest="changesFile"),
 797                optparse.make_option("--silent", dest="silent", action="store_true"),
 798                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 799                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 800                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 801                                     help="Import into refs/heads/ , not refs/remotes"),
 802                optparse.make_option("--max-changes", dest="maxChanges"),
 803                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 804                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 805                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 806                                     help="Only sync files that are included in the Perforce Client Spec")
 807        ]
 808        self.description = """Imports from Perforce into a git repository.\n
 809    example:
 810    //depot/my/project/ -- to import the current head
 811    //depot/my/project/@all -- to import everything
 812    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 813
 814    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 815
 816        self.usage += " //depot/path[@revRange]"
 817        self.silent = False
 818        self.createdBranches = Set()
 819        self.committedChanges = Set()
 820        self.branch = ""
 821        self.detectBranches = False
 822        self.detectLabels = False
 823        self.changesFile = ""
 824        self.syncWithOrigin = True
 825        self.verbose = False
 826        self.importIntoRemotes = True
 827        self.maxChanges = ""
 828        self.isWindows = (platform.system() == "Windows")
 829        self.keepRepoPath = False
 830        self.depotPaths = None
 831        self.p4BranchesInGit = []
 832        self.cloneExclude = []
 833        self.useClientSpec = False
 834        self.clientSpecDirs = []
 835
 836        if gitConfig("git-p4.syncFromOrigin") == "false":
 837            self.syncWithOrigin = False
 838
 839    def extractFilesFromCommit(self, commit):
 840        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 841                             for path in self.cloneExclude]
 842        files = []
 843        fnum = 0
 844        while commit.has_key("depotFile%s" % fnum):
 845            path =  commit["depotFile%s" % fnum]
 846
 847            if [p for p in self.cloneExclude
 848                if path.startswith (p)]:
 849                found = False
 850            else:
 851                found = [p for p in self.depotPaths
 852                         if path.startswith (p)]
 853            if not found:
 854                fnum = fnum + 1
 855                continue
 856
 857            file = {}
 858            file["path"] = path
 859            file["rev"] = commit["rev%s" % fnum]
 860            file["action"] = commit["action%s" % fnum]
 861            file["type"] = commit["type%s" % fnum]
 862            files.append(file)
 863            fnum = fnum + 1
 864        return files
 865
 866    def stripRepoPath(self, path, prefixes):
 867        if self.keepRepoPath:
 868            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 869
 870        for p in prefixes:
 871            if path.startswith(p):
 872                path = path[len(p):]
 873
 874        return path
 875
 876    def splitFilesIntoBranches(self, commit):
 877        branches = {}
 878        fnum = 0
 879        while commit.has_key("depotFile%s" % fnum):
 880            path =  commit["depotFile%s" % fnum]
 881            found = [p for p in self.depotPaths
 882                     if path.startswith (p)]
 883            if not found:
 884                fnum = fnum + 1
 885                continue
 886
 887            file = {}
 888            file["path"] = path
 889            file["rev"] = commit["rev%s" % fnum]
 890            file["action"] = commit["action%s" % fnum]
 891            file["type"] = commit["type%s" % fnum]
 892            fnum = fnum + 1
 893
 894            relPath = self.stripRepoPath(path, self.depotPaths)
 895
 896            for branch in self.knownBranches.keys():
 897
 898                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 899                if relPath.startswith(branch + "/"):
 900                    if branch not in branches:
 901                        branches[branch] = []
 902                    branches[branch].append(file)
 903                    break
 904
 905        return branches
 906
 907    ## Should move this out, doesn't use SELF.
 908    def readP4Files(self, files):
 909        filesForCommit = []
 910        filesToRead = []
 911
 912        for f in files:
 913            includeFile = True
 914            for val in self.clientSpecDirs:
 915                if f['path'].startswith(val[0]):
 916                    if val[1] <= 0:
 917                        includeFile = False
 918                    break
 919
 920            if includeFile:
 921                filesForCommit.append(f)
 922                if f['action'] != 'delete':
 923                    filesToRead.append(f)
 924
 925        filedata = []
 926        if len(filesToRead) > 0:
 927            filedata = p4CmdList('-x - print',
 928                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 929                                                  for f in filesToRead]),
 930                                 stdin_mode='w+')
 931
 932            if "p4ExitCode" in filedata[0]:
 933                die("Problems executing p4. Error: [%d]."
 934                    % (filedata[0]['p4ExitCode']));
 935
 936        j = 0;
 937        contents = {}
 938        while j < len(filedata):
 939            stat = filedata[j]
 940            j += 1
 941            text = [];
 942            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 943                text.append(filedata[j]['data'])
 944                j += 1
 945            text = ''.join(text)
 946
 947            if not stat.has_key('depotFile'):
 948                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 949                continue
 950
 951            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 952                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 953            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 954                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 955
 956            contents[stat['depotFile']] = text
 957
 958        for f in filesForCommit:
 959            path = f['path']
 960            if contents.has_key(path):
 961                f['data'] = contents[path]
 962
 963        return filesForCommit
 964
 965    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 966        epoch = details["time"]
 967        author = details["user"]
 968
 969        if self.verbose:
 970            print "commit into %s" % branch
 971
 972        # start with reading files; if that fails, we should not
 973        # create a commit.
 974        new_files = []
 975        for f in files:
 976            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 977                new_files.append (f)
 978            else:
 979                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 980        files = self.readP4Files(new_files)
 981
 982        self.gitStream.write("commit %s\n" % branch)
 983#        gitStream.write("mark :%s\n" % details["change"])
 984        self.committedChanges.add(int(details["change"]))
 985        committer = ""
 986        if author not in self.users:
 987            self.getUserMapFromPerforceServer()
 988        if author in self.users:
 989            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 990        else:
 991            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 992
 993        self.gitStream.write("committer %s\n" % committer)
 994
 995        self.gitStream.write("data <<EOT\n")
 996        self.gitStream.write(details["desc"])
 997        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 998                             % (','.join (branchPrefixes), details["change"]))
 999        if len(details['options']) > 0:
1000            self.gitStream.write(": options = %s" % details['options'])
1001        self.gitStream.write("]\nEOT\n\n")
1002
1003        if len(parent) > 0:
1004            if self.verbose:
1005                print "parent %s" % parent
1006            self.gitStream.write("from %s\n" % parent)
1007
1008        for file in files:
1009            if file["type"] == "apple":
1010                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1011                continue
1012
1013            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1014            if file["action"] == "delete":
1015                self.gitStream.write("D %s\n" % relPath)
1016            else:
1017                data = file['data']
1018
1019                mode = "644"
1020                if isP4Exec(file["type"]):
1021                    mode = "755"
1022                elif file["type"] == "symlink":
1023                    mode = "120000"
1024                    # p4 print on a symlink contains "target\n", so strip it off
1025                    data = data[:-1]
1026
1027                if self.isWindows and file["type"].endswith("text"):
1028                    data = data.replace("\r\n", "\n")
1029
1030                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1031                self.gitStream.write("data %s\n" % len(data))
1032                self.gitStream.write(data)
1033                self.gitStream.write("\n")
1034
1035        self.gitStream.write("\n")
1036
1037        change = int(details["change"])
1038
1039        if self.labels.has_key(change):
1040            label = self.labels[change]
1041            labelDetails = label[0]
1042            labelRevisions = label[1]
1043            if self.verbose:
1044                print "Change %s is labelled %s" % (change, labelDetails)
1045
1046            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1047                                                    for p in branchPrefixes]))
1048
1049            if len(files) == len(labelRevisions):
1050
1051                cleanedFiles = {}
1052                for info in files:
1053                    if info["action"] == "delete":
1054                        continue
1055                    cleanedFiles[info["depotFile"]] = info["rev"]
1056
1057                if cleanedFiles == labelRevisions:
1058                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1059                    self.gitStream.write("from %s\n" % branch)
1060
1061                    owner = labelDetails["Owner"]
1062                    tagger = ""
1063                    if author in self.users:
1064                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1065                    else:
1066                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1067                    self.gitStream.write("tagger %s\n" % tagger)
1068                    self.gitStream.write("data <<EOT\n")
1069                    self.gitStream.write(labelDetails["Description"])
1070                    self.gitStream.write("EOT\n\n")
1071
1072                else:
1073                    if not self.silent:
1074                        print ("Tag %s does not match with change %s: files do not match."
1075                               % (labelDetails["label"], change))
1076
1077            else:
1078                if not self.silent:
1079                    print ("Tag %s does not match with change %s: file count is different."
1080                           % (labelDetails["label"], change))
1081
1082    def getUserCacheFilename(self):
1083        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1084        return home + "/.gitp4-usercache.txt"
1085
1086    def getUserMapFromPerforceServer(self):
1087        if self.userMapFromPerforceServer:
1088            return
1089        self.users = {}
1090
1091        for output in p4CmdList("users"):
1092            if not output.has_key("User"):
1093                continue
1094            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1095
1096
1097        s = ''
1098        for (key, val) in self.users.items():
1099            s += "%s\t%s\n" % (key, val)
1100
1101        open(self.getUserCacheFilename(), "wb").write(s)
1102        self.userMapFromPerforceServer = True
1103
1104    def loadUserMapFromCache(self):
1105        self.users = {}
1106        self.userMapFromPerforceServer = False
1107        try:
1108            cache = open(self.getUserCacheFilename(), "rb")
1109            lines = cache.readlines()
1110            cache.close()
1111            for line in lines:
1112                entry = line.strip().split("\t")
1113                self.users[entry[0]] = entry[1]
1114        except IOError:
1115            self.getUserMapFromPerforceServer()
1116
1117    def getLabels(self):
1118        self.labels = {}
1119
1120        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1121        if len(l) > 0 and not self.silent:
1122            print "Finding files belonging to labels in %s" % `self.depotPaths`
1123
1124        for output in l:
1125            label = output["label"]
1126            revisions = {}
1127            newestChange = 0
1128            if self.verbose:
1129                print "Querying files for label %s" % label
1130            for file in p4CmdList("files "
1131                                  +  ' '.join (["%s...@%s" % (p, label)
1132                                                for p in self.depotPaths])):
1133                revisions[file["depotFile"]] = file["rev"]
1134                change = int(file["change"])
1135                if change > newestChange:
1136                    newestChange = change
1137
1138            self.labels[newestChange] = [output, revisions]
1139
1140        if self.verbose:
1141            print "Label changes: %s" % self.labels.keys()
1142
1143    def guessProjectName(self):
1144        for p in self.depotPaths:
1145            if p.endswith("/"):
1146                p = p[:-1]
1147            p = p[p.strip().rfind("/") + 1:]
1148            if not p.endswith("/"):
1149               p += "/"
1150            return p
1151
1152    def getBranchMapping(self):
1153        lostAndFoundBranches = set()
1154
1155        for info in p4CmdList("branches"):
1156            details = p4Cmd("branch -o %s" % info["branch"])
1157            viewIdx = 0
1158            while details.has_key("View%s" % viewIdx):
1159                paths = details["View%s" % viewIdx].split(" ")
1160                viewIdx = viewIdx + 1
1161                # require standard //depot/foo/... //depot/bar/... mapping
1162                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1163                    continue
1164                source = paths[0]
1165                destination = paths[1]
1166                ## HACK
1167                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1168                    source = source[len(self.depotPaths[0]):-4]
1169                    destination = destination[len(self.depotPaths[0]):-4]
1170
1171                    if destination in self.knownBranches:
1172                        if not self.silent:
1173                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1174                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1175                        continue
1176
1177                    self.knownBranches[destination] = source
1178
1179                    lostAndFoundBranches.discard(destination)
1180
1181                    if source not in self.knownBranches:
1182                        lostAndFoundBranches.add(source)
1183
1184
1185        for branch in lostAndFoundBranches:
1186            self.knownBranches[branch] = branch
1187
1188    def getBranchMappingFromGitBranches(self):
1189        branches = p4BranchesInGit(self.importIntoRemotes)
1190        for branch in branches.keys():
1191            if branch == "master":
1192                branch = "main"
1193            else:
1194                branch = branch[len(self.projectName):]
1195            self.knownBranches[branch] = branch
1196
1197    def listExistingP4GitBranches(self):
1198        # branches holds mapping from name to commit
1199        branches = p4BranchesInGit(self.importIntoRemotes)
1200        self.p4BranchesInGit = branches.keys()
1201        for branch in branches.keys():
1202            self.initialParents[self.refPrefix + branch] = branches[branch]
1203
1204    def updateOptionDict(self, d):
1205        option_keys = {}
1206        if self.keepRepoPath:
1207            option_keys['keepRepoPath'] = 1
1208
1209        d["options"] = ' '.join(sorted(option_keys.keys()))
1210
1211    def readOptions(self, d):
1212        self.keepRepoPath = (d.has_key('options')
1213                             and ('keepRepoPath' in d['options']))
1214
1215    def gitRefForBranch(self, branch):
1216        if branch == "main":
1217            return self.refPrefix + "master"
1218
1219        if len(branch) <= 0:
1220            return branch
1221
1222        return self.refPrefix + self.projectName + branch
1223
1224    def gitCommitByP4Change(self, ref, change):
1225        if self.verbose:
1226            print "looking in ref " + ref + " for change %s using bisect..." % change
1227
1228        earliestCommit = ""
1229        latestCommit = parseRevision(ref)
1230
1231        while True:
1232            if self.verbose:
1233                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1234            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1235            if len(next) == 0:
1236                if self.verbose:
1237                    print "argh"
1238                return ""
1239            log = extractLogMessageFromGitCommit(next)
1240            settings = extractSettingsGitLog(log)
1241            currentChange = int(settings['change'])
1242            if self.verbose:
1243                print "current change %s" % currentChange
1244
1245            if currentChange == change:
1246                if self.verbose:
1247                    print "found %s" % next
1248                return next
1249
1250            if currentChange < change:
1251                earliestCommit = "^%s" % next
1252            else:
1253                latestCommit = "%s" % next
1254
1255        return ""
1256
1257    def importNewBranch(self, branch, maxChange):
1258        # make fast-import flush all changes to disk and update the refs using the checkpoint
1259        # command so that we can try to find the branch parent in the git history
1260        self.gitStream.write("checkpoint\n\n");
1261        self.gitStream.flush();
1262        branchPrefix = self.depotPaths[0] + branch + "/"
1263        range = "@1,%s" % maxChange
1264        #print "prefix" + branchPrefix
1265        changes = p4ChangesForPaths([branchPrefix], range)
1266        if len(changes) <= 0:
1267            return False
1268        firstChange = changes[0]
1269        #print "first change in branch: %s" % firstChange
1270        sourceBranch = self.knownBranches[branch]
1271        sourceDepotPath = self.depotPaths[0] + sourceBranch
1272        sourceRef = self.gitRefForBranch(sourceBranch)
1273        #print "source " + sourceBranch
1274
1275        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1276        #print "branch parent: %s" % branchParentChange
1277        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1278        if len(gitParent) > 0:
1279            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1280            #print "parent git commit: %s" % gitParent
1281
1282        self.importChanges(changes)
1283        return True
1284
1285    def importChanges(self, changes):
1286        cnt = 1
1287        for change in changes:
1288            description = p4Cmd("describe %s" % change)
1289            self.updateOptionDict(description)
1290
1291            if not self.silent:
1292                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1293                sys.stdout.flush()
1294            cnt = cnt + 1
1295
1296            try:
1297                if self.detectBranches:
1298                    branches = self.splitFilesIntoBranches(description)
1299                    for branch in branches.keys():
1300                        ## HACK  --hwn
1301                        branchPrefix = self.depotPaths[0] + branch + "/"
1302
1303                        parent = ""
1304
1305                        filesForCommit = branches[branch]
1306
1307                        if self.verbose:
1308                            print "branch is %s" % branch
1309
1310                        self.updatedBranches.add(branch)
1311
1312                        if branch not in self.createdBranches:
1313                            self.createdBranches.add(branch)
1314                            parent = self.knownBranches[branch]
1315                            if parent == branch:
1316                                parent = ""
1317                            else:
1318                                fullBranch = self.projectName + branch
1319                                if fullBranch not in self.p4BranchesInGit:
1320                                    if not self.silent:
1321                                        print("\n    Importing new branch %s" % fullBranch);
1322                                    if self.importNewBranch(branch, change - 1):
1323                                        parent = ""
1324                                        self.p4BranchesInGit.append(fullBranch)
1325                                    if not self.silent:
1326                                        print("\n    Resuming with change %s" % change);
1327
1328                                if self.verbose:
1329                                    print "parent determined through known branches: %s" % parent
1330
1331                        branch = self.gitRefForBranch(branch)
1332                        parent = self.gitRefForBranch(parent)
1333
1334                        if self.verbose:
1335                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1336
1337                        if len(parent) == 0 and branch in self.initialParents:
1338                            parent = self.initialParents[branch]
1339                            del self.initialParents[branch]
1340
1341                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1342                else:
1343                    files = self.extractFilesFromCommit(description)
1344                    self.commit(description, files, self.branch, self.depotPaths,
1345                                self.initialParent)
1346                    self.initialParent = ""
1347            except IOError:
1348                print self.gitError.read()
1349                sys.exit(1)
1350
1351    def importHeadRevision(self, revision):
1352        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1353
1354        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1355        details["desc"] = ("Initial import of %s from the state at revision %s"
1356                           % (' '.join(self.depotPaths), revision))
1357        details["change"] = revision
1358        newestRevision = 0
1359
1360        fileCnt = 0
1361        for info in p4CmdList("files "
1362                              +  ' '.join(["%s...%s"
1363                                           % (p, revision)
1364                                           for p in self.depotPaths])):
1365
1366            if info['code'] == 'error':
1367                sys.stderr.write("p4 returned an error: %s\n"
1368                                 % info['data'])
1369                sys.exit(1)
1370
1371
1372            change = int(info["change"])
1373            if change > newestRevision:
1374                newestRevision = change
1375
1376            if info["action"] == "delete":
1377                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1378                #fileCnt = fileCnt + 1
1379                continue
1380
1381            for prop in ["depotFile", "rev", "action", "type" ]:
1382                details["%s%s" % (prop, fileCnt)] = info[prop]
1383
1384            fileCnt = fileCnt + 1
1385
1386        details["change"] = newestRevision
1387        self.updateOptionDict(details)
1388        try:
1389            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1390        except IOError:
1391            print "IO error with git fast-import. Is your git version recent enough?"
1392            print self.gitError.read()
1393
1394
1395    def getClientSpec(self):
1396        specList = p4CmdList( "client -o" )
1397        temp = {}
1398        for entry in specList:
1399            for k,v in entry.iteritems():
1400                if k.startswith("View"):
1401                    if v.startswith('"'):
1402                        start = 1
1403                    else:
1404                        start = 0
1405                    index = v.find("...")
1406                    v = v[start:index]
1407                    if v.startswith("-"):
1408                        v = v[1:]
1409                        temp[v] = -len(v)
1410                    else:
1411                        temp[v] = len(v)
1412        self.clientSpecDirs = temp.items()
1413        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1414
1415    def run(self, args):
1416        self.depotPaths = []
1417        self.changeRange = ""
1418        self.initialParent = ""
1419        self.previousDepotPaths = []
1420
1421        # map from branch depot path to parent branch
1422        self.knownBranches = {}
1423        self.initialParents = {}
1424        self.hasOrigin = originP4BranchesExist()
1425        if not self.syncWithOrigin:
1426            self.hasOrigin = False
1427
1428        if self.importIntoRemotes:
1429            self.refPrefix = "refs/remotes/p4/"
1430        else:
1431            self.refPrefix = "refs/heads/p4/"
1432
1433        if self.syncWithOrigin and self.hasOrigin:
1434            if not self.silent:
1435                print "Syncing with origin first by calling git fetch origin"
1436            system("git fetch origin")
1437
1438        if len(self.branch) == 0:
1439            self.branch = self.refPrefix + "master"
1440            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1441                system("git update-ref %s refs/heads/p4" % self.branch)
1442                system("git branch -D p4");
1443            # create it /after/ importing, when master exists
1444            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1445                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1446
1447        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1448            self.getClientSpec()
1449
1450        # TODO: should always look at previous commits,
1451        # merge with previous imports, if possible.
1452        if args == []:
1453            if self.hasOrigin:
1454                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1455            self.listExistingP4GitBranches()
1456
1457            if len(self.p4BranchesInGit) > 1:
1458                if not self.silent:
1459                    print "Importing from/into multiple branches"
1460                self.detectBranches = True
1461
1462            if self.verbose:
1463                print "branches: %s" % self.p4BranchesInGit
1464
1465            p4Change = 0
1466            for branch in self.p4BranchesInGit:
1467                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1468
1469                settings = extractSettingsGitLog(logMsg)
1470
1471                self.readOptions(settings)
1472                if (settings.has_key('depot-paths')
1473                    and settings.has_key ('change')):
1474                    change = int(settings['change']) + 1
1475                    p4Change = max(p4Change, change)
1476
1477                    depotPaths = sorted(settings['depot-paths'])
1478                    if self.previousDepotPaths == []:
1479                        self.previousDepotPaths = depotPaths
1480                    else:
1481                        paths = []
1482                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1483                            for i in range(0, min(len(cur), len(prev))):
1484                                if cur[i] <> prev[i]:
1485                                    i = i - 1
1486                                    break
1487
1488                            paths.append (cur[:i + 1])
1489
1490                        self.previousDepotPaths = paths
1491
1492            if p4Change > 0:
1493                self.depotPaths = sorted(self.previousDepotPaths)
1494                self.changeRange = "@%s,#head" % p4Change
1495                if not self.detectBranches:
1496                    self.initialParent = parseRevision(self.branch)
1497                if not self.silent and not self.detectBranches:
1498                    print "Performing incremental import into %s git branch" % self.branch
1499
1500        if not self.branch.startswith("refs/"):
1501            self.branch = "refs/heads/" + self.branch
1502
1503        if len(args) == 0 and self.depotPaths:
1504            if not self.silent:
1505                print "Depot paths: %s" % ' '.join(self.depotPaths)
1506        else:
1507            if self.depotPaths and self.depotPaths != args:
1508                print ("previous import used depot path %s and now %s was specified. "
1509                       "This doesn't work!" % (' '.join (self.depotPaths),
1510                                               ' '.join (args)))
1511                sys.exit(1)
1512
1513            self.depotPaths = sorted(args)
1514
1515        revision = ""
1516        self.users = {}
1517
1518        newPaths = []
1519        for p in self.depotPaths:
1520            if p.find("@") != -1:
1521                atIdx = p.index("@")
1522                self.changeRange = p[atIdx:]
1523                if self.changeRange == "@all":
1524                    self.changeRange = ""
1525                elif ',' not in self.changeRange:
1526                    revision = self.changeRange
1527                    self.changeRange = ""
1528                p = p[:atIdx]
1529            elif p.find("#") != -1:
1530                hashIdx = p.index("#")
1531                revision = p[hashIdx:]
1532                p = p[:hashIdx]
1533            elif self.previousDepotPaths == []:
1534                revision = "#head"
1535
1536            p = re.sub ("\.\.\.$", "", p)
1537            if not p.endswith("/"):
1538                p += "/"
1539
1540            newPaths.append(p)
1541
1542        self.depotPaths = newPaths
1543
1544
1545        self.loadUserMapFromCache()
1546        self.labels = {}
1547        if self.detectLabels:
1548            self.getLabels();
1549
1550        if self.detectBranches:
1551            ## FIXME - what's a P4 projectName ?
1552            self.projectName = self.guessProjectName()
1553
1554            if self.hasOrigin:
1555                self.getBranchMappingFromGitBranches()
1556            else:
1557                self.getBranchMapping()
1558            if self.verbose:
1559                print "p4-git branches: %s" % self.p4BranchesInGit
1560                print "initial parents: %s" % self.initialParents
1561            for b in self.p4BranchesInGit:
1562                if b != "master":
1563
1564                    ## FIXME
1565                    b = b[len(self.projectName):]
1566                self.createdBranches.add(b)
1567
1568        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1569
1570        importProcess = subprocess.Popen(["git", "fast-import"],
1571                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1572                                         stderr=subprocess.PIPE);
1573        self.gitOutput = importProcess.stdout
1574        self.gitStream = importProcess.stdin
1575        self.gitError = importProcess.stderr
1576
1577        if revision:
1578            self.importHeadRevision(revision)
1579        else:
1580            changes = []
1581
1582            if len(self.changesFile) > 0:
1583                output = open(self.changesFile).readlines()
1584                changeSet = Set()
1585                for line in output:
1586                    changeSet.add(int(line))
1587
1588                for change in changeSet:
1589                    changes.append(change)
1590
1591                changes.sort()
1592            else:
1593                if self.verbose:
1594                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1595                                                              self.changeRange)
1596                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1597
1598                if len(self.maxChanges) > 0:
1599                    changes = changes[:min(int(self.maxChanges), len(changes))]
1600
1601            if len(changes) == 0:
1602                if not self.silent:
1603                    print "No changes to import!"
1604                return True
1605
1606            if not self.silent and not self.detectBranches:
1607                print "Import destination: %s" % self.branch
1608
1609            self.updatedBranches = set()
1610
1611            self.importChanges(changes)
1612
1613            if not self.silent:
1614                print ""
1615                if len(self.updatedBranches) > 0:
1616                    sys.stdout.write("Updated branches: ")
1617                    for b in self.updatedBranches:
1618                        sys.stdout.write("%s " % b)
1619                    sys.stdout.write("\n")
1620
1621        self.gitStream.close()
1622        if importProcess.wait() != 0:
1623            die("fast-import failed: %s" % self.gitError.read())
1624        self.gitOutput.close()
1625        self.gitError.close()
1626
1627        return True
1628
1629class P4Rebase(Command):
1630    def __init__(self):
1631        Command.__init__(self)
1632        self.options = [ ]
1633        self.description = ("Fetches the latest revision from perforce and "
1634                            + "rebases the current work (branch) against it")
1635        self.verbose = False
1636
1637    def run(self, args):
1638        sync = P4Sync()
1639        sync.run([])
1640
1641        return self.rebase()
1642
1643    def rebase(self):
1644        if os.system("git update-index --refresh") != 0:
1645            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1646        if len(read_pipe("git diff-index HEAD --")) > 0:
1647            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1648
1649        [upstream, settings] = findUpstreamBranchPoint()
1650        if len(upstream) == 0:
1651            die("Cannot find upstream branchpoint for rebase")
1652
1653        # the branchpoint may be p4/foo~3, so strip off the parent
1654        upstream = re.sub("~[0-9]+$", "", upstream)
1655
1656        print "Rebasing the current branch onto %s" % upstream
1657        oldHead = read_pipe("git rev-parse HEAD").strip()
1658        system("git rebase %s" % upstream)
1659        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1660        return True
1661
1662class P4Clone(P4Sync):
1663    def __init__(self):
1664        P4Sync.__init__(self)
1665        self.description = "Creates a new git repository and imports from Perforce into it"
1666        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1667        self.options += [
1668            optparse.make_option("--destination", dest="cloneDestination",
1669                                 action='store', default=None,
1670                                 help="where to leave result of the clone"),
1671            optparse.make_option("-/", dest="cloneExclude",
1672                                 action="append", type="string",
1673                                 help="exclude depot path")
1674        ]
1675        self.cloneDestination = None
1676        self.needsGit = False
1677
1678    # This is required for the "append" cloneExclude action
1679    def ensure_value(self, attr, value):
1680        if not hasattr(self, attr) or getattr(self, attr) is None:
1681            setattr(self, attr, value)
1682        return getattr(self, attr)
1683
1684    def defaultDestination(self, args):
1685        ## TODO: use common prefix of args?
1686        depotPath = args[0]
1687        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1688        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1689        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1690        depotDir = re.sub(r"/$", "", depotDir)
1691        return os.path.split(depotDir)[1]
1692
1693    def run(self, args):
1694        if len(args) < 1:
1695            return False
1696
1697        if self.keepRepoPath and not self.cloneDestination:
1698            sys.stderr.write("Must specify destination for --keep-path\n")
1699            sys.exit(1)
1700
1701        depotPaths = args
1702
1703        if not self.cloneDestination and len(depotPaths) > 1:
1704            self.cloneDestination = depotPaths[-1]
1705            depotPaths = depotPaths[:-1]
1706
1707        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1708        for p in depotPaths:
1709            if not p.startswith("//"):
1710                return False
1711
1712        if not self.cloneDestination:
1713            self.cloneDestination = self.defaultDestination(args)
1714
1715        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1716        if not os.path.exists(self.cloneDestination):
1717            os.makedirs(self.cloneDestination)
1718        os.chdir(self.cloneDestination)
1719        system("git init")
1720        self.gitdir = os.getcwd() + "/.git"
1721        if not P4Sync.run(self, depotPaths):
1722            return False
1723        if self.branch != "master":
1724            if gitBranchExists("refs/remotes/p4/master"):
1725                system("git branch master refs/remotes/p4/master")
1726                system("git checkout -f")
1727            else:
1728                print "Could not detect main branch. No checkout/master branch created."
1729
1730        return True
1731
1732class P4Branches(Command):
1733    def __init__(self):
1734        Command.__init__(self)
1735        self.options = [ ]
1736        self.description = ("Shows the git branches that hold imports and their "
1737                            + "corresponding perforce depot paths")
1738        self.verbose = False
1739
1740    def run(self, args):
1741        if originP4BranchesExist():
1742            createOrUpdateBranchesFromOrigin()
1743
1744        cmdline = "git rev-parse --symbolic "
1745        cmdline += " --remotes"
1746
1747        for line in read_pipe_lines(cmdline):
1748            line = line.strip()
1749
1750            if not line.startswith('p4/') or line == "p4/HEAD":
1751                continue
1752            branch = line
1753
1754            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1755            settings = extractSettingsGitLog(log)
1756
1757            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1758        return True
1759
1760class HelpFormatter(optparse.IndentedHelpFormatter):
1761    def __init__(self):
1762        optparse.IndentedHelpFormatter.__init__(self)
1763
1764    def format_description(self, description):
1765        if description:
1766            return description + "\n"
1767        else:
1768            return ""
1769
1770def printUsage(commands):
1771    print "usage: %s <command> [options]" % sys.argv[0]
1772    print ""
1773    print "valid commands: %s" % ", ".join(commands)
1774    print ""
1775    print "Try %s <command> --help for command specific help." % sys.argv[0]
1776    print ""
1777
1778commands = {
1779    "debug" : P4Debug,
1780    "submit" : P4Submit,
1781    "commit" : P4Submit,
1782    "sync" : P4Sync,
1783    "rebase" : P4Rebase,
1784    "clone" : P4Clone,
1785    "rollback" : P4RollBack,
1786    "branches" : P4Branches
1787}
1788
1789
1790def main():
1791    if len(sys.argv[1:]) == 0:
1792        printUsage(commands.keys())
1793        sys.exit(2)
1794
1795    cmd = ""
1796    cmdName = sys.argv[1]
1797    try:
1798        klass = commands[cmdName]
1799        cmd = klass()
1800    except KeyError:
1801        print "unknown command %s" % cmdName
1802        print ""
1803        printUsage(commands.keys())
1804        sys.exit(2)
1805
1806    options = cmd.options
1807    cmd.gitdir = os.environ.get("GIT_DIR", None)
1808
1809    args = sys.argv[2:]
1810
1811    if len(options) > 0:
1812        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1813
1814        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1815                                       options,
1816                                       description = cmd.description,
1817                                       formatter = HelpFormatter())
1818
1819        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1820    global verbose
1821    verbose = cmd.verbose
1822    if cmd.needsGit:
1823        if cmd.gitdir == None:
1824            cmd.gitdir = os.path.abspath(".git")
1825            if not isValidGitDir(cmd.gitdir):
1826                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1827                if os.path.exists(cmd.gitdir):
1828                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1829                    if len(cdup) > 0:
1830                        os.chdir(cdup);
1831
1832        if not isValidGitDir(cmd.gitdir):
1833            if isValidGitDir(cmd.gitdir + "/.git"):
1834                cmd.gitdir += "/.git"
1835            else:
1836                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1837
1838        os.environ["GIT_DIR"] = cmd.gitdir
1839
1840    if not cmd.run(args):
1841        parser.print_help()
1842
1843
1844if __name__ == '__main__':
1845    main()