Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit If we are in verbose mode, output what we are about to run (or return) (ee06427)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s %s" % ("p4", cmd)
  28    if verbose:
  29        print real_cmd
  30    return real_cmd
  31
  32def die(msg):
  33    if verbose:
  34        raise Exception(msg)
  35    else:
  36        sys.stderr.write(msg + "\n")
  37        sys.exit(1)
  38
  39def write_pipe(c, str):
  40    if verbose:
  41        sys.stderr.write('Writing pipe: %s\n' % c)
  42
  43    pipe = os.popen(c, 'w')
  44    val = pipe.write(str)
  45    if pipe.close():
  46        die('Command failed: %s' % c)
  47
  48    return val
  49
  50def read_pipe(c, ignore_error=False):
  51    if verbose:
  52        sys.stderr.write('Reading pipe: %s\n' % c)
  53
  54    pipe = os.popen(c, 'rb')
  55    val = pipe.read()
  56    if pipe.close() and not ignore_error:
  57        die('Command failed: %s' % c)
  58
  59    return val
  60
  61
  62def read_pipe_lines(c):
  63    if verbose:
  64        sys.stderr.write('Reading pipe: %s\n' % c)
  65    ## todo: check return status
  66    pipe = os.popen(c, 'rb')
  67    val = pipe.readlines()
  68    if pipe.close():
  69        die('Command failed: %s' % c)
  70
  71    return val
  72
  73def p4_read_pipe_lines(c):
  74    """Specifically invoke p4 on the command supplied. """
  75    real_cmd = "%s %s" % ("p4", c)
  76    if verbose:
  77        print real_cmd
  78    return read_pipe_lines(real_cmd)
  79
  80def system(cmd):
  81    if verbose:
  82        sys.stderr.write("executing %s\n" % cmd)
  83    if os.system(cmd) != 0:
  84        die("command failed: %s" % cmd)
  85
  86def p4_system(cmd):
  87    """Specifically invoke p4 as the system command. """
  88    real_cmd = "%s %s" % ("p4", cmd)
  89    if verbose:
  90        print real_cmd
  91    return system(real_cmd)
  92
  93def isP4Exec(kind):
  94    """Determine if a Perforce 'kind' should have execute permission
  95
  96    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  97    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  98    a plus sign, it is also executable"""
  99    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 100
 101def setP4ExecBit(file, mode):
 102    # Reopens an already open file and changes the execute bit to match
 103    # the execute bit setting in the passed in mode.
 104
 105    p4Type = "+x"
 106
 107    if not isModeExec(mode):
 108        p4Type = getP4OpenedType(file)
 109        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 110        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 111        if p4Type[-1] == "+":
 112            p4Type = p4Type[0:-1]
 113
 114    p4_system("reopen -t %s %s" % (p4Type, file))
 115
 116def getP4OpenedType(file):
 117    # Returns the perforce file type for the given file.
 118
 119    result = read_pipe("p4 opened %s" % file)
 120    match = re.match(".*\((.+)\)\r?$", result)
 121    if match:
 122        return match.group(1)
 123    else:
 124        die("Could not determine file type for %s (result: '%s')" % (file, result))
 125
 126def diffTreePattern():
 127    # This is a simple generator for the diff tree regex pattern. This could be
 128    # a class variable if this and parseDiffTreeEntry were a part of a class.
 129    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 130    while True:
 131        yield pattern
 132
 133def parseDiffTreeEntry(entry):
 134    """Parses a single diff tree entry into its component elements.
 135
 136    See git-diff-tree(1) manpage for details about the format of the diff
 137    output. This method returns a dictionary with the following elements:
 138
 139    src_mode - The mode of the source file
 140    dst_mode - The mode of the destination file
 141    src_sha1 - The sha1 for the source file
 142    dst_sha1 - The sha1 fr the destination file
 143    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 144    status_score - The score for the status (applicable for 'C' and 'R'
 145                   statuses). This is None if there is no score.
 146    src - The path for the source file.
 147    dst - The path for the destination file. This is only present for
 148          copy or renames. If it is not present, this is None.
 149
 150    If the pattern is not matched, None is returned."""
 151
 152    match = diffTreePattern().next().match(entry)
 153    if match:
 154        return {
 155            'src_mode': match.group(1),
 156            'dst_mode': match.group(2),
 157            'src_sha1': match.group(3),
 158            'dst_sha1': match.group(4),
 159            'status': match.group(5),
 160            'status_score': match.group(6),
 161            'src': match.group(7),
 162            'dst': match.group(10)
 163        }
 164    return None
 165
 166def isModeExec(mode):
 167    # Returns True if the given git mode represents an executable file,
 168    # otherwise False.
 169    return mode[-3:] == "755"
 170
 171def isModeExecChanged(src_mode, dst_mode):
 172    return isModeExec(src_mode) != isModeExec(dst_mode)
 173
 174def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 175    cmd = "p4 -G %s" % cmd
 176    if verbose:
 177        sys.stderr.write("Opening pipe: %s\n" % cmd)
 178
 179    # Use a temporary file to avoid deadlocks without
 180    # subprocess.communicate(), which would put another copy
 181    # of stdout into memory.
 182    stdin_file = None
 183    if stdin is not None:
 184        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 185        stdin_file.write(stdin)
 186        stdin_file.flush()
 187        stdin_file.seek(0)
 188
 189    p4 = subprocess.Popen(cmd, shell=True,
 190                          stdin=stdin_file,
 191                          stdout=subprocess.PIPE)
 192
 193    result = []
 194    try:
 195        while True:
 196            entry = marshal.load(p4.stdout)
 197            result.append(entry)
 198    except EOFError:
 199        pass
 200    exitCode = p4.wait()
 201    if exitCode != 0:
 202        entry = {}
 203        entry["p4ExitCode"] = exitCode
 204        result.append(entry)
 205
 206    return result
 207
 208def p4Cmd(cmd):
 209    list = p4CmdList(cmd)
 210    result = {}
 211    for entry in list:
 212        result.update(entry)
 213    return result;
 214
 215def p4Where(depotPath):
 216    if not depotPath.endswith("/"):
 217        depotPath += "/"
 218    output = p4Cmd("where %s..." % depotPath)
 219    if output["code"] == "error":
 220        return ""
 221    clientPath = ""
 222    if "path" in output:
 223        clientPath = output.get("path")
 224    elif "data" in output:
 225        data = output.get("data")
 226        lastSpace = data.rfind(" ")
 227        clientPath = data[lastSpace + 1:]
 228
 229    if clientPath.endswith("..."):
 230        clientPath = clientPath[:-3]
 231    return clientPath
 232
 233def currentGitBranch():
 234    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 235
 236def isValidGitDir(path):
 237    if (os.path.exists(path + "/HEAD")
 238        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 239        return True;
 240    return False
 241
 242def parseRevision(ref):
 243    return read_pipe("git rev-parse %s" % ref).strip()
 244
 245def extractLogMessageFromGitCommit(commit):
 246    logMessage = ""
 247
 248    ## fixme: title is first line of commit, not 1st paragraph.
 249    foundTitle = False
 250    for log in read_pipe_lines("git cat-file commit %s" % commit):
 251       if not foundTitle:
 252           if len(log) == 1:
 253               foundTitle = True
 254           continue
 255
 256       logMessage += log
 257    return logMessage
 258
 259def extractSettingsGitLog(log):
 260    values = {}
 261    for line in log.split("\n"):
 262        line = line.strip()
 263        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 264        if not m:
 265            continue
 266
 267        assignments = m.group(1).split (':')
 268        for a in assignments:
 269            vals = a.split ('=')
 270            key = vals[0].strip()
 271            val = ('='.join (vals[1:])).strip()
 272            if val.endswith ('\"') and val.startswith('"'):
 273                val = val[1:-1]
 274
 275            values[key] = val
 276
 277    paths = values.get("depot-paths")
 278    if not paths:
 279        paths = values.get("depot-path")
 280    if paths:
 281        values['depot-paths'] = paths.split(',')
 282    return values
 283
 284def gitBranchExists(branch):
 285    proc = subprocess.Popen(["git", "rev-parse", branch],
 286                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 287    return proc.wait() == 0;
 288
 289def gitConfig(key):
 290    return read_pipe("git config %s" % key, ignore_error=True).strip()
 291
 292def p4BranchesInGit(branchesAreInRemotes = True):
 293    branches = {}
 294
 295    cmdline = "git rev-parse --symbolic "
 296    if branchesAreInRemotes:
 297        cmdline += " --remotes"
 298    else:
 299        cmdline += " --branches"
 300
 301    for line in read_pipe_lines(cmdline):
 302        line = line.strip()
 303
 304        ## only import to p4/
 305        if not line.startswith('p4/') or line == "p4/HEAD":
 306            continue
 307        branch = line
 308
 309        # strip off p4
 310        branch = re.sub ("^p4/", "", line)
 311
 312        branches[branch] = parseRevision(line)
 313    return branches
 314
 315def findUpstreamBranchPoint(head = "HEAD"):
 316    branches = p4BranchesInGit()
 317    # map from depot-path to branch name
 318    branchByDepotPath = {}
 319    for branch in branches.keys():
 320        tip = branches[branch]
 321        log = extractLogMessageFromGitCommit(tip)
 322        settings = extractSettingsGitLog(log)
 323        if settings.has_key("depot-paths"):
 324            paths = ",".join(settings["depot-paths"])
 325            branchByDepotPath[paths] = "remotes/p4/" + branch
 326
 327    settings = None
 328    parent = 0
 329    while parent < 65535:
 330        commit = head + "~%s" % parent
 331        log = extractLogMessageFromGitCommit(commit)
 332        settings = extractSettingsGitLog(log)
 333        if settings.has_key("depot-paths"):
 334            paths = ",".join(settings["depot-paths"])
 335            if branchByDepotPath.has_key(paths):
 336                return [branchByDepotPath[paths], settings]
 337
 338        parent = parent + 1
 339
 340    return ["", settings]
 341
 342def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 343    if not silent:
 344        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 345               % localRefPrefix)
 346
 347    originPrefix = "origin/p4/"
 348
 349    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 350        line = line.strip()
 351        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 352            continue
 353
 354        headName = line[len(originPrefix):]
 355        remoteHead = localRefPrefix + headName
 356        originHead = line
 357
 358        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 359        if (not original.has_key('depot-paths')
 360            or not original.has_key('change')):
 361            continue
 362
 363        update = False
 364        if not gitBranchExists(remoteHead):
 365            if verbose:
 366                print "creating %s" % remoteHead
 367            update = True
 368        else:
 369            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 370            if settings.has_key('change') > 0:
 371                if settings['depot-paths'] == original['depot-paths']:
 372                    originP4Change = int(original['change'])
 373                    p4Change = int(settings['change'])
 374                    if originP4Change > p4Change:
 375                        print ("%s (%s) is newer than %s (%s). "
 376                               "Updating p4 branch from origin."
 377                               % (originHead, originP4Change,
 378                                  remoteHead, p4Change))
 379                        update = True
 380                else:
 381                    print ("Ignoring: %s was imported from %s while "
 382                           "%s was imported from %s"
 383                           % (originHead, ','.join(original['depot-paths']),
 384                              remoteHead, ','.join(settings['depot-paths'])))
 385
 386        if update:
 387            system("git update-ref %s %s" % (remoteHead, originHead))
 388
 389def originP4BranchesExist():
 390        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 391
 392def p4ChangesForPaths(depotPaths, changeRange):
 393    assert depotPaths
 394    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 395                                                        for p in depotPaths]))
 396
 397    changes = []
 398    for line in output:
 399        changeNum = line.split(" ")[1]
 400        changes.append(int(changeNum))
 401
 402    changes.sort()
 403    return changes
 404
 405class Command:
 406    def __init__(self):
 407        self.usage = "usage: %prog [options]"
 408        self.needsGit = True
 409
 410class P4Debug(Command):
 411    def __init__(self):
 412        Command.__init__(self)
 413        self.options = [
 414            optparse.make_option("--verbose", dest="verbose", action="store_true",
 415                                 default=False),
 416            ]
 417        self.description = "A tool to debug the output of p4 -G."
 418        self.needsGit = False
 419        self.verbose = False
 420
 421    def run(self, args):
 422        j = 0
 423        for output in p4CmdList(" ".join(args)):
 424            print 'Element: %d' % j
 425            j += 1
 426            print output
 427        return True
 428
 429class P4RollBack(Command):
 430    def __init__(self):
 431        Command.__init__(self)
 432        self.options = [
 433            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 434            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 435        ]
 436        self.description = "A tool to debug the multi-branch import. Don't use :)"
 437        self.verbose = False
 438        self.rollbackLocalBranches = False
 439
 440    def run(self, args):
 441        if len(args) != 1:
 442            return False
 443        maxChange = int(args[0])
 444
 445        if "p4ExitCode" in p4Cmd("changes -m 1"):
 446            die("Problems executing p4");
 447
 448        if self.rollbackLocalBranches:
 449            refPrefix = "refs/heads/"
 450            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 451        else:
 452            refPrefix = "refs/remotes/"
 453            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 454
 455        for line in lines:
 456            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 457                line = line.strip()
 458                ref = refPrefix + line
 459                log = extractLogMessageFromGitCommit(ref)
 460                settings = extractSettingsGitLog(log)
 461
 462                depotPaths = settings['depot-paths']
 463                change = settings['change']
 464
 465                changed = False
 466
 467                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 468                                                           for p in depotPaths]))) == 0:
 469                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 470                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 471                    continue
 472
 473                while change and int(change) > maxChange:
 474                    changed = True
 475                    if self.verbose:
 476                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 477                    system("git update-ref %s \"%s^\"" % (ref, ref))
 478                    log = extractLogMessageFromGitCommit(ref)
 479                    settings =  extractSettingsGitLog(log)
 480
 481
 482                    depotPaths = settings['depot-paths']
 483                    change = settings['change']
 484
 485                if changed:
 486                    print "%s rewound to %s" % (ref, change)
 487
 488        return True
 489
 490class P4Submit(Command):
 491    def __init__(self):
 492        Command.__init__(self)
 493        self.options = [
 494                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 495                optparse.make_option("--origin", dest="origin"),
 496                optparse.make_option("-M", dest="detectRename", action="store_true"),
 497        ]
 498        self.description = "Submit changes from git to the perforce depot."
 499        self.usage += " [name of git branch to submit into perforce depot]"
 500        self.interactive = True
 501        self.origin = ""
 502        self.detectRename = False
 503        self.verbose = False
 504        self.isWindows = (platform.system() == "Windows")
 505
 506    def check(self):
 507        if len(p4CmdList("opened ...")) > 0:
 508            die("You have files opened with perforce! Close them before starting the sync.")
 509
 510    # replaces everything between 'Description:' and the next P4 submit template field with the
 511    # commit message
 512    def prepareLogMessage(self, template, message):
 513        result = ""
 514
 515        inDescriptionSection = False
 516
 517        for line in template.split("\n"):
 518            if line.startswith("#"):
 519                result += line + "\n"
 520                continue
 521
 522            if inDescriptionSection:
 523                if line.startswith("Files:"):
 524                    inDescriptionSection = False
 525                else:
 526                    continue
 527            else:
 528                if line.startswith("Description:"):
 529                    inDescriptionSection = True
 530                    line += "\n"
 531                    for messageLine in message.split("\n"):
 532                        line += "\t" + messageLine + "\n"
 533
 534            result += line + "\n"
 535
 536        return result
 537
 538    def prepareSubmitTemplate(self):
 539        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 540        template = ""
 541        inFilesSection = False
 542        for line in p4_read_pipe_lines("change -o"):
 543            if line.endswith("\r\n"):
 544                line = line[:-2] + "\n"
 545            if inFilesSection:
 546                if line.startswith("\t"):
 547                    # path starts and ends with a tab
 548                    path = line[1:]
 549                    lastTab = path.rfind("\t")
 550                    if lastTab != -1:
 551                        path = path[:lastTab]
 552                        if not path.startswith(self.depotPath):
 553                            continue
 554                else:
 555                    inFilesSection = False
 556            else:
 557                if line.startswith("Files:"):
 558                    inFilesSection = True
 559
 560            template += line
 561
 562        return template
 563
 564    def applyCommit(self, id):
 565        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 566        diffOpts = ("", "-M")[self.detectRename]
 567        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 568        filesToAdd = set()
 569        filesToDelete = set()
 570        editedFiles = set()
 571        filesToChangeExecBit = {}
 572        for line in diff:
 573            diff = parseDiffTreeEntry(line)
 574            modifier = diff['status']
 575            path = diff['src']
 576            if modifier == "M":
 577                p4_system("edit \"%s\"" % path)
 578                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 579                    filesToChangeExecBit[path] = diff['dst_mode']
 580                editedFiles.add(path)
 581            elif modifier == "A":
 582                filesToAdd.add(path)
 583                filesToChangeExecBit[path] = diff['dst_mode']
 584                if path in filesToDelete:
 585                    filesToDelete.remove(path)
 586            elif modifier == "D":
 587                filesToDelete.add(path)
 588                if path in filesToAdd:
 589                    filesToAdd.remove(path)
 590            elif modifier == "R":
 591                src, dest = diff['src'], diff['dst']
 592                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 593                p4_system("edit \"%s\"" % (dest))
 594                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 595                    filesToChangeExecBit[dest] = diff['dst_mode']
 596                os.unlink(dest)
 597                editedFiles.add(dest)
 598                filesToDelete.add(src)
 599            else:
 600                die("unknown modifier %s for %s" % (modifier, path))
 601
 602        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 603        patchcmd = diffcmd + " | git apply "
 604        tryPatchCmd = patchcmd + "--check -"
 605        applyPatchCmd = patchcmd + "--check --apply -"
 606
 607        if os.system(tryPatchCmd) != 0:
 608            print "Unfortunately applying the change failed!"
 609            print "What do you want to do?"
 610            response = "x"
 611            while response != "s" and response != "a" and response != "w":
 612                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 613                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 614            if response == "s":
 615                print "Skipping! Good luck with the next patches..."
 616                for f in editedFiles:
 617                    p4_system("revert \"%s\"" % f);
 618                for f in filesToAdd:
 619                    system("rm %s" %f)
 620                return
 621            elif response == "a":
 622                os.system(applyPatchCmd)
 623                if len(filesToAdd) > 0:
 624                    print "You may also want to call p4 add on the following files:"
 625                    print " ".join(filesToAdd)
 626                if len(filesToDelete):
 627                    print "The following files should be scheduled for deletion with p4 delete:"
 628                    print " ".join(filesToDelete)
 629                die("Please resolve and submit the conflict manually and "
 630                    + "continue afterwards with git-p4 submit --continue")
 631            elif response == "w":
 632                system(diffcmd + " > patch.txt")
 633                print "Patch saved to patch.txt in %s !" % self.clientPath
 634                die("Please resolve and submit the conflict manually and "
 635                    "continue afterwards with git-p4 submit --continue")
 636
 637        system(applyPatchCmd)
 638
 639        for f in filesToAdd:
 640            p4_system("add \"%s\"" % f)
 641        for f in filesToDelete:
 642            p4_system("revert \"%s\"" % f)
 643            p4_system("delete \"%s\"" % f)
 644
 645        # Set/clear executable bits
 646        for f in filesToChangeExecBit.keys():
 647            mode = filesToChangeExecBit[f]
 648            setP4ExecBit(f, mode)
 649
 650        logMessage = extractLogMessageFromGitCommit(id)
 651        logMessage = logMessage.strip()
 652
 653        template = self.prepareSubmitTemplate()
 654
 655        if self.interactive:
 656            submitTemplate = self.prepareLogMessage(template, logMessage)
 657            if os.environ.has_key("P4DIFF"):
 658                del(os.environ["P4DIFF"])
 659            diff = read_pipe("p4 diff -du ...")
 660
 661            newdiff = ""
 662            for newFile in filesToAdd:
 663                newdiff += "==== new file ====\n"
 664                newdiff += "--- /dev/null\n"
 665                newdiff += "+++ %s\n" % newFile
 666                f = open(newFile, "r")
 667                for line in f.readlines():
 668                    newdiff += "+" + line
 669                f.close()
 670
 671            separatorLine = "######## everything below this line is just the diff #######\n"
 672
 673            [handle, fileName] = tempfile.mkstemp()
 674            tmpFile = os.fdopen(handle, "w+")
 675            if self.isWindows:
 676                submitTemplate = submitTemplate.replace("\n", "\r\n")
 677                separatorLine = separatorLine.replace("\n", "\r\n")
 678                newdiff = newdiff.replace("\n", "\r\n")
 679            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 680            tmpFile.close()
 681            defaultEditor = "vi"
 682            if platform.system() == "Windows":
 683                defaultEditor = "notepad"
 684            if os.environ.has_key("P4EDITOR"):
 685                editor = os.environ.get("P4EDITOR")
 686            else:
 687                editor = os.environ.get("EDITOR", defaultEditor);
 688            system(editor + " " + fileName)
 689            tmpFile = open(fileName, "rb")
 690            message = tmpFile.read()
 691            tmpFile.close()
 692            os.remove(fileName)
 693            submitTemplate = message[:message.index(separatorLine)]
 694            if self.isWindows:
 695                submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697            write_pipe("p4 submit -i", submitTemplate)
 698        else:
 699            fileName = "submit.txt"
 700            file = open(fileName, "w+")
 701            file.write(self.prepareLogMessage(template, logMessage))
 702            file.close()
 703            print ("Perforce submit template written as %s. "
 704                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 705                   % (fileName, fileName))
 706
 707    def run(self, args):
 708        if len(args) == 0:
 709            self.master = currentGitBranch()
 710            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 711                die("Detecting current git branch failed!")
 712        elif len(args) == 1:
 713            self.master = args[0]
 714        else:
 715            return False
 716
 717        allowSubmit = gitConfig("git-p4.allowSubmit")
 718        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 719            die("%s is not in git-p4.allowSubmit" % self.master)
 720
 721        [upstream, settings] = findUpstreamBranchPoint()
 722        self.depotPath = settings['depot-paths'][0]
 723        if len(self.origin) == 0:
 724            self.origin = upstream
 725
 726        if self.verbose:
 727            print "Origin branch is " + self.origin
 728
 729        if len(self.depotPath) == 0:
 730            print "Internal error: cannot locate perforce depot path from existing branches"
 731            sys.exit(128)
 732
 733        self.clientPath = p4Where(self.depotPath)
 734
 735        if len(self.clientPath) == 0:
 736            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 737            sys.exit(128)
 738
 739        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 740        self.oldWorkingDirectory = os.getcwd()
 741
 742        os.chdir(self.clientPath)
 743        print "Syncronizing p4 checkout..."
 744        p4_system("sync ...")
 745
 746        self.check()
 747
 748        commits = []
 749        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 750            commits.append(line.strip())
 751        commits.reverse()
 752
 753        while len(commits) > 0:
 754            commit = commits[0]
 755            commits = commits[1:]
 756            self.applyCommit(commit)
 757            if not self.interactive:
 758                break
 759
 760        if len(commits) == 0:
 761            print "All changes applied!"
 762            os.chdir(self.oldWorkingDirectory)
 763
 764            sync = P4Sync()
 765            sync.run([])
 766
 767            rebase = P4Rebase()
 768            rebase.rebase()
 769
 770        return True
 771
 772class P4Sync(Command):
 773    def __init__(self):
 774        Command.__init__(self)
 775        self.options = [
 776                optparse.make_option("--branch", dest="branch"),
 777                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 778                optparse.make_option("--changesfile", dest="changesFile"),
 779                optparse.make_option("--silent", dest="silent", action="store_true"),
 780                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 781                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 782                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 783                                     help="Import into refs/heads/ , not refs/remotes"),
 784                optparse.make_option("--max-changes", dest="maxChanges"),
 785                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 786                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 787                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 788                                     help="Only sync files that are included in the Perforce Client Spec")
 789        ]
 790        self.description = """Imports from Perforce into a git repository.\n
 791    example:
 792    //depot/my/project/ -- to import the current head
 793    //depot/my/project/@all -- to import everything
 794    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 795
 796    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 797
 798        self.usage += " //depot/path[@revRange]"
 799        self.silent = False
 800        self.createdBranches = Set()
 801        self.committedChanges = Set()
 802        self.branch = ""
 803        self.detectBranches = False
 804        self.detectLabels = False
 805        self.changesFile = ""
 806        self.syncWithOrigin = True
 807        self.verbose = False
 808        self.importIntoRemotes = True
 809        self.maxChanges = ""
 810        self.isWindows = (platform.system() == "Windows")
 811        self.keepRepoPath = False
 812        self.depotPaths = None
 813        self.p4BranchesInGit = []
 814        self.cloneExclude = []
 815        self.useClientSpec = False
 816        self.clientSpecDirs = []
 817
 818        if gitConfig("git-p4.syncFromOrigin") == "false":
 819            self.syncWithOrigin = False
 820
 821    def extractFilesFromCommit(self, commit):
 822        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 823                             for path in self.cloneExclude]
 824        files = []
 825        fnum = 0
 826        while commit.has_key("depotFile%s" % fnum):
 827            path =  commit["depotFile%s" % fnum]
 828
 829            if [p for p in self.cloneExclude
 830                if path.startswith (p)]:
 831                found = False
 832            else:
 833                found = [p for p in self.depotPaths
 834                         if path.startswith (p)]
 835            if not found:
 836                fnum = fnum + 1
 837                continue
 838
 839            file = {}
 840            file["path"] = path
 841            file["rev"] = commit["rev%s" % fnum]
 842            file["action"] = commit["action%s" % fnum]
 843            file["type"] = commit["type%s" % fnum]
 844            files.append(file)
 845            fnum = fnum + 1
 846        return files
 847
 848    def stripRepoPath(self, path, prefixes):
 849        if self.keepRepoPath:
 850            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 851
 852        for p in prefixes:
 853            if path.startswith(p):
 854                path = path[len(p):]
 855
 856        return path
 857
 858    def splitFilesIntoBranches(self, commit):
 859        branches = {}
 860        fnum = 0
 861        while commit.has_key("depotFile%s" % fnum):
 862            path =  commit["depotFile%s" % fnum]
 863            found = [p for p in self.depotPaths
 864                     if path.startswith (p)]
 865            if not found:
 866                fnum = fnum + 1
 867                continue
 868
 869            file = {}
 870            file["path"] = path
 871            file["rev"] = commit["rev%s" % fnum]
 872            file["action"] = commit["action%s" % fnum]
 873            file["type"] = commit["type%s" % fnum]
 874            fnum = fnum + 1
 875
 876            relPath = self.stripRepoPath(path, self.depotPaths)
 877
 878            for branch in self.knownBranches.keys():
 879
 880                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 881                if relPath.startswith(branch + "/"):
 882                    if branch not in branches:
 883                        branches[branch] = []
 884                    branches[branch].append(file)
 885                    break
 886
 887        return branches
 888
 889    ## Should move this out, doesn't use SELF.
 890    def readP4Files(self, files):
 891        filesForCommit = []
 892        filesToRead = []
 893
 894        for f in files:
 895            includeFile = True
 896            for val in self.clientSpecDirs:
 897                if f['path'].startswith(val[0]):
 898                    if val[1] <= 0:
 899                        includeFile = False
 900                    break
 901
 902            if includeFile:
 903                filesForCommit.append(f)
 904                if f['action'] != 'delete':
 905                    filesToRead.append(f)
 906
 907        filedata = []
 908        if len(filesToRead) > 0:
 909            filedata = p4CmdList('-x - print',
 910                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 911                                                  for f in filesToRead]),
 912                                 stdin_mode='w+')
 913
 914            if "p4ExitCode" in filedata[0]:
 915                die("Problems executing p4. Error: [%d]."
 916                    % (filedata[0]['p4ExitCode']));
 917
 918        j = 0;
 919        contents = {}
 920        while j < len(filedata):
 921            stat = filedata[j]
 922            j += 1
 923            text = [];
 924            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 925                text.append(filedata[j]['data'])
 926                j += 1
 927            text = ''.join(text)
 928
 929            if not stat.has_key('depotFile'):
 930                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 931                continue
 932
 933            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 934                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 935            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 936                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 937
 938            contents[stat['depotFile']] = text
 939
 940        for f in filesForCommit:
 941            path = f['path']
 942            if contents.has_key(path):
 943                f['data'] = contents[path]
 944
 945        return filesForCommit
 946
 947    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 948        epoch = details["time"]
 949        author = details["user"]
 950
 951        if self.verbose:
 952            print "commit into %s" % branch
 953
 954        # start with reading files; if that fails, we should not
 955        # create a commit.
 956        new_files = []
 957        for f in files:
 958            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 959                new_files.append (f)
 960            else:
 961                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 962        files = self.readP4Files(new_files)
 963
 964        self.gitStream.write("commit %s\n" % branch)
 965#        gitStream.write("mark :%s\n" % details["change"])
 966        self.committedChanges.add(int(details["change"]))
 967        committer = ""
 968        if author not in self.users:
 969            self.getUserMapFromPerforceServer()
 970        if author in self.users:
 971            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 972        else:
 973            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 974
 975        self.gitStream.write("committer %s\n" % committer)
 976
 977        self.gitStream.write("data <<EOT\n")
 978        self.gitStream.write(details["desc"])
 979        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 980                             % (','.join (branchPrefixes), details["change"]))
 981        if len(details['options']) > 0:
 982            self.gitStream.write(": options = %s" % details['options'])
 983        self.gitStream.write("]\nEOT\n\n")
 984
 985        if len(parent) > 0:
 986            if self.verbose:
 987                print "parent %s" % parent
 988            self.gitStream.write("from %s\n" % parent)
 989
 990        for file in files:
 991            if file["type"] == "apple":
 992                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 993                continue
 994
 995            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 996            if file["action"] == "delete":
 997                self.gitStream.write("D %s\n" % relPath)
 998            else:
 999                data = file['data']
1000
1001                mode = "644"
1002                if isP4Exec(file["type"]):
1003                    mode = "755"
1004                elif file["type"] == "symlink":
1005                    mode = "120000"
1006                    # p4 print on a symlink contains "target\n", so strip it off
1007                    data = data[:-1]
1008
1009                if self.isWindows and file["type"].endswith("text"):
1010                    data = data.replace("\r\n", "\n")
1011
1012                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1013                self.gitStream.write("data %s\n" % len(data))
1014                self.gitStream.write(data)
1015                self.gitStream.write("\n")
1016
1017        self.gitStream.write("\n")
1018
1019        change = int(details["change"])
1020
1021        if self.labels.has_key(change):
1022            label = self.labels[change]
1023            labelDetails = label[0]
1024            labelRevisions = label[1]
1025            if self.verbose:
1026                print "Change %s is labelled %s" % (change, labelDetails)
1027
1028            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1029                                                    for p in branchPrefixes]))
1030
1031            if len(files) == len(labelRevisions):
1032
1033                cleanedFiles = {}
1034                for info in files:
1035                    if info["action"] == "delete":
1036                        continue
1037                    cleanedFiles[info["depotFile"]] = info["rev"]
1038
1039                if cleanedFiles == labelRevisions:
1040                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1041                    self.gitStream.write("from %s\n" % branch)
1042
1043                    owner = labelDetails["Owner"]
1044                    tagger = ""
1045                    if author in self.users:
1046                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1047                    else:
1048                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1049                    self.gitStream.write("tagger %s\n" % tagger)
1050                    self.gitStream.write("data <<EOT\n")
1051                    self.gitStream.write(labelDetails["Description"])
1052                    self.gitStream.write("EOT\n\n")
1053
1054                else:
1055                    if not self.silent:
1056                        print ("Tag %s does not match with change %s: files do not match."
1057                               % (labelDetails["label"], change))
1058
1059            else:
1060                if not self.silent:
1061                    print ("Tag %s does not match with change %s: file count is different."
1062                           % (labelDetails["label"], change))
1063
1064    def getUserCacheFilename(self):
1065        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1066        return home + "/.gitp4-usercache.txt"
1067
1068    def getUserMapFromPerforceServer(self):
1069        if self.userMapFromPerforceServer:
1070            return
1071        self.users = {}
1072
1073        for output in p4CmdList("users"):
1074            if not output.has_key("User"):
1075                continue
1076            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1077
1078
1079        s = ''
1080        for (key, val) in self.users.items():
1081            s += "%s\t%s\n" % (key, val)
1082
1083        open(self.getUserCacheFilename(), "wb").write(s)
1084        self.userMapFromPerforceServer = True
1085
1086    def loadUserMapFromCache(self):
1087        self.users = {}
1088        self.userMapFromPerforceServer = False
1089        try:
1090            cache = open(self.getUserCacheFilename(), "rb")
1091            lines = cache.readlines()
1092            cache.close()
1093            for line in lines:
1094                entry = line.strip().split("\t")
1095                self.users[entry[0]] = entry[1]
1096        except IOError:
1097            self.getUserMapFromPerforceServer()
1098
1099    def getLabels(self):
1100        self.labels = {}
1101
1102        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1103        if len(l) > 0 and not self.silent:
1104            print "Finding files belonging to labels in %s" % `self.depotPaths`
1105
1106        for output in l:
1107            label = output["label"]
1108            revisions = {}
1109            newestChange = 0
1110            if self.verbose:
1111                print "Querying files for label %s" % label
1112            for file in p4CmdList("files "
1113                                  +  ' '.join (["%s...@%s" % (p, label)
1114                                                for p in self.depotPaths])):
1115                revisions[file["depotFile"]] = file["rev"]
1116                change = int(file["change"])
1117                if change > newestChange:
1118                    newestChange = change
1119
1120            self.labels[newestChange] = [output, revisions]
1121
1122        if self.verbose:
1123            print "Label changes: %s" % self.labels.keys()
1124
1125    def guessProjectName(self):
1126        for p in self.depotPaths:
1127            if p.endswith("/"):
1128                p = p[:-1]
1129            p = p[p.strip().rfind("/") + 1:]
1130            if not p.endswith("/"):
1131               p += "/"
1132            return p
1133
1134    def getBranchMapping(self):
1135        lostAndFoundBranches = set()
1136
1137        for info in p4CmdList("branches"):
1138            details = p4Cmd("branch -o %s" % info["branch"])
1139            viewIdx = 0
1140            while details.has_key("View%s" % viewIdx):
1141                paths = details["View%s" % viewIdx].split(" ")
1142                viewIdx = viewIdx + 1
1143                # require standard //depot/foo/... //depot/bar/... mapping
1144                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1145                    continue
1146                source = paths[0]
1147                destination = paths[1]
1148                ## HACK
1149                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1150                    source = source[len(self.depotPaths[0]):-4]
1151                    destination = destination[len(self.depotPaths[0]):-4]
1152
1153                    if destination in self.knownBranches:
1154                        if not self.silent:
1155                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1156                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1157                        continue
1158
1159                    self.knownBranches[destination] = source
1160
1161                    lostAndFoundBranches.discard(destination)
1162
1163                    if source not in self.knownBranches:
1164                        lostAndFoundBranches.add(source)
1165
1166
1167        for branch in lostAndFoundBranches:
1168            self.knownBranches[branch] = branch
1169
1170    def getBranchMappingFromGitBranches(self):
1171        branches = p4BranchesInGit(self.importIntoRemotes)
1172        for branch in branches.keys():
1173            if branch == "master":
1174                branch = "main"
1175            else:
1176                branch = branch[len(self.projectName):]
1177            self.knownBranches[branch] = branch
1178
1179    def listExistingP4GitBranches(self):
1180        # branches holds mapping from name to commit
1181        branches = p4BranchesInGit(self.importIntoRemotes)
1182        self.p4BranchesInGit = branches.keys()
1183        for branch in branches.keys():
1184            self.initialParents[self.refPrefix + branch] = branches[branch]
1185
1186    def updateOptionDict(self, d):
1187        option_keys = {}
1188        if self.keepRepoPath:
1189            option_keys['keepRepoPath'] = 1
1190
1191        d["options"] = ' '.join(sorted(option_keys.keys()))
1192
1193    def readOptions(self, d):
1194        self.keepRepoPath = (d.has_key('options')
1195                             and ('keepRepoPath' in d['options']))
1196
1197    def gitRefForBranch(self, branch):
1198        if branch == "main":
1199            return self.refPrefix + "master"
1200
1201        if len(branch) <= 0:
1202            return branch
1203
1204        return self.refPrefix + self.projectName + branch
1205
1206    def gitCommitByP4Change(self, ref, change):
1207        if self.verbose:
1208            print "looking in ref " + ref + " for change %s using bisect..." % change
1209
1210        earliestCommit = ""
1211        latestCommit = parseRevision(ref)
1212
1213        while True:
1214            if self.verbose:
1215                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1216            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1217            if len(next) == 0:
1218                if self.verbose:
1219                    print "argh"
1220                return ""
1221            log = extractLogMessageFromGitCommit(next)
1222            settings = extractSettingsGitLog(log)
1223            currentChange = int(settings['change'])
1224            if self.verbose:
1225                print "current change %s" % currentChange
1226
1227            if currentChange == change:
1228                if self.verbose:
1229                    print "found %s" % next
1230                return next
1231
1232            if currentChange < change:
1233                earliestCommit = "^%s" % next
1234            else:
1235                latestCommit = "%s" % next
1236
1237        return ""
1238
1239    def importNewBranch(self, branch, maxChange):
1240        # make fast-import flush all changes to disk and update the refs using the checkpoint
1241        # command so that we can try to find the branch parent in the git history
1242        self.gitStream.write("checkpoint\n\n");
1243        self.gitStream.flush();
1244        branchPrefix = self.depotPaths[0] + branch + "/"
1245        range = "@1,%s" % maxChange
1246        #print "prefix" + branchPrefix
1247        changes = p4ChangesForPaths([branchPrefix], range)
1248        if len(changes) <= 0:
1249            return False
1250        firstChange = changes[0]
1251        #print "first change in branch: %s" % firstChange
1252        sourceBranch = self.knownBranches[branch]
1253        sourceDepotPath = self.depotPaths[0] + sourceBranch
1254        sourceRef = self.gitRefForBranch(sourceBranch)
1255        #print "source " + sourceBranch
1256
1257        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1258        #print "branch parent: %s" % branchParentChange
1259        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1260        if len(gitParent) > 0:
1261            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1262            #print "parent git commit: %s" % gitParent
1263
1264        self.importChanges(changes)
1265        return True
1266
1267    def importChanges(self, changes):
1268        cnt = 1
1269        for change in changes:
1270            description = p4Cmd("describe %s" % change)
1271            self.updateOptionDict(description)
1272
1273            if not self.silent:
1274                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1275                sys.stdout.flush()
1276            cnt = cnt + 1
1277
1278            try:
1279                if self.detectBranches:
1280                    branches = self.splitFilesIntoBranches(description)
1281                    for branch in branches.keys():
1282                        ## HACK  --hwn
1283                        branchPrefix = self.depotPaths[0] + branch + "/"
1284
1285                        parent = ""
1286
1287                        filesForCommit = branches[branch]
1288
1289                        if self.verbose:
1290                            print "branch is %s" % branch
1291
1292                        self.updatedBranches.add(branch)
1293
1294                        if branch not in self.createdBranches:
1295                            self.createdBranches.add(branch)
1296                            parent = self.knownBranches[branch]
1297                            if parent == branch:
1298                                parent = ""
1299                            else:
1300                                fullBranch = self.projectName + branch
1301                                if fullBranch not in self.p4BranchesInGit:
1302                                    if not self.silent:
1303                                        print("\n    Importing new branch %s" % fullBranch);
1304                                    if self.importNewBranch(branch, change - 1):
1305                                        parent = ""
1306                                        self.p4BranchesInGit.append(fullBranch)
1307                                    if not self.silent:
1308                                        print("\n    Resuming with change %s" % change);
1309
1310                                if self.verbose:
1311                                    print "parent determined through known branches: %s" % parent
1312
1313                        branch = self.gitRefForBranch(branch)
1314                        parent = self.gitRefForBranch(parent)
1315
1316                        if self.verbose:
1317                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1318
1319                        if len(parent) == 0 and branch in self.initialParents:
1320                            parent = self.initialParents[branch]
1321                            del self.initialParents[branch]
1322
1323                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1324                else:
1325                    files = self.extractFilesFromCommit(description)
1326                    self.commit(description, files, self.branch, self.depotPaths,
1327                                self.initialParent)
1328                    self.initialParent = ""
1329            except IOError:
1330                print self.gitError.read()
1331                sys.exit(1)
1332
1333    def importHeadRevision(self, revision):
1334        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1335
1336        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1337        details["desc"] = ("Initial import of %s from the state at revision %s"
1338                           % (' '.join(self.depotPaths), revision))
1339        details["change"] = revision
1340        newestRevision = 0
1341
1342        fileCnt = 0
1343        for info in p4CmdList("files "
1344                              +  ' '.join(["%s...%s"
1345                                           % (p, revision)
1346                                           for p in self.depotPaths])):
1347
1348            if info['code'] == 'error':
1349                sys.stderr.write("p4 returned an error: %s\n"
1350                                 % info['data'])
1351                sys.exit(1)
1352
1353
1354            change = int(info["change"])
1355            if change > newestRevision:
1356                newestRevision = change
1357
1358            if info["action"] == "delete":
1359                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1360                #fileCnt = fileCnt + 1
1361                continue
1362
1363            for prop in ["depotFile", "rev", "action", "type" ]:
1364                details["%s%s" % (prop, fileCnt)] = info[prop]
1365
1366            fileCnt = fileCnt + 1
1367
1368        details["change"] = newestRevision
1369        self.updateOptionDict(details)
1370        try:
1371            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1372        except IOError:
1373            print "IO error with git fast-import. Is your git version recent enough?"
1374            print self.gitError.read()
1375
1376
1377    def getClientSpec(self):
1378        specList = p4CmdList( "client -o" )
1379        temp = {}
1380        for entry in specList:
1381            for k,v in entry.iteritems():
1382                if k.startswith("View"):
1383                    if v.startswith('"'):
1384                        start = 1
1385                    else:
1386                        start = 0
1387                    index = v.find("...")
1388                    v = v[start:index]
1389                    if v.startswith("-"):
1390                        v = v[1:]
1391                        temp[v] = -len(v)
1392                    else:
1393                        temp[v] = len(v)
1394        self.clientSpecDirs = temp.items()
1395        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1396
1397    def run(self, args):
1398        self.depotPaths = []
1399        self.changeRange = ""
1400        self.initialParent = ""
1401        self.previousDepotPaths = []
1402
1403        # map from branch depot path to parent branch
1404        self.knownBranches = {}
1405        self.initialParents = {}
1406        self.hasOrigin = originP4BranchesExist()
1407        if not self.syncWithOrigin:
1408            self.hasOrigin = False
1409
1410        if self.importIntoRemotes:
1411            self.refPrefix = "refs/remotes/p4/"
1412        else:
1413            self.refPrefix = "refs/heads/p4/"
1414
1415        if self.syncWithOrigin and self.hasOrigin:
1416            if not self.silent:
1417                print "Syncing with origin first by calling git fetch origin"
1418            system("git fetch origin")
1419
1420        if len(self.branch) == 0:
1421            self.branch = self.refPrefix + "master"
1422            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1423                system("git update-ref %s refs/heads/p4" % self.branch)
1424                system("git branch -D p4");
1425            # create it /after/ importing, when master exists
1426            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1427                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1428
1429        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1430            self.getClientSpec()
1431
1432        # TODO: should always look at previous commits,
1433        # merge with previous imports, if possible.
1434        if args == []:
1435            if self.hasOrigin:
1436                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1437            self.listExistingP4GitBranches()
1438
1439            if len(self.p4BranchesInGit) > 1:
1440                if not self.silent:
1441                    print "Importing from/into multiple branches"
1442                self.detectBranches = True
1443
1444            if self.verbose:
1445                print "branches: %s" % self.p4BranchesInGit
1446
1447            p4Change = 0
1448            for branch in self.p4BranchesInGit:
1449                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1450
1451                settings = extractSettingsGitLog(logMsg)
1452
1453                self.readOptions(settings)
1454                if (settings.has_key('depot-paths')
1455                    and settings.has_key ('change')):
1456                    change = int(settings['change']) + 1
1457                    p4Change = max(p4Change, change)
1458
1459                    depotPaths = sorted(settings['depot-paths'])
1460                    if self.previousDepotPaths == []:
1461                        self.previousDepotPaths = depotPaths
1462                    else:
1463                        paths = []
1464                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1465                            for i in range(0, min(len(cur), len(prev))):
1466                                if cur[i] <> prev[i]:
1467                                    i = i - 1
1468                                    break
1469
1470                            paths.append (cur[:i + 1])
1471
1472                        self.previousDepotPaths = paths
1473
1474            if p4Change > 0:
1475                self.depotPaths = sorted(self.previousDepotPaths)
1476                self.changeRange = "@%s,#head" % p4Change
1477                if not self.detectBranches:
1478                    self.initialParent = parseRevision(self.branch)
1479                if not self.silent and not self.detectBranches:
1480                    print "Performing incremental import into %s git branch" % self.branch
1481
1482        if not self.branch.startswith("refs/"):
1483            self.branch = "refs/heads/" + self.branch
1484
1485        if len(args) == 0 and self.depotPaths:
1486            if not self.silent:
1487                print "Depot paths: %s" % ' '.join(self.depotPaths)
1488        else:
1489            if self.depotPaths and self.depotPaths != args:
1490                print ("previous import used depot path %s and now %s was specified. "
1491                       "This doesn't work!" % (' '.join (self.depotPaths),
1492                                               ' '.join (args)))
1493                sys.exit(1)
1494
1495            self.depotPaths = sorted(args)
1496
1497        revision = ""
1498        self.users = {}
1499
1500        newPaths = []
1501        for p in self.depotPaths:
1502            if p.find("@") != -1:
1503                atIdx = p.index("@")
1504                self.changeRange = p[atIdx:]
1505                if self.changeRange == "@all":
1506                    self.changeRange = ""
1507                elif ',' not in self.changeRange:
1508                    revision = self.changeRange
1509                    self.changeRange = ""
1510                p = p[:atIdx]
1511            elif p.find("#") != -1:
1512                hashIdx = p.index("#")
1513                revision = p[hashIdx:]
1514                p = p[:hashIdx]
1515            elif self.previousDepotPaths == []:
1516                revision = "#head"
1517
1518            p = re.sub ("\.\.\.$", "", p)
1519            if not p.endswith("/"):
1520                p += "/"
1521
1522            newPaths.append(p)
1523
1524        self.depotPaths = newPaths
1525
1526
1527        self.loadUserMapFromCache()
1528        self.labels = {}
1529        if self.detectLabels:
1530            self.getLabels();
1531
1532        if self.detectBranches:
1533            ## FIXME - what's a P4 projectName ?
1534            self.projectName = self.guessProjectName()
1535
1536            if self.hasOrigin:
1537                self.getBranchMappingFromGitBranches()
1538            else:
1539                self.getBranchMapping()
1540            if self.verbose:
1541                print "p4-git branches: %s" % self.p4BranchesInGit
1542                print "initial parents: %s" % self.initialParents
1543            for b in self.p4BranchesInGit:
1544                if b != "master":
1545
1546                    ## FIXME
1547                    b = b[len(self.projectName):]
1548                self.createdBranches.add(b)
1549
1550        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1551
1552        importProcess = subprocess.Popen(["git", "fast-import"],
1553                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1554                                         stderr=subprocess.PIPE);
1555        self.gitOutput = importProcess.stdout
1556        self.gitStream = importProcess.stdin
1557        self.gitError = importProcess.stderr
1558
1559        if revision:
1560            self.importHeadRevision(revision)
1561        else:
1562            changes = []
1563
1564            if len(self.changesFile) > 0:
1565                output = open(self.changesFile).readlines()
1566                changeSet = Set()
1567                for line in output:
1568                    changeSet.add(int(line))
1569
1570                for change in changeSet:
1571                    changes.append(change)
1572
1573                changes.sort()
1574            else:
1575                if self.verbose:
1576                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1577                                                              self.changeRange)
1578                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1579
1580                if len(self.maxChanges) > 0:
1581                    changes = changes[:min(int(self.maxChanges), len(changes))]
1582
1583            if len(changes) == 0:
1584                if not self.silent:
1585                    print "No changes to import!"
1586                return True
1587
1588            if not self.silent and not self.detectBranches:
1589                print "Import destination: %s" % self.branch
1590
1591            self.updatedBranches = set()
1592
1593            self.importChanges(changes)
1594
1595            if not self.silent:
1596                print ""
1597                if len(self.updatedBranches) > 0:
1598                    sys.stdout.write("Updated branches: ")
1599                    for b in self.updatedBranches:
1600                        sys.stdout.write("%s " % b)
1601                    sys.stdout.write("\n")
1602
1603        self.gitStream.close()
1604        if importProcess.wait() != 0:
1605            die("fast-import failed: %s" % self.gitError.read())
1606        self.gitOutput.close()
1607        self.gitError.close()
1608
1609        return True
1610
1611class P4Rebase(Command):
1612    def __init__(self):
1613        Command.__init__(self)
1614        self.options = [ ]
1615        self.description = ("Fetches the latest revision from perforce and "
1616                            + "rebases the current work (branch) against it")
1617        self.verbose = False
1618
1619    def run(self, args):
1620        sync = P4Sync()
1621        sync.run([])
1622
1623        return self.rebase()
1624
1625    def rebase(self):
1626        if os.system("git update-index --refresh") != 0:
1627            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1628        if len(read_pipe("git diff-index HEAD --")) > 0:
1629            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1630
1631        [upstream, settings] = findUpstreamBranchPoint()
1632        if len(upstream) == 0:
1633            die("Cannot find upstream branchpoint for rebase")
1634
1635        # the branchpoint may be p4/foo~3, so strip off the parent
1636        upstream = re.sub("~[0-9]+$", "", upstream)
1637
1638        print "Rebasing the current branch onto %s" % upstream
1639        oldHead = read_pipe("git rev-parse HEAD").strip()
1640        system("git rebase %s" % upstream)
1641        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1642        return True
1643
1644class P4Clone(P4Sync):
1645    def __init__(self):
1646        P4Sync.__init__(self)
1647        self.description = "Creates a new git repository and imports from Perforce into it"
1648        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1649        self.options += [
1650            optparse.make_option("--destination", dest="cloneDestination",
1651                                 action='store', default=None,
1652                                 help="where to leave result of the clone"),
1653            optparse.make_option("-/", dest="cloneExclude",
1654                                 action="append", type="string",
1655                                 help="exclude depot path")
1656        ]
1657        self.cloneDestination = None
1658        self.needsGit = False
1659
1660    # This is required for the "append" cloneExclude action
1661    def ensure_value(self, attr, value):
1662        if not hasattr(self, attr) or getattr(self, attr) is None:
1663            setattr(self, attr, value)
1664        return getattr(self, attr)
1665
1666    def defaultDestination(self, args):
1667        ## TODO: use common prefix of args?
1668        depotPath = args[0]
1669        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1670        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1671        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1672        depotDir = re.sub(r"/$", "", depotDir)
1673        return os.path.split(depotDir)[1]
1674
1675    def run(self, args):
1676        if len(args) < 1:
1677            return False
1678
1679        if self.keepRepoPath and not self.cloneDestination:
1680            sys.stderr.write("Must specify destination for --keep-path\n")
1681            sys.exit(1)
1682
1683        depotPaths = args
1684
1685        if not self.cloneDestination and len(depotPaths) > 1:
1686            self.cloneDestination = depotPaths[-1]
1687            depotPaths = depotPaths[:-1]
1688
1689        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1690        for p in depotPaths:
1691            if not p.startswith("//"):
1692                return False
1693
1694        if not self.cloneDestination:
1695            self.cloneDestination = self.defaultDestination(args)
1696
1697        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1698        if not os.path.exists(self.cloneDestination):
1699            os.makedirs(self.cloneDestination)
1700        os.chdir(self.cloneDestination)
1701        system("git init")
1702        self.gitdir = os.getcwd() + "/.git"
1703        if not P4Sync.run(self, depotPaths):
1704            return False
1705        if self.branch != "master":
1706            if gitBranchExists("refs/remotes/p4/master"):
1707                system("git branch master refs/remotes/p4/master")
1708                system("git checkout -f")
1709            else:
1710                print "Could not detect main branch. No checkout/master branch created."
1711
1712        return True
1713
1714class P4Branches(Command):
1715    def __init__(self):
1716        Command.__init__(self)
1717        self.options = [ ]
1718        self.description = ("Shows the git branches that hold imports and their "
1719                            + "corresponding perforce depot paths")
1720        self.verbose = False
1721
1722    def run(self, args):
1723        if originP4BranchesExist():
1724            createOrUpdateBranchesFromOrigin()
1725
1726        cmdline = "git rev-parse --symbolic "
1727        cmdline += " --remotes"
1728
1729        for line in read_pipe_lines(cmdline):
1730            line = line.strip()
1731
1732            if not line.startswith('p4/') or line == "p4/HEAD":
1733                continue
1734            branch = line
1735
1736            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1737            settings = extractSettingsGitLog(log)
1738
1739            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1740        return True
1741
1742class HelpFormatter(optparse.IndentedHelpFormatter):
1743    def __init__(self):
1744        optparse.IndentedHelpFormatter.__init__(self)
1745
1746    def format_description(self, description):
1747        if description:
1748            return description + "\n"
1749        else:
1750            return ""
1751
1752def printUsage(commands):
1753    print "usage: %s <command> [options]" % sys.argv[0]
1754    print ""
1755    print "valid commands: %s" % ", ".join(commands)
1756    print ""
1757    print "Try %s <command> --help for command specific help." % sys.argv[0]
1758    print ""
1759
1760commands = {
1761    "debug" : P4Debug,
1762    "submit" : P4Submit,
1763    "commit" : P4Submit,
1764    "sync" : P4Sync,
1765    "rebase" : P4Rebase,
1766    "clone" : P4Clone,
1767    "rollback" : P4RollBack,
1768    "branches" : P4Branches
1769}
1770
1771
1772def main():
1773    if len(sys.argv[1:]) == 0:
1774        printUsage(commands.keys())
1775        sys.exit(2)
1776
1777    cmd = ""
1778    cmdName = sys.argv[1]
1779    try:
1780        klass = commands[cmdName]
1781        cmd = klass()
1782    except KeyError:
1783        print "unknown command %s" % cmdName
1784        print ""
1785        printUsage(commands.keys())
1786        sys.exit(2)
1787
1788    options = cmd.options
1789    cmd.gitdir = os.environ.get("GIT_DIR", None)
1790
1791    args = sys.argv[2:]
1792
1793    if len(options) > 0:
1794        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1795
1796        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1797                                       options,
1798                                       description = cmd.description,
1799                                       formatter = HelpFormatter())
1800
1801        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1802    global verbose
1803    verbose = cmd.verbose
1804    if cmd.needsGit:
1805        if cmd.gitdir == None:
1806            cmd.gitdir = os.path.abspath(".git")
1807            if not isValidGitDir(cmd.gitdir):
1808                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1809                if os.path.exists(cmd.gitdir):
1810                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1811                    if len(cdup) > 0:
1812                        os.chdir(cdup);
1813
1814        if not isValidGitDir(cmd.gitdir):
1815            if isValidGitDir(cmd.gitdir + "/.git"):
1816                cmd.gitdir += "/.git"
1817            else:
1818                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1819
1820        os.environ["GIT_DIR"] = cmd.gitdir
1821
1822    if not cmd.run(args):
1823        parser.print_help()
1824
1825
1826if __name__ == '__main__':
1827    main()