Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'js/parallel-test' (6da9d5b)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(c, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    output = p4Cmd("where %s..." % depotPath)
 249    if output["code"] == "error":
 250        return ""
 251    clientPath = ""
 252    if "path" in output:
 253        clientPath = output.get("path")
 254    elif "data" in output:
 255        data = output.get("data")
 256        lastSpace = data.rfind(" ")
 257        clientPath = data[lastSpace + 1:]
 258
 259    if clientPath.endswith("..."):
 260        clientPath = clientPath[:-3]
 261    return clientPath
 262
 263def currentGitBranch():
 264    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266def isValidGitDir(path):
 267    if (os.path.exists(path + "/HEAD")
 268        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269        return True;
 270    return False
 271
 272def parseRevision(ref):
 273    return read_pipe("git rev-parse %s" % ref).strip()
 274
 275def extractLogMessageFromGitCommit(commit):
 276    logMessage = ""
 277
 278    ## fixme: title is first line of commit, not 1st paragraph.
 279    foundTitle = False
 280    for log in read_pipe_lines("git cat-file commit %s" % commit):
 281       if not foundTitle:
 282           if len(log) == 1:
 283               foundTitle = True
 284           continue
 285
 286       logMessage += log
 287    return logMessage
 288
 289def extractSettingsGitLog(log):
 290    values = {}
 291    for line in log.split("\n"):
 292        line = line.strip()
 293        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294        if not m:
 295            continue
 296
 297        assignments = m.group(1).split (':')
 298        for a in assignments:
 299            vals = a.split ('=')
 300            key = vals[0].strip()
 301            val = ('='.join (vals[1:])).strip()
 302            if val.endswith ('\"') and val.startswith('"'):
 303                val = val[1:-1]
 304
 305            values[key] = val
 306
 307    paths = values.get("depot-paths")
 308    if not paths:
 309        paths = values.get("depot-path")
 310    if paths:
 311        values['depot-paths'] = paths.split(',')
 312    return values
 313
 314def gitBranchExists(branch):
 315    proc = subprocess.Popen(["git", "rev-parse", branch],
 316                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317    return proc.wait() == 0;
 318
 319def gitConfig(key):
 320    return read_pipe("git config %s" % key, ignore_error=True).strip()
 321
 322def p4BranchesInGit(branchesAreInRemotes = True):
 323    branches = {}
 324
 325    cmdline = "git rev-parse --symbolic "
 326    if branchesAreInRemotes:
 327        cmdline += " --remotes"
 328    else:
 329        cmdline += " --branches"
 330
 331    for line in read_pipe_lines(cmdline):
 332        line = line.strip()
 333
 334        ## only import to p4/
 335        if not line.startswith('p4/') or line == "p4/HEAD":
 336            continue
 337        branch = line
 338
 339        # strip off p4
 340        branch = re.sub ("^p4/", "", line)
 341
 342        branches[branch] = parseRevision(line)
 343    return branches
 344
 345def findUpstreamBranchPoint(head = "HEAD"):
 346    branches = p4BranchesInGit()
 347    # map from depot-path to branch name
 348    branchByDepotPath = {}
 349    for branch in branches.keys():
 350        tip = branches[branch]
 351        log = extractLogMessageFromGitCommit(tip)
 352        settings = extractSettingsGitLog(log)
 353        if settings.has_key("depot-paths"):
 354            paths = ",".join(settings["depot-paths"])
 355            branchByDepotPath[paths] = "remotes/p4/" + branch
 356
 357    settings = None
 358    parent = 0
 359    while parent < 65535:
 360        commit = head + "~%s" % parent
 361        log = extractLogMessageFromGitCommit(commit)
 362        settings = extractSettingsGitLog(log)
 363        if settings.has_key("depot-paths"):
 364            paths = ",".join(settings["depot-paths"])
 365            if branchByDepotPath.has_key(paths):
 366                return [branchByDepotPath[paths], settings]
 367
 368        parent = parent + 1
 369
 370    return ["", settings]
 371
 372def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 373    if not silent:
 374        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 375               % localRefPrefix)
 376
 377    originPrefix = "origin/p4/"
 378
 379    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 380        line = line.strip()
 381        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 382            continue
 383
 384        headName = line[len(originPrefix):]
 385        remoteHead = localRefPrefix + headName
 386        originHead = line
 387
 388        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 389        if (not original.has_key('depot-paths')
 390            or not original.has_key('change')):
 391            continue
 392
 393        update = False
 394        if not gitBranchExists(remoteHead):
 395            if verbose:
 396                print "creating %s" % remoteHead
 397            update = True
 398        else:
 399            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 400            if settings.has_key('change') > 0:
 401                if settings['depot-paths'] == original['depot-paths']:
 402                    originP4Change = int(original['change'])
 403                    p4Change = int(settings['change'])
 404                    if originP4Change > p4Change:
 405                        print ("%s (%s) is newer than %s (%s). "
 406                               "Updating p4 branch from origin."
 407                               % (originHead, originP4Change,
 408                                  remoteHead, p4Change))
 409                        update = True
 410                else:
 411                    print ("Ignoring: %s was imported from %s while "
 412                           "%s was imported from %s"
 413                           % (originHead, ','.join(original['depot-paths']),
 414                              remoteHead, ','.join(settings['depot-paths'])))
 415
 416        if update:
 417            system("git update-ref %s %s" % (remoteHead, originHead))
 418
 419def originP4BranchesExist():
 420        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 421
 422def p4ChangesForPaths(depotPaths, changeRange):
 423    assert depotPaths
 424    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 425                                                        for p in depotPaths]))
 426
 427    changes = []
 428    for line in output:
 429        changeNum = line.split(" ")[1]
 430        changes.append(int(changeNum))
 431
 432    changes.sort()
 433    return changes
 434
 435class Command:
 436    def __init__(self):
 437        self.usage = "usage: %prog [options]"
 438        self.needsGit = True
 439
 440class P4Debug(Command):
 441    def __init__(self):
 442        Command.__init__(self)
 443        self.options = [
 444            optparse.make_option("--verbose", dest="verbose", action="store_true",
 445                                 default=False),
 446            ]
 447        self.description = "A tool to debug the output of p4 -G."
 448        self.needsGit = False
 449        self.verbose = False
 450
 451    def run(self, args):
 452        j = 0
 453        for output in p4CmdList(" ".join(args)):
 454            print 'Element: %d' % j
 455            j += 1
 456            print output
 457        return True
 458
 459class P4RollBack(Command):
 460    def __init__(self):
 461        Command.__init__(self)
 462        self.options = [
 463            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 464            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 465        ]
 466        self.description = "A tool to debug the multi-branch import. Don't use :)"
 467        self.verbose = False
 468        self.rollbackLocalBranches = False
 469
 470    def run(self, args):
 471        if len(args) != 1:
 472            return False
 473        maxChange = int(args[0])
 474
 475        if "p4ExitCode" in p4Cmd("changes -m 1"):
 476            die("Problems executing p4");
 477
 478        if self.rollbackLocalBranches:
 479            refPrefix = "refs/heads/"
 480            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 481        else:
 482            refPrefix = "refs/remotes/"
 483            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 484
 485        for line in lines:
 486            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 487                line = line.strip()
 488                ref = refPrefix + line
 489                log = extractLogMessageFromGitCommit(ref)
 490                settings = extractSettingsGitLog(log)
 491
 492                depotPaths = settings['depot-paths']
 493                change = settings['change']
 494
 495                changed = False
 496
 497                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 498                                                           for p in depotPaths]))) == 0:
 499                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 500                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 501                    continue
 502
 503                while change and int(change) > maxChange:
 504                    changed = True
 505                    if self.verbose:
 506                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 507                    system("git update-ref %s \"%s^\"" % (ref, ref))
 508                    log = extractLogMessageFromGitCommit(ref)
 509                    settings =  extractSettingsGitLog(log)
 510
 511
 512                    depotPaths = settings['depot-paths']
 513                    change = settings['change']
 514
 515                if changed:
 516                    print "%s rewound to %s" % (ref, change)
 517
 518        return True
 519
 520class P4Submit(Command):
 521    def __init__(self):
 522        Command.__init__(self)
 523        self.options = [
 524                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 525                optparse.make_option("--origin", dest="origin"),
 526                optparse.make_option("-M", dest="detectRename", action="store_true"),
 527        ]
 528        self.description = "Submit changes from git to the perforce depot."
 529        self.usage += " [name of git branch to submit into perforce depot]"
 530        self.interactive = True
 531        self.origin = ""
 532        self.detectRename = False
 533        self.verbose = False
 534        self.isWindows = (platform.system() == "Windows")
 535
 536    def check(self):
 537        if len(p4CmdList("opened ...")) > 0:
 538            die("You have files opened with perforce! Close them before starting the sync.")
 539
 540    # replaces everything between 'Description:' and the next P4 submit template field with the
 541    # commit message
 542    def prepareLogMessage(self, template, message):
 543        result = ""
 544
 545        inDescriptionSection = False
 546
 547        for line in template.split("\n"):
 548            if line.startswith("#"):
 549                result += line + "\n"
 550                continue
 551
 552            if inDescriptionSection:
 553                if line.startswith("Files:"):
 554                    inDescriptionSection = False
 555                else:
 556                    continue
 557            else:
 558                if line.startswith("Description:"):
 559                    inDescriptionSection = True
 560                    line += "\n"
 561                    for messageLine in message.split("\n"):
 562                        line += "\t" + messageLine + "\n"
 563
 564            result += line + "\n"
 565
 566        return result
 567
 568    def prepareSubmitTemplate(self):
 569        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 570        template = ""
 571        inFilesSection = False
 572        for line in p4_read_pipe_lines("change -o"):
 573            if line.endswith("\r\n"):
 574                line = line[:-2] + "\n"
 575            if inFilesSection:
 576                if line.startswith("\t"):
 577                    # path starts and ends with a tab
 578                    path = line[1:]
 579                    lastTab = path.rfind("\t")
 580                    if lastTab != -1:
 581                        path = path[:lastTab]
 582                        if not path.startswith(self.depotPath):
 583                            continue
 584                else:
 585                    inFilesSection = False
 586            else:
 587                if line.startswith("Files:"):
 588                    inFilesSection = True
 589
 590            template += line
 591
 592        return template
 593
 594    def applyCommit(self, id):
 595        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 596        diffOpts = ("", "-M")[self.detectRename]
 597        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 598        filesToAdd = set()
 599        filesToDelete = set()
 600        editedFiles = set()
 601        filesToChangeExecBit = {}
 602        for line in diff:
 603            diff = parseDiffTreeEntry(line)
 604            modifier = diff['status']
 605            path = diff['src']
 606            if modifier == "M":
 607                p4_system("edit \"%s\"" % path)
 608                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 609                    filesToChangeExecBit[path] = diff['dst_mode']
 610                editedFiles.add(path)
 611            elif modifier == "A":
 612                filesToAdd.add(path)
 613                filesToChangeExecBit[path] = diff['dst_mode']
 614                if path in filesToDelete:
 615                    filesToDelete.remove(path)
 616            elif modifier == "D":
 617                filesToDelete.add(path)
 618                if path in filesToAdd:
 619                    filesToAdd.remove(path)
 620            elif modifier == "R":
 621                src, dest = diff['src'], diff['dst']
 622                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 623                p4_system("edit \"%s\"" % (dest))
 624                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 625                    filesToChangeExecBit[dest] = diff['dst_mode']
 626                os.unlink(dest)
 627                editedFiles.add(dest)
 628                filesToDelete.add(src)
 629            else:
 630                die("unknown modifier %s for %s" % (modifier, path))
 631
 632        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 633        patchcmd = diffcmd + " | git apply "
 634        tryPatchCmd = patchcmd + "--check -"
 635        applyPatchCmd = patchcmd + "--check --apply -"
 636
 637        if os.system(tryPatchCmd) != 0:
 638            print "Unfortunately applying the change failed!"
 639            print "What do you want to do?"
 640            response = "x"
 641            while response != "s" and response != "a" and response != "w":
 642                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 643                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 644            if response == "s":
 645                print "Skipping! Good luck with the next patches..."
 646                for f in editedFiles:
 647                    p4_system("revert \"%s\"" % f);
 648                for f in filesToAdd:
 649                    system("rm %s" %f)
 650                return
 651            elif response == "a":
 652                os.system(applyPatchCmd)
 653                if len(filesToAdd) > 0:
 654                    print "You may also want to call p4 add on the following files:"
 655                    print " ".join(filesToAdd)
 656                if len(filesToDelete):
 657                    print "The following files should be scheduled for deletion with p4 delete:"
 658                    print " ".join(filesToDelete)
 659                die("Please resolve and submit the conflict manually and "
 660                    + "continue afterwards with git-p4 submit --continue")
 661            elif response == "w":
 662                system(diffcmd + " > patch.txt")
 663                print "Patch saved to patch.txt in %s !" % self.clientPath
 664                die("Please resolve and submit the conflict manually and "
 665                    "continue afterwards with git-p4 submit --continue")
 666
 667        system(applyPatchCmd)
 668
 669        for f in filesToAdd:
 670            p4_system("add \"%s\"" % f)
 671        for f in filesToDelete:
 672            p4_system("revert \"%s\"" % f)
 673            p4_system("delete \"%s\"" % f)
 674
 675        # Set/clear executable bits
 676        for f in filesToChangeExecBit.keys():
 677            mode = filesToChangeExecBit[f]
 678            setP4ExecBit(f, mode)
 679
 680        logMessage = extractLogMessageFromGitCommit(id)
 681        logMessage = logMessage.strip()
 682
 683        template = self.prepareSubmitTemplate()
 684
 685        if self.interactive:
 686            submitTemplate = self.prepareLogMessage(template, logMessage)
 687            if os.environ.has_key("P4DIFF"):
 688                del(os.environ["P4DIFF"])
 689            diff = p4_read_pipe("diff -du ...")
 690
 691            newdiff = ""
 692            for newFile in filesToAdd:
 693                newdiff += "==== new file ====\n"
 694                newdiff += "--- /dev/null\n"
 695                newdiff += "+++ %s\n" % newFile
 696                f = open(newFile, "r")
 697                for line in f.readlines():
 698                    newdiff += "+" + line
 699                f.close()
 700
 701            separatorLine = "######## everything below this line is just the diff #######\n"
 702
 703            [handle, fileName] = tempfile.mkstemp()
 704            tmpFile = os.fdopen(handle, "w+")
 705            if self.isWindows:
 706                submitTemplate = submitTemplate.replace("\n", "\r\n")
 707                separatorLine = separatorLine.replace("\n", "\r\n")
 708                newdiff = newdiff.replace("\n", "\r\n")
 709            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 710            tmpFile.close()
 711            defaultEditor = "vi"
 712            if platform.system() == "Windows":
 713                defaultEditor = "notepad"
 714            if os.environ.has_key("P4EDITOR"):
 715                editor = os.environ.get("P4EDITOR")
 716            else:
 717                editor = os.environ.get("EDITOR", defaultEditor);
 718            system(editor + " " + fileName)
 719            tmpFile = open(fileName, "rb")
 720            message = tmpFile.read()
 721            tmpFile.close()
 722            os.remove(fileName)
 723            submitTemplate = message[:message.index(separatorLine)]
 724            if self.isWindows:
 725                submitTemplate = submitTemplate.replace("\r\n", "\n")
 726
 727            p4_write_pipe("submit -i", submitTemplate)
 728        else:
 729            fileName = "submit.txt"
 730            file = open(fileName, "w+")
 731            file.write(self.prepareLogMessage(template, logMessage))
 732            file.close()
 733            print ("Perforce submit template written as %s. "
 734                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 735                   % (fileName, fileName))
 736
 737    def run(self, args):
 738        if len(args) == 0:
 739            self.master = currentGitBranch()
 740            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 741                die("Detecting current git branch failed!")
 742        elif len(args) == 1:
 743            self.master = args[0]
 744        else:
 745            return False
 746
 747        allowSubmit = gitConfig("git-p4.allowSubmit")
 748        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 749            die("%s is not in git-p4.allowSubmit" % self.master)
 750
 751        [upstream, settings] = findUpstreamBranchPoint()
 752        self.depotPath = settings['depot-paths'][0]
 753        if len(self.origin) == 0:
 754            self.origin = upstream
 755
 756        if self.verbose:
 757            print "Origin branch is " + self.origin
 758
 759        if len(self.depotPath) == 0:
 760            print "Internal error: cannot locate perforce depot path from existing branches"
 761            sys.exit(128)
 762
 763        self.clientPath = p4Where(self.depotPath)
 764
 765        if len(self.clientPath) == 0:
 766            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 767            sys.exit(128)
 768
 769        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 770        self.oldWorkingDirectory = os.getcwd()
 771
 772        chdir(self.clientPath)
 773        print "Syncronizing p4 checkout..."
 774        p4_system("sync ...")
 775
 776        self.check()
 777
 778        commits = []
 779        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 780            commits.append(line.strip())
 781        commits.reverse()
 782
 783        while len(commits) > 0:
 784            commit = commits[0]
 785            commits = commits[1:]
 786            self.applyCommit(commit)
 787            if not self.interactive:
 788                break
 789
 790        if len(commits) == 0:
 791            print "All changes applied!"
 792            chdir(self.oldWorkingDirectory)
 793
 794            sync = P4Sync()
 795            sync.run([])
 796
 797            rebase = P4Rebase()
 798            rebase.rebase()
 799
 800        return True
 801
 802class P4Sync(Command):
 803    def __init__(self):
 804        Command.__init__(self)
 805        self.options = [
 806                optparse.make_option("--branch", dest="branch"),
 807                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 808                optparse.make_option("--changesfile", dest="changesFile"),
 809                optparse.make_option("--silent", dest="silent", action="store_true"),
 810                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 811                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 812                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 813                                     help="Import into refs/heads/ , not refs/remotes"),
 814                optparse.make_option("--max-changes", dest="maxChanges"),
 815                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 816                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 817                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 818                                     help="Only sync files that are included in the Perforce Client Spec")
 819        ]
 820        self.description = """Imports from Perforce into a git repository.\n
 821    example:
 822    //depot/my/project/ -- to import the current head
 823    //depot/my/project/@all -- to import everything
 824    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 825
 826    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 827
 828        self.usage += " //depot/path[@revRange]"
 829        self.silent = False
 830        self.createdBranches = Set()
 831        self.committedChanges = Set()
 832        self.branch = ""
 833        self.detectBranches = False
 834        self.detectLabels = False
 835        self.changesFile = ""
 836        self.syncWithOrigin = True
 837        self.verbose = False
 838        self.importIntoRemotes = True
 839        self.maxChanges = ""
 840        self.isWindows = (platform.system() == "Windows")
 841        self.keepRepoPath = False
 842        self.depotPaths = None
 843        self.p4BranchesInGit = []
 844        self.cloneExclude = []
 845        self.useClientSpec = False
 846        self.clientSpecDirs = []
 847
 848        if gitConfig("git-p4.syncFromOrigin") == "false":
 849            self.syncWithOrigin = False
 850
 851    def extractFilesFromCommit(self, commit):
 852        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 853                             for path in self.cloneExclude]
 854        files = []
 855        fnum = 0
 856        while commit.has_key("depotFile%s" % fnum):
 857            path =  commit["depotFile%s" % fnum]
 858
 859            if [p for p in self.cloneExclude
 860                if path.startswith (p)]:
 861                found = False
 862            else:
 863                found = [p for p in self.depotPaths
 864                         if path.startswith (p)]
 865            if not found:
 866                fnum = fnum + 1
 867                continue
 868
 869            file = {}
 870            file["path"] = path
 871            file["rev"] = commit["rev%s" % fnum]
 872            file["action"] = commit["action%s" % fnum]
 873            file["type"] = commit["type%s" % fnum]
 874            files.append(file)
 875            fnum = fnum + 1
 876        return files
 877
 878    def stripRepoPath(self, path, prefixes):
 879        if self.keepRepoPath:
 880            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 881
 882        for p in prefixes:
 883            if path.startswith(p):
 884                path = path[len(p):]
 885
 886        return path
 887
 888    def splitFilesIntoBranches(self, commit):
 889        branches = {}
 890        fnum = 0
 891        while commit.has_key("depotFile%s" % fnum):
 892            path =  commit["depotFile%s" % fnum]
 893            found = [p for p in self.depotPaths
 894                     if path.startswith (p)]
 895            if not found:
 896                fnum = fnum + 1
 897                continue
 898
 899            file = {}
 900            file["path"] = path
 901            file["rev"] = commit["rev%s" % fnum]
 902            file["action"] = commit["action%s" % fnum]
 903            file["type"] = commit["type%s" % fnum]
 904            fnum = fnum + 1
 905
 906            relPath = self.stripRepoPath(path, self.depotPaths)
 907
 908            for branch in self.knownBranches.keys():
 909
 910                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 911                if relPath.startswith(branch + "/"):
 912                    if branch not in branches:
 913                        branches[branch] = []
 914                    branches[branch].append(file)
 915                    break
 916
 917        return branches
 918
 919    ## Should move this out, doesn't use SELF.
 920    def readP4Files(self, files):
 921        filesForCommit = []
 922        filesToRead = []
 923
 924        for f in files:
 925            includeFile = True
 926            for val in self.clientSpecDirs:
 927                if f['path'].startswith(val[0]):
 928                    if val[1] <= 0:
 929                        includeFile = False
 930                    break
 931
 932            if includeFile:
 933                filesForCommit.append(f)
 934                if f['action'] != 'delete':
 935                    filesToRead.append(f)
 936
 937        filedata = []
 938        if len(filesToRead) > 0:
 939            filedata = p4CmdList('-x - print',
 940                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 941                                                  for f in filesToRead]),
 942                                 stdin_mode='w+')
 943
 944            if "p4ExitCode" in filedata[0]:
 945                die("Problems executing p4. Error: [%d]."
 946                    % (filedata[0]['p4ExitCode']));
 947
 948        j = 0;
 949        contents = {}
 950        while j < len(filedata):
 951            stat = filedata[j]
 952            j += 1
 953            text = [];
 954            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 955                text.append(filedata[j]['data'])
 956                j += 1
 957            text = ''.join(text)
 958
 959            if not stat.has_key('depotFile'):
 960                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 961                continue
 962
 963            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 964                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 965            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 966                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 967
 968            contents[stat['depotFile']] = text
 969
 970        for f in filesForCommit:
 971            path = f['path']
 972            if contents.has_key(path):
 973                f['data'] = contents[path]
 974
 975        return filesForCommit
 976
 977    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 978        epoch = details["time"]
 979        author = details["user"]
 980
 981        if self.verbose:
 982            print "commit into %s" % branch
 983
 984        # start with reading files; if that fails, we should not
 985        # create a commit.
 986        new_files = []
 987        for f in files:
 988            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 989                new_files.append (f)
 990            else:
 991                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 992        files = self.readP4Files(new_files)
 993
 994        self.gitStream.write("commit %s\n" % branch)
 995#        gitStream.write("mark :%s\n" % details["change"])
 996        self.committedChanges.add(int(details["change"]))
 997        committer = ""
 998        if author not in self.users:
 999            self.getUserMapFromPerforceServer()
1000        if author in self.users:
1001            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1002        else:
1003            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1004
1005        self.gitStream.write("committer %s\n" % committer)
1006
1007        self.gitStream.write("data <<EOT\n")
1008        self.gitStream.write(details["desc"])
1009        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1010                             % (','.join (branchPrefixes), details["change"]))
1011        if len(details['options']) > 0:
1012            self.gitStream.write(": options = %s" % details['options'])
1013        self.gitStream.write("]\nEOT\n\n")
1014
1015        if len(parent) > 0:
1016            if self.verbose:
1017                print "parent %s" % parent
1018            self.gitStream.write("from %s\n" % parent)
1019
1020        for file in files:
1021            if file["type"] == "apple":
1022                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1023                continue
1024
1025            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1026            if file["action"] == "delete":
1027                self.gitStream.write("D %s\n" % relPath)
1028            else:
1029                data = file['data']
1030
1031                mode = "644"
1032                if isP4Exec(file["type"]):
1033                    mode = "755"
1034                elif file["type"] == "symlink":
1035                    mode = "120000"
1036                    # p4 print on a symlink contains "target\n", so strip it off
1037                    data = data[:-1]
1038
1039                if self.isWindows and file["type"].endswith("text"):
1040                    data = data.replace("\r\n", "\n")
1041
1042                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1043                self.gitStream.write("data %s\n" % len(data))
1044                self.gitStream.write(data)
1045                self.gitStream.write("\n")
1046
1047        self.gitStream.write("\n")
1048
1049        change = int(details["change"])
1050
1051        if self.labels.has_key(change):
1052            label = self.labels[change]
1053            labelDetails = label[0]
1054            labelRevisions = label[1]
1055            if self.verbose:
1056                print "Change %s is labelled %s" % (change, labelDetails)
1057
1058            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1059                                                    for p in branchPrefixes]))
1060
1061            if len(files) == len(labelRevisions):
1062
1063                cleanedFiles = {}
1064                for info in files:
1065                    if info["action"] == "delete":
1066                        continue
1067                    cleanedFiles[info["depotFile"]] = info["rev"]
1068
1069                if cleanedFiles == labelRevisions:
1070                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1071                    self.gitStream.write("from %s\n" % branch)
1072
1073                    owner = labelDetails["Owner"]
1074                    tagger = ""
1075                    if author in self.users:
1076                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1077                    else:
1078                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1079                    self.gitStream.write("tagger %s\n" % tagger)
1080                    self.gitStream.write("data <<EOT\n")
1081                    self.gitStream.write(labelDetails["Description"])
1082                    self.gitStream.write("EOT\n\n")
1083
1084                else:
1085                    if not self.silent:
1086                        print ("Tag %s does not match with change %s: files do not match."
1087                               % (labelDetails["label"], change))
1088
1089            else:
1090                if not self.silent:
1091                    print ("Tag %s does not match with change %s: file count is different."
1092                           % (labelDetails["label"], change))
1093
1094    def getUserCacheFilename(self):
1095        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1096        return home + "/.gitp4-usercache.txt"
1097
1098    def getUserMapFromPerforceServer(self):
1099        if self.userMapFromPerforceServer:
1100            return
1101        self.users = {}
1102
1103        for output in p4CmdList("users"):
1104            if not output.has_key("User"):
1105                continue
1106            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1107
1108
1109        s = ''
1110        for (key, val) in self.users.items():
1111            s += "%s\t%s\n" % (key, val)
1112
1113        open(self.getUserCacheFilename(), "wb").write(s)
1114        self.userMapFromPerforceServer = True
1115
1116    def loadUserMapFromCache(self):
1117        self.users = {}
1118        self.userMapFromPerforceServer = False
1119        try:
1120            cache = open(self.getUserCacheFilename(), "rb")
1121            lines = cache.readlines()
1122            cache.close()
1123            for line in lines:
1124                entry = line.strip().split("\t")
1125                self.users[entry[0]] = entry[1]
1126        except IOError:
1127            self.getUserMapFromPerforceServer()
1128
1129    def getLabels(self):
1130        self.labels = {}
1131
1132        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1133        if len(l) > 0 and not self.silent:
1134            print "Finding files belonging to labels in %s" % `self.depotPaths`
1135
1136        for output in l:
1137            label = output["label"]
1138            revisions = {}
1139            newestChange = 0
1140            if self.verbose:
1141                print "Querying files for label %s" % label
1142            for file in p4CmdList("files "
1143                                  +  ' '.join (["%s...@%s" % (p, label)
1144                                                for p in self.depotPaths])):
1145                revisions[file["depotFile"]] = file["rev"]
1146                change = int(file["change"])
1147                if change > newestChange:
1148                    newestChange = change
1149
1150            self.labels[newestChange] = [output, revisions]
1151
1152        if self.verbose:
1153            print "Label changes: %s" % self.labels.keys()
1154
1155    def guessProjectName(self):
1156        for p in self.depotPaths:
1157            if p.endswith("/"):
1158                p = p[:-1]
1159            p = p[p.strip().rfind("/") + 1:]
1160            if not p.endswith("/"):
1161               p += "/"
1162            return p
1163
1164    def getBranchMapping(self):
1165        lostAndFoundBranches = set()
1166
1167        for info in p4CmdList("branches"):
1168            details = p4Cmd("branch -o %s" % info["branch"])
1169            viewIdx = 0
1170            while details.has_key("View%s" % viewIdx):
1171                paths = details["View%s" % viewIdx].split(" ")
1172                viewIdx = viewIdx + 1
1173                # require standard //depot/foo/... //depot/bar/... mapping
1174                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1175                    continue
1176                source = paths[0]
1177                destination = paths[1]
1178                ## HACK
1179                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1180                    source = source[len(self.depotPaths[0]):-4]
1181                    destination = destination[len(self.depotPaths[0]):-4]
1182
1183                    if destination in self.knownBranches:
1184                        if not self.silent:
1185                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1186                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1187                        continue
1188
1189                    self.knownBranches[destination] = source
1190
1191                    lostAndFoundBranches.discard(destination)
1192
1193                    if source not in self.knownBranches:
1194                        lostAndFoundBranches.add(source)
1195
1196
1197        for branch in lostAndFoundBranches:
1198            self.knownBranches[branch] = branch
1199
1200    def getBranchMappingFromGitBranches(self):
1201        branches = p4BranchesInGit(self.importIntoRemotes)
1202        for branch in branches.keys():
1203            if branch == "master":
1204                branch = "main"
1205            else:
1206                branch = branch[len(self.projectName):]
1207            self.knownBranches[branch] = branch
1208
1209    def listExistingP4GitBranches(self):
1210        # branches holds mapping from name to commit
1211        branches = p4BranchesInGit(self.importIntoRemotes)
1212        self.p4BranchesInGit = branches.keys()
1213        for branch in branches.keys():
1214            self.initialParents[self.refPrefix + branch] = branches[branch]
1215
1216    def updateOptionDict(self, d):
1217        option_keys = {}
1218        if self.keepRepoPath:
1219            option_keys['keepRepoPath'] = 1
1220
1221        d["options"] = ' '.join(sorted(option_keys.keys()))
1222
1223    def readOptions(self, d):
1224        self.keepRepoPath = (d.has_key('options')
1225                             and ('keepRepoPath' in d['options']))
1226
1227    def gitRefForBranch(self, branch):
1228        if branch == "main":
1229            return self.refPrefix + "master"
1230
1231        if len(branch) <= 0:
1232            return branch
1233
1234        return self.refPrefix + self.projectName + branch
1235
1236    def gitCommitByP4Change(self, ref, change):
1237        if self.verbose:
1238            print "looking in ref " + ref + " for change %s using bisect..." % change
1239
1240        earliestCommit = ""
1241        latestCommit = parseRevision(ref)
1242
1243        while True:
1244            if self.verbose:
1245                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1246            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1247            if len(next) == 0:
1248                if self.verbose:
1249                    print "argh"
1250                return ""
1251            log = extractLogMessageFromGitCommit(next)
1252            settings = extractSettingsGitLog(log)
1253            currentChange = int(settings['change'])
1254            if self.verbose:
1255                print "current change %s" % currentChange
1256
1257            if currentChange == change:
1258                if self.verbose:
1259                    print "found %s" % next
1260                return next
1261
1262            if currentChange < change:
1263                earliestCommit = "^%s" % next
1264            else:
1265                latestCommit = "%s" % next
1266
1267        return ""
1268
1269    def importNewBranch(self, branch, maxChange):
1270        # make fast-import flush all changes to disk and update the refs using the checkpoint
1271        # command so that we can try to find the branch parent in the git history
1272        self.gitStream.write("checkpoint\n\n");
1273        self.gitStream.flush();
1274        branchPrefix = self.depotPaths[0] + branch + "/"
1275        range = "@1,%s" % maxChange
1276        #print "prefix" + branchPrefix
1277        changes = p4ChangesForPaths([branchPrefix], range)
1278        if len(changes) <= 0:
1279            return False
1280        firstChange = changes[0]
1281        #print "first change in branch: %s" % firstChange
1282        sourceBranch = self.knownBranches[branch]
1283        sourceDepotPath = self.depotPaths[0] + sourceBranch
1284        sourceRef = self.gitRefForBranch(sourceBranch)
1285        #print "source " + sourceBranch
1286
1287        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1288        #print "branch parent: %s" % branchParentChange
1289        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1290        if len(gitParent) > 0:
1291            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1292            #print "parent git commit: %s" % gitParent
1293
1294        self.importChanges(changes)
1295        return True
1296
1297    def importChanges(self, changes):
1298        cnt = 1
1299        for change in changes:
1300            description = p4Cmd("describe %s" % change)
1301            self.updateOptionDict(description)
1302
1303            if not self.silent:
1304                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1305                sys.stdout.flush()
1306            cnt = cnt + 1
1307
1308            try:
1309                if self.detectBranches:
1310                    branches = self.splitFilesIntoBranches(description)
1311                    for branch in branches.keys():
1312                        ## HACK  --hwn
1313                        branchPrefix = self.depotPaths[0] + branch + "/"
1314
1315                        parent = ""
1316
1317                        filesForCommit = branches[branch]
1318
1319                        if self.verbose:
1320                            print "branch is %s" % branch
1321
1322                        self.updatedBranches.add(branch)
1323
1324                        if branch not in self.createdBranches:
1325                            self.createdBranches.add(branch)
1326                            parent = self.knownBranches[branch]
1327                            if parent == branch:
1328                                parent = ""
1329                            else:
1330                                fullBranch = self.projectName + branch
1331                                if fullBranch not in self.p4BranchesInGit:
1332                                    if not self.silent:
1333                                        print("\n    Importing new branch %s" % fullBranch);
1334                                    if self.importNewBranch(branch, change - 1):
1335                                        parent = ""
1336                                        self.p4BranchesInGit.append(fullBranch)
1337                                    if not self.silent:
1338                                        print("\n    Resuming with change %s" % change);
1339
1340                                if self.verbose:
1341                                    print "parent determined through known branches: %s" % parent
1342
1343                        branch = self.gitRefForBranch(branch)
1344                        parent = self.gitRefForBranch(parent)
1345
1346                        if self.verbose:
1347                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1348
1349                        if len(parent) == 0 and branch in self.initialParents:
1350                            parent = self.initialParents[branch]
1351                            del self.initialParents[branch]
1352
1353                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1354                else:
1355                    files = self.extractFilesFromCommit(description)
1356                    self.commit(description, files, self.branch, self.depotPaths,
1357                                self.initialParent)
1358                    self.initialParent = ""
1359            except IOError:
1360                print self.gitError.read()
1361                sys.exit(1)
1362
1363    def importHeadRevision(self, revision):
1364        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1365
1366        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1367        details["desc"] = ("Initial import of %s from the state at revision %s"
1368                           % (' '.join(self.depotPaths), revision))
1369        details["change"] = revision
1370        newestRevision = 0
1371
1372        fileCnt = 0
1373        for info in p4CmdList("files "
1374                              +  ' '.join(["%s...%s"
1375                                           % (p, revision)
1376                                           for p in self.depotPaths])):
1377
1378            if info['code'] == 'error':
1379                sys.stderr.write("p4 returned an error: %s\n"
1380                                 % info['data'])
1381                sys.exit(1)
1382
1383
1384            change = int(info["change"])
1385            if change > newestRevision:
1386                newestRevision = change
1387
1388            if info["action"] == "delete":
1389                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1390                #fileCnt = fileCnt + 1
1391                continue
1392
1393            for prop in ["depotFile", "rev", "action", "type" ]:
1394                details["%s%s" % (prop, fileCnt)] = info[prop]
1395
1396            fileCnt = fileCnt + 1
1397
1398        details["change"] = newestRevision
1399        self.updateOptionDict(details)
1400        try:
1401            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1402        except IOError:
1403            print "IO error with git fast-import. Is your git version recent enough?"
1404            print self.gitError.read()
1405
1406
1407    def getClientSpec(self):
1408        specList = p4CmdList( "client -o" )
1409        temp = {}
1410        for entry in specList:
1411            for k,v in entry.iteritems():
1412                if k.startswith("View"):
1413                    if v.startswith('"'):
1414                        start = 1
1415                    else:
1416                        start = 0
1417                    index = v.find("...")
1418                    v = v[start:index]
1419                    if v.startswith("-"):
1420                        v = v[1:]
1421                        temp[v] = -len(v)
1422                    else:
1423                        temp[v] = len(v)
1424        self.clientSpecDirs = temp.items()
1425        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1426
1427    def run(self, args):
1428        self.depotPaths = []
1429        self.changeRange = ""
1430        self.initialParent = ""
1431        self.previousDepotPaths = []
1432
1433        # map from branch depot path to parent branch
1434        self.knownBranches = {}
1435        self.initialParents = {}
1436        self.hasOrigin = originP4BranchesExist()
1437        if not self.syncWithOrigin:
1438            self.hasOrigin = False
1439
1440        if self.importIntoRemotes:
1441            self.refPrefix = "refs/remotes/p4/"
1442        else:
1443            self.refPrefix = "refs/heads/p4/"
1444
1445        if self.syncWithOrigin and self.hasOrigin:
1446            if not self.silent:
1447                print "Syncing with origin first by calling git fetch origin"
1448            system("git fetch origin")
1449
1450        if len(self.branch) == 0:
1451            self.branch = self.refPrefix + "master"
1452            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1453                system("git update-ref %s refs/heads/p4" % self.branch)
1454                system("git branch -D p4");
1455            # create it /after/ importing, when master exists
1456            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1457                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1458
1459        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1460            self.getClientSpec()
1461
1462        # TODO: should always look at previous commits,
1463        # merge with previous imports, if possible.
1464        if args == []:
1465            if self.hasOrigin:
1466                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1467            self.listExistingP4GitBranches()
1468
1469            if len(self.p4BranchesInGit) > 1:
1470                if not self.silent:
1471                    print "Importing from/into multiple branches"
1472                self.detectBranches = True
1473
1474            if self.verbose:
1475                print "branches: %s" % self.p4BranchesInGit
1476
1477            p4Change = 0
1478            for branch in self.p4BranchesInGit:
1479                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1480
1481                settings = extractSettingsGitLog(logMsg)
1482
1483                self.readOptions(settings)
1484                if (settings.has_key('depot-paths')
1485                    and settings.has_key ('change')):
1486                    change = int(settings['change']) + 1
1487                    p4Change = max(p4Change, change)
1488
1489                    depotPaths = sorted(settings['depot-paths'])
1490                    if self.previousDepotPaths == []:
1491                        self.previousDepotPaths = depotPaths
1492                    else:
1493                        paths = []
1494                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1495                            for i in range(0, min(len(cur), len(prev))):
1496                                if cur[i] <> prev[i]:
1497                                    i = i - 1
1498                                    break
1499
1500                            paths.append (cur[:i + 1])
1501
1502                        self.previousDepotPaths = paths
1503
1504            if p4Change > 0:
1505                self.depotPaths = sorted(self.previousDepotPaths)
1506                self.changeRange = "@%s,#head" % p4Change
1507                if not self.detectBranches:
1508                    self.initialParent = parseRevision(self.branch)
1509                if not self.silent and not self.detectBranches:
1510                    print "Performing incremental import into %s git branch" % self.branch
1511
1512        if not self.branch.startswith("refs/"):
1513            self.branch = "refs/heads/" + self.branch
1514
1515        if len(args) == 0 and self.depotPaths:
1516            if not self.silent:
1517                print "Depot paths: %s" % ' '.join(self.depotPaths)
1518        else:
1519            if self.depotPaths and self.depotPaths != args:
1520                print ("previous import used depot path %s and now %s was specified. "
1521                       "This doesn't work!" % (' '.join (self.depotPaths),
1522                                               ' '.join (args)))
1523                sys.exit(1)
1524
1525            self.depotPaths = sorted(args)
1526
1527        revision = ""
1528        self.users = {}
1529
1530        newPaths = []
1531        for p in self.depotPaths:
1532            if p.find("@") != -1:
1533                atIdx = p.index("@")
1534                self.changeRange = p[atIdx:]
1535                if self.changeRange == "@all":
1536                    self.changeRange = ""
1537                elif ',' not in self.changeRange:
1538                    revision = self.changeRange
1539                    self.changeRange = ""
1540                p = p[:atIdx]
1541            elif p.find("#") != -1:
1542                hashIdx = p.index("#")
1543                revision = p[hashIdx:]
1544                p = p[:hashIdx]
1545            elif self.previousDepotPaths == []:
1546                revision = "#head"
1547
1548            p = re.sub ("\.\.\.$", "", p)
1549            if not p.endswith("/"):
1550                p += "/"
1551
1552            newPaths.append(p)
1553
1554        self.depotPaths = newPaths
1555
1556
1557        self.loadUserMapFromCache()
1558        self.labels = {}
1559        if self.detectLabels:
1560            self.getLabels();
1561
1562        if self.detectBranches:
1563            ## FIXME - what's a P4 projectName ?
1564            self.projectName = self.guessProjectName()
1565
1566            if self.hasOrigin:
1567                self.getBranchMappingFromGitBranches()
1568            else:
1569                self.getBranchMapping()
1570            if self.verbose:
1571                print "p4-git branches: %s" % self.p4BranchesInGit
1572                print "initial parents: %s" % self.initialParents
1573            for b in self.p4BranchesInGit:
1574                if b != "master":
1575
1576                    ## FIXME
1577                    b = b[len(self.projectName):]
1578                self.createdBranches.add(b)
1579
1580        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1581
1582        importProcess = subprocess.Popen(["git", "fast-import"],
1583                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1584                                         stderr=subprocess.PIPE);
1585        self.gitOutput = importProcess.stdout
1586        self.gitStream = importProcess.stdin
1587        self.gitError = importProcess.stderr
1588
1589        if revision:
1590            self.importHeadRevision(revision)
1591        else:
1592            changes = []
1593
1594            if len(self.changesFile) > 0:
1595                output = open(self.changesFile).readlines()
1596                changeSet = Set()
1597                for line in output:
1598                    changeSet.add(int(line))
1599
1600                for change in changeSet:
1601                    changes.append(change)
1602
1603                changes.sort()
1604            else:
1605                if self.verbose:
1606                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1607                                                              self.changeRange)
1608                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1609
1610                if len(self.maxChanges) > 0:
1611                    changes = changes[:min(int(self.maxChanges), len(changes))]
1612
1613            if len(changes) == 0:
1614                if not self.silent:
1615                    print "No changes to import!"
1616                return True
1617
1618            if not self.silent and not self.detectBranches:
1619                print "Import destination: %s" % self.branch
1620
1621            self.updatedBranches = set()
1622
1623            self.importChanges(changes)
1624
1625            if not self.silent:
1626                print ""
1627                if len(self.updatedBranches) > 0:
1628                    sys.stdout.write("Updated branches: ")
1629                    for b in self.updatedBranches:
1630                        sys.stdout.write("%s " % b)
1631                    sys.stdout.write("\n")
1632
1633        self.gitStream.close()
1634        if importProcess.wait() != 0:
1635            die("fast-import failed: %s" % self.gitError.read())
1636        self.gitOutput.close()
1637        self.gitError.close()
1638
1639        return True
1640
1641class P4Rebase(Command):
1642    def __init__(self):
1643        Command.__init__(self)
1644        self.options = [ ]
1645        self.description = ("Fetches the latest revision from perforce and "
1646                            + "rebases the current work (branch) against it")
1647        self.verbose = False
1648
1649    def run(self, args):
1650        sync = P4Sync()
1651        sync.run([])
1652
1653        return self.rebase()
1654
1655    def rebase(self):
1656        if os.system("git update-index --refresh") != 0:
1657            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1658        if len(read_pipe("git diff-index HEAD --")) > 0:
1659            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1660
1661        [upstream, settings] = findUpstreamBranchPoint()
1662        if len(upstream) == 0:
1663            die("Cannot find upstream branchpoint for rebase")
1664
1665        # the branchpoint may be p4/foo~3, so strip off the parent
1666        upstream = re.sub("~[0-9]+$", "", upstream)
1667
1668        print "Rebasing the current branch onto %s" % upstream
1669        oldHead = read_pipe("git rev-parse HEAD").strip()
1670        system("git rebase %s" % upstream)
1671        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1672        return True
1673
1674class P4Clone(P4Sync):
1675    def __init__(self):
1676        P4Sync.__init__(self)
1677        self.description = "Creates a new git repository and imports from Perforce into it"
1678        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1679        self.options += [
1680            optparse.make_option("--destination", dest="cloneDestination",
1681                                 action='store', default=None,
1682                                 help="where to leave result of the clone"),
1683            optparse.make_option("-/", dest="cloneExclude",
1684                                 action="append", type="string",
1685                                 help="exclude depot path")
1686        ]
1687        self.cloneDestination = None
1688        self.needsGit = False
1689
1690    # This is required for the "append" cloneExclude action
1691    def ensure_value(self, attr, value):
1692        if not hasattr(self, attr) or getattr(self, attr) is None:
1693            setattr(self, attr, value)
1694        return getattr(self, attr)
1695
1696    def defaultDestination(self, args):
1697        ## TODO: use common prefix of args?
1698        depotPath = args[0]
1699        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1700        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1701        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1702        depotDir = re.sub(r"/$", "", depotDir)
1703        return os.path.split(depotDir)[1]
1704
1705    def run(self, args):
1706        if len(args) < 1:
1707            return False
1708
1709        if self.keepRepoPath and not self.cloneDestination:
1710            sys.stderr.write("Must specify destination for --keep-path\n")
1711            sys.exit(1)
1712
1713        depotPaths = args
1714
1715        if not self.cloneDestination and len(depotPaths) > 1:
1716            self.cloneDestination = depotPaths[-1]
1717            depotPaths = depotPaths[:-1]
1718
1719        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1720        for p in depotPaths:
1721            if not p.startswith("//"):
1722                return False
1723
1724        if not self.cloneDestination:
1725            self.cloneDestination = self.defaultDestination(args)
1726
1727        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1728        if not os.path.exists(self.cloneDestination):
1729            os.makedirs(self.cloneDestination)
1730        chdir(self.cloneDestination)
1731        system("git init")
1732        self.gitdir = os.getcwd() + "/.git"
1733        if not P4Sync.run(self, depotPaths):
1734            return False
1735        if self.branch != "master":
1736            if gitBranchExists("refs/remotes/p4/master"):
1737                system("git branch master refs/remotes/p4/master")
1738                system("git checkout -f")
1739            else:
1740                print "Could not detect main branch. No checkout/master branch created."
1741
1742        return True
1743
1744class P4Branches(Command):
1745    def __init__(self):
1746        Command.__init__(self)
1747        self.options = [ ]
1748        self.description = ("Shows the git branches that hold imports and their "
1749                            + "corresponding perforce depot paths")
1750        self.verbose = False
1751
1752    def run(self, args):
1753        if originP4BranchesExist():
1754            createOrUpdateBranchesFromOrigin()
1755
1756        cmdline = "git rev-parse --symbolic "
1757        cmdline += " --remotes"
1758
1759        for line in read_pipe_lines(cmdline):
1760            line = line.strip()
1761
1762            if not line.startswith('p4/') or line == "p4/HEAD":
1763                continue
1764            branch = line
1765
1766            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1767            settings = extractSettingsGitLog(log)
1768
1769            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1770        return True
1771
1772class HelpFormatter(optparse.IndentedHelpFormatter):
1773    def __init__(self):
1774        optparse.IndentedHelpFormatter.__init__(self)
1775
1776    def format_description(self, description):
1777        if description:
1778            return description + "\n"
1779        else:
1780            return ""
1781
1782def printUsage(commands):
1783    print "usage: %s <command> [options]" % sys.argv[0]
1784    print ""
1785    print "valid commands: %s" % ", ".join(commands)
1786    print ""
1787    print "Try %s <command> --help for command specific help." % sys.argv[0]
1788    print ""
1789
1790commands = {
1791    "debug" : P4Debug,
1792    "submit" : P4Submit,
1793    "commit" : P4Submit,
1794    "sync" : P4Sync,
1795    "rebase" : P4Rebase,
1796    "clone" : P4Clone,
1797    "rollback" : P4RollBack,
1798    "branches" : P4Branches
1799}
1800
1801
1802def main():
1803    if len(sys.argv[1:]) == 0:
1804        printUsage(commands.keys())
1805        sys.exit(2)
1806
1807    cmd = ""
1808    cmdName = sys.argv[1]
1809    try:
1810        klass = commands[cmdName]
1811        cmd = klass()
1812    except KeyError:
1813        print "unknown command %s" % cmdName
1814        print ""
1815        printUsage(commands.keys())
1816        sys.exit(2)
1817
1818    options = cmd.options
1819    cmd.gitdir = os.environ.get("GIT_DIR", None)
1820
1821    args = sys.argv[2:]
1822
1823    if len(options) > 0:
1824        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1825
1826        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1827                                       options,
1828                                       description = cmd.description,
1829                                       formatter = HelpFormatter())
1830
1831        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1832    global verbose
1833    verbose = cmd.verbose
1834    if cmd.needsGit:
1835        if cmd.gitdir == None:
1836            cmd.gitdir = os.path.abspath(".git")
1837            if not isValidGitDir(cmd.gitdir):
1838                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1839                if os.path.exists(cmd.gitdir):
1840                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1841                    if len(cdup) > 0:
1842                        chdir(cdup);
1843
1844        if not isValidGitDir(cmd.gitdir):
1845            if isValidGitDir(cmd.gitdir + "/.git"):
1846                cmd.gitdir += "/.git"
1847            else:
1848                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1849
1850        os.environ["GIT_DIR"] = cmd.gitdir
1851
1852    if not cmd.run(args):
1853        parser.print_help()
1854
1855
1856if __name__ == '__main__':
1857    main()