Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' (445cac1)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    output = p4Cmd("where %s..." % depotPath)
 249    if output["code"] == "error":
 250        return ""
 251    clientPath = ""
 252    if "path" in output:
 253        clientPath = output.get("path")
 254    elif "data" in output:
 255        data = output.get("data")
 256        lastSpace = data.rfind(" ")
 257        clientPath = data[lastSpace + 1:]
 258
 259    if clientPath.endswith("..."):
 260        clientPath = clientPath[:-3]
 261    return clientPath
 262
 263def currentGitBranch():
 264    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 265
 266def isValidGitDir(path):
 267    if (os.path.exists(path + "/HEAD")
 268        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 269        return True;
 270    return False
 271
 272def parseRevision(ref):
 273    return read_pipe("git rev-parse %s" % ref).strip()
 274
 275def extractLogMessageFromGitCommit(commit):
 276    logMessage = ""
 277
 278    ## fixme: title is first line of commit, not 1st paragraph.
 279    foundTitle = False
 280    for log in read_pipe_lines("git cat-file commit %s" % commit):
 281       if not foundTitle:
 282           if len(log) == 1:
 283               foundTitle = True
 284           continue
 285
 286       logMessage += log
 287    return logMessage
 288
 289def extractSettingsGitLog(log):
 290    values = {}
 291    for line in log.split("\n"):
 292        line = line.strip()
 293        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 294        if not m:
 295            continue
 296
 297        assignments = m.group(1).split (':')
 298        for a in assignments:
 299            vals = a.split ('=')
 300            key = vals[0].strip()
 301            val = ('='.join (vals[1:])).strip()
 302            if val.endswith ('\"') and val.startswith('"'):
 303                val = val[1:-1]
 304
 305            values[key] = val
 306
 307    paths = values.get("depot-paths")
 308    if not paths:
 309        paths = values.get("depot-path")
 310    if paths:
 311        values['depot-paths'] = paths.split(',')
 312    return values
 313
 314def gitBranchExists(branch):
 315    proc = subprocess.Popen(["git", "rev-parse", branch],
 316                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 317    return proc.wait() == 0;
 318
 319def gitConfig(key):
 320    return read_pipe("git config %s" % key, ignore_error=True).strip()
 321
 322def p4BranchesInGit(branchesAreInRemotes = True):
 323    branches = {}
 324
 325    cmdline = "git rev-parse --symbolic "
 326    if branchesAreInRemotes:
 327        cmdline += " --remotes"
 328    else:
 329        cmdline += " --branches"
 330
 331    for line in read_pipe_lines(cmdline):
 332        line = line.strip()
 333
 334        ## only import to p4/
 335        if not line.startswith('p4/') or line == "p4/HEAD":
 336            continue
 337        branch = line
 338
 339        # strip off p4
 340        branch = re.sub ("^p4/", "", line)
 341
 342        branches[branch] = parseRevision(line)
 343    return branches
 344
 345def findUpstreamBranchPoint(head = "HEAD"):
 346    branches = p4BranchesInGit()
 347    # map from depot-path to branch name
 348    branchByDepotPath = {}
 349    for branch in branches.keys():
 350        tip = branches[branch]
 351        log = extractLogMessageFromGitCommit(tip)
 352        settings = extractSettingsGitLog(log)
 353        if settings.has_key("depot-paths"):
 354            paths = ",".join(settings["depot-paths"])
 355            branchByDepotPath[paths] = "remotes/p4/" + branch
 356
 357    settings = None
 358    parent = 0
 359    while parent < 65535:
 360        commit = head + "~%s" % parent
 361        log = extractLogMessageFromGitCommit(commit)
 362        settings = extractSettingsGitLog(log)
 363        if settings.has_key("depot-paths"):
 364            paths = ",".join(settings["depot-paths"])
 365            if branchByDepotPath.has_key(paths):
 366                return [branchByDepotPath[paths], settings]
 367
 368        parent = parent + 1
 369
 370    return ["", settings]
 371
 372def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 373    if not silent:
 374        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 375               % localRefPrefix)
 376
 377    originPrefix = "origin/p4/"
 378
 379    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 380        line = line.strip()
 381        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 382            continue
 383
 384        headName = line[len(originPrefix):]
 385        remoteHead = localRefPrefix + headName
 386        originHead = line
 387
 388        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 389        if (not original.has_key('depot-paths')
 390            or not original.has_key('change')):
 391            continue
 392
 393        update = False
 394        if not gitBranchExists(remoteHead):
 395            if verbose:
 396                print "creating %s" % remoteHead
 397            update = True
 398        else:
 399            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 400            if settings.has_key('change') > 0:
 401                if settings['depot-paths'] == original['depot-paths']:
 402                    originP4Change = int(original['change'])
 403                    p4Change = int(settings['change'])
 404                    if originP4Change > p4Change:
 405                        print ("%s (%s) is newer than %s (%s). "
 406                               "Updating p4 branch from origin."
 407                               % (originHead, originP4Change,
 408                                  remoteHead, p4Change))
 409                        update = True
 410                else:
 411                    print ("Ignoring: %s was imported from %s while "
 412                           "%s was imported from %s"
 413                           % (originHead, ','.join(original['depot-paths']),
 414                              remoteHead, ','.join(settings['depot-paths'])))
 415
 416        if update:
 417            system("git update-ref %s %s" % (remoteHead, originHead))
 418
 419def originP4BranchesExist():
 420        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 421
 422def p4ChangesForPaths(depotPaths, changeRange):
 423    assert depotPaths
 424    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 425                                                        for p in depotPaths]))
 426
 427    changes = []
 428    for line in output:
 429        changeNum = line.split(" ")[1]
 430        changes.append(int(changeNum))
 431
 432    changes.sort()
 433    return changes
 434
 435class Command:
 436    def __init__(self):
 437        self.usage = "usage: %prog [options]"
 438        self.needsGit = True
 439
 440class P4Debug(Command):
 441    def __init__(self):
 442        Command.__init__(self)
 443        self.options = [
 444            optparse.make_option("--verbose", dest="verbose", action="store_true",
 445                                 default=False),
 446            ]
 447        self.description = "A tool to debug the output of p4 -G."
 448        self.needsGit = False
 449        self.verbose = False
 450
 451    def run(self, args):
 452        j = 0
 453        for output in p4CmdList(" ".join(args)):
 454            print 'Element: %d' % j
 455            j += 1
 456            print output
 457        return True
 458
 459class P4RollBack(Command):
 460    def __init__(self):
 461        Command.__init__(self)
 462        self.options = [
 463            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 464            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 465        ]
 466        self.description = "A tool to debug the multi-branch import. Don't use :)"
 467        self.verbose = False
 468        self.rollbackLocalBranches = False
 469
 470    def run(self, args):
 471        if len(args) != 1:
 472            return False
 473        maxChange = int(args[0])
 474
 475        if "p4ExitCode" in p4Cmd("changes -m 1"):
 476            die("Problems executing p4");
 477
 478        if self.rollbackLocalBranches:
 479            refPrefix = "refs/heads/"
 480            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 481        else:
 482            refPrefix = "refs/remotes/"
 483            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 484
 485        for line in lines:
 486            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 487                line = line.strip()
 488                ref = refPrefix + line
 489                log = extractLogMessageFromGitCommit(ref)
 490                settings = extractSettingsGitLog(log)
 491
 492                depotPaths = settings['depot-paths']
 493                change = settings['change']
 494
 495                changed = False
 496
 497                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 498                                                           for p in depotPaths]))) == 0:
 499                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 500                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 501                    continue
 502
 503                while change and int(change) > maxChange:
 504                    changed = True
 505                    if self.verbose:
 506                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 507                    system("git update-ref %s \"%s^\"" % (ref, ref))
 508                    log = extractLogMessageFromGitCommit(ref)
 509                    settings =  extractSettingsGitLog(log)
 510
 511
 512                    depotPaths = settings['depot-paths']
 513                    change = settings['change']
 514
 515                if changed:
 516                    print "%s rewound to %s" % (ref, change)
 517
 518        return True
 519
 520class P4Submit(Command):
 521    def __init__(self):
 522        Command.__init__(self)
 523        self.options = [
 524                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 525                optparse.make_option("--origin", dest="origin"),
 526                optparse.make_option("-M", dest="detectRename", action="store_true"),
 527        ]
 528        self.description = "Submit changes from git to the perforce depot."
 529        self.usage += " [name of git branch to submit into perforce depot]"
 530        self.interactive = True
 531        self.origin = ""
 532        self.detectRename = False
 533        self.verbose = False
 534        self.isWindows = (platform.system() == "Windows")
 535
 536    def check(self):
 537        if len(p4CmdList("opened ...")) > 0:
 538            die("You have files opened with perforce! Close them before starting the sync.")
 539
 540    # replaces everything between 'Description:' and the next P4 submit template field with the
 541    # commit message
 542    def prepareLogMessage(self, template, message):
 543        result = ""
 544
 545        inDescriptionSection = False
 546
 547        for line in template.split("\n"):
 548            if line.startswith("#"):
 549                result += line + "\n"
 550                continue
 551
 552            if inDescriptionSection:
 553                if line.startswith("Files:"):
 554                    inDescriptionSection = False
 555                else:
 556                    continue
 557            else:
 558                if line.startswith("Description:"):
 559                    inDescriptionSection = True
 560                    line += "\n"
 561                    for messageLine in message.split("\n"):
 562                        line += "\t" + messageLine + "\n"
 563
 564            result += line + "\n"
 565
 566        return result
 567
 568    def prepareSubmitTemplate(self):
 569        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 570        template = ""
 571        inFilesSection = False
 572        for line in p4_read_pipe_lines("change -o"):
 573            if line.endswith("\r\n"):
 574                line = line[:-2] + "\n"
 575            if inFilesSection:
 576                if line.startswith("\t"):
 577                    # path starts and ends with a tab
 578                    path = line[1:]
 579                    lastTab = path.rfind("\t")
 580                    if lastTab != -1:
 581                        path = path[:lastTab]
 582                        if not path.startswith(self.depotPath):
 583                            continue
 584                else:
 585                    inFilesSection = False
 586            else:
 587                if line.startswith("Files:"):
 588                    inFilesSection = True
 589
 590            template += line
 591
 592        return template
 593
 594    def applyCommit(self, id):
 595        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 596        diffOpts = ("", "-M")[self.detectRename]
 597        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 598        filesToAdd = set()
 599        filesToDelete = set()
 600        editedFiles = set()
 601        filesToChangeExecBit = {}
 602        for line in diff:
 603            diff = parseDiffTreeEntry(line)
 604            modifier = diff['status']
 605            path = diff['src']
 606            if modifier == "M":
 607                p4_system("edit \"%s\"" % path)
 608                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 609                    filesToChangeExecBit[path] = diff['dst_mode']
 610                editedFiles.add(path)
 611            elif modifier == "A":
 612                filesToAdd.add(path)
 613                filesToChangeExecBit[path] = diff['dst_mode']
 614                if path in filesToDelete:
 615                    filesToDelete.remove(path)
 616            elif modifier == "D":
 617                filesToDelete.add(path)
 618                if path in filesToAdd:
 619                    filesToAdd.remove(path)
 620            elif modifier == "R":
 621                src, dest = diff['src'], diff['dst']
 622                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 623                p4_system("edit \"%s\"" % (dest))
 624                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 625                    filesToChangeExecBit[dest] = diff['dst_mode']
 626                os.unlink(dest)
 627                editedFiles.add(dest)
 628                filesToDelete.add(src)
 629            else:
 630                die("unknown modifier %s for %s" % (modifier, path))
 631
 632        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 633        patchcmd = diffcmd + " | git apply "
 634        tryPatchCmd = patchcmd + "--check -"
 635        applyPatchCmd = patchcmd + "--check --apply -"
 636
 637        if os.system(tryPatchCmd) != 0:
 638            print "Unfortunately applying the change failed!"
 639            print "What do you want to do?"
 640            response = "x"
 641            while response != "s" and response != "a" and response != "w":
 642                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 643                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 644            if response == "s":
 645                print "Skipping! Good luck with the next patches..."
 646                for f in editedFiles:
 647                    p4_system("revert \"%s\"" % f);
 648                for f in filesToAdd:
 649                    system("rm %s" %f)
 650                return
 651            elif response == "a":
 652                os.system(applyPatchCmd)
 653                if len(filesToAdd) > 0:
 654                    print "You may also want to call p4 add on the following files:"
 655                    print " ".join(filesToAdd)
 656                if len(filesToDelete):
 657                    print "The following files should be scheduled for deletion with p4 delete:"
 658                    print " ".join(filesToDelete)
 659                die("Please resolve and submit the conflict manually and "
 660                    + "continue afterwards with git-p4 submit --continue")
 661            elif response == "w":
 662                system(diffcmd + " > patch.txt")
 663                print "Patch saved to patch.txt in %s !" % self.clientPath
 664                die("Please resolve and submit the conflict manually and "
 665                    "continue afterwards with git-p4 submit --continue")
 666
 667        system(applyPatchCmd)
 668
 669        for f in filesToAdd:
 670            p4_system("add \"%s\"" % f)
 671        for f in filesToDelete:
 672            p4_system("revert \"%s\"" % f)
 673            p4_system("delete \"%s\"" % f)
 674
 675        # Set/clear executable bits
 676        for f in filesToChangeExecBit.keys():
 677            mode = filesToChangeExecBit[f]
 678            setP4ExecBit(f, mode)
 679
 680        logMessage = extractLogMessageFromGitCommit(id)
 681        logMessage = logMessage.strip()
 682
 683        template = self.prepareSubmitTemplate()
 684
 685        if self.interactive:
 686            submitTemplate = self.prepareLogMessage(template, logMessage)
 687            if os.environ.has_key("P4DIFF"):
 688                del(os.environ["P4DIFF"])
 689            diff = p4_read_pipe("diff -du ...")
 690
 691            newdiff = ""
 692            for newFile in filesToAdd:
 693                newdiff += "==== new file ====\n"
 694                newdiff += "--- /dev/null\n"
 695                newdiff += "+++ %s\n" % newFile
 696                f = open(newFile, "r")
 697                for line in f.readlines():
 698                    newdiff += "+" + line
 699                f.close()
 700
 701            separatorLine = "######## everything below this line is just the diff #######\n"
 702
 703            [handle, fileName] = tempfile.mkstemp()
 704            tmpFile = os.fdopen(handle, "w+")
 705            if self.isWindows:
 706                submitTemplate = submitTemplate.replace("\n", "\r\n")
 707                separatorLine = separatorLine.replace("\n", "\r\n")
 708                newdiff = newdiff.replace("\n", "\r\n")
 709            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 710            tmpFile.close()
 711            mtime = os.stat(fileName).st_mtime
 712            defaultEditor = "vi"
 713            if platform.system() == "Windows":
 714                defaultEditor = "notepad"
 715            if os.environ.has_key("P4EDITOR"):
 716                editor = os.environ.get("P4EDITOR")
 717            else:
 718                editor = os.environ.get("EDITOR", defaultEditor);
 719            system(editor + " " + fileName)
 720
 721            response = "y"
 722            if os.stat(fileName).st_mtime <= mtime:
 723                response = "x"
 724                while response != "y" and response != "n":
 725                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 726
 727            if response == "y":
 728                tmpFile = open(fileName, "rb")
 729                message = tmpFile.read()
 730                tmpFile.close()
 731                submitTemplate = message[:message.index(separatorLine)]
 732                if self.isWindows:
 733                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 734                p4_write_pipe("submit -i", submitTemplate)
 735            else:
 736                for f in editedFiles:
 737                    p4_system("revert \"%s\"" % f);
 738                for f in filesToAdd:
 739                    p4_system("revert \"%s\"" % f);
 740                    system("rm %s" %f)
 741
 742            os.remove(fileName)
 743        else:
 744            fileName = "submit.txt"
 745            file = open(fileName, "w+")
 746            file.write(self.prepareLogMessage(template, logMessage))
 747            file.close()
 748            print ("Perforce submit template written as %s. "
 749                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 750                   % (fileName, fileName))
 751
 752    def run(self, args):
 753        if len(args) == 0:
 754            self.master = currentGitBranch()
 755            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 756                die("Detecting current git branch failed!")
 757        elif len(args) == 1:
 758            self.master = args[0]
 759        else:
 760            return False
 761
 762        allowSubmit = gitConfig("git-p4.allowSubmit")
 763        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 764            die("%s is not in git-p4.allowSubmit" % self.master)
 765
 766        [upstream, settings] = findUpstreamBranchPoint()
 767        self.depotPath = settings['depot-paths'][0]
 768        if len(self.origin) == 0:
 769            self.origin = upstream
 770
 771        if self.verbose:
 772            print "Origin branch is " + self.origin
 773
 774        if len(self.depotPath) == 0:
 775            print "Internal error: cannot locate perforce depot path from existing branches"
 776            sys.exit(128)
 777
 778        self.clientPath = p4Where(self.depotPath)
 779
 780        if len(self.clientPath) == 0:
 781            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 782            sys.exit(128)
 783
 784        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 785        self.oldWorkingDirectory = os.getcwd()
 786
 787        chdir(self.clientPath)
 788        print "Syncronizing p4 checkout..."
 789        p4_system("sync ...")
 790
 791        self.check()
 792
 793        commits = []
 794        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 795            commits.append(line.strip())
 796        commits.reverse()
 797
 798        while len(commits) > 0:
 799            commit = commits[0]
 800            commits = commits[1:]
 801            self.applyCommit(commit)
 802            if not self.interactive:
 803                break
 804
 805        if len(commits) == 0:
 806            print "All changes applied!"
 807            chdir(self.oldWorkingDirectory)
 808
 809            sync = P4Sync()
 810            sync.run([])
 811
 812            rebase = P4Rebase()
 813            rebase.rebase()
 814
 815        return True
 816
 817class P4Sync(Command):
 818    def __init__(self):
 819        Command.__init__(self)
 820        self.options = [
 821                optparse.make_option("--branch", dest="branch"),
 822                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 823                optparse.make_option("--changesfile", dest="changesFile"),
 824                optparse.make_option("--silent", dest="silent", action="store_true"),
 825                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 826                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 827                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 828                                     help="Import into refs/heads/ , not refs/remotes"),
 829                optparse.make_option("--max-changes", dest="maxChanges"),
 830                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 831                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 832                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 833                                     help="Only sync files that are included in the Perforce Client Spec")
 834        ]
 835        self.description = """Imports from Perforce into a git repository.\n
 836    example:
 837    //depot/my/project/ -- to import the current head
 838    //depot/my/project/@all -- to import everything
 839    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 840
 841    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 842
 843        self.usage += " //depot/path[@revRange]"
 844        self.silent = False
 845        self.createdBranches = Set()
 846        self.committedChanges = Set()
 847        self.branch = ""
 848        self.detectBranches = False
 849        self.detectLabels = False
 850        self.changesFile = ""
 851        self.syncWithOrigin = True
 852        self.verbose = False
 853        self.importIntoRemotes = True
 854        self.maxChanges = ""
 855        self.isWindows = (platform.system() == "Windows")
 856        self.keepRepoPath = False
 857        self.depotPaths = None
 858        self.p4BranchesInGit = []
 859        self.cloneExclude = []
 860        self.useClientSpec = False
 861        self.clientSpecDirs = []
 862
 863        if gitConfig("git-p4.syncFromOrigin") == "false":
 864            self.syncWithOrigin = False
 865
 866    def extractFilesFromCommit(self, commit):
 867        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 868                             for path in self.cloneExclude]
 869        files = []
 870        fnum = 0
 871        while commit.has_key("depotFile%s" % fnum):
 872            path =  commit["depotFile%s" % fnum]
 873
 874            if [p for p in self.cloneExclude
 875                if path.startswith (p)]:
 876                found = False
 877            else:
 878                found = [p for p in self.depotPaths
 879                         if path.startswith (p)]
 880            if not found:
 881                fnum = fnum + 1
 882                continue
 883
 884            file = {}
 885            file["path"] = path
 886            file["rev"] = commit["rev%s" % fnum]
 887            file["action"] = commit["action%s" % fnum]
 888            file["type"] = commit["type%s" % fnum]
 889            files.append(file)
 890            fnum = fnum + 1
 891        return files
 892
 893    def stripRepoPath(self, path, prefixes):
 894        if self.keepRepoPath:
 895            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 896
 897        for p in prefixes:
 898            if path.startswith(p):
 899                path = path[len(p):]
 900
 901        return path
 902
 903    def splitFilesIntoBranches(self, commit):
 904        branches = {}
 905        fnum = 0
 906        while commit.has_key("depotFile%s" % fnum):
 907            path =  commit["depotFile%s" % fnum]
 908            found = [p for p in self.depotPaths
 909                     if path.startswith (p)]
 910            if not found:
 911                fnum = fnum + 1
 912                continue
 913
 914            file = {}
 915            file["path"] = path
 916            file["rev"] = commit["rev%s" % fnum]
 917            file["action"] = commit["action%s" % fnum]
 918            file["type"] = commit["type%s" % fnum]
 919            fnum = fnum + 1
 920
 921            relPath = self.stripRepoPath(path, self.depotPaths)
 922
 923            for branch in self.knownBranches.keys():
 924
 925                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 926                if relPath.startswith(branch + "/"):
 927                    if branch not in branches:
 928                        branches[branch] = []
 929                    branches[branch].append(file)
 930                    break
 931
 932        return branches
 933
 934    ## Should move this out, doesn't use SELF.
 935    def readP4Files(self, files):
 936        filesForCommit = []
 937        filesToRead = []
 938
 939        for f in files:
 940            includeFile = True
 941            for val in self.clientSpecDirs:
 942                if f['path'].startswith(val[0]):
 943                    if val[1] <= 0:
 944                        includeFile = False
 945                    break
 946
 947            if includeFile:
 948                filesForCommit.append(f)
 949                if f['action'] != 'delete':
 950                    filesToRead.append(f)
 951
 952        filedata = []
 953        if len(filesToRead) > 0:
 954            filedata = p4CmdList('-x - print',
 955                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 956                                                  for f in filesToRead]),
 957                                 stdin_mode='w+')
 958
 959            if "p4ExitCode" in filedata[0]:
 960                die("Problems executing p4. Error: [%d]."
 961                    % (filedata[0]['p4ExitCode']));
 962
 963        j = 0;
 964        contents = {}
 965        while j < len(filedata):
 966            stat = filedata[j]
 967            j += 1
 968            text = [];
 969            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 970                text.append(filedata[j]['data'])
 971                j += 1
 972            text = ''.join(text)
 973
 974            if not stat.has_key('depotFile'):
 975                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 976                continue
 977
 978            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 979                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 980            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 981                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 982
 983            contents[stat['depotFile']] = text
 984
 985        for f in filesForCommit:
 986            path = f['path']
 987            if contents.has_key(path):
 988                f['data'] = contents[path]
 989
 990        return filesForCommit
 991
 992    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 993        epoch = details["time"]
 994        author = details["user"]
 995
 996        if self.verbose:
 997            print "commit into %s" % branch
 998
 999        # start with reading files; if that fails, we should not
1000        # create a commit.
1001        new_files = []
1002        for f in files:
1003            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1004                new_files.append (f)
1005            else:
1006                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1007        files = self.readP4Files(new_files)
1008
1009        self.gitStream.write("commit %s\n" % branch)
1010#        gitStream.write("mark :%s\n" % details["change"])
1011        self.committedChanges.add(int(details["change"]))
1012        committer = ""
1013        if author not in self.users:
1014            self.getUserMapFromPerforceServer()
1015        if author in self.users:
1016            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1017        else:
1018            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1019
1020        self.gitStream.write("committer %s\n" % committer)
1021
1022        self.gitStream.write("data <<EOT\n")
1023        self.gitStream.write(details["desc"])
1024        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1025                             % (','.join (branchPrefixes), details["change"]))
1026        if len(details['options']) > 0:
1027            self.gitStream.write(": options = %s" % details['options'])
1028        self.gitStream.write("]\nEOT\n\n")
1029
1030        if len(parent) > 0:
1031            if self.verbose:
1032                print "parent %s" % parent
1033            self.gitStream.write("from %s\n" % parent)
1034
1035        for file in files:
1036            if file["type"] == "apple":
1037                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1038                continue
1039
1040            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1041            if file["action"] == "delete":
1042                self.gitStream.write("D %s\n" % relPath)
1043            else:
1044                data = file['data']
1045
1046                mode = "644"
1047                if isP4Exec(file["type"]):
1048                    mode = "755"
1049                elif file["type"] == "symlink":
1050                    mode = "120000"
1051                    # p4 print on a symlink contains "target\n", so strip it off
1052                    data = data[:-1]
1053
1054                if self.isWindows and file["type"].endswith("text"):
1055                    data = data.replace("\r\n", "\n")
1056
1057                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1058                self.gitStream.write("data %s\n" % len(data))
1059                self.gitStream.write(data)
1060                self.gitStream.write("\n")
1061
1062        self.gitStream.write("\n")
1063
1064        change = int(details["change"])
1065
1066        if self.labels.has_key(change):
1067            label = self.labels[change]
1068            labelDetails = label[0]
1069            labelRevisions = label[1]
1070            if self.verbose:
1071                print "Change %s is labelled %s" % (change, labelDetails)
1072
1073            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1074                                                    for p in branchPrefixes]))
1075
1076            if len(files) == len(labelRevisions):
1077
1078                cleanedFiles = {}
1079                for info in files:
1080                    if info["action"] == "delete":
1081                        continue
1082                    cleanedFiles[info["depotFile"]] = info["rev"]
1083
1084                if cleanedFiles == labelRevisions:
1085                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1086                    self.gitStream.write("from %s\n" % branch)
1087
1088                    owner = labelDetails["Owner"]
1089                    tagger = ""
1090                    if author in self.users:
1091                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1092                    else:
1093                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1094                    self.gitStream.write("tagger %s\n" % tagger)
1095                    self.gitStream.write("data <<EOT\n")
1096                    self.gitStream.write(labelDetails["Description"])
1097                    self.gitStream.write("EOT\n\n")
1098
1099                else:
1100                    if not self.silent:
1101                        print ("Tag %s does not match with change %s: files do not match."
1102                               % (labelDetails["label"], change))
1103
1104            else:
1105                if not self.silent:
1106                    print ("Tag %s does not match with change %s: file count is different."
1107                           % (labelDetails["label"], change))
1108
1109    def getUserCacheFilename(self):
1110        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1111        return home + "/.gitp4-usercache.txt"
1112
1113    def getUserMapFromPerforceServer(self):
1114        if self.userMapFromPerforceServer:
1115            return
1116        self.users = {}
1117
1118        for output in p4CmdList("users"):
1119            if not output.has_key("User"):
1120                continue
1121            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1122
1123
1124        s = ''
1125        for (key, val) in self.users.items():
1126            s += "%s\t%s\n" % (key, val)
1127
1128        open(self.getUserCacheFilename(), "wb").write(s)
1129        self.userMapFromPerforceServer = True
1130
1131    def loadUserMapFromCache(self):
1132        self.users = {}
1133        self.userMapFromPerforceServer = False
1134        try:
1135            cache = open(self.getUserCacheFilename(), "rb")
1136            lines = cache.readlines()
1137            cache.close()
1138            for line in lines:
1139                entry = line.strip().split("\t")
1140                self.users[entry[0]] = entry[1]
1141        except IOError:
1142            self.getUserMapFromPerforceServer()
1143
1144    def getLabels(self):
1145        self.labels = {}
1146
1147        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1148        if len(l) > 0 and not self.silent:
1149            print "Finding files belonging to labels in %s" % `self.depotPaths`
1150
1151        for output in l:
1152            label = output["label"]
1153            revisions = {}
1154            newestChange = 0
1155            if self.verbose:
1156                print "Querying files for label %s" % label
1157            for file in p4CmdList("files "
1158                                  +  ' '.join (["%s...@%s" % (p, label)
1159                                                for p in self.depotPaths])):
1160                revisions[file["depotFile"]] = file["rev"]
1161                change = int(file["change"])
1162                if change > newestChange:
1163                    newestChange = change
1164
1165            self.labels[newestChange] = [output, revisions]
1166
1167        if self.verbose:
1168            print "Label changes: %s" % self.labels.keys()
1169
1170    def guessProjectName(self):
1171        for p in self.depotPaths:
1172            if p.endswith("/"):
1173                p = p[:-1]
1174            p = p[p.strip().rfind("/") + 1:]
1175            if not p.endswith("/"):
1176               p += "/"
1177            return p
1178
1179    def getBranchMapping(self):
1180        lostAndFoundBranches = set()
1181
1182        for info in p4CmdList("branches"):
1183            details = p4Cmd("branch -o %s" % info["branch"])
1184            viewIdx = 0
1185            while details.has_key("View%s" % viewIdx):
1186                paths = details["View%s" % viewIdx].split(" ")
1187                viewIdx = viewIdx + 1
1188                # require standard //depot/foo/... //depot/bar/... mapping
1189                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1190                    continue
1191                source = paths[0]
1192                destination = paths[1]
1193                ## HACK
1194                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1195                    source = source[len(self.depotPaths[0]):-4]
1196                    destination = destination[len(self.depotPaths[0]):-4]
1197
1198                    if destination in self.knownBranches:
1199                        if not self.silent:
1200                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1201                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1202                        continue
1203
1204                    self.knownBranches[destination] = source
1205
1206                    lostAndFoundBranches.discard(destination)
1207
1208                    if source not in self.knownBranches:
1209                        lostAndFoundBranches.add(source)
1210
1211
1212        for branch in lostAndFoundBranches:
1213            self.knownBranches[branch] = branch
1214
1215    def getBranchMappingFromGitBranches(self):
1216        branches = p4BranchesInGit(self.importIntoRemotes)
1217        for branch in branches.keys():
1218            if branch == "master":
1219                branch = "main"
1220            else:
1221                branch = branch[len(self.projectName):]
1222            self.knownBranches[branch] = branch
1223
1224    def listExistingP4GitBranches(self):
1225        # branches holds mapping from name to commit
1226        branches = p4BranchesInGit(self.importIntoRemotes)
1227        self.p4BranchesInGit = branches.keys()
1228        for branch in branches.keys():
1229            self.initialParents[self.refPrefix + branch] = branches[branch]
1230
1231    def updateOptionDict(self, d):
1232        option_keys = {}
1233        if self.keepRepoPath:
1234            option_keys['keepRepoPath'] = 1
1235
1236        d["options"] = ' '.join(sorted(option_keys.keys()))
1237
1238    def readOptions(self, d):
1239        self.keepRepoPath = (d.has_key('options')
1240                             and ('keepRepoPath' in d['options']))
1241
1242    def gitRefForBranch(self, branch):
1243        if branch == "main":
1244            return self.refPrefix + "master"
1245
1246        if len(branch) <= 0:
1247            return branch
1248
1249        return self.refPrefix + self.projectName + branch
1250
1251    def gitCommitByP4Change(self, ref, change):
1252        if self.verbose:
1253            print "looking in ref " + ref + " for change %s using bisect..." % change
1254
1255        earliestCommit = ""
1256        latestCommit = parseRevision(ref)
1257
1258        while True:
1259            if self.verbose:
1260                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1261            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1262            if len(next) == 0:
1263                if self.verbose:
1264                    print "argh"
1265                return ""
1266            log = extractLogMessageFromGitCommit(next)
1267            settings = extractSettingsGitLog(log)
1268            currentChange = int(settings['change'])
1269            if self.verbose:
1270                print "current change %s" % currentChange
1271
1272            if currentChange == change:
1273                if self.verbose:
1274                    print "found %s" % next
1275                return next
1276
1277            if currentChange < change:
1278                earliestCommit = "^%s" % next
1279            else:
1280                latestCommit = "%s" % next
1281
1282        return ""
1283
1284    def importNewBranch(self, branch, maxChange):
1285        # make fast-import flush all changes to disk and update the refs using the checkpoint
1286        # command so that we can try to find the branch parent in the git history
1287        self.gitStream.write("checkpoint\n\n");
1288        self.gitStream.flush();
1289        branchPrefix = self.depotPaths[0] + branch + "/"
1290        range = "@1,%s" % maxChange
1291        #print "prefix" + branchPrefix
1292        changes = p4ChangesForPaths([branchPrefix], range)
1293        if len(changes) <= 0:
1294            return False
1295        firstChange = changes[0]
1296        #print "first change in branch: %s" % firstChange
1297        sourceBranch = self.knownBranches[branch]
1298        sourceDepotPath = self.depotPaths[0] + sourceBranch
1299        sourceRef = self.gitRefForBranch(sourceBranch)
1300        #print "source " + sourceBranch
1301
1302        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1303        #print "branch parent: %s" % branchParentChange
1304        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1305        if len(gitParent) > 0:
1306            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1307            #print "parent git commit: %s" % gitParent
1308
1309        self.importChanges(changes)
1310        return True
1311
1312    def importChanges(self, changes):
1313        cnt = 1
1314        for change in changes:
1315            description = p4Cmd("describe %s" % change)
1316            self.updateOptionDict(description)
1317
1318            if not self.silent:
1319                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1320                sys.stdout.flush()
1321            cnt = cnt + 1
1322
1323            try:
1324                if self.detectBranches:
1325                    branches = self.splitFilesIntoBranches(description)
1326                    for branch in branches.keys():
1327                        ## HACK  --hwn
1328                        branchPrefix = self.depotPaths[0] + branch + "/"
1329
1330                        parent = ""
1331
1332                        filesForCommit = branches[branch]
1333
1334                        if self.verbose:
1335                            print "branch is %s" % branch
1336
1337                        self.updatedBranches.add(branch)
1338
1339                        if branch not in self.createdBranches:
1340                            self.createdBranches.add(branch)
1341                            parent = self.knownBranches[branch]
1342                            if parent == branch:
1343                                parent = ""
1344                            else:
1345                                fullBranch = self.projectName + branch
1346                                if fullBranch not in self.p4BranchesInGit:
1347                                    if not self.silent:
1348                                        print("\n    Importing new branch %s" % fullBranch);
1349                                    if self.importNewBranch(branch, change - 1):
1350                                        parent = ""
1351                                        self.p4BranchesInGit.append(fullBranch)
1352                                    if not self.silent:
1353                                        print("\n    Resuming with change %s" % change);
1354
1355                                if self.verbose:
1356                                    print "parent determined through known branches: %s" % parent
1357
1358                        branch = self.gitRefForBranch(branch)
1359                        parent = self.gitRefForBranch(parent)
1360
1361                        if self.verbose:
1362                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1363
1364                        if len(parent) == 0 and branch in self.initialParents:
1365                            parent = self.initialParents[branch]
1366                            del self.initialParents[branch]
1367
1368                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1369                else:
1370                    files = self.extractFilesFromCommit(description)
1371                    self.commit(description, files, self.branch, self.depotPaths,
1372                                self.initialParent)
1373                    self.initialParent = ""
1374            except IOError:
1375                print self.gitError.read()
1376                sys.exit(1)
1377
1378    def importHeadRevision(self, revision):
1379        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1380
1381        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1382        details["desc"] = ("Initial import of %s from the state at revision %s"
1383                           % (' '.join(self.depotPaths), revision))
1384        details["change"] = revision
1385        newestRevision = 0
1386
1387        fileCnt = 0
1388        for info in p4CmdList("files "
1389                              +  ' '.join(["%s...%s"
1390                                           % (p, revision)
1391                                           for p in self.depotPaths])):
1392
1393            if info['code'] == 'error':
1394                sys.stderr.write("p4 returned an error: %s\n"
1395                                 % info['data'])
1396                sys.exit(1)
1397
1398
1399            change = int(info["change"])
1400            if change > newestRevision:
1401                newestRevision = change
1402
1403            if info["action"] == "delete":
1404                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1405                #fileCnt = fileCnt + 1
1406                continue
1407
1408            for prop in ["depotFile", "rev", "action", "type" ]:
1409                details["%s%s" % (prop, fileCnt)] = info[prop]
1410
1411            fileCnt = fileCnt + 1
1412
1413        details["change"] = newestRevision
1414        self.updateOptionDict(details)
1415        try:
1416            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1417        except IOError:
1418            print "IO error with git fast-import. Is your git version recent enough?"
1419            print self.gitError.read()
1420
1421
1422    def getClientSpec(self):
1423        specList = p4CmdList( "client -o" )
1424        temp = {}
1425        for entry in specList:
1426            for k,v in entry.iteritems():
1427                if k.startswith("View"):
1428                    if v.startswith('"'):
1429                        start = 1
1430                    else:
1431                        start = 0
1432                    index = v.find("...")
1433                    v = v[start:index]
1434                    if v.startswith("-"):
1435                        v = v[1:]
1436                        temp[v] = -len(v)
1437                    else:
1438                        temp[v] = len(v)
1439        self.clientSpecDirs = temp.items()
1440        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1441
1442    def run(self, args):
1443        self.depotPaths = []
1444        self.changeRange = ""
1445        self.initialParent = ""
1446        self.previousDepotPaths = []
1447
1448        # map from branch depot path to parent branch
1449        self.knownBranches = {}
1450        self.initialParents = {}
1451        self.hasOrigin = originP4BranchesExist()
1452        if not self.syncWithOrigin:
1453            self.hasOrigin = False
1454
1455        if self.importIntoRemotes:
1456            self.refPrefix = "refs/remotes/p4/"
1457        else:
1458            self.refPrefix = "refs/heads/p4/"
1459
1460        if self.syncWithOrigin and self.hasOrigin:
1461            if not self.silent:
1462                print "Syncing with origin first by calling git fetch origin"
1463            system("git fetch origin")
1464
1465        if len(self.branch) == 0:
1466            self.branch = self.refPrefix + "master"
1467            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1468                system("git update-ref %s refs/heads/p4" % self.branch)
1469                system("git branch -D p4");
1470            # create it /after/ importing, when master exists
1471            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1472                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1473
1474        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1475            self.getClientSpec()
1476
1477        # TODO: should always look at previous commits,
1478        # merge with previous imports, if possible.
1479        if args == []:
1480            if self.hasOrigin:
1481                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1482            self.listExistingP4GitBranches()
1483
1484            if len(self.p4BranchesInGit) > 1:
1485                if not self.silent:
1486                    print "Importing from/into multiple branches"
1487                self.detectBranches = True
1488
1489            if self.verbose:
1490                print "branches: %s" % self.p4BranchesInGit
1491
1492            p4Change = 0
1493            for branch in self.p4BranchesInGit:
1494                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1495
1496                settings = extractSettingsGitLog(logMsg)
1497
1498                self.readOptions(settings)
1499                if (settings.has_key('depot-paths')
1500                    and settings.has_key ('change')):
1501                    change = int(settings['change']) + 1
1502                    p4Change = max(p4Change, change)
1503
1504                    depotPaths = sorted(settings['depot-paths'])
1505                    if self.previousDepotPaths == []:
1506                        self.previousDepotPaths = depotPaths
1507                    else:
1508                        paths = []
1509                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1510                            for i in range(0, min(len(cur), len(prev))):
1511                                if cur[i] <> prev[i]:
1512                                    i = i - 1
1513                                    break
1514
1515                            paths.append (cur[:i + 1])
1516
1517                        self.previousDepotPaths = paths
1518
1519            if p4Change > 0:
1520                self.depotPaths = sorted(self.previousDepotPaths)
1521                self.changeRange = "@%s,#head" % p4Change
1522                if not self.detectBranches:
1523                    self.initialParent = parseRevision(self.branch)
1524                if not self.silent and not self.detectBranches:
1525                    print "Performing incremental import into %s git branch" % self.branch
1526
1527        if not self.branch.startswith("refs/"):
1528            self.branch = "refs/heads/" + self.branch
1529
1530        if len(args) == 0 and self.depotPaths:
1531            if not self.silent:
1532                print "Depot paths: %s" % ' '.join(self.depotPaths)
1533        else:
1534            if self.depotPaths and self.depotPaths != args:
1535                print ("previous import used depot path %s and now %s was specified. "
1536                       "This doesn't work!" % (' '.join (self.depotPaths),
1537                                               ' '.join (args)))
1538                sys.exit(1)
1539
1540            self.depotPaths = sorted(args)
1541
1542        revision = ""
1543        self.users = {}
1544
1545        newPaths = []
1546        for p in self.depotPaths:
1547            if p.find("@") != -1:
1548                atIdx = p.index("@")
1549                self.changeRange = p[atIdx:]
1550                if self.changeRange == "@all":
1551                    self.changeRange = ""
1552                elif ',' not in self.changeRange:
1553                    revision = self.changeRange
1554                    self.changeRange = ""
1555                p = p[:atIdx]
1556            elif p.find("#") != -1:
1557                hashIdx = p.index("#")
1558                revision = p[hashIdx:]
1559                p = p[:hashIdx]
1560            elif self.previousDepotPaths == []:
1561                revision = "#head"
1562
1563            p = re.sub ("\.\.\.$", "", p)
1564            if not p.endswith("/"):
1565                p += "/"
1566
1567            newPaths.append(p)
1568
1569        self.depotPaths = newPaths
1570
1571
1572        self.loadUserMapFromCache()
1573        self.labels = {}
1574        if self.detectLabels:
1575            self.getLabels();
1576
1577        if self.detectBranches:
1578            ## FIXME - what's a P4 projectName ?
1579            self.projectName = self.guessProjectName()
1580
1581            if self.hasOrigin:
1582                self.getBranchMappingFromGitBranches()
1583            else:
1584                self.getBranchMapping()
1585            if self.verbose:
1586                print "p4-git branches: %s" % self.p4BranchesInGit
1587                print "initial parents: %s" % self.initialParents
1588            for b in self.p4BranchesInGit:
1589                if b != "master":
1590
1591                    ## FIXME
1592                    b = b[len(self.projectName):]
1593                self.createdBranches.add(b)
1594
1595        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1596
1597        importProcess = subprocess.Popen(["git", "fast-import"],
1598                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1599                                         stderr=subprocess.PIPE);
1600        self.gitOutput = importProcess.stdout
1601        self.gitStream = importProcess.stdin
1602        self.gitError = importProcess.stderr
1603
1604        if revision:
1605            self.importHeadRevision(revision)
1606        else:
1607            changes = []
1608
1609            if len(self.changesFile) > 0:
1610                output = open(self.changesFile).readlines()
1611                changeSet = Set()
1612                for line in output:
1613                    changeSet.add(int(line))
1614
1615                for change in changeSet:
1616                    changes.append(change)
1617
1618                changes.sort()
1619            else:
1620                if self.verbose:
1621                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1622                                                              self.changeRange)
1623                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1624
1625                if len(self.maxChanges) > 0:
1626                    changes = changes[:min(int(self.maxChanges), len(changes))]
1627
1628            if len(changes) == 0:
1629                if not self.silent:
1630                    print "No changes to import!"
1631                return True
1632
1633            if not self.silent and not self.detectBranches:
1634                print "Import destination: %s" % self.branch
1635
1636            self.updatedBranches = set()
1637
1638            self.importChanges(changes)
1639
1640            if not self.silent:
1641                print ""
1642                if len(self.updatedBranches) > 0:
1643                    sys.stdout.write("Updated branches: ")
1644                    for b in self.updatedBranches:
1645                        sys.stdout.write("%s " % b)
1646                    sys.stdout.write("\n")
1647
1648        self.gitStream.close()
1649        if importProcess.wait() != 0:
1650            die("fast-import failed: %s" % self.gitError.read())
1651        self.gitOutput.close()
1652        self.gitError.close()
1653
1654        return True
1655
1656class P4Rebase(Command):
1657    def __init__(self):
1658        Command.__init__(self)
1659        self.options = [ ]
1660        self.description = ("Fetches the latest revision from perforce and "
1661                            + "rebases the current work (branch) against it")
1662        self.verbose = False
1663
1664    def run(self, args):
1665        sync = P4Sync()
1666        sync.run([])
1667
1668        return self.rebase()
1669
1670    def rebase(self):
1671        if os.system("git update-index --refresh") != 0:
1672            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1673        if len(read_pipe("git diff-index HEAD --")) > 0:
1674            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1675
1676        [upstream, settings] = findUpstreamBranchPoint()
1677        if len(upstream) == 0:
1678            die("Cannot find upstream branchpoint for rebase")
1679
1680        # the branchpoint may be p4/foo~3, so strip off the parent
1681        upstream = re.sub("~[0-9]+$", "", upstream)
1682
1683        print "Rebasing the current branch onto %s" % upstream
1684        oldHead = read_pipe("git rev-parse HEAD").strip()
1685        system("git rebase %s" % upstream)
1686        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1687        return True
1688
1689class P4Clone(P4Sync):
1690    def __init__(self):
1691        P4Sync.__init__(self)
1692        self.description = "Creates a new git repository and imports from Perforce into it"
1693        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1694        self.options += [
1695            optparse.make_option("--destination", dest="cloneDestination",
1696                                 action='store', default=None,
1697                                 help="where to leave result of the clone"),
1698            optparse.make_option("-/", dest="cloneExclude",
1699                                 action="append", type="string",
1700                                 help="exclude depot path")
1701        ]
1702        self.cloneDestination = None
1703        self.needsGit = False
1704
1705    # This is required for the "append" cloneExclude action
1706    def ensure_value(self, attr, value):
1707        if not hasattr(self, attr) or getattr(self, attr) is None:
1708            setattr(self, attr, value)
1709        return getattr(self, attr)
1710
1711    def defaultDestination(self, args):
1712        ## TODO: use common prefix of args?
1713        depotPath = args[0]
1714        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1715        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1716        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1717        depotDir = re.sub(r"/$", "", depotDir)
1718        return os.path.split(depotDir)[1]
1719
1720    def run(self, args):
1721        if len(args) < 1:
1722            return False
1723
1724        if self.keepRepoPath and not self.cloneDestination:
1725            sys.stderr.write("Must specify destination for --keep-path\n")
1726            sys.exit(1)
1727
1728        depotPaths = args
1729
1730        if not self.cloneDestination and len(depotPaths) > 1:
1731            self.cloneDestination = depotPaths[-1]
1732            depotPaths = depotPaths[:-1]
1733
1734        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1735        for p in depotPaths:
1736            if not p.startswith("//"):
1737                return False
1738
1739        if not self.cloneDestination:
1740            self.cloneDestination = self.defaultDestination(args)
1741
1742        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1743        if not os.path.exists(self.cloneDestination):
1744            os.makedirs(self.cloneDestination)
1745        chdir(self.cloneDestination)
1746        system("git init")
1747        self.gitdir = os.getcwd() + "/.git"
1748        if not P4Sync.run(self, depotPaths):
1749            return False
1750        if self.branch != "master":
1751            if self.importIntoRemotes:
1752                masterbranch = "refs/remotes/p4/master"
1753            else:
1754                masterbranch = "refs/heads/p4/master"
1755            if gitBranchExists(masterbranch):
1756                system("git branch master %s" % masterbranch)
1757                system("git checkout -f")
1758            else:
1759                print "Could not detect main branch. No checkout/master branch created."
1760
1761        return True
1762
1763class P4Branches(Command):
1764    def __init__(self):
1765        Command.__init__(self)
1766        self.options = [ ]
1767        self.description = ("Shows the git branches that hold imports and their "
1768                            + "corresponding perforce depot paths")
1769        self.verbose = False
1770
1771    def run(self, args):
1772        if originP4BranchesExist():
1773            createOrUpdateBranchesFromOrigin()
1774
1775        cmdline = "git rev-parse --symbolic "
1776        cmdline += " --remotes"
1777
1778        for line in read_pipe_lines(cmdline):
1779            line = line.strip()
1780
1781            if not line.startswith('p4/') or line == "p4/HEAD":
1782                continue
1783            branch = line
1784
1785            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1786            settings = extractSettingsGitLog(log)
1787
1788            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1789        return True
1790
1791class HelpFormatter(optparse.IndentedHelpFormatter):
1792    def __init__(self):
1793        optparse.IndentedHelpFormatter.__init__(self)
1794
1795    def format_description(self, description):
1796        if description:
1797            return description + "\n"
1798        else:
1799            return ""
1800
1801def printUsage(commands):
1802    print "usage: %s <command> [options]" % sys.argv[0]
1803    print ""
1804    print "valid commands: %s" % ", ".join(commands)
1805    print ""
1806    print "Try %s <command> --help for command specific help." % sys.argv[0]
1807    print ""
1808
1809commands = {
1810    "debug" : P4Debug,
1811    "submit" : P4Submit,
1812    "commit" : P4Submit,
1813    "sync" : P4Sync,
1814    "rebase" : P4Rebase,
1815    "clone" : P4Clone,
1816    "rollback" : P4RollBack,
1817    "branches" : P4Branches
1818}
1819
1820
1821def main():
1822    if len(sys.argv[1:]) == 0:
1823        printUsage(commands.keys())
1824        sys.exit(2)
1825
1826    cmd = ""
1827    cmdName = sys.argv[1]
1828    try:
1829        klass = commands[cmdName]
1830        cmd = klass()
1831    except KeyError:
1832        print "unknown command %s" % cmdName
1833        print ""
1834        printUsage(commands.keys())
1835        sys.exit(2)
1836
1837    options = cmd.options
1838    cmd.gitdir = os.environ.get("GIT_DIR", None)
1839
1840    args = sys.argv[2:]
1841
1842    if len(options) > 0:
1843        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1844
1845        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1846                                       options,
1847                                       description = cmd.description,
1848                                       formatter = HelpFormatter())
1849
1850        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1851    global verbose
1852    verbose = cmd.verbose
1853    if cmd.needsGit:
1854        if cmd.gitdir == None:
1855            cmd.gitdir = os.path.abspath(".git")
1856            if not isValidGitDir(cmd.gitdir):
1857                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1858                if os.path.exists(cmd.gitdir):
1859                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1860                    if len(cdup) > 0:
1861                        chdir(cdup);
1862
1863        if not isValidGitDir(cmd.gitdir):
1864            if isValidGitDir(cmd.gitdir + "/.git"):
1865                cmd.gitdir += "/.git"
1866            else:
1867                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1868
1869        os.environ["GIT_DIR"] = cmd.gitdir
1870
1871    if not cmd.run(args):
1872        parser.print_help()
1873
1874
1875if __name__ == '__main__':
1876    main()