Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'lt/block-sha1' (106a365)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            if cb is not None:
 228                cb(entry)
 229            else:
 230                result.append(entry)
 231    except EOFError:
 232        pass
 233    exitCode = p4.wait()
 234    if exitCode != 0:
 235        entry = {}
 236        entry["p4ExitCode"] = exitCode
 237        result.append(entry)
 238
 239    return result
 240
 241def p4Cmd(cmd):
 242    list = p4CmdList(cmd)
 243    result = {}
 244    for entry in list:
 245        result.update(entry)
 246    return result;
 247
 248def p4Where(depotPath):
 249    if not depotPath.endswith("/"):
 250        depotPath += "/"
 251    depotPath = depotPath + "..."
 252    outputList = p4CmdList("where %s" % depotPath)
 253    output = None
 254    for entry in outputList:
 255        if "depotFile" in entry:
 256            if entry["depotFile"] == depotPath:
 257                output = entry
 258                break
 259        elif "data" in entry:
 260            data = entry.get("data")
 261            space = data.find(" ")
 262            if data[:space] == depotPath:
 263                output = entry
 264                break
 265    if output == None:
 266        return ""
 267    if output["code"] == "error":
 268        return ""
 269    clientPath = ""
 270    if "path" in output:
 271        clientPath = output.get("path")
 272    elif "data" in output:
 273        data = output.get("data")
 274        lastSpace = data.rfind(" ")
 275        clientPath = data[lastSpace + 1:]
 276
 277    if clientPath.endswith("..."):
 278        clientPath = clientPath[:-3]
 279    return clientPath
 280
 281def currentGitBranch():
 282    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 283
 284def isValidGitDir(path):
 285    if (os.path.exists(path + "/HEAD")
 286        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 287        return True;
 288    return False
 289
 290def parseRevision(ref):
 291    return read_pipe("git rev-parse %s" % ref).strip()
 292
 293def extractLogMessageFromGitCommit(commit):
 294    logMessage = ""
 295
 296    ## fixme: title is first line of commit, not 1st paragraph.
 297    foundTitle = False
 298    for log in read_pipe_lines("git cat-file commit %s" % commit):
 299       if not foundTitle:
 300           if len(log) == 1:
 301               foundTitle = True
 302           continue
 303
 304       logMessage += log
 305    return logMessage
 306
 307def extractSettingsGitLog(log):
 308    values = {}
 309    for line in log.split("\n"):
 310        line = line.strip()
 311        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 312        if not m:
 313            continue
 314
 315        assignments = m.group(1).split (':')
 316        for a in assignments:
 317            vals = a.split ('=')
 318            key = vals[0].strip()
 319            val = ('='.join (vals[1:])).strip()
 320            if val.endswith ('\"') and val.startswith('"'):
 321                val = val[1:-1]
 322
 323            values[key] = val
 324
 325    paths = values.get("depot-paths")
 326    if not paths:
 327        paths = values.get("depot-path")
 328    if paths:
 329        values['depot-paths'] = paths.split(',')
 330    return values
 331
 332def gitBranchExists(branch):
 333    proc = subprocess.Popen(["git", "rev-parse", branch],
 334                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 335    return proc.wait() == 0;
 336
 337_gitConfig = {}
 338def gitConfig(key):
 339    if not _gitConfig.has_key(key):
 340        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 341    return _gitConfig[key]
 342
 343def p4BranchesInGit(branchesAreInRemotes = True):
 344    branches = {}
 345
 346    cmdline = "git rev-parse --symbolic "
 347    if branchesAreInRemotes:
 348        cmdline += " --remotes"
 349    else:
 350        cmdline += " --branches"
 351
 352    for line in read_pipe_lines(cmdline):
 353        line = line.strip()
 354
 355        ## only import to p4/
 356        if not line.startswith('p4/') or line == "p4/HEAD":
 357            continue
 358        branch = line
 359
 360        # strip off p4
 361        branch = re.sub ("^p4/", "", line)
 362
 363        branches[branch] = parseRevision(line)
 364    return branches
 365
 366def findUpstreamBranchPoint(head = "HEAD"):
 367    branches = p4BranchesInGit()
 368    # map from depot-path to branch name
 369    branchByDepotPath = {}
 370    for branch in branches.keys():
 371        tip = branches[branch]
 372        log = extractLogMessageFromGitCommit(tip)
 373        settings = extractSettingsGitLog(log)
 374        if settings.has_key("depot-paths"):
 375            paths = ",".join(settings["depot-paths"])
 376            branchByDepotPath[paths] = "remotes/p4/" + branch
 377
 378    settings = None
 379    parent = 0
 380    while parent < 65535:
 381        commit = head + "~%s" % parent
 382        log = extractLogMessageFromGitCommit(commit)
 383        settings = extractSettingsGitLog(log)
 384        if settings.has_key("depot-paths"):
 385            paths = ",".join(settings["depot-paths"])
 386            if branchByDepotPath.has_key(paths):
 387                return [branchByDepotPath[paths], settings]
 388
 389        parent = parent + 1
 390
 391    return ["", settings]
 392
 393def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 394    if not silent:
 395        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 396               % localRefPrefix)
 397
 398    originPrefix = "origin/p4/"
 399
 400    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 401        line = line.strip()
 402        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 403            continue
 404
 405        headName = line[len(originPrefix):]
 406        remoteHead = localRefPrefix + headName
 407        originHead = line
 408
 409        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 410        if (not original.has_key('depot-paths')
 411            or not original.has_key('change')):
 412            continue
 413
 414        update = False
 415        if not gitBranchExists(remoteHead):
 416            if verbose:
 417                print "creating %s" % remoteHead
 418            update = True
 419        else:
 420            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 421            if settings.has_key('change') > 0:
 422                if settings['depot-paths'] == original['depot-paths']:
 423                    originP4Change = int(original['change'])
 424                    p4Change = int(settings['change'])
 425                    if originP4Change > p4Change:
 426                        print ("%s (%s) is newer than %s (%s). "
 427                               "Updating p4 branch from origin."
 428                               % (originHead, originP4Change,
 429                                  remoteHead, p4Change))
 430                        update = True
 431                else:
 432                    print ("Ignoring: %s was imported from %s while "
 433                           "%s was imported from %s"
 434                           % (originHead, ','.join(original['depot-paths']),
 435                              remoteHead, ','.join(settings['depot-paths'])))
 436
 437        if update:
 438            system("git update-ref %s %s" % (remoteHead, originHead))
 439
 440def originP4BranchesExist():
 441        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 442
 443def p4ChangesForPaths(depotPaths, changeRange):
 444    assert depotPaths
 445    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 446                                                        for p in depotPaths]))
 447
 448    changes = {}
 449    for line in output:
 450        changeNum = int(line.split(" ")[1])
 451        changes[changeNum] = True
 452
 453    changelist = changes.keys()
 454    changelist.sort()
 455    return changelist
 456
 457class Command:
 458    def __init__(self):
 459        self.usage = "usage: %prog [options]"
 460        self.needsGit = True
 461
 462class P4Debug(Command):
 463    def __init__(self):
 464        Command.__init__(self)
 465        self.options = [
 466            optparse.make_option("--verbose", dest="verbose", action="store_true",
 467                                 default=False),
 468            ]
 469        self.description = "A tool to debug the output of p4 -G."
 470        self.needsGit = False
 471        self.verbose = False
 472
 473    def run(self, args):
 474        j = 0
 475        for output in p4CmdList(" ".join(args)):
 476            print 'Element: %d' % j
 477            j += 1
 478            print output
 479        return True
 480
 481class P4RollBack(Command):
 482    def __init__(self):
 483        Command.__init__(self)
 484        self.options = [
 485            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 486            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 487        ]
 488        self.description = "A tool to debug the multi-branch import. Don't use :)"
 489        self.verbose = False
 490        self.rollbackLocalBranches = False
 491
 492    def run(self, args):
 493        if len(args) != 1:
 494            return False
 495        maxChange = int(args[0])
 496
 497        if "p4ExitCode" in p4Cmd("changes -m 1"):
 498            die("Problems executing p4");
 499
 500        if self.rollbackLocalBranches:
 501            refPrefix = "refs/heads/"
 502            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 503        else:
 504            refPrefix = "refs/remotes/"
 505            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 506
 507        for line in lines:
 508            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 509                line = line.strip()
 510                ref = refPrefix + line
 511                log = extractLogMessageFromGitCommit(ref)
 512                settings = extractSettingsGitLog(log)
 513
 514                depotPaths = settings['depot-paths']
 515                change = settings['change']
 516
 517                changed = False
 518
 519                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 520                                                           for p in depotPaths]))) == 0:
 521                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 522                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 523                    continue
 524
 525                while change and int(change) > maxChange:
 526                    changed = True
 527                    if self.verbose:
 528                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 529                    system("git update-ref %s \"%s^\"" % (ref, ref))
 530                    log = extractLogMessageFromGitCommit(ref)
 531                    settings =  extractSettingsGitLog(log)
 532
 533
 534                    depotPaths = settings['depot-paths']
 535                    change = settings['change']
 536
 537                if changed:
 538                    print "%s rewound to %s" % (ref, change)
 539
 540        return True
 541
 542class P4Submit(Command):
 543    def __init__(self):
 544        Command.__init__(self)
 545        self.options = [
 546                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 547                optparse.make_option("--origin", dest="origin"),
 548                optparse.make_option("-M", dest="detectRename", action="store_true"),
 549        ]
 550        self.description = "Submit changes from git to the perforce depot."
 551        self.usage += " [name of git branch to submit into perforce depot]"
 552        self.interactive = True
 553        self.origin = ""
 554        self.detectRename = False
 555        self.verbose = False
 556        self.isWindows = (platform.system() == "Windows")
 557
 558    def check(self):
 559        if len(p4CmdList("opened ...")) > 0:
 560            die("You have files opened with perforce! Close them before starting the sync.")
 561
 562    # replaces everything between 'Description:' and the next P4 submit template field with the
 563    # commit message
 564    def prepareLogMessage(self, template, message):
 565        result = ""
 566
 567        inDescriptionSection = False
 568
 569        for line in template.split("\n"):
 570            if line.startswith("#"):
 571                result += line + "\n"
 572                continue
 573
 574            if inDescriptionSection:
 575                if line.startswith("Files:"):
 576                    inDescriptionSection = False
 577                else:
 578                    continue
 579            else:
 580                if line.startswith("Description:"):
 581                    inDescriptionSection = True
 582                    line += "\n"
 583                    for messageLine in message.split("\n"):
 584                        line += "\t" + messageLine + "\n"
 585
 586            result += line + "\n"
 587
 588        return result
 589
 590    def prepareSubmitTemplate(self):
 591        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 592        template = ""
 593        inFilesSection = False
 594        for line in p4_read_pipe_lines("change -o"):
 595            if line.endswith("\r\n"):
 596                line = line[:-2] + "\n"
 597            if inFilesSection:
 598                if line.startswith("\t"):
 599                    # path starts and ends with a tab
 600                    path = line[1:]
 601                    lastTab = path.rfind("\t")
 602                    if lastTab != -1:
 603                        path = path[:lastTab]
 604                        if not path.startswith(self.depotPath):
 605                            continue
 606                else:
 607                    inFilesSection = False
 608            else:
 609                if line.startswith("Files:"):
 610                    inFilesSection = True
 611
 612            template += line
 613
 614        return template
 615
 616    def applyCommit(self, id):
 617        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 618        diffOpts = ("", "-M")[self.detectRename]
 619        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 620        filesToAdd = set()
 621        filesToDelete = set()
 622        editedFiles = set()
 623        filesToChangeExecBit = {}
 624        for line in diff:
 625            diff = parseDiffTreeEntry(line)
 626            modifier = diff['status']
 627            path = diff['src']
 628            if modifier == "M":
 629                p4_system("edit \"%s\"" % path)
 630                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 631                    filesToChangeExecBit[path] = diff['dst_mode']
 632                editedFiles.add(path)
 633            elif modifier == "A":
 634                filesToAdd.add(path)
 635                filesToChangeExecBit[path] = diff['dst_mode']
 636                if path in filesToDelete:
 637                    filesToDelete.remove(path)
 638            elif modifier == "D":
 639                filesToDelete.add(path)
 640                if path in filesToAdd:
 641                    filesToAdd.remove(path)
 642            elif modifier == "R":
 643                src, dest = diff['src'], diff['dst']
 644                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 645                p4_system("edit \"%s\"" % (dest))
 646                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 647                    filesToChangeExecBit[dest] = diff['dst_mode']
 648                os.unlink(dest)
 649                editedFiles.add(dest)
 650                filesToDelete.add(src)
 651            else:
 652                die("unknown modifier %s for %s" % (modifier, path))
 653
 654        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 655        patchcmd = diffcmd + " | git apply "
 656        tryPatchCmd = patchcmd + "--check -"
 657        applyPatchCmd = patchcmd + "--check --apply -"
 658
 659        if os.system(tryPatchCmd) != 0:
 660            print "Unfortunately applying the change failed!"
 661            print "What do you want to do?"
 662            response = "x"
 663            while response != "s" and response != "a" and response != "w":
 664                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 665                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 666            if response == "s":
 667                print "Skipping! Good luck with the next patches..."
 668                for f in editedFiles:
 669                    p4_system("revert \"%s\"" % f);
 670                for f in filesToAdd:
 671                    system("rm %s" %f)
 672                return
 673            elif response == "a":
 674                os.system(applyPatchCmd)
 675                if len(filesToAdd) > 0:
 676                    print "You may also want to call p4 add on the following files:"
 677                    print " ".join(filesToAdd)
 678                if len(filesToDelete):
 679                    print "The following files should be scheduled for deletion with p4 delete:"
 680                    print " ".join(filesToDelete)
 681                die("Please resolve and submit the conflict manually and "
 682                    + "continue afterwards with git-p4 submit --continue")
 683            elif response == "w":
 684                system(diffcmd + " > patch.txt")
 685                print "Patch saved to patch.txt in %s !" % self.clientPath
 686                die("Please resolve and submit the conflict manually and "
 687                    "continue afterwards with git-p4 submit --continue")
 688
 689        system(applyPatchCmd)
 690
 691        for f in filesToAdd:
 692            p4_system("add \"%s\"" % f)
 693        for f in filesToDelete:
 694            p4_system("revert \"%s\"" % f)
 695            p4_system("delete \"%s\"" % f)
 696
 697        # Set/clear executable bits
 698        for f in filesToChangeExecBit.keys():
 699            mode = filesToChangeExecBit[f]
 700            setP4ExecBit(f, mode)
 701
 702        logMessage = extractLogMessageFromGitCommit(id)
 703        logMessage = logMessage.strip()
 704
 705        template = self.prepareSubmitTemplate()
 706
 707        if self.interactive:
 708            submitTemplate = self.prepareLogMessage(template, logMessage)
 709            if os.environ.has_key("P4DIFF"):
 710                del(os.environ["P4DIFF"])
 711            diff = p4_read_pipe("diff -du ...")
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            defaultEditor = "vi"
 735            if platform.system() == "Windows":
 736                defaultEditor = "notepad"
 737            if os.environ.has_key("P4EDITOR"):
 738                editor = os.environ.get("P4EDITOR")
 739            else:
 740                editor = os.environ.get("EDITOR", defaultEditor);
 741            system(editor + " " + fileName)
 742
 743            response = "y"
 744            if os.stat(fileName).st_mtime <= mtime:
 745                response = "x"
 746                while response != "y" and response != "n":
 747                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 748
 749            if response == "y":
 750                tmpFile = open(fileName, "rb")
 751                message = tmpFile.read()
 752                tmpFile.close()
 753                submitTemplate = message[:message.index(separatorLine)]
 754                if self.isWindows:
 755                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 756                p4_write_pipe("submit -i", submitTemplate)
 757            else:
 758                for f in editedFiles:
 759                    p4_system("revert \"%s\"" % f);
 760                for f in filesToAdd:
 761                    p4_system("revert \"%s\"" % f);
 762                    system("rm %s" %f)
 763
 764            os.remove(fileName)
 765        else:
 766            fileName = "submit.txt"
 767            file = open(fileName, "w+")
 768            file.write(self.prepareLogMessage(template, logMessage))
 769            file.close()
 770            print ("Perforce submit template written as %s. "
 771                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 772                   % (fileName, fileName))
 773
 774    def run(self, args):
 775        if len(args) == 0:
 776            self.master = currentGitBranch()
 777            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 778                die("Detecting current git branch failed!")
 779        elif len(args) == 1:
 780            self.master = args[0]
 781        else:
 782            return False
 783
 784        allowSubmit = gitConfig("git-p4.allowSubmit")
 785        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 786            die("%s is not in git-p4.allowSubmit" % self.master)
 787
 788        [upstream, settings] = findUpstreamBranchPoint()
 789        self.depotPath = settings['depot-paths'][0]
 790        if len(self.origin) == 0:
 791            self.origin = upstream
 792
 793        if self.verbose:
 794            print "Origin branch is " + self.origin
 795
 796        if len(self.depotPath) == 0:
 797            print "Internal error: cannot locate perforce depot path from existing branches"
 798            sys.exit(128)
 799
 800        self.clientPath = p4Where(self.depotPath)
 801
 802        if len(self.clientPath) == 0:
 803            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 804            sys.exit(128)
 805
 806        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 807        self.oldWorkingDirectory = os.getcwd()
 808
 809        chdir(self.clientPath)
 810        print "Syncronizing p4 checkout..."
 811        p4_system("sync ...")
 812
 813        self.check()
 814
 815        commits = []
 816        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 817            commits.append(line.strip())
 818        commits.reverse()
 819
 820        while len(commits) > 0:
 821            commit = commits[0]
 822            commits = commits[1:]
 823            self.applyCommit(commit)
 824            if not self.interactive:
 825                break
 826
 827        if len(commits) == 0:
 828            print "All changes applied!"
 829            chdir(self.oldWorkingDirectory)
 830
 831            sync = P4Sync()
 832            sync.run([])
 833
 834            rebase = P4Rebase()
 835            rebase.rebase()
 836
 837        return True
 838
 839class P4Sync(Command):
 840    def __init__(self):
 841        Command.__init__(self)
 842        self.options = [
 843                optparse.make_option("--branch", dest="branch"),
 844                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                optparse.make_option("--changesfile", dest="changesFile"),
 846                optparse.make_option("--silent", dest="silent", action="store_true"),
 847                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                     help="Import into refs/heads/ , not refs/remotes"),
 851                optparse.make_option("--max-changes", dest="maxChanges"),
 852                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 854                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 855                                     help="Only sync files that are included in the Perforce Client Spec")
 856        ]
 857        self.description = """Imports from Perforce into a git repository.\n
 858    example:
 859    //depot/my/project/ -- to import the current head
 860    //depot/my/project/@all -- to import everything
 861    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 862
 863    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 864
 865        self.usage += " //depot/path[@revRange]"
 866        self.silent = False
 867        self.createdBranches = Set()
 868        self.committedChanges = Set()
 869        self.branch = ""
 870        self.detectBranches = False
 871        self.detectLabels = False
 872        self.changesFile = ""
 873        self.syncWithOrigin = True
 874        self.verbose = False
 875        self.importIntoRemotes = True
 876        self.maxChanges = ""
 877        self.isWindows = (platform.system() == "Windows")
 878        self.keepRepoPath = False
 879        self.depotPaths = None
 880        self.p4BranchesInGit = []
 881        self.cloneExclude = []
 882        self.useClientSpec = False
 883        self.clientSpecDirs = []
 884
 885        if gitConfig("git-p4.syncFromOrigin") == "false":
 886            self.syncWithOrigin = False
 887
 888    def extractFilesFromCommit(self, commit):
 889        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 890                             for path in self.cloneExclude]
 891        files = []
 892        fnum = 0
 893        while commit.has_key("depotFile%s" % fnum):
 894            path =  commit["depotFile%s" % fnum]
 895
 896            if [p for p in self.cloneExclude
 897                if path.startswith (p)]:
 898                found = False
 899            else:
 900                found = [p for p in self.depotPaths
 901                         if path.startswith (p)]
 902            if not found:
 903                fnum = fnum + 1
 904                continue
 905
 906            file = {}
 907            file["path"] = path
 908            file["rev"] = commit["rev%s" % fnum]
 909            file["action"] = commit["action%s" % fnum]
 910            file["type"] = commit["type%s" % fnum]
 911            files.append(file)
 912            fnum = fnum + 1
 913        return files
 914
 915    def stripRepoPath(self, path, prefixes):
 916        if self.keepRepoPath:
 917            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 918
 919        for p in prefixes:
 920            if path.startswith(p):
 921                path = path[len(p):]
 922
 923        return path
 924
 925    def splitFilesIntoBranches(self, commit):
 926        branches = {}
 927        fnum = 0
 928        while commit.has_key("depotFile%s" % fnum):
 929            path =  commit["depotFile%s" % fnum]
 930            found = [p for p in self.depotPaths
 931                     if path.startswith (p)]
 932            if not found:
 933                fnum = fnum + 1
 934                continue
 935
 936            file = {}
 937            file["path"] = path
 938            file["rev"] = commit["rev%s" % fnum]
 939            file["action"] = commit["action%s" % fnum]
 940            file["type"] = commit["type%s" % fnum]
 941            fnum = fnum + 1
 942
 943            relPath = self.stripRepoPath(path, self.depotPaths)
 944
 945            for branch in self.knownBranches.keys():
 946
 947                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 948                if relPath.startswith(branch + "/"):
 949                    if branch not in branches:
 950                        branches[branch] = []
 951                    branches[branch].append(file)
 952                    break
 953
 954        return branches
 955
 956    # output one file from the P4 stream
 957    # - helper for streamP4Files
 958
 959    def streamOneP4File(self, file, contents):
 960        if file["type"] == "apple":
 961            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 962                file['depotFile']
 963            return
 964
 965        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 966        if verbose:
 967            sys.stderr.write("%s\n" % relPath)
 968
 969        mode = "644"
 970        if isP4Exec(file["type"]):
 971            mode = "755"
 972        elif file["type"] == "symlink":
 973            mode = "120000"
 974            # p4 print on a symlink contains "target\n", so strip it off
 975            last = contents.pop()
 976            last = last[:-1]
 977            contents.append(last)
 978
 979        if self.isWindows and file["type"].endswith("text"):
 980            mangled = []
 981            for data in contents:
 982                data = data.replace("\r\n", "\n")
 983                mangled.append(data)
 984            contents = mangled
 985
 986        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 987            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 988        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 989            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 990
 991        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 992
 993        # total length...
 994        length = 0
 995        for d in contents:
 996            length = length + len(d)
 997
 998        self.gitStream.write("data %d\n" % length)
 999        for d in contents:
1000            self.gitStream.write(d)
1001        self.gitStream.write("\n")
1002
1003    def streamOneP4Deletion(self, file):
1004        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1005        if verbose:
1006            sys.stderr.write("delete %s\n" % relPath)
1007        self.gitStream.write("D %s\n" % relPath)
1008
1009    # handle another chunk of streaming data
1010    def streamP4FilesCb(self, marshalled):
1011
1012        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1013            # start of a new file - output the old one first
1014            self.streamOneP4File(self.stream_file, self.stream_contents)
1015            self.stream_file = {}
1016            self.stream_contents = []
1017            self.stream_have_file_info = False
1018
1019        # pick up the new file information... for the
1020        # 'data' field we need to append to our array
1021        for k in marshalled.keys():
1022            if k == 'data':
1023                self.stream_contents.append(marshalled['data'])
1024            else:
1025                self.stream_file[k] = marshalled[k]
1026
1027        self.stream_have_file_info = True
1028
1029    # Stream directly from "p4 files" into "git fast-import"
1030    def streamP4Files(self, files):
1031        filesForCommit = []
1032        filesToRead = []
1033        filesToDelete = []
1034
1035        for f in files:
1036            includeFile = True
1037            for val in self.clientSpecDirs:
1038                if f['path'].startswith(val[0]):
1039                    if val[1] <= 0:
1040                        includeFile = False
1041                    break
1042
1043            if includeFile:
1044                filesForCommit.append(f)
1045                if f['action'] not in ('delete', 'purge'):
1046                    filesToRead.append(f)
1047                else:
1048                    filesToDelete.append(f)
1049
1050        # deleted files...
1051        for f in filesToDelete:
1052            self.streamOneP4Deletion(f)
1053
1054        if len(filesToRead) > 0:
1055            self.stream_file = {}
1056            self.stream_contents = []
1057            self.stream_have_file_info = False
1058
1059            # curry self argument
1060            def streamP4FilesCbSelf(entry):
1061                self.streamP4FilesCb(entry)
1062
1063            p4CmdList("-x - print",
1064                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1065                                                  for f in filesToRead]),
1066                cb=streamP4FilesCbSelf)
1067
1068            # do the last chunk
1069            if self.stream_file.has_key('depotFile'):
1070                self.streamOneP4File(self.stream_file, self.stream_contents)
1071
1072    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1073        epoch = details["time"]
1074        author = details["user"]
1075        self.branchPrefixes = branchPrefixes
1076
1077        if self.verbose:
1078            print "commit into %s" % branch
1079
1080        # start with reading files; if that fails, we should not
1081        # create a commit.
1082        new_files = []
1083        for f in files:
1084            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1085                new_files.append (f)
1086            else:
1087                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1088
1089        self.gitStream.write("commit %s\n" % branch)
1090#        gitStream.write("mark :%s\n" % details["change"])
1091        self.committedChanges.add(int(details["change"]))
1092        committer = ""
1093        if author not in self.users:
1094            self.getUserMapFromPerforceServer()
1095        if author in self.users:
1096            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1097        else:
1098            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1099
1100        self.gitStream.write("committer %s\n" % committer)
1101
1102        self.gitStream.write("data <<EOT\n")
1103        self.gitStream.write(details["desc"])
1104        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1105                             % (','.join (branchPrefixes), details["change"]))
1106        if len(details['options']) > 0:
1107            self.gitStream.write(": options = %s" % details['options'])
1108        self.gitStream.write("]\nEOT\n\n")
1109
1110        if len(parent) > 0:
1111            if self.verbose:
1112                print "parent %s" % parent
1113            self.gitStream.write("from %s\n" % parent)
1114
1115        self.streamP4Files(new_files)
1116        self.gitStream.write("\n")
1117
1118        change = int(details["change"])
1119
1120        if self.labels.has_key(change):
1121            label = self.labels[change]
1122            labelDetails = label[0]
1123            labelRevisions = label[1]
1124            if self.verbose:
1125                print "Change %s is labelled %s" % (change, labelDetails)
1126
1127            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1128                                                    for p in branchPrefixes]))
1129
1130            if len(files) == len(labelRevisions):
1131
1132                cleanedFiles = {}
1133                for info in files:
1134                    if info["action"] in ("delete", "purge"):
1135                        continue
1136                    cleanedFiles[info["depotFile"]] = info["rev"]
1137
1138                if cleanedFiles == labelRevisions:
1139                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1140                    self.gitStream.write("from %s\n" % branch)
1141
1142                    owner = labelDetails["Owner"]
1143                    tagger = ""
1144                    if author in self.users:
1145                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1146                    else:
1147                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1148                    self.gitStream.write("tagger %s\n" % tagger)
1149                    self.gitStream.write("data <<EOT\n")
1150                    self.gitStream.write(labelDetails["Description"])
1151                    self.gitStream.write("EOT\n\n")
1152
1153                else:
1154                    if not self.silent:
1155                        print ("Tag %s does not match with change %s: files do not match."
1156                               % (labelDetails["label"], change))
1157
1158            else:
1159                if not self.silent:
1160                    print ("Tag %s does not match with change %s: file count is different."
1161                           % (labelDetails["label"], change))
1162
1163    def getUserCacheFilename(self):
1164        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1165        return home + "/.gitp4-usercache.txt"
1166
1167    def getUserMapFromPerforceServer(self):
1168        if self.userMapFromPerforceServer:
1169            return
1170        self.users = {}
1171
1172        for output in p4CmdList("users"):
1173            if not output.has_key("User"):
1174                continue
1175            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1176
1177
1178        s = ''
1179        for (key, val) in self.users.items():
1180            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1181
1182        open(self.getUserCacheFilename(), "wb").write(s)
1183        self.userMapFromPerforceServer = True
1184
1185    def loadUserMapFromCache(self):
1186        self.users = {}
1187        self.userMapFromPerforceServer = False
1188        try:
1189            cache = open(self.getUserCacheFilename(), "rb")
1190            lines = cache.readlines()
1191            cache.close()
1192            for line in lines:
1193                entry = line.strip().split("\t")
1194                self.users[entry[0]] = entry[1]
1195        except IOError:
1196            self.getUserMapFromPerforceServer()
1197
1198    def getLabels(self):
1199        self.labels = {}
1200
1201        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1202        if len(l) > 0 and not self.silent:
1203            print "Finding files belonging to labels in %s" % `self.depotPaths`
1204
1205        for output in l:
1206            label = output["label"]
1207            revisions = {}
1208            newestChange = 0
1209            if self.verbose:
1210                print "Querying files for label %s" % label
1211            for file in p4CmdList("files "
1212                                  +  ' '.join (["%s...@%s" % (p, label)
1213                                                for p in self.depotPaths])):
1214                revisions[file["depotFile"]] = file["rev"]
1215                change = int(file["change"])
1216                if change > newestChange:
1217                    newestChange = change
1218
1219            self.labels[newestChange] = [output, revisions]
1220
1221        if self.verbose:
1222            print "Label changes: %s" % self.labels.keys()
1223
1224    def guessProjectName(self):
1225        for p in self.depotPaths:
1226            if p.endswith("/"):
1227                p = p[:-1]
1228            p = p[p.strip().rfind("/") + 1:]
1229            if not p.endswith("/"):
1230               p += "/"
1231            return p
1232
1233    def getBranchMapping(self):
1234        lostAndFoundBranches = set()
1235
1236        for info in p4CmdList("branches"):
1237            details = p4Cmd("branch -o %s" % info["branch"])
1238            viewIdx = 0
1239            while details.has_key("View%s" % viewIdx):
1240                paths = details["View%s" % viewIdx].split(" ")
1241                viewIdx = viewIdx + 1
1242                # require standard //depot/foo/... //depot/bar/... mapping
1243                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1244                    continue
1245                source = paths[0]
1246                destination = paths[1]
1247                ## HACK
1248                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1249                    source = source[len(self.depotPaths[0]):-4]
1250                    destination = destination[len(self.depotPaths[0]):-4]
1251
1252                    if destination in self.knownBranches:
1253                        if not self.silent:
1254                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1255                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1256                        continue
1257
1258                    self.knownBranches[destination] = source
1259
1260                    lostAndFoundBranches.discard(destination)
1261
1262                    if source not in self.knownBranches:
1263                        lostAndFoundBranches.add(source)
1264
1265
1266        for branch in lostAndFoundBranches:
1267            self.knownBranches[branch] = branch
1268
1269    def getBranchMappingFromGitBranches(self):
1270        branches = p4BranchesInGit(self.importIntoRemotes)
1271        for branch in branches.keys():
1272            if branch == "master":
1273                branch = "main"
1274            else:
1275                branch = branch[len(self.projectName):]
1276            self.knownBranches[branch] = branch
1277
1278    def listExistingP4GitBranches(self):
1279        # branches holds mapping from name to commit
1280        branches = p4BranchesInGit(self.importIntoRemotes)
1281        self.p4BranchesInGit = branches.keys()
1282        for branch in branches.keys():
1283            self.initialParents[self.refPrefix + branch] = branches[branch]
1284
1285    def updateOptionDict(self, d):
1286        option_keys = {}
1287        if self.keepRepoPath:
1288            option_keys['keepRepoPath'] = 1
1289
1290        d["options"] = ' '.join(sorted(option_keys.keys()))
1291
1292    def readOptions(self, d):
1293        self.keepRepoPath = (d.has_key('options')
1294                             and ('keepRepoPath' in d['options']))
1295
1296    def gitRefForBranch(self, branch):
1297        if branch == "main":
1298            return self.refPrefix + "master"
1299
1300        if len(branch) <= 0:
1301            return branch
1302
1303        return self.refPrefix + self.projectName + branch
1304
1305    def gitCommitByP4Change(self, ref, change):
1306        if self.verbose:
1307            print "looking in ref " + ref + " for change %s using bisect..." % change
1308
1309        earliestCommit = ""
1310        latestCommit = parseRevision(ref)
1311
1312        while True:
1313            if self.verbose:
1314                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1315            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1316            if len(next) == 0:
1317                if self.verbose:
1318                    print "argh"
1319                return ""
1320            log = extractLogMessageFromGitCommit(next)
1321            settings = extractSettingsGitLog(log)
1322            currentChange = int(settings['change'])
1323            if self.verbose:
1324                print "current change %s" % currentChange
1325
1326            if currentChange == change:
1327                if self.verbose:
1328                    print "found %s" % next
1329                return next
1330
1331            if currentChange < change:
1332                earliestCommit = "^%s" % next
1333            else:
1334                latestCommit = "%s" % next
1335
1336        return ""
1337
1338    def importNewBranch(self, branch, maxChange):
1339        # make fast-import flush all changes to disk and update the refs using the checkpoint
1340        # command so that we can try to find the branch parent in the git history
1341        self.gitStream.write("checkpoint\n\n");
1342        self.gitStream.flush();
1343        branchPrefix = self.depotPaths[0] + branch + "/"
1344        range = "@1,%s" % maxChange
1345        #print "prefix" + branchPrefix
1346        changes = p4ChangesForPaths([branchPrefix], range)
1347        if len(changes) <= 0:
1348            return False
1349        firstChange = changes[0]
1350        #print "first change in branch: %s" % firstChange
1351        sourceBranch = self.knownBranches[branch]
1352        sourceDepotPath = self.depotPaths[0] + sourceBranch
1353        sourceRef = self.gitRefForBranch(sourceBranch)
1354        #print "source " + sourceBranch
1355
1356        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1357        #print "branch parent: %s" % branchParentChange
1358        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1359        if len(gitParent) > 0:
1360            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1361            #print "parent git commit: %s" % gitParent
1362
1363        self.importChanges(changes)
1364        return True
1365
1366    def importChanges(self, changes):
1367        cnt = 1
1368        for change in changes:
1369            description = p4Cmd("describe %s" % change)
1370            self.updateOptionDict(description)
1371
1372            if not self.silent:
1373                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1374                sys.stdout.flush()
1375            cnt = cnt + 1
1376
1377            try:
1378                if self.detectBranches:
1379                    branches = self.splitFilesIntoBranches(description)
1380                    for branch in branches.keys():
1381                        ## HACK  --hwn
1382                        branchPrefix = self.depotPaths[0] + branch + "/"
1383
1384                        parent = ""
1385
1386                        filesForCommit = branches[branch]
1387
1388                        if self.verbose:
1389                            print "branch is %s" % branch
1390
1391                        self.updatedBranches.add(branch)
1392
1393                        if branch not in self.createdBranches:
1394                            self.createdBranches.add(branch)
1395                            parent = self.knownBranches[branch]
1396                            if parent == branch:
1397                                parent = ""
1398                            else:
1399                                fullBranch = self.projectName + branch
1400                                if fullBranch not in self.p4BranchesInGit:
1401                                    if not self.silent:
1402                                        print("\n    Importing new branch %s" % fullBranch);
1403                                    if self.importNewBranch(branch, change - 1):
1404                                        parent = ""
1405                                        self.p4BranchesInGit.append(fullBranch)
1406                                    if not self.silent:
1407                                        print("\n    Resuming with change %s" % change);
1408
1409                                if self.verbose:
1410                                    print "parent determined through known branches: %s" % parent
1411
1412                        branch = self.gitRefForBranch(branch)
1413                        parent = self.gitRefForBranch(parent)
1414
1415                        if self.verbose:
1416                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1417
1418                        if len(parent) == 0 and branch in self.initialParents:
1419                            parent = self.initialParents[branch]
1420                            del self.initialParents[branch]
1421
1422                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1423                else:
1424                    files = self.extractFilesFromCommit(description)
1425                    self.commit(description, files, self.branch, self.depotPaths,
1426                                self.initialParent)
1427                    self.initialParent = ""
1428            except IOError:
1429                print self.gitError.read()
1430                sys.exit(1)
1431
1432    def importHeadRevision(self, revision):
1433        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1434
1435        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1436        details["desc"] = ("Initial import of %s from the state at revision %s"
1437                           % (' '.join(self.depotPaths), revision))
1438        details["change"] = revision
1439        newestRevision = 0
1440
1441        fileCnt = 0
1442        for info in p4CmdList("files "
1443                              +  ' '.join(["%s...%s"
1444                                           % (p, revision)
1445                                           for p in self.depotPaths])):
1446
1447            if info['code'] == 'error':
1448                sys.stderr.write("p4 returned an error: %s\n"
1449                                 % info['data'])
1450                sys.exit(1)
1451
1452
1453            change = int(info["change"])
1454            if change > newestRevision:
1455                newestRevision = change
1456
1457            if info["action"] in ("delete", "purge"):
1458                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1459                #fileCnt = fileCnt + 1
1460                continue
1461
1462            for prop in ["depotFile", "rev", "action", "type" ]:
1463                details["%s%s" % (prop, fileCnt)] = info[prop]
1464
1465            fileCnt = fileCnt + 1
1466
1467        details["change"] = newestRevision
1468        self.updateOptionDict(details)
1469        try:
1470            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1471        except IOError:
1472            print "IO error with git fast-import. Is your git version recent enough?"
1473            print self.gitError.read()
1474
1475
1476    def getClientSpec(self):
1477        specList = p4CmdList( "client -o" )
1478        temp = {}
1479        for entry in specList:
1480            for k,v in entry.iteritems():
1481                if k.startswith("View"):
1482                    if v.startswith('"'):
1483                        start = 1
1484                    else:
1485                        start = 0
1486                    index = v.find("...")
1487                    v = v[start:index]
1488                    if v.startswith("-"):
1489                        v = v[1:]
1490                        temp[v] = -len(v)
1491                    else:
1492                        temp[v] = len(v)
1493        self.clientSpecDirs = temp.items()
1494        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1495
1496    def run(self, args):
1497        self.depotPaths = []
1498        self.changeRange = ""
1499        self.initialParent = ""
1500        self.previousDepotPaths = []
1501
1502        # map from branch depot path to parent branch
1503        self.knownBranches = {}
1504        self.initialParents = {}
1505        self.hasOrigin = originP4BranchesExist()
1506        if not self.syncWithOrigin:
1507            self.hasOrigin = False
1508
1509        if self.importIntoRemotes:
1510            self.refPrefix = "refs/remotes/p4/"
1511        else:
1512            self.refPrefix = "refs/heads/p4/"
1513
1514        if self.syncWithOrigin and self.hasOrigin:
1515            if not self.silent:
1516                print "Syncing with origin first by calling git fetch origin"
1517            system("git fetch origin")
1518
1519        if len(self.branch) == 0:
1520            self.branch = self.refPrefix + "master"
1521            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1522                system("git update-ref %s refs/heads/p4" % self.branch)
1523                system("git branch -D p4");
1524            # create it /after/ importing, when master exists
1525            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1526                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1527
1528        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1529            self.getClientSpec()
1530
1531        # TODO: should always look at previous commits,
1532        # merge with previous imports, if possible.
1533        if args == []:
1534            if self.hasOrigin:
1535                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1536            self.listExistingP4GitBranches()
1537
1538            if len(self.p4BranchesInGit) > 1:
1539                if not self.silent:
1540                    print "Importing from/into multiple branches"
1541                self.detectBranches = True
1542
1543            if self.verbose:
1544                print "branches: %s" % self.p4BranchesInGit
1545
1546            p4Change = 0
1547            for branch in self.p4BranchesInGit:
1548                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1549
1550                settings = extractSettingsGitLog(logMsg)
1551
1552                self.readOptions(settings)
1553                if (settings.has_key('depot-paths')
1554                    and settings.has_key ('change')):
1555                    change = int(settings['change']) + 1
1556                    p4Change = max(p4Change, change)
1557
1558                    depotPaths = sorted(settings['depot-paths'])
1559                    if self.previousDepotPaths == []:
1560                        self.previousDepotPaths = depotPaths
1561                    else:
1562                        paths = []
1563                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1564                            for i in range(0, min(len(cur), len(prev))):
1565                                if cur[i] <> prev[i]:
1566                                    i = i - 1
1567                                    break
1568
1569                            paths.append (cur[:i + 1])
1570
1571                        self.previousDepotPaths = paths
1572
1573            if p4Change > 0:
1574                self.depotPaths = sorted(self.previousDepotPaths)
1575                self.changeRange = "@%s,#head" % p4Change
1576                if not self.detectBranches:
1577                    self.initialParent = parseRevision(self.branch)
1578                if not self.silent and not self.detectBranches:
1579                    print "Performing incremental import into %s git branch" % self.branch
1580
1581        if not self.branch.startswith("refs/"):
1582            self.branch = "refs/heads/" + self.branch
1583
1584        if len(args) == 0 and self.depotPaths:
1585            if not self.silent:
1586                print "Depot paths: %s" % ' '.join(self.depotPaths)
1587        else:
1588            if self.depotPaths and self.depotPaths != args:
1589                print ("previous import used depot path %s and now %s was specified. "
1590                       "This doesn't work!" % (' '.join (self.depotPaths),
1591                                               ' '.join (args)))
1592                sys.exit(1)
1593
1594            self.depotPaths = sorted(args)
1595
1596        revision = ""
1597        self.users = {}
1598
1599        newPaths = []
1600        for p in self.depotPaths:
1601            if p.find("@") != -1:
1602                atIdx = p.index("@")
1603                self.changeRange = p[atIdx:]
1604                if self.changeRange == "@all":
1605                    self.changeRange = ""
1606                elif ',' not in self.changeRange:
1607                    revision = self.changeRange
1608                    self.changeRange = ""
1609                p = p[:atIdx]
1610            elif p.find("#") != -1:
1611                hashIdx = p.index("#")
1612                revision = p[hashIdx:]
1613                p = p[:hashIdx]
1614            elif self.previousDepotPaths == []:
1615                revision = "#head"
1616
1617            p = re.sub ("\.\.\.$", "", p)
1618            if not p.endswith("/"):
1619                p += "/"
1620
1621            newPaths.append(p)
1622
1623        self.depotPaths = newPaths
1624
1625
1626        self.loadUserMapFromCache()
1627        self.labels = {}
1628        if self.detectLabels:
1629            self.getLabels();
1630
1631        if self.detectBranches:
1632            ## FIXME - what's a P4 projectName ?
1633            self.projectName = self.guessProjectName()
1634
1635            if self.hasOrigin:
1636                self.getBranchMappingFromGitBranches()
1637            else:
1638                self.getBranchMapping()
1639            if self.verbose:
1640                print "p4-git branches: %s" % self.p4BranchesInGit
1641                print "initial parents: %s" % self.initialParents
1642            for b in self.p4BranchesInGit:
1643                if b != "master":
1644
1645                    ## FIXME
1646                    b = b[len(self.projectName):]
1647                self.createdBranches.add(b)
1648
1649        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1650
1651        importProcess = subprocess.Popen(["git", "fast-import"],
1652                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1653                                         stderr=subprocess.PIPE);
1654        self.gitOutput = importProcess.stdout
1655        self.gitStream = importProcess.stdin
1656        self.gitError = importProcess.stderr
1657
1658        if revision:
1659            self.importHeadRevision(revision)
1660        else:
1661            changes = []
1662
1663            if len(self.changesFile) > 0:
1664                output = open(self.changesFile).readlines()
1665                changeSet = Set()
1666                for line in output:
1667                    changeSet.add(int(line))
1668
1669                for change in changeSet:
1670                    changes.append(change)
1671
1672                changes.sort()
1673            else:
1674                if self.verbose:
1675                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1676                                                              self.changeRange)
1677                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1678
1679                if len(self.maxChanges) > 0:
1680                    changes = changes[:min(int(self.maxChanges), len(changes))]
1681
1682            if len(changes) == 0:
1683                if not self.silent:
1684                    print "No changes to import!"
1685                return True
1686
1687            if not self.silent and not self.detectBranches:
1688                print "Import destination: %s" % self.branch
1689
1690            self.updatedBranches = set()
1691
1692            self.importChanges(changes)
1693
1694            if not self.silent:
1695                print ""
1696                if len(self.updatedBranches) > 0:
1697                    sys.stdout.write("Updated branches: ")
1698                    for b in self.updatedBranches:
1699                        sys.stdout.write("%s " % b)
1700                    sys.stdout.write("\n")
1701
1702        self.gitStream.close()
1703        if importProcess.wait() != 0:
1704            die("fast-import failed: %s" % self.gitError.read())
1705        self.gitOutput.close()
1706        self.gitError.close()
1707
1708        return True
1709
1710class P4Rebase(Command):
1711    def __init__(self):
1712        Command.__init__(self)
1713        self.options = [ ]
1714        self.description = ("Fetches the latest revision from perforce and "
1715                            + "rebases the current work (branch) against it")
1716        self.verbose = False
1717
1718    def run(self, args):
1719        sync = P4Sync()
1720        sync.run([])
1721
1722        return self.rebase()
1723
1724    def rebase(self):
1725        if os.system("git update-index --refresh") != 0:
1726            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1727        if len(read_pipe("git diff-index HEAD --")) > 0:
1728            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1729
1730        [upstream, settings] = findUpstreamBranchPoint()
1731        if len(upstream) == 0:
1732            die("Cannot find upstream branchpoint for rebase")
1733
1734        # the branchpoint may be p4/foo~3, so strip off the parent
1735        upstream = re.sub("~[0-9]+$", "", upstream)
1736
1737        print "Rebasing the current branch onto %s" % upstream
1738        oldHead = read_pipe("git rev-parse HEAD").strip()
1739        system("git rebase %s" % upstream)
1740        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1741        return True
1742
1743class P4Clone(P4Sync):
1744    def __init__(self):
1745        P4Sync.__init__(self)
1746        self.description = "Creates a new git repository and imports from Perforce into it"
1747        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1748        self.options += [
1749            optparse.make_option("--destination", dest="cloneDestination",
1750                                 action='store', default=None,
1751                                 help="where to leave result of the clone"),
1752            optparse.make_option("-/", dest="cloneExclude",
1753                                 action="append", type="string",
1754                                 help="exclude depot path")
1755        ]
1756        self.cloneDestination = None
1757        self.needsGit = False
1758
1759    # This is required for the "append" cloneExclude action
1760    def ensure_value(self, attr, value):
1761        if not hasattr(self, attr) or getattr(self, attr) is None:
1762            setattr(self, attr, value)
1763        return getattr(self, attr)
1764
1765    def defaultDestination(self, args):
1766        ## TODO: use common prefix of args?
1767        depotPath = args[0]
1768        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1769        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1770        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1771        depotDir = re.sub(r"/$", "", depotDir)
1772        return os.path.split(depotDir)[1]
1773
1774    def run(self, args):
1775        if len(args) < 1:
1776            return False
1777
1778        if self.keepRepoPath and not self.cloneDestination:
1779            sys.stderr.write("Must specify destination for --keep-path\n")
1780            sys.exit(1)
1781
1782        depotPaths = args
1783
1784        if not self.cloneDestination and len(depotPaths) > 1:
1785            self.cloneDestination = depotPaths[-1]
1786            depotPaths = depotPaths[:-1]
1787
1788        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1789        for p in depotPaths:
1790            if not p.startswith("//"):
1791                return False
1792
1793        if not self.cloneDestination:
1794            self.cloneDestination = self.defaultDestination(args)
1795
1796        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1797        if not os.path.exists(self.cloneDestination):
1798            os.makedirs(self.cloneDestination)
1799        chdir(self.cloneDestination)
1800        system("git init")
1801        self.gitdir = os.getcwd() + "/.git"
1802        if not P4Sync.run(self, depotPaths):
1803            return False
1804        if self.branch != "master":
1805            if self.importIntoRemotes:
1806                masterbranch = "refs/remotes/p4/master"
1807            else:
1808                masterbranch = "refs/heads/p4/master"
1809            if gitBranchExists(masterbranch):
1810                system("git branch master %s" % masterbranch)
1811                system("git checkout -f")
1812            else:
1813                print "Could not detect main branch. No checkout/master branch created."
1814
1815        return True
1816
1817class P4Branches(Command):
1818    def __init__(self):
1819        Command.__init__(self)
1820        self.options = [ ]
1821        self.description = ("Shows the git branches that hold imports and their "
1822                            + "corresponding perforce depot paths")
1823        self.verbose = False
1824
1825    def run(self, args):
1826        if originP4BranchesExist():
1827            createOrUpdateBranchesFromOrigin()
1828
1829        cmdline = "git rev-parse --symbolic "
1830        cmdline += " --remotes"
1831
1832        for line in read_pipe_lines(cmdline):
1833            line = line.strip()
1834
1835            if not line.startswith('p4/') or line == "p4/HEAD":
1836                continue
1837            branch = line
1838
1839            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1840            settings = extractSettingsGitLog(log)
1841
1842            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1843        return True
1844
1845class HelpFormatter(optparse.IndentedHelpFormatter):
1846    def __init__(self):
1847        optparse.IndentedHelpFormatter.__init__(self)
1848
1849    def format_description(self, description):
1850        if description:
1851            return description + "\n"
1852        else:
1853            return ""
1854
1855def printUsage(commands):
1856    print "usage: %s <command> [options]" % sys.argv[0]
1857    print ""
1858    print "valid commands: %s" % ", ".join(commands)
1859    print ""
1860    print "Try %s <command> --help for command specific help." % sys.argv[0]
1861    print ""
1862
1863commands = {
1864    "debug" : P4Debug,
1865    "submit" : P4Submit,
1866    "commit" : P4Submit,
1867    "sync" : P4Sync,
1868    "rebase" : P4Rebase,
1869    "clone" : P4Clone,
1870    "rollback" : P4RollBack,
1871    "branches" : P4Branches
1872}
1873
1874
1875def main():
1876    if len(sys.argv[1:]) == 0:
1877        printUsage(commands.keys())
1878        sys.exit(2)
1879
1880    cmd = ""
1881    cmdName = sys.argv[1]
1882    try:
1883        klass = commands[cmdName]
1884        cmd = klass()
1885    except KeyError:
1886        print "unknown command %s" % cmdName
1887        print ""
1888        printUsage(commands.keys())
1889        sys.exit(2)
1890
1891    options = cmd.options
1892    cmd.gitdir = os.environ.get("GIT_DIR", None)
1893
1894    args = sys.argv[2:]
1895
1896    if len(options) > 0:
1897        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1898
1899        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1900                                       options,
1901                                       description = cmd.description,
1902                                       formatter = HelpFormatter())
1903
1904        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1905    global verbose
1906    verbose = cmd.verbose
1907    if cmd.needsGit:
1908        if cmd.gitdir == None:
1909            cmd.gitdir = os.path.abspath(".git")
1910            if not isValidGitDir(cmd.gitdir):
1911                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1912                if os.path.exists(cmd.gitdir):
1913                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1914                    if len(cdup) > 0:
1915                        chdir(cdup);
1916
1917        if not isValidGitDir(cmd.gitdir):
1918            if isValidGitDir(cmd.gitdir + "/.git"):
1919                cmd.gitdir += "/.git"
1920            else:
1921                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1922
1923        os.environ["GIT_DIR"] = cmd.gitdir
1924
1925    if not cmd.run(args):
1926        parser.print_help()
1927
1928
1929if __name__ == '__main__':
1930    main()