Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' (1562445)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    depotPath = depotPath + "..."
 249    outputList = p4CmdList("where %s" % depotPath)
 250    output = None
 251    for entry in outputList:
 252        if "depotFile" in entry:
 253            if entry["depotFile"] == depotPath:
 254                output = entry
 255                break
 256        elif "data" in entry:
 257            data = entry.get("data")
 258            space = data.find(" ")
 259            if data[:space] == depotPath:
 260                output = entry
 261                break
 262    if output == None:
 263        return ""
 264    if output["code"] == "error":
 265        return ""
 266    clientPath = ""
 267    if "path" in output:
 268        clientPath = output.get("path")
 269    elif "data" in output:
 270        data = output.get("data")
 271        lastSpace = data.rfind(" ")
 272        clientPath = data[lastSpace + 1:]
 273
 274    if clientPath.endswith("..."):
 275        clientPath = clientPath[:-3]
 276    return clientPath
 277
 278def currentGitBranch():
 279    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 280
 281def isValidGitDir(path):
 282    if (os.path.exists(path + "/HEAD")
 283        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 284        return True;
 285    return False
 286
 287def parseRevision(ref):
 288    return read_pipe("git rev-parse %s" % ref).strip()
 289
 290def extractLogMessageFromGitCommit(commit):
 291    logMessage = ""
 292
 293    ## fixme: title is first line of commit, not 1st paragraph.
 294    foundTitle = False
 295    for log in read_pipe_lines("git cat-file commit %s" % commit):
 296       if not foundTitle:
 297           if len(log) == 1:
 298               foundTitle = True
 299           continue
 300
 301       logMessage += log
 302    return logMessage
 303
 304def extractSettingsGitLog(log):
 305    values = {}
 306    for line in log.split("\n"):
 307        line = line.strip()
 308        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 309        if not m:
 310            continue
 311
 312        assignments = m.group(1).split (':')
 313        for a in assignments:
 314            vals = a.split ('=')
 315            key = vals[0].strip()
 316            val = ('='.join (vals[1:])).strip()
 317            if val.endswith ('\"') and val.startswith('"'):
 318                val = val[1:-1]
 319
 320            values[key] = val
 321
 322    paths = values.get("depot-paths")
 323    if not paths:
 324        paths = values.get("depot-path")
 325    if paths:
 326        values['depot-paths'] = paths.split(',')
 327    return values
 328
 329def gitBranchExists(branch):
 330    proc = subprocess.Popen(["git", "rev-parse", branch],
 331                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 332    return proc.wait() == 0;
 333
 334_gitConfig = {}
 335def gitConfig(key):
 336    if not _gitConfig.has_key(key):
 337        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 338    return _gitConfig[key]
 339
 340def p4BranchesInGit(branchesAreInRemotes = True):
 341    branches = {}
 342
 343    cmdline = "git rev-parse --symbolic "
 344    if branchesAreInRemotes:
 345        cmdline += " --remotes"
 346    else:
 347        cmdline += " --branches"
 348
 349    for line in read_pipe_lines(cmdline):
 350        line = line.strip()
 351
 352        ## only import to p4/
 353        if not line.startswith('p4/') or line == "p4/HEAD":
 354            continue
 355        branch = line
 356
 357        # strip off p4
 358        branch = re.sub ("^p4/", "", line)
 359
 360        branches[branch] = parseRevision(line)
 361    return branches
 362
 363def findUpstreamBranchPoint(head = "HEAD"):
 364    branches = p4BranchesInGit()
 365    # map from depot-path to branch name
 366    branchByDepotPath = {}
 367    for branch in branches.keys():
 368        tip = branches[branch]
 369        log = extractLogMessageFromGitCommit(tip)
 370        settings = extractSettingsGitLog(log)
 371        if settings.has_key("depot-paths"):
 372            paths = ",".join(settings["depot-paths"])
 373            branchByDepotPath[paths] = "remotes/p4/" + branch
 374
 375    settings = None
 376    parent = 0
 377    while parent < 65535:
 378        commit = head + "~%s" % parent
 379        log = extractLogMessageFromGitCommit(commit)
 380        settings = extractSettingsGitLog(log)
 381        if settings.has_key("depot-paths"):
 382            paths = ",".join(settings["depot-paths"])
 383            if branchByDepotPath.has_key(paths):
 384                return [branchByDepotPath[paths], settings]
 385
 386        parent = parent + 1
 387
 388    return ["", settings]
 389
 390def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 391    if not silent:
 392        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 393               % localRefPrefix)
 394
 395    originPrefix = "origin/p4/"
 396
 397    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 398        line = line.strip()
 399        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 400            continue
 401
 402        headName = line[len(originPrefix):]
 403        remoteHead = localRefPrefix + headName
 404        originHead = line
 405
 406        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 407        if (not original.has_key('depot-paths')
 408            or not original.has_key('change')):
 409            continue
 410
 411        update = False
 412        if not gitBranchExists(remoteHead):
 413            if verbose:
 414                print "creating %s" % remoteHead
 415            update = True
 416        else:
 417            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 418            if settings.has_key('change') > 0:
 419                if settings['depot-paths'] == original['depot-paths']:
 420                    originP4Change = int(original['change'])
 421                    p4Change = int(settings['change'])
 422                    if originP4Change > p4Change:
 423                        print ("%s (%s) is newer than %s (%s). "
 424                               "Updating p4 branch from origin."
 425                               % (originHead, originP4Change,
 426                                  remoteHead, p4Change))
 427                        update = True
 428                else:
 429                    print ("Ignoring: %s was imported from %s while "
 430                           "%s was imported from %s"
 431                           % (originHead, ','.join(original['depot-paths']),
 432                              remoteHead, ','.join(settings['depot-paths'])))
 433
 434        if update:
 435            system("git update-ref %s %s" % (remoteHead, originHead))
 436
 437def originP4BranchesExist():
 438        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 439
 440def p4ChangesForPaths(depotPaths, changeRange):
 441    assert depotPaths
 442    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 443                                                        for p in depotPaths]))
 444
 445    changes = []
 446    for line in output:
 447        changeNum = line.split(" ")[1]
 448        changes.append(int(changeNum))
 449
 450    changes.sort()
 451    return changes
 452
 453class Command:
 454    def __init__(self):
 455        self.usage = "usage: %prog [options]"
 456        self.needsGit = True
 457
 458class P4Debug(Command):
 459    def __init__(self):
 460        Command.__init__(self)
 461        self.options = [
 462            optparse.make_option("--verbose", dest="verbose", action="store_true",
 463                                 default=False),
 464            ]
 465        self.description = "A tool to debug the output of p4 -G."
 466        self.needsGit = False
 467        self.verbose = False
 468
 469    def run(self, args):
 470        j = 0
 471        for output in p4CmdList(" ".join(args)):
 472            print 'Element: %d' % j
 473            j += 1
 474            print output
 475        return True
 476
 477class P4RollBack(Command):
 478    def __init__(self):
 479        Command.__init__(self)
 480        self.options = [
 481            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 482            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 483        ]
 484        self.description = "A tool to debug the multi-branch import. Don't use :)"
 485        self.verbose = False
 486        self.rollbackLocalBranches = False
 487
 488    def run(self, args):
 489        if len(args) != 1:
 490            return False
 491        maxChange = int(args[0])
 492
 493        if "p4ExitCode" in p4Cmd("changes -m 1"):
 494            die("Problems executing p4");
 495
 496        if self.rollbackLocalBranches:
 497            refPrefix = "refs/heads/"
 498            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 499        else:
 500            refPrefix = "refs/remotes/"
 501            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 502
 503        for line in lines:
 504            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 505                line = line.strip()
 506                ref = refPrefix + line
 507                log = extractLogMessageFromGitCommit(ref)
 508                settings = extractSettingsGitLog(log)
 509
 510                depotPaths = settings['depot-paths']
 511                change = settings['change']
 512
 513                changed = False
 514
 515                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 516                                                           for p in depotPaths]))) == 0:
 517                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 518                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 519                    continue
 520
 521                while change and int(change) > maxChange:
 522                    changed = True
 523                    if self.verbose:
 524                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 525                    system("git update-ref %s \"%s^\"" % (ref, ref))
 526                    log = extractLogMessageFromGitCommit(ref)
 527                    settings =  extractSettingsGitLog(log)
 528
 529
 530                    depotPaths = settings['depot-paths']
 531                    change = settings['change']
 532
 533                if changed:
 534                    print "%s rewound to %s" % (ref, change)
 535
 536        return True
 537
 538class P4Submit(Command):
 539    def __init__(self):
 540        Command.__init__(self)
 541        self.options = [
 542                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 543                optparse.make_option("--origin", dest="origin"),
 544                optparse.make_option("-M", dest="detectRename", action="store_true"),
 545        ]
 546        self.description = "Submit changes from git to the perforce depot."
 547        self.usage += " [name of git branch to submit into perforce depot]"
 548        self.interactive = True
 549        self.origin = ""
 550        self.detectRename = False
 551        self.verbose = False
 552        self.isWindows = (platform.system() == "Windows")
 553
 554    def check(self):
 555        if len(p4CmdList("opened ...")) > 0:
 556            die("You have files opened with perforce! Close them before starting the sync.")
 557
 558    # replaces everything between 'Description:' and the next P4 submit template field with the
 559    # commit message
 560    def prepareLogMessage(self, template, message):
 561        result = ""
 562
 563        inDescriptionSection = False
 564
 565        for line in template.split("\n"):
 566            if line.startswith("#"):
 567                result += line + "\n"
 568                continue
 569
 570            if inDescriptionSection:
 571                if line.startswith("Files:"):
 572                    inDescriptionSection = False
 573                else:
 574                    continue
 575            else:
 576                if line.startswith("Description:"):
 577                    inDescriptionSection = True
 578                    line += "\n"
 579                    for messageLine in message.split("\n"):
 580                        line += "\t" + messageLine + "\n"
 581
 582            result += line + "\n"
 583
 584        return result
 585
 586    def prepareSubmitTemplate(self):
 587        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 588        template = ""
 589        inFilesSection = False
 590        for line in p4_read_pipe_lines("change -o"):
 591            if line.endswith("\r\n"):
 592                line = line[:-2] + "\n"
 593            if inFilesSection:
 594                if line.startswith("\t"):
 595                    # path starts and ends with a tab
 596                    path = line[1:]
 597                    lastTab = path.rfind("\t")
 598                    if lastTab != -1:
 599                        path = path[:lastTab]
 600                        if not path.startswith(self.depotPath):
 601                            continue
 602                else:
 603                    inFilesSection = False
 604            else:
 605                if line.startswith("Files:"):
 606                    inFilesSection = True
 607
 608            template += line
 609
 610        return template
 611
 612    def applyCommit(self, id):
 613        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 614        diffOpts = ("", "-M")[self.detectRename]
 615        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 616        filesToAdd = set()
 617        filesToDelete = set()
 618        editedFiles = set()
 619        filesToChangeExecBit = {}
 620        for line in diff:
 621            diff = parseDiffTreeEntry(line)
 622            modifier = diff['status']
 623            path = diff['src']
 624            if modifier == "M":
 625                p4_system("edit \"%s\"" % path)
 626                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 627                    filesToChangeExecBit[path] = diff['dst_mode']
 628                editedFiles.add(path)
 629            elif modifier == "A":
 630                filesToAdd.add(path)
 631                filesToChangeExecBit[path] = diff['dst_mode']
 632                if path in filesToDelete:
 633                    filesToDelete.remove(path)
 634            elif modifier == "D":
 635                filesToDelete.add(path)
 636                if path in filesToAdd:
 637                    filesToAdd.remove(path)
 638            elif modifier == "R":
 639                src, dest = diff['src'], diff['dst']
 640                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 641                p4_system("edit \"%s\"" % (dest))
 642                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 643                    filesToChangeExecBit[dest] = diff['dst_mode']
 644                os.unlink(dest)
 645                editedFiles.add(dest)
 646                filesToDelete.add(src)
 647            else:
 648                die("unknown modifier %s for %s" % (modifier, path))
 649
 650        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 651        patchcmd = diffcmd + " | git apply "
 652        tryPatchCmd = patchcmd + "--check -"
 653        applyPatchCmd = patchcmd + "--check --apply -"
 654
 655        if os.system(tryPatchCmd) != 0:
 656            print "Unfortunately applying the change failed!"
 657            print "What do you want to do?"
 658            response = "x"
 659            while response != "s" and response != "a" and response != "w":
 660                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 661                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 662            if response == "s":
 663                print "Skipping! Good luck with the next patches..."
 664                for f in editedFiles:
 665                    p4_system("revert \"%s\"" % f);
 666                for f in filesToAdd:
 667                    system("rm %s" %f)
 668                return
 669            elif response == "a":
 670                os.system(applyPatchCmd)
 671                if len(filesToAdd) > 0:
 672                    print "You may also want to call p4 add on the following files:"
 673                    print " ".join(filesToAdd)
 674                if len(filesToDelete):
 675                    print "The following files should be scheduled for deletion with p4 delete:"
 676                    print " ".join(filesToDelete)
 677                die("Please resolve and submit the conflict manually and "
 678                    + "continue afterwards with git-p4 submit --continue")
 679            elif response == "w":
 680                system(diffcmd + " > patch.txt")
 681                print "Patch saved to patch.txt in %s !" % self.clientPath
 682                die("Please resolve and submit the conflict manually and "
 683                    "continue afterwards with git-p4 submit --continue")
 684
 685        system(applyPatchCmd)
 686
 687        for f in filesToAdd:
 688            p4_system("add \"%s\"" % f)
 689        for f in filesToDelete:
 690            p4_system("revert \"%s\"" % f)
 691            p4_system("delete \"%s\"" % f)
 692
 693        # Set/clear executable bits
 694        for f in filesToChangeExecBit.keys():
 695            mode = filesToChangeExecBit[f]
 696            setP4ExecBit(f, mode)
 697
 698        logMessage = extractLogMessageFromGitCommit(id)
 699        logMessage = logMessage.strip()
 700
 701        template = self.prepareSubmitTemplate()
 702
 703        if self.interactive:
 704            submitTemplate = self.prepareLogMessage(template, logMessage)
 705            if os.environ.has_key("P4DIFF"):
 706                del(os.environ["P4DIFF"])
 707            diff = p4_read_pipe("diff -du ...")
 708
 709            newdiff = ""
 710            for newFile in filesToAdd:
 711                newdiff += "==== new file ====\n"
 712                newdiff += "--- /dev/null\n"
 713                newdiff += "+++ %s\n" % newFile
 714                f = open(newFile, "r")
 715                for line in f.readlines():
 716                    newdiff += "+" + line
 717                f.close()
 718
 719            separatorLine = "######## everything below this line is just the diff #######\n"
 720
 721            [handle, fileName] = tempfile.mkstemp()
 722            tmpFile = os.fdopen(handle, "w+")
 723            if self.isWindows:
 724                submitTemplate = submitTemplate.replace("\n", "\r\n")
 725                separatorLine = separatorLine.replace("\n", "\r\n")
 726                newdiff = newdiff.replace("\n", "\r\n")
 727            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 728            tmpFile.close()
 729            mtime = os.stat(fileName).st_mtime
 730            defaultEditor = "vi"
 731            if platform.system() == "Windows":
 732                defaultEditor = "notepad"
 733            if os.environ.has_key("P4EDITOR"):
 734                editor = os.environ.get("P4EDITOR")
 735            else:
 736                editor = os.environ.get("EDITOR", defaultEditor);
 737            system(editor + " " + fileName)
 738
 739            response = "y"
 740            if os.stat(fileName).st_mtime <= mtime:
 741                response = "x"
 742                while response != "y" and response != "n":
 743                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 744
 745            if response == "y":
 746                tmpFile = open(fileName, "rb")
 747                message = tmpFile.read()
 748                tmpFile.close()
 749                submitTemplate = message[:message.index(separatorLine)]
 750                if self.isWindows:
 751                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 752                p4_write_pipe("submit -i", submitTemplate)
 753            else:
 754                for f in editedFiles:
 755                    p4_system("revert \"%s\"" % f);
 756                for f in filesToAdd:
 757                    p4_system("revert \"%s\"" % f);
 758                    system("rm %s" %f)
 759
 760            os.remove(fileName)
 761        else:
 762            fileName = "submit.txt"
 763            file = open(fileName, "w+")
 764            file.write(self.prepareLogMessage(template, logMessage))
 765            file.close()
 766            print ("Perforce submit template written as %s. "
 767                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 768                   % (fileName, fileName))
 769
 770    def run(self, args):
 771        if len(args) == 0:
 772            self.master = currentGitBranch()
 773            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 774                die("Detecting current git branch failed!")
 775        elif len(args) == 1:
 776            self.master = args[0]
 777        else:
 778            return False
 779
 780        allowSubmit = gitConfig("git-p4.allowSubmit")
 781        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 782            die("%s is not in git-p4.allowSubmit" % self.master)
 783
 784        [upstream, settings] = findUpstreamBranchPoint()
 785        self.depotPath = settings['depot-paths'][0]
 786        if len(self.origin) == 0:
 787            self.origin = upstream
 788
 789        if self.verbose:
 790            print "Origin branch is " + self.origin
 791
 792        if len(self.depotPath) == 0:
 793            print "Internal error: cannot locate perforce depot path from existing branches"
 794            sys.exit(128)
 795
 796        self.clientPath = p4Where(self.depotPath)
 797
 798        if len(self.clientPath) == 0:
 799            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 800            sys.exit(128)
 801
 802        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 803        self.oldWorkingDirectory = os.getcwd()
 804
 805        chdir(self.clientPath)
 806        print "Syncronizing p4 checkout..."
 807        p4_system("sync ...")
 808
 809        self.check()
 810
 811        commits = []
 812        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 813            commits.append(line.strip())
 814        commits.reverse()
 815
 816        while len(commits) > 0:
 817            commit = commits[0]
 818            commits = commits[1:]
 819            self.applyCommit(commit)
 820            if not self.interactive:
 821                break
 822
 823        if len(commits) == 0:
 824            print "All changes applied!"
 825            chdir(self.oldWorkingDirectory)
 826
 827            sync = P4Sync()
 828            sync.run([])
 829
 830            rebase = P4Rebase()
 831            rebase.rebase()
 832
 833        return True
 834
 835class P4Sync(Command):
 836    def __init__(self):
 837        Command.__init__(self)
 838        self.options = [
 839                optparse.make_option("--branch", dest="branch"),
 840                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 841                optparse.make_option("--changesfile", dest="changesFile"),
 842                optparse.make_option("--silent", dest="silent", action="store_true"),
 843                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 844                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 845                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 846                                     help="Import into refs/heads/ , not refs/remotes"),
 847                optparse.make_option("--max-changes", dest="maxChanges"),
 848                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 849                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 850                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 851                                     help="Only sync files that are included in the Perforce Client Spec")
 852        ]
 853        self.description = """Imports from Perforce into a git repository.\n
 854    example:
 855    //depot/my/project/ -- to import the current head
 856    //depot/my/project/@all -- to import everything
 857    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 858
 859    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 860
 861        self.usage += " //depot/path[@revRange]"
 862        self.silent = False
 863        self.createdBranches = Set()
 864        self.committedChanges = Set()
 865        self.branch = ""
 866        self.detectBranches = False
 867        self.detectLabels = False
 868        self.changesFile = ""
 869        self.syncWithOrigin = True
 870        self.verbose = False
 871        self.importIntoRemotes = True
 872        self.maxChanges = ""
 873        self.isWindows = (platform.system() == "Windows")
 874        self.keepRepoPath = False
 875        self.depotPaths = None
 876        self.p4BranchesInGit = []
 877        self.cloneExclude = []
 878        self.useClientSpec = False
 879        self.clientSpecDirs = []
 880
 881        if gitConfig("git-p4.syncFromOrigin") == "false":
 882            self.syncWithOrigin = False
 883
 884    def extractFilesFromCommit(self, commit):
 885        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 886                             for path in self.cloneExclude]
 887        files = []
 888        fnum = 0
 889        while commit.has_key("depotFile%s" % fnum):
 890            path =  commit["depotFile%s" % fnum]
 891
 892            if [p for p in self.cloneExclude
 893                if path.startswith (p)]:
 894                found = False
 895            else:
 896                found = [p for p in self.depotPaths
 897                         if path.startswith (p)]
 898            if not found:
 899                fnum = fnum + 1
 900                continue
 901
 902            file = {}
 903            file["path"] = path
 904            file["rev"] = commit["rev%s" % fnum]
 905            file["action"] = commit["action%s" % fnum]
 906            file["type"] = commit["type%s" % fnum]
 907            files.append(file)
 908            fnum = fnum + 1
 909        return files
 910
 911    def stripRepoPath(self, path, prefixes):
 912        if self.keepRepoPath:
 913            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 914
 915        for p in prefixes:
 916            if path.startswith(p):
 917                path = path[len(p):]
 918
 919        return path
 920
 921    def splitFilesIntoBranches(self, commit):
 922        branches = {}
 923        fnum = 0
 924        while commit.has_key("depotFile%s" % fnum):
 925            path =  commit["depotFile%s" % fnum]
 926            found = [p for p in self.depotPaths
 927                     if path.startswith (p)]
 928            if not found:
 929                fnum = fnum + 1
 930                continue
 931
 932            file = {}
 933            file["path"] = path
 934            file["rev"] = commit["rev%s" % fnum]
 935            file["action"] = commit["action%s" % fnum]
 936            file["type"] = commit["type%s" % fnum]
 937            fnum = fnum + 1
 938
 939            relPath = self.stripRepoPath(path, self.depotPaths)
 940
 941            for branch in self.knownBranches.keys():
 942
 943                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 944                if relPath.startswith(branch + "/"):
 945                    if branch not in branches:
 946                        branches[branch] = []
 947                    branches[branch].append(file)
 948                    break
 949
 950        return branches
 951
 952    ## Should move this out, doesn't use SELF.
 953    def readP4Files(self, files):
 954        filesForCommit = []
 955        filesToRead = []
 956
 957        for f in files:
 958            includeFile = True
 959            for val in self.clientSpecDirs:
 960                if f['path'].startswith(val[0]):
 961                    if val[1] <= 0:
 962                        includeFile = False
 963                    break
 964
 965            if includeFile:
 966                filesForCommit.append(f)
 967                if f['action'] not in ('delete', 'purge'):
 968                    filesToRead.append(f)
 969
 970        filedata = []
 971        if len(filesToRead) > 0:
 972            filedata = p4CmdList('-x - print',
 973                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 974                                                  for f in filesToRead]),
 975                                 stdin_mode='w+')
 976
 977            if "p4ExitCode" in filedata[0]:
 978                die("Problems executing p4. Error: [%d]."
 979                    % (filedata[0]['p4ExitCode']));
 980
 981        j = 0;
 982        contents = {}
 983        while j < len(filedata):
 984            stat = filedata[j]
 985            j += 1
 986            text = ''
 987            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 988                text += filedata[j]['data']
 989                del filedata[j]['data']
 990                j += 1
 991
 992            if not stat.has_key('depotFile'):
 993                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 994                continue
 995
 996            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 997                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 998            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 999                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
1000
1001            contents[stat['depotFile']] = text
1002
1003        for f in filesForCommit:
1004            path = f['path']
1005            if contents.has_key(path):
1006                f['data'] = contents[path]
1007
1008        return filesForCommit
1009
1010    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1011        epoch = details["time"]
1012        author = details["user"]
1013
1014        if self.verbose:
1015            print "commit into %s" % branch
1016
1017        # start with reading files; if that fails, we should not
1018        # create a commit.
1019        new_files = []
1020        for f in files:
1021            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1022                new_files.append (f)
1023            else:
1024                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1025        files = self.readP4Files(new_files)
1026
1027        self.gitStream.write("commit %s\n" % branch)
1028#        gitStream.write("mark :%s\n" % details["change"])
1029        self.committedChanges.add(int(details["change"]))
1030        committer = ""
1031        if author not in self.users:
1032            self.getUserMapFromPerforceServer()
1033        if author in self.users:
1034            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1035        else:
1036            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1037
1038        self.gitStream.write("committer %s\n" % committer)
1039
1040        self.gitStream.write("data <<EOT\n")
1041        self.gitStream.write(details["desc"])
1042        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1043                             % (','.join (branchPrefixes), details["change"]))
1044        if len(details['options']) > 0:
1045            self.gitStream.write(": options = %s" % details['options'])
1046        self.gitStream.write("]\nEOT\n\n")
1047
1048        if len(parent) > 0:
1049            if self.verbose:
1050                print "parent %s" % parent
1051            self.gitStream.write("from %s\n" % parent)
1052
1053        for file in files:
1054            if file["type"] == "apple":
1055                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1056                continue
1057
1058            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1059            if file["action"] in ("delete", "purge"):
1060                self.gitStream.write("D %s\n" % relPath)
1061            else:
1062                data = file['data']
1063
1064                mode = "644"
1065                if isP4Exec(file["type"]):
1066                    mode = "755"
1067                elif file["type"] == "symlink":
1068                    mode = "120000"
1069                    # p4 print on a symlink contains "target\n", so strip it off
1070                    data = data[:-1]
1071
1072                if self.isWindows and file["type"].endswith("text"):
1073                    data = data.replace("\r\n", "\n")
1074
1075                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1076                self.gitStream.write("data %s\n" % len(data))
1077                self.gitStream.write(data)
1078                self.gitStream.write("\n")
1079
1080        self.gitStream.write("\n")
1081
1082        change = int(details["change"])
1083
1084        if self.labels.has_key(change):
1085            label = self.labels[change]
1086            labelDetails = label[0]
1087            labelRevisions = label[1]
1088            if self.verbose:
1089                print "Change %s is labelled %s" % (change, labelDetails)
1090
1091            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1092                                                    for p in branchPrefixes]))
1093
1094            if len(files) == len(labelRevisions):
1095
1096                cleanedFiles = {}
1097                for info in files:
1098                    if info["action"] in ("delete", "purge"):
1099                        continue
1100                    cleanedFiles[info["depotFile"]] = info["rev"]
1101
1102                if cleanedFiles == labelRevisions:
1103                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1104                    self.gitStream.write("from %s\n" % branch)
1105
1106                    owner = labelDetails["Owner"]
1107                    tagger = ""
1108                    if author in self.users:
1109                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1110                    else:
1111                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1112                    self.gitStream.write("tagger %s\n" % tagger)
1113                    self.gitStream.write("data <<EOT\n")
1114                    self.gitStream.write(labelDetails["Description"])
1115                    self.gitStream.write("EOT\n\n")
1116
1117                else:
1118                    if not self.silent:
1119                        print ("Tag %s does not match with change %s: files do not match."
1120                               % (labelDetails["label"], change))
1121
1122            else:
1123                if not self.silent:
1124                    print ("Tag %s does not match with change %s: file count is different."
1125                           % (labelDetails["label"], change))
1126
1127    def getUserCacheFilename(self):
1128        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1129        return home + "/.gitp4-usercache.txt"
1130
1131    def getUserMapFromPerforceServer(self):
1132        if self.userMapFromPerforceServer:
1133            return
1134        self.users = {}
1135
1136        for output in p4CmdList("users"):
1137            if not output.has_key("User"):
1138                continue
1139            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1140
1141
1142        s = ''
1143        for (key, val) in self.users.items():
1144            s += "%s\t%s\n" % (key, val)
1145
1146        open(self.getUserCacheFilename(), "wb").write(s)
1147        self.userMapFromPerforceServer = True
1148
1149    def loadUserMapFromCache(self):
1150        self.users = {}
1151        self.userMapFromPerforceServer = False
1152        try:
1153            cache = open(self.getUserCacheFilename(), "rb")
1154            lines = cache.readlines()
1155            cache.close()
1156            for line in lines:
1157                entry = line.strip().split("\t")
1158                self.users[entry[0]] = entry[1]
1159        except IOError:
1160            self.getUserMapFromPerforceServer()
1161
1162    def getLabels(self):
1163        self.labels = {}
1164
1165        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1166        if len(l) > 0 and not self.silent:
1167            print "Finding files belonging to labels in %s" % `self.depotPaths`
1168
1169        for output in l:
1170            label = output["label"]
1171            revisions = {}
1172            newestChange = 0
1173            if self.verbose:
1174                print "Querying files for label %s" % label
1175            for file in p4CmdList("files "
1176                                  +  ' '.join (["%s...@%s" % (p, label)
1177                                                for p in self.depotPaths])):
1178                revisions[file["depotFile"]] = file["rev"]
1179                change = int(file["change"])
1180                if change > newestChange:
1181                    newestChange = change
1182
1183            self.labels[newestChange] = [output, revisions]
1184
1185        if self.verbose:
1186            print "Label changes: %s" % self.labels.keys()
1187
1188    def guessProjectName(self):
1189        for p in self.depotPaths:
1190            if p.endswith("/"):
1191                p = p[:-1]
1192            p = p[p.strip().rfind("/") + 1:]
1193            if not p.endswith("/"):
1194               p += "/"
1195            return p
1196
1197    def getBranchMapping(self):
1198        lostAndFoundBranches = set()
1199
1200        for info in p4CmdList("branches"):
1201            details = p4Cmd("branch -o %s" % info["branch"])
1202            viewIdx = 0
1203            while details.has_key("View%s" % viewIdx):
1204                paths = details["View%s" % viewIdx].split(" ")
1205                viewIdx = viewIdx + 1
1206                # require standard //depot/foo/... //depot/bar/... mapping
1207                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1208                    continue
1209                source = paths[0]
1210                destination = paths[1]
1211                ## HACK
1212                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1213                    source = source[len(self.depotPaths[0]):-4]
1214                    destination = destination[len(self.depotPaths[0]):-4]
1215
1216                    if destination in self.knownBranches:
1217                        if not self.silent:
1218                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1219                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1220                        continue
1221
1222                    self.knownBranches[destination] = source
1223
1224                    lostAndFoundBranches.discard(destination)
1225
1226                    if source not in self.knownBranches:
1227                        lostAndFoundBranches.add(source)
1228
1229
1230        for branch in lostAndFoundBranches:
1231            self.knownBranches[branch] = branch
1232
1233    def getBranchMappingFromGitBranches(self):
1234        branches = p4BranchesInGit(self.importIntoRemotes)
1235        for branch in branches.keys():
1236            if branch == "master":
1237                branch = "main"
1238            else:
1239                branch = branch[len(self.projectName):]
1240            self.knownBranches[branch] = branch
1241
1242    def listExistingP4GitBranches(self):
1243        # branches holds mapping from name to commit
1244        branches = p4BranchesInGit(self.importIntoRemotes)
1245        self.p4BranchesInGit = branches.keys()
1246        for branch in branches.keys():
1247            self.initialParents[self.refPrefix + branch] = branches[branch]
1248
1249    def updateOptionDict(self, d):
1250        option_keys = {}
1251        if self.keepRepoPath:
1252            option_keys['keepRepoPath'] = 1
1253
1254        d["options"] = ' '.join(sorted(option_keys.keys()))
1255
1256    def readOptions(self, d):
1257        self.keepRepoPath = (d.has_key('options')
1258                             and ('keepRepoPath' in d['options']))
1259
1260    def gitRefForBranch(self, branch):
1261        if branch == "main":
1262            return self.refPrefix + "master"
1263
1264        if len(branch) <= 0:
1265            return branch
1266
1267        return self.refPrefix + self.projectName + branch
1268
1269    def gitCommitByP4Change(self, ref, change):
1270        if self.verbose:
1271            print "looking in ref " + ref + " for change %s using bisect..." % change
1272
1273        earliestCommit = ""
1274        latestCommit = parseRevision(ref)
1275
1276        while True:
1277            if self.verbose:
1278                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1279            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1280            if len(next) == 0:
1281                if self.verbose:
1282                    print "argh"
1283                return ""
1284            log = extractLogMessageFromGitCommit(next)
1285            settings = extractSettingsGitLog(log)
1286            currentChange = int(settings['change'])
1287            if self.verbose:
1288                print "current change %s" % currentChange
1289
1290            if currentChange == change:
1291                if self.verbose:
1292                    print "found %s" % next
1293                return next
1294
1295            if currentChange < change:
1296                earliestCommit = "^%s" % next
1297            else:
1298                latestCommit = "%s" % next
1299
1300        return ""
1301
1302    def importNewBranch(self, branch, maxChange):
1303        # make fast-import flush all changes to disk and update the refs using the checkpoint
1304        # command so that we can try to find the branch parent in the git history
1305        self.gitStream.write("checkpoint\n\n");
1306        self.gitStream.flush();
1307        branchPrefix = self.depotPaths[0] + branch + "/"
1308        range = "@1,%s" % maxChange
1309        #print "prefix" + branchPrefix
1310        changes = p4ChangesForPaths([branchPrefix], range)
1311        if len(changes) <= 0:
1312            return False
1313        firstChange = changes[0]
1314        #print "first change in branch: %s" % firstChange
1315        sourceBranch = self.knownBranches[branch]
1316        sourceDepotPath = self.depotPaths[0] + sourceBranch
1317        sourceRef = self.gitRefForBranch(sourceBranch)
1318        #print "source " + sourceBranch
1319
1320        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1321        #print "branch parent: %s" % branchParentChange
1322        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1323        if len(gitParent) > 0:
1324            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1325            #print "parent git commit: %s" % gitParent
1326
1327        self.importChanges(changes)
1328        return True
1329
1330    def importChanges(self, changes):
1331        cnt = 1
1332        for change in changes:
1333            description = p4Cmd("describe %s" % change)
1334            self.updateOptionDict(description)
1335
1336            if not self.silent:
1337                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1338                sys.stdout.flush()
1339            cnt = cnt + 1
1340
1341            try:
1342                if self.detectBranches:
1343                    branches = self.splitFilesIntoBranches(description)
1344                    for branch in branches.keys():
1345                        ## HACK  --hwn
1346                        branchPrefix = self.depotPaths[0] + branch + "/"
1347
1348                        parent = ""
1349
1350                        filesForCommit = branches[branch]
1351
1352                        if self.verbose:
1353                            print "branch is %s" % branch
1354
1355                        self.updatedBranches.add(branch)
1356
1357                        if branch not in self.createdBranches:
1358                            self.createdBranches.add(branch)
1359                            parent = self.knownBranches[branch]
1360                            if parent == branch:
1361                                parent = ""
1362                            else:
1363                                fullBranch = self.projectName + branch
1364                                if fullBranch not in self.p4BranchesInGit:
1365                                    if not self.silent:
1366                                        print("\n    Importing new branch %s" % fullBranch);
1367                                    if self.importNewBranch(branch, change - 1):
1368                                        parent = ""
1369                                        self.p4BranchesInGit.append(fullBranch)
1370                                    if not self.silent:
1371                                        print("\n    Resuming with change %s" % change);
1372
1373                                if self.verbose:
1374                                    print "parent determined through known branches: %s" % parent
1375
1376                        branch = self.gitRefForBranch(branch)
1377                        parent = self.gitRefForBranch(parent)
1378
1379                        if self.verbose:
1380                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1381
1382                        if len(parent) == 0 and branch in self.initialParents:
1383                            parent = self.initialParents[branch]
1384                            del self.initialParents[branch]
1385
1386                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1387                else:
1388                    files = self.extractFilesFromCommit(description)
1389                    self.commit(description, files, self.branch, self.depotPaths,
1390                                self.initialParent)
1391                    self.initialParent = ""
1392            except IOError:
1393                print self.gitError.read()
1394                sys.exit(1)
1395
1396    def importHeadRevision(self, revision):
1397        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1398
1399        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1400        details["desc"] = ("Initial import of %s from the state at revision %s"
1401                           % (' '.join(self.depotPaths), revision))
1402        details["change"] = revision
1403        newestRevision = 0
1404
1405        fileCnt = 0
1406        for info in p4CmdList("files "
1407                              +  ' '.join(["%s...%s"
1408                                           % (p, revision)
1409                                           for p in self.depotPaths])):
1410
1411            if info['code'] == 'error':
1412                sys.stderr.write("p4 returned an error: %s\n"
1413                                 % info['data'])
1414                sys.exit(1)
1415
1416
1417            change = int(info["change"])
1418            if change > newestRevision:
1419                newestRevision = change
1420
1421            if info["action"] in ("delete", "purge"):
1422                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1423                #fileCnt = fileCnt + 1
1424                continue
1425
1426            for prop in ["depotFile", "rev", "action", "type" ]:
1427                details["%s%s" % (prop, fileCnt)] = info[prop]
1428
1429            fileCnt = fileCnt + 1
1430
1431        details["change"] = newestRevision
1432        self.updateOptionDict(details)
1433        try:
1434            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1435        except IOError:
1436            print "IO error with git fast-import. Is your git version recent enough?"
1437            print self.gitError.read()
1438
1439
1440    def getClientSpec(self):
1441        specList = p4CmdList( "client -o" )
1442        temp = {}
1443        for entry in specList:
1444            for k,v in entry.iteritems():
1445                if k.startswith("View"):
1446                    if v.startswith('"'):
1447                        start = 1
1448                    else:
1449                        start = 0
1450                    index = v.find("...")
1451                    v = v[start:index]
1452                    if v.startswith("-"):
1453                        v = v[1:]
1454                        temp[v] = -len(v)
1455                    else:
1456                        temp[v] = len(v)
1457        self.clientSpecDirs = temp.items()
1458        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1459
1460    def run(self, args):
1461        self.depotPaths = []
1462        self.changeRange = ""
1463        self.initialParent = ""
1464        self.previousDepotPaths = []
1465
1466        # map from branch depot path to parent branch
1467        self.knownBranches = {}
1468        self.initialParents = {}
1469        self.hasOrigin = originP4BranchesExist()
1470        if not self.syncWithOrigin:
1471            self.hasOrigin = False
1472
1473        if self.importIntoRemotes:
1474            self.refPrefix = "refs/remotes/p4/"
1475        else:
1476            self.refPrefix = "refs/heads/p4/"
1477
1478        if self.syncWithOrigin and self.hasOrigin:
1479            if not self.silent:
1480                print "Syncing with origin first by calling git fetch origin"
1481            system("git fetch origin")
1482
1483        if len(self.branch) == 0:
1484            self.branch = self.refPrefix + "master"
1485            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1486                system("git update-ref %s refs/heads/p4" % self.branch)
1487                system("git branch -D p4");
1488            # create it /after/ importing, when master exists
1489            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1490                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1491
1492        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1493            self.getClientSpec()
1494
1495        # TODO: should always look at previous commits,
1496        # merge with previous imports, if possible.
1497        if args == []:
1498            if self.hasOrigin:
1499                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1500            self.listExistingP4GitBranches()
1501
1502            if len(self.p4BranchesInGit) > 1:
1503                if not self.silent:
1504                    print "Importing from/into multiple branches"
1505                self.detectBranches = True
1506
1507            if self.verbose:
1508                print "branches: %s" % self.p4BranchesInGit
1509
1510            p4Change = 0
1511            for branch in self.p4BranchesInGit:
1512                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1513
1514                settings = extractSettingsGitLog(logMsg)
1515
1516                self.readOptions(settings)
1517                if (settings.has_key('depot-paths')
1518                    and settings.has_key ('change')):
1519                    change = int(settings['change']) + 1
1520                    p4Change = max(p4Change, change)
1521
1522                    depotPaths = sorted(settings['depot-paths'])
1523                    if self.previousDepotPaths == []:
1524                        self.previousDepotPaths = depotPaths
1525                    else:
1526                        paths = []
1527                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1528                            for i in range(0, min(len(cur), len(prev))):
1529                                if cur[i] <> prev[i]:
1530                                    i = i - 1
1531                                    break
1532
1533                            paths.append (cur[:i + 1])
1534
1535                        self.previousDepotPaths = paths
1536
1537            if p4Change > 0:
1538                self.depotPaths = sorted(self.previousDepotPaths)
1539                self.changeRange = "@%s,#head" % p4Change
1540                if not self.detectBranches:
1541                    self.initialParent = parseRevision(self.branch)
1542                if not self.silent and not self.detectBranches:
1543                    print "Performing incremental import into %s git branch" % self.branch
1544
1545        if not self.branch.startswith("refs/"):
1546            self.branch = "refs/heads/" + self.branch
1547
1548        if len(args) == 0 and self.depotPaths:
1549            if not self.silent:
1550                print "Depot paths: %s" % ' '.join(self.depotPaths)
1551        else:
1552            if self.depotPaths and self.depotPaths != args:
1553                print ("previous import used depot path %s and now %s was specified. "
1554                       "This doesn't work!" % (' '.join (self.depotPaths),
1555                                               ' '.join (args)))
1556                sys.exit(1)
1557
1558            self.depotPaths = sorted(args)
1559
1560        revision = ""
1561        self.users = {}
1562
1563        newPaths = []
1564        for p in self.depotPaths:
1565            if p.find("@") != -1:
1566                atIdx = p.index("@")
1567                self.changeRange = p[atIdx:]
1568                if self.changeRange == "@all":
1569                    self.changeRange = ""
1570                elif ',' not in self.changeRange:
1571                    revision = self.changeRange
1572                    self.changeRange = ""
1573                p = p[:atIdx]
1574            elif p.find("#") != -1:
1575                hashIdx = p.index("#")
1576                revision = p[hashIdx:]
1577                p = p[:hashIdx]
1578            elif self.previousDepotPaths == []:
1579                revision = "#head"
1580
1581            p = re.sub ("\.\.\.$", "", p)
1582            if not p.endswith("/"):
1583                p += "/"
1584
1585            newPaths.append(p)
1586
1587        self.depotPaths = newPaths
1588
1589
1590        self.loadUserMapFromCache()
1591        self.labels = {}
1592        if self.detectLabels:
1593            self.getLabels();
1594
1595        if self.detectBranches:
1596            ## FIXME - what's a P4 projectName ?
1597            self.projectName = self.guessProjectName()
1598
1599            if self.hasOrigin:
1600                self.getBranchMappingFromGitBranches()
1601            else:
1602                self.getBranchMapping()
1603            if self.verbose:
1604                print "p4-git branches: %s" % self.p4BranchesInGit
1605                print "initial parents: %s" % self.initialParents
1606            for b in self.p4BranchesInGit:
1607                if b != "master":
1608
1609                    ## FIXME
1610                    b = b[len(self.projectName):]
1611                self.createdBranches.add(b)
1612
1613        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1614
1615        importProcess = subprocess.Popen(["git", "fast-import"],
1616                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1617                                         stderr=subprocess.PIPE);
1618        self.gitOutput = importProcess.stdout
1619        self.gitStream = importProcess.stdin
1620        self.gitError = importProcess.stderr
1621
1622        if revision:
1623            self.importHeadRevision(revision)
1624        else:
1625            changes = []
1626
1627            if len(self.changesFile) > 0:
1628                output = open(self.changesFile).readlines()
1629                changeSet = Set()
1630                for line in output:
1631                    changeSet.add(int(line))
1632
1633                for change in changeSet:
1634                    changes.append(change)
1635
1636                changes.sort()
1637            else:
1638                if self.verbose:
1639                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1640                                                              self.changeRange)
1641                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1642
1643                if len(self.maxChanges) > 0:
1644                    changes = changes[:min(int(self.maxChanges), len(changes))]
1645
1646            if len(changes) == 0:
1647                if not self.silent:
1648                    print "No changes to import!"
1649                return True
1650
1651            if not self.silent and not self.detectBranches:
1652                print "Import destination: %s" % self.branch
1653
1654            self.updatedBranches = set()
1655
1656            self.importChanges(changes)
1657
1658            if not self.silent:
1659                print ""
1660                if len(self.updatedBranches) > 0:
1661                    sys.stdout.write("Updated branches: ")
1662                    for b in self.updatedBranches:
1663                        sys.stdout.write("%s " % b)
1664                    sys.stdout.write("\n")
1665
1666        self.gitStream.close()
1667        if importProcess.wait() != 0:
1668            die("fast-import failed: %s" % self.gitError.read())
1669        self.gitOutput.close()
1670        self.gitError.close()
1671
1672        return True
1673
1674class P4Rebase(Command):
1675    def __init__(self):
1676        Command.__init__(self)
1677        self.options = [ ]
1678        self.description = ("Fetches the latest revision from perforce and "
1679                            + "rebases the current work (branch) against it")
1680        self.verbose = False
1681
1682    def run(self, args):
1683        sync = P4Sync()
1684        sync.run([])
1685
1686        return self.rebase()
1687
1688    def rebase(self):
1689        if os.system("git update-index --refresh") != 0:
1690            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1691        if len(read_pipe("git diff-index HEAD --")) > 0:
1692            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1693
1694        [upstream, settings] = findUpstreamBranchPoint()
1695        if len(upstream) == 0:
1696            die("Cannot find upstream branchpoint for rebase")
1697
1698        # the branchpoint may be p4/foo~3, so strip off the parent
1699        upstream = re.sub("~[0-9]+$", "", upstream)
1700
1701        print "Rebasing the current branch onto %s" % upstream
1702        oldHead = read_pipe("git rev-parse HEAD").strip()
1703        system("git rebase %s" % upstream)
1704        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1705        return True
1706
1707class P4Clone(P4Sync):
1708    def __init__(self):
1709        P4Sync.__init__(self)
1710        self.description = "Creates a new git repository and imports from Perforce into it"
1711        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1712        self.options += [
1713            optparse.make_option("--destination", dest="cloneDestination",
1714                                 action='store', default=None,
1715                                 help="where to leave result of the clone"),
1716            optparse.make_option("-/", dest="cloneExclude",
1717                                 action="append", type="string",
1718                                 help="exclude depot path")
1719        ]
1720        self.cloneDestination = None
1721        self.needsGit = False
1722
1723    # This is required for the "append" cloneExclude action
1724    def ensure_value(self, attr, value):
1725        if not hasattr(self, attr) or getattr(self, attr) is None:
1726            setattr(self, attr, value)
1727        return getattr(self, attr)
1728
1729    def defaultDestination(self, args):
1730        ## TODO: use common prefix of args?
1731        depotPath = args[0]
1732        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1733        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1734        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1735        depotDir = re.sub(r"/$", "", depotDir)
1736        return os.path.split(depotDir)[1]
1737
1738    def run(self, args):
1739        if len(args) < 1:
1740            return False
1741
1742        if self.keepRepoPath and not self.cloneDestination:
1743            sys.stderr.write("Must specify destination for --keep-path\n")
1744            sys.exit(1)
1745
1746        depotPaths = args
1747
1748        if not self.cloneDestination and len(depotPaths) > 1:
1749            self.cloneDestination = depotPaths[-1]
1750            depotPaths = depotPaths[:-1]
1751
1752        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1753        for p in depotPaths:
1754            if not p.startswith("//"):
1755                return False
1756
1757        if not self.cloneDestination:
1758            self.cloneDestination = self.defaultDestination(args)
1759
1760        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1761        if not os.path.exists(self.cloneDestination):
1762            os.makedirs(self.cloneDestination)
1763        chdir(self.cloneDestination)
1764        system("git init")
1765        self.gitdir = os.getcwd() + "/.git"
1766        if not P4Sync.run(self, depotPaths):
1767            return False
1768        if self.branch != "master":
1769            if self.importIntoRemotes:
1770                masterbranch = "refs/remotes/p4/master"
1771            else:
1772                masterbranch = "refs/heads/p4/master"
1773            if gitBranchExists(masterbranch):
1774                system("git branch master %s" % masterbranch)
1775                system("git checkout -f")
1776            else:
1777                print "Could not detect main branch. No checkout/master branch created."
1778
1779        return True
1780
1781class P4Branches(Command):
1782    def __init__(self):
1783        Command.__init__(self)
1784        self.options = [ ]
1785        self.description = ("Shows the git branches that hold imports and their "
1786                            + "corresponding perforce depot paths")
1787        self.verbose = False
1788
1789    def run(self, args):
1790        if originP4BranchesExist():
1791            createOrUpdateBranchesFromOrigin()
1792
1793        cmdline = "git rev-parse --symbolic "
1794        cmdline += " --remotes"
1795
1796        for line in read_pipe_lines(cmdline):
1797            line = line.strip()
1798
1799            if not line.startswith('p4/') or line == "p4/HEAD":
1800                continue
1801            branch = line
1802
1803            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1804            settings = extractSettingsGitLog(log)
1805
1806            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1807        return True
1808
1809class HelpFormatter(optparse.IndentedHelpFormatter):
1810    def __init__(self):
1811        optparse.IndentedHelpFormatter.__init__(self)
1812
1813    def format_description(self, description):
1814        if description:
1815            return description + "\n"
1816        else:
1817            return ""
1818
1819def printUsage(commands):
1820    print "usage: %s <command> [options]" % sys.argv[0]
1821    print ""
1822    print "valid commands: %s" % ", ".join(commands)
1823    print ""
1824    print "Try %s <command> --help for command specific help." % sys.argv[0]
1825    print ""
1826
1827commands = {
1828    "debug" : P4Debug,
1829    "submit" : P4Submit,
1830    "commit" : P4Submit,
1831    "sync" : P4Sync,
1832    "rebase" : P4Rebase,
1833    "clone" : P4Clone,
1834    "rollback" : P4RollBack,
1835    "branches" : P4Branches
1836}
1837
1838
1839def main():
1840    if len(sys.argv[1:]) == 0:
1841        printUsage(commands.keys())
1842        sys.exit(2)
1843
1844    cmd = ""
1845    cmdName = sys.argv[1]
1846    try:
1847        klass = commands[cmdName]
1848        cmd = klass()
1849    except KeyError:
1850        print "unknown command %s" % cmdName
1851        print ""
1852        printUsage(commands.keys())
1853        sys.exit(2)
1854
1855    options = cmd.options
1856    cmd.gitdir = os.environ.get("GIT_DIR", None)
1857
1858    args = sys.argv[2:]
1859
1860    if len(options) > 0:
1861        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1862
1863        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1864                                       options,
1865                                       description = cmd.description,
1866                                       formatter = HelpFormatter())
1867
1868        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1869    global verbose
1870    verbose = cmd.verbose
1871    if cmd.needsGit:
1872        if cmd.gitdir == None:
1873            cmd.gitdir = os.path.abspath(".git")
1874            if not isValidGitDir(cmd.gitdir):
1875                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1876                if os.path.exists(cmd.gitdir):
1877                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1878                    if len(cdup) > 0:
1879                        chdir(cdup);
1880
1881        if not isValidGitDir(cmd.gitdir):
1882            if isValidGitDir(cmd.gitdir + "/.git"):
1883                cmd.gitdir += "/.git"
1884            else:
1885                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1886
1887        os.environ["GIT_DIR"] = cmd.gitdir
1888
1889    if not cmd.run(args):
1890        parser.print_help()
1891
1892
1893if __name__ == '__main__':
1894    main()