contrib / fast-import / git-p4on commit Merge branch 'tg/tag-points-at' (76bdcf0)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = ["p4"]
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += ["-u",user]
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += ["-P", password]
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += ["-p", port]
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += ["-h", host]
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += ["-c", client]
  46
  47
  48    if isinstance(cmd,basestring):
  49        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50    else:
  51        real_cmd += cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56    # not using the shell, we have to set it ourselves.  This path could
  57    # be relative, so go there first, then figure out where we ended up.
  58    os.chdir(dir)
  59    os.environ['PWD'] = os.getcwd()
  60
  61def die(msg):
  62    if verbose:
  63        raise Exception(msg)
  64    else:
  65        sys.stderr.write(msg + "\n")
  66        sys.exit(1)
  67
  68def write_pipe(c, stdin):
  69    if verbose:
  70        sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72    expand = isinstance(c,basestring)
  73    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74    pipe = p.stdin
  75    val = pipe.write(stdin)
  76    pipe.close()
  77    if p.wait():
  78        die('Command failed: %s' % str(c))
  79
  80    return val
  81
  82def p4_write_pipe(c, stdin):
  83    real_cmd = p4_build_cmd(c)
  84    return write_pipe(real_cmd, stdin)
  85
  86def read_pipe(c, ignore_error=False):
  87    if verbose:
  88        sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90    expand = isinstance(c,basestring)
  91    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92    pipe = p.stdout
  93    val = pipe.read()
  94    if p.wait() and not ignore_error:
  95        die('Command failed: %s' % str(c))
  96
  97    return val
  98
  99def p4_read_pipe(c, ignore_error=False):
 100    real_cmd = p4_build_cmd(c)
 101    return read_pipe(real_cmd, ignore_error)
 102
 103def read_pipe_lines(c):
 104    if verbose:
 105        sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107    expand = isinstance(c, basestring)
 108    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109    pipe = p.stdout
 110    val = pipe.readlines()
 111    if pipe.close() or p.wait():
 112        die('Command failed: %s' % str(c))
 113
 114    return val
 115
 116def p4_read_pipe_lines(c):
 117    """Specifically invoke p4 on the command supplied. """
 118    real_cmd = p4_build_cmd(c)
 119    return read_pipe_lines(real_cmd)
 120
 121def system(cmd):
 122    expand = isinstance(cmd,basestring)
 123    if verbose:
 124        sys.stderr.write("executing %s\n" % str(cmd))
 125    subprocess.check_call(cmd, shell=expand)
 126
 127def p4_system(cmd):
 128    """Specifically invoke p4 as the system command. """
 129    real_cmd = p4_build_cmd(cmd)
 130    expand = isinstance(real_cmd, basestring)
 131    subprocess.check_call(real_cmd, shell=expand)
 132
 133def p4_integrate(src, dest):
 134    p4_system(["integrate", "-Dt", src, dest])
 135
 136def p4_sync(path):
 137    p4_system(["sync", path])
 138
 139def p4_add(f):
 140    p4_system(["add", f])
 141
 142def p4_delete(f):
 143    p4_system(["delete", f])
 144
 145def p4_edit(f):
 146    p4_system(["edit", f])
 147
 148def p4_revert(f):
 149    p4_system(["revert", f])
 150
 151def p4_reopen(type, file):
 152    p4_system(["reopen", "-t", type, file])
 153
 154#
 155# Canonicalize the p4 type and return a tuple of the
 156# base type, plus any modifiers.  See "p4 help filetypes"
 157# for a list and explanation.
 158#
 159def split_p4_type(p4type):
 160
 161    p4_filetypes_historical = {
 162        "ctempobj": "binary+Sw",
 163        "ctext": "text+C",
 164        "cxtext": "text+Cx",
 165        "ktext": "text+k",
 166        "kxtext": "text+kx",
 167        "ltext": "text+F",
 168        "tempobj": "binary+FSw",
 169        "ubinary": "binary+F",
 170        "uresource": "resource+F",
 171        "uxbinary": "binary+Fx",
 172        "xbinary": "binary+x",
 173        "xltext": "text+Fx",
 174        "xtempobj": "binary+Swx",
 175        "xtext": "text+x",
 176        "xunicode": "unicode+x",
 177        "xutf16": "utf16+x",
 178    }
 179    if p4type in p4_filetypes_historical:
 180        p4type = p4_filetypes_historical[p4type]
 181    mods = ""
 182    s = p4type.split("+")
 183    base = s[0]
 184    mods = ""
 185    if len(s) > 1:
 186        mods = s[1]
 187    return (base, mods)
 188
 189
 190def setP4ExecBit(file, mode):
 191    # Reopens an already open file and changes the execute bit to match
 192    # the execute bit setting in the passed in mode.
 193
 194    p4Type = "+x"
 195
 196    if not isModeExec(mode):
 197        p4Type = getP4OpenedType(file)
 198        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200        if p4Type[-1] == "+":
 201            p4Type = p4Type[0:-1]
 202
 203    p4_reopen(p4Type, file)
 204
 205def getP4OpenedType(file):
 206    # Returns the perforce file type for the given file.
 207
 208    result = p4_read_pipe(["opened", file])
 209    match = re.match(".*\((.+)\)\r?$", result)
 210    if match:
 211        return match.group(1)
 212    else:
 213        die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215def diffTreePattern():
 216    # This is a simple generator for the diff tree regex pattern. This could be
 217    # a class variable if this and parseDiffTreeEntry were a part of a class.
 218    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219    while True:
 220        yield pattern
 221
 222def parseDiffTreeEntry(entry):
 223    """Parses a single diff tree entry into its component elements.
 224
 225    See git-diff-tree(1) manpage for details about the format of the diff
 226    output. This method returns a dictionary with the following elements:
 227
 228    src_mode - The mode of the source file
 229    dst_mode - The mode of the destination file
 230    src_sha1 - The sha1 for the source file
 231    dst_sha1 - The sha1 fr the destination file
 232    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233    status_score - The score for the status (applicable for 'C' and 'R'
 234                   statuses). This is None if there is no score.
 235    src - The path for the source file.
 236    dst - The path for the destination file. This is only present for
 237          copy or renames. If it is not present, this is None.
 238
 239    If the pattern is not matched, None is returned."""
 240
 241    match = diffTreePattern().next().match(entry)
 242    if match:
 243        return {
 244            'src_mode': match.group(1),
 245            'dst_mode': match.group(2),
 246            'src_sha1': match.group(3),
 247            'dst_sha1': match.group(4),
 248            'status': match.group(5),
 249            'status_score': match.group(6),
 250            'src': match.group(7),
 251            'dst': match.group(10)
 252        }
 253    return None
 254
 255def isModeExec(mode):
 256    # Returns True if the given git mode represents an executable file,
 257    # otherwise False.
 258    return mode[-3:] == "755"
 259
 260def isModeExecChanged(src_mode, dst_mode):
 261    return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265    if isinstance(cmd,basestring):
 266        cmd = "-G " + cmd
 267        expand = True
 268    else:
 269        cmd = ["-G"] + cmd
 270        expand = False
 271
 272    cmd = p4_build_cmd(cmd)
 273    if verbose:
 274        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276    # Use a temporary file to avoid deadlocks without
 277    # subprocess.communicate(), which would put another copy
 278    # of stdout into memory.
 279    stdin_file = None
 280    if stdin is not None:
 281        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282        if isinstance(stdin,basestring):
 283            stdin_file.write(stdin)
 284        else:
 285            for i in stdin:
 286                stdin_file.write(i + '\n')
 287        stdin_file.flush()
 288        stdin_file.seek(0)
 289
 290    p4 = subprocess.Popen(cmd,
 291                          shell=expand,
 292                          stdin=stdin_file,
 293                          stdout=subprocess.PIPE)
 294
 295    result = []
 296    try:
 297        while True:
 298            entry = marshal.load(p4.stdout)
 299            if cb is not None:
 300                cb(entry)
 301            else:
 302                result.append(entry)
 303    except EOFError:
 304        pass
 305    exitCode = p4.wait()
 306    if exitCode != 0:
 307        entry = {}
 308        entry["p4ExitCode"] = exitCode
 309        result.append(entry)
 310
 311    return result
 312
 313def p4Cmd(cmd):
 314    list = p4CmdList(cmd)
 315    result = {}
 316    for entry in list:
 317        result.update(entry)
 318    return result;
 319
 320def p4Where(depotPath):
 321    if not depotPath.endswith("/"):
 322        depotPath += "/"
 323    depotPath = depotPath + "..."
 324    outputList = p4CmdList(["where", depotPath])
 325    output = None
 326    for entry in outputList:
 327        if "depotFile" in entry:
 328            if entry["depotFile"] == depotPath:
 329                output = entry
 330                break
 331        elif "data" in entry:
 332            data = entry.get("data")
 333            space = data.find(" ")
 334            if data[:space] == depotPath:
 335                output = entry
 336                break
 337    if output == None:
 338        return ""
 339    if output["code"] == "error":
 340        return ""
 341    clientPath = ""
 342    if "path" in output:
 343        clientPath = output.get("path")
 344    elif "data" in output:
 345        data = output.get("data")
 346        lastSpace = data.rfind(" ")
 347        clientPath = data[lastSpace + 1:]
 348
 349    if clientPath.endswith("..."):
 350        clientPath = clientPath[:-3]
 351    return clientPath
 352
 353def currentGitBranch():
 354    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356def isValidGitDir(path):
 357    if (os.path.exists(path + "/HEAD")
 358        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359        return True;
 360    return False
 361
 362def parseRevision(ref):
 363    return read_pipe("git rev-parse %s" % ref).strip()
 364
 365def branchExists(ref):
 366    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                     ignore_error=True)
 368    return len(rev) > 0
 369
 370def extractLogMessageFromGitCommit(commit):
 371    logMessage = ""
 372
 373    ## fixme: title is first line of commit, not 1st paragraph.
 374    foundTitle = False
 375    for log in read_pipe_lines("git cat-file commit %s" % commit):
 376       if not foundTitle:
 377           if len(log) == 1:
 378               foundTitle = True
 379           continue
 380
 381       logMessage += log
 382    return logMessage
 383
 384def extractSettingsGitLog(log):
 385    values = {}
 386    for line in log.split("\n"):
 387        line = line.strip()
 388        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389        if not m:
 390            continue
 391
 392        assignments = m.group(1).split (':')
 393        for a in assignments:
 394            vals = a.split ('=')
 395            key = vals[0].strip()
 396            val = ('='.join (vals[1:])).strip()
 397            if val.endswith ('\"') and val.startswith('"'):
 398                val = val[1:-1]
 399
 400            values[key] = val
 401
 402    paths = values.get("depot-paths")
 403    if not paths:
 404        paths = values.get("depot-path")
 405    if paths:
 406        values['depot-paths'] = paths.split(',')
 407    return values
 408
 409def gitBranchExists(branch):
 410    proc = subprocess.Popen(["git", "rev-parse", branch],
 411                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412    return proc.wait() == 0;
 413
 414_gitConfig = {}
 415def gitConfig(key, args = None): # set args to "--bool", for instance
 416    if not _gitConfig.has_key(key):
 417        argsFilter = ""
 418        if args != None:
 419            argsFilter = "%s " % args
 420        cmd = "git config %s%s" % (argsFilter, key)
 421        _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422    return _gitConfig[key]
 423
 424def gitConfigList(key):
 425    if not _gitConfig.has_key(key):
 426        _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427    return _gitConfig[key]
 428
 429def p4BranchesInGit(branchesAreInRemotes = True):
 430    branches = {}
 431
 432    cmdline = "git rev-parse --symbolic "
 433    if branchesAreInRemotes:
 434        cmdline += " --remotes"
 435    else:
 436        cmdline += " --branches"
 437
 438    for line in read_pipe_lines(cmdline):
 439        line = line.strip()
 440
 441        ## only import to p4/
 442        if not line.startswith('p4/') or line == "p4/HEAD":
 443            continue
 444        branch = line
 445
 446        # strip off p4
 447        branch = re.sub ("^p4/", "", line)
 448
 449        branches[branch] = parseRevision(line)
 450    return branches
 451
 452def findUpstreamBranchPoint(head = "HEAD"):
 453    branches = p4BranchesInGit()
 454    # map from depot-path to branch name
 455    branchByDepotPath = {}
 456    for branch in branches.keys():
 457        tip = branches[branch]
 458        log = extractLogMessageFromGitCommit(tip)
 459        settings = extractSettingsGitLog(log)
 460        if settings.has_key("depot-paths"):
 461            paths = ",".join(settings["depot-paths"])
 462            branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464    settings = None
 465    parent = 0
 466    while parent < 65535:
 467        commit = head + "~%s" % parent
 468        log = extractLogMessageFromGitCommit(commit)
 469        settings = extractSettingsGitLog(log)
 470        if settings.has_key("depot-paths"):
 471            paths = ",".join(settings["depot-paths"])
 472            if branchByDepotPath.has_key(paths):
 473                return [branchByDepotPath[paths], settings]
 474
 475        parent = parent + 1
 476
 477    return ["", settings]
 478
 479def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480    if not silent:
 481        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482               % localRefPrefix)
 483
 484    originPrefix = "origin/p4/"
 485
 486    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487        line = line.strip()
 488        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489            continue
 490
 491        headName = line[len(originPrefix):]
 492        remoteHead = localRefPrefix + headName
 493        originHead = line
 494
 495        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496        if (not original.has_key('depot-paths')
 497            or not original.has_key('change')):
 498            continue
 499
 500        update = False
 501        if not gitBranchExists(remoteHead):
 502            if verbose:
 503                print "creating %s" % remoteHead
 504            update = True
 505        else:
 506            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507            if settings.has_key('change') > 0:
 508                if settings['depot-paths'] == original['depot-paths']:
 509                    originP4Change = int(original['change'])
 510                    p4Change = int(settings['change'])
 511                    if originP4Change > p4Change:
 512                        print ("%s (%s) is newer than %s (%s). "
 513                               "Updating p4 branch from origin."
 514                               % (originHead, originP4Change,
 515                                  remoteHead, p4Change))
 516                        update = True
 517                else:
 518                    print ("Ignoring: %s was imported from %s while "
 519                           "%s was imported from %s"
 520                           % (originHead, ','.join(original['depot-paths']),
 521                              remoteHead, ','.join(settings['depot-paths'])))
 522
 523        if update:
 524            system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526def originP4BranchesExist():
 527        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529def p4ChangesForPaths(depotPaths, changeRange):
 530    assert depotPaths
 531    cmd = ['changes']
 532    for p in depotPaths:
 533        cmd += ["%s...%s" % (p, changeRange)]
 534    output = p4_read_pipe_lines(cmd)
 535
 536    changes = {}
 537    for line in output:
 538        changeNum = int(line.split(" ")[1])
 539        changes[changeNum] = True
 540
 541    changelist = changes.keys()
 542    changelist.sort()
 543    return changelist
 544
 545def p4PathStartsWith(path, prefix):
 546    # This method tries to remedy a potential mixed-case issue:
 547    #
 548    # If UserA adds  //depot/DirA/file1
 549    # and UserB adds //depot/dira/file2
 550    #
 551    # we may or may not have a problem. If you have core.ignorecase=true,
 552    # we treat DirA and dira as the same directory
 553    ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554    if ignorecase:
 555        return path.lower().startswith(prefix.lower())
 556    return path.startswith(prefix)
 557
 558class Command:
 559    def __init__(self):
 560        self.usage = "usage: %prog [options]"
 561        self.needsGit = True
 562
 563class P4UserMap:
 564    def __init__(self):
 565        self.userMapFromPerforceServer = False
 566        self.myP4UserId = None
 567
 568    def p4UserId(self):
 569        if self.myP4UserId:
 570            return self.myP4UserId
 571
 572        results = p4CmdList("user -o")
 573        for r in results:
 574            if r.has_key('User'):
 575                self.myP4UserId = r['User']
 576                return r['User']
 577        die("Could not find your p4 user id")
 578
 579    def p4UserIsMe(self, p4User):
 580        # return True if the given p4 user is actually me
 581        me = self.p4UserId()
 582        if not p4User or p4User != me:
 583            return False
 584        else:
 585            return True
 586
 587    def getUserCacheFilename(self):
 588        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 589        return home + "/.gitp4-usercache.txt"
 590
 591    def getUserMapFromPerforceServer(self):
 592        if self.userMapFromPerforceServer:
 593            return
 594        self.users = {}
 595        self.emails = {}
 596
 597        for output in p4CmdList("users"):
 598            if not output.has_key("User"):
 599                continue
 600            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 601            self.emails[output["Email"]] = output["User"]
 602
 603
 604        s = ''
 605        for (key, val) in self.users.items():
 606            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 607
 608        open(self.getUserCacheFilename(), "wb").write(s)
 609        self.userMapFromPerforceServer = True
 610
 611    def loadUserMapFromCache(self):
 612        self.users = {}
 613        self.userMapFromPerforceServer = False
 614        try:
 615            cache = open(self.getUserCacheFilename(), "rb")
 616            lines = cache.readlines()
 617            cache.close()
 618            for line in lines:
 619                entry = line.strip().split("\t")
 620                self.users[entry[0]] = entry[1]
 621        except IOError:
 622            self.getUserMapFromPerforceServer()
 623
 624class P4Debug(Command):
 625    def __init__(self):
 626        Command.__init__(self)
 627        self.options = [
 628            optparse.make_option("--verbose", dest="verbose", action="store_true",
 629                                 default=False),
 630            ]
 631        self.description = "A tool to debug the output of p4 -G."
 632        self.needsGit = False
 633        self.verbose = False
 634
 635    def run(self, args):
 636        j = 0
 637        for output in p4CmdList(args):
 638            print 'Element: %d' % j
 639            j += 1
 640            print output
 641        return True
 642
 643class P4RollBack(Command):
 644    def __init__(self):
 645        Command.__init__(self)
 646        self.options = [
 647            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 648            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 649        ]
 650        self.description = "A tool to debug the multi-branch import. Don't use :)"
 651        self.verbose = False
 652        self.rollbackLocalBranches = False
 653
 654    def run(self, args):
 655        if len(args) != 1:
 656            return False
 657        maxChange = int(args[0])
 658
 659        if "p4ExitCode" in p4Cmd("changes -m 1"):
 660            die("Problems executing p4");
 661
 662        if self.rollbackLocalBranches:
 663            refPrefix = "refs/heads/"
 664            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 665        else:
 666            refPrefix = "refs/remotes/"
 667            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 668
 669        for line in lines:
 670            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 671                line = line.strip()
 672                ref = refPrefix + line
 673                log = extractLogMessageFromGitCommit(ref)
 674                settings = extractSettingsGitLog(log)
 675
 676                depotPaths = settings['depot-paths']
 677                change = settings['change']
 678
 679                changed = False
 680
 681                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 682                                                           for p in depotPaths]))) == 0:
 683                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 684                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 685                    continue
 686
 687                while change and int(change) > maxChange:
 688                    changed = True
 689                    if self.verbose:
 690                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 691                    system("git update-ref %s \"%s^\"" % (ref, ref))
 692                    log = extractLogMessageFromGitCommit(ref)
 693                    settings =  extractSettingsGitLog(log)
 694
 695
 696                    depotPaths = settings['depot-paths']
 697                    change = settings['change']
 698
 699                if changed:
 700                    print "%s rewound to %s" % (ref, change)
 701
 702        return True
 703
 704class P4Submit(Command, P4UserMap):
 705    def __init__(self):
 706        Command.__init__(self)
 707        P4UserMap.__init__(self)
 708        self.options = [
 709                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 710                optparse.make_option("--origin", dest="origin"),
 711                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 712                # preserve the user, requires relevant p4 permissions
 713                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 714        ]
 715        self.description = "Submit changes from git to the perforce depot."
 716        self.usage += " [name of git branch to submit into perforce depot]"
 717        self.interactive = True
 718        self.origin = ""
 719        self.detectRenames = False
 720        self.verbose = False
 721        self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 722        self.isWindows = (platform.system() == "Windows")
 723
 724    def check(self):
 725        if len(p4CmdList("opened ...")) > 0:
 726            die("You have files opened with perforce! Close them before starting the sync.")
 727
 728    # replaces everything between 'Description:' and the next P4 submit template field with the
 729    # commit message
 730    def prepareLogMessage(self, template, message):
 731        result = ""
 732
 733        inDescriptionSection = False
 734
 735        for line in template.split("\n"):
 736            if line.startswith("#"):
 737                result += line + "\n"
 738                continue
 739
 740            if inDescriptionSection:
 741                if line.startswith("Files:") or line.startswith("Jobs:"):
 742                    inDescriptionSection = False
 743                else:
 744                    continue
 745            else:
 746                if line.startswith("Description:"):
 747                    inDescriptionSection = True
 748                    line += "\n"
 749                    for messageLine in message.split("\n"):
 750                        line += "\t" + messageLine + "\n"
 751
 752            result += line + "\n"
 753
 754        return result
 755
 756    def p4UserForCommit(self,id):
 757        # Return the tuple (perforce user,git email) for a given git commit id
 758        self.getUserMapFromPerforceServer()
 759        gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 760        gitEmail = gitEmail.strip()
 761        if not self.emails.has_key(gitEmail):
 762            return (None,gitEmail)
 763        else:
 764            return (self.emails[gitEmail],gitEmail)
 765
 766    def checkValidP4Users(self,commits):
 767        # check if any git authors cannot be mapped to p4 users
 768        for id in commits:
 769            (user,email) = self.p4UserForCommit(id)
 770            if not user:
 771                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 772                if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 773                    print "%s" % msg
 774                else:
 775                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 776
 777    def lastP4Changelist(self):
 778        # Get back the last changelist number submitted in this client spec. This
 779        # then gets used to patch up the username in the change. If the same
 780        # client spec is being used by multiple processes then this might go
 781        # wrong.
 782        results = p4CmdList("client -o")        # find the current client
 783        client = None
 784        for r in results:
 785            if r.has_key('Client'):
 786                client = r['Client']
 787                break
 788        if not client:
 789            die("could not get client spec")
 790        results = p4CmdList(["changes", "-c", client, "-m", "1"])
 791        for r in results:
 792            if r.has_key('change'):
 793                return r['change']
 794        die("Could not get changelist number for last submit - cannot patch up user details")
 795
 796    def modifyChangelistUser(self, changelist, newUser):
 797        # fixup the user field of a changelist after it has been submitted.
 798        changes = p4CmdList("change -o %s" % changelist)
 799        if len(changes) != 1:
 800            die("Bad output from p4 change modifying %s to user %s" %
 801                (changelist, newUser))
 802
 803        c = changes[0]
 804        if c['User'] == newUser: return   # nothing to do
 805        c['User'] = newUser
 806        input = marshal.dumps(c)
 807
 808        result = p4CmdList("change -f -i", stdin=input)
 809        for r in result:
 810            if r.has_key('code'):
 811                if r['code'] == 'error':
 812                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 813            if r.has_key('data'):
 814                print("Updated user field for changelist %s to %s" % (changelist, newUser))
 815                return
 816        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 817
 818    def canChangeChangelists(self):
 819        # check to see if we have p4 admin or super-user permissions, either of
 820        # which are required to modify changelists.
 821        results = p4CmdList(["protects", self.depotPath])
 822        for r in results:
 823            if r.has_key('perm'):
 824                if r['perm'] == 'admin':
 825                    return 1
 826                if r['perm'] == 'super':
 827                    return 1
 828        return 0
 829
 830    def prepareSubmitTemplate(self):
 831        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832        template = ""
 833        inFilesSection = False
 834        for line in p4_read_pipe_lines(['change', '-o']):
 835            if line.endswith("\r\n"):
 836                line = line[:-2] + "\n"
 837            if inFilesSection:
 838                if line.startswith("\t"):
 839                    # path starts and ends with a tab
 840                    path = line[1:]
 841                    lastTab = path.rfind("\t")
 842                    if lastTab != -1:
 843                        path = path[:lastTab]
 844                        if not p4PathStartsWith(path, self.depotPath):
 845                            continue
 846                else:
 847                    inFilesSection = False
 848            else:
 849                if line.startswith("Files:"):
 850                    inFilesSection = True
 851
 852            template += line
 853
 854        return template
 855
 856    def edit_template(self, template_file):
 857        """Invoke the editor to let the user change the submission
 858           message.  Return true if okay to continue with the submit."""
 859
 860        # if configured to skip the editing part, just submit
 861        if gitConfig("git-p4.skipSubmitEdit") == "true":
 862            return True
 863
 864        # look at the modification time, to check later if the user saved
 865        # the file
 866        mtime = os.stat(template_file).st_mtime
 867
 868        # invoke the editor
 869        if os.environ.has_key("P4EDITOR"):
 870            editor = os.environ.get("P4EDITOR")
 871        else:
 872            editor = read_pipe("git var GIT_EDITOR").strip()
 873        system(editor + " " + template_file)
 874
 875        # If the file was not saved, prompt to see if this patch should
 876        # be skipped.  But skip this verification step if configured so.
 877        if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878            return True
 879
 880        # modification time updated means user saved the file
 881        if os.stat(template_file).st_mtime > mtime:
 882            return True
 883
 884        while True:
 885            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886            if response == 'y':
 887                return True
 888            if response == 'n':
 889                return False
 890
 891    def applyCommit(self, id):
 892        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894        (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896        if not self.detectRenames:
 897            # If not explicitly set check the config variable
 898            self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900        if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901            diffOpts = ""
 902        elif self.detectRenames.lower() == "true":
 903            diffOpts = "-M"
 904        else:
 905            diffOpts = "-M%s" % self.detectRenames
 906
 907        detectCopies = gitConfig("git-p4.detectCopies")
 908        if detectCopies.lower() == "true":
 909            diffOpts += " -C"
 910        elif detectCopies != "" and detectCopies.lower() != "false":
 911            diffOpts += " -C%s" % detectCopies
 912
 913        if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914            diffOpts += " --find-copies-harder"
 915
 916        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917        filesToAdd = set()
 918        filesToDelete = set()
 919        editedFiles = set()
 920        filesToChangeExecBit = {}
 921        for line in diff:
 922            diff = parseDiffTreeEntry(line)
 923            modifier = diff['status']
 924            path = diff['src']
 925            if modifier == "M":
 926                p4_edit(path)
 927                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                    filesToChangeExecBit[path] = diff['dst_mode']
 929                editedFiles.add(path)
 930            elif modifier == "A":
 931                filesToAdd.add(path)
 932                filesToChangeExecBit[path] = diff['dst_mode']
 933                if path in filesToDelete:
 934                    filesToDelete.remove(path)
 935            elif modifier == "D":
 936                filesToDelete.add(path)
 937                if path in filesToAdd:
 938                    filesToAdd.remove(path)
 939            elif modifier == "C":
 940                src, dest = diff['src'], diff['dst']
 941                p4_integrate(src, dest)
 942                if diff['src_sha1'] != diff['dst_sha1']:
 943                    p4_edit(dest)
 944                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                    p4_edit(dest)
 946                    filesToChangeExecBit[dest] = diff['dst_mode']
 947                os.unlink(dest)
 948                editedFiles.add(dest)
 949            elif modifier == "R":
 950                src, dest = diff['src'], diff['dst']
 951                p4_integrate(src, dest)
 952                if diff['src_sha1'] != diff['dst_sha1']:
 953                    p4_edit(dest)
 954                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                    p4_edit(dest)
 956                    filesToChangeExecBit[dest] = diff['dst_mode']
 957                os.unlink(dest)
 958                editedFiles.add(dest)
 959                filesToDelete.add(src)
 960            else:
 961                die("unknown modifier %s for %s" % (modifier, path))
 962
 963        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964        patchcmd = diffcmd + " | git apply "
 965        tryPatchCmd = patchcmd + "--check -"
 966        applyPatchCmd = patchcmd + "--check --apply -"
 967
 968        if os.system(tryPatchCmd) != 0:
 969            print "Unfortunately applying the change failed!"
 970            print "What do you want to do?"
 971            response = "x"
 972            while response != "s" and response != "a" and response != "w":
 973                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975            if response == "s":
 976                print "Skipping! Good luck with the next patches..."
 977                for f in editedFiles:
 978                    p4_revert(f)
 979                for f in filesToAdd:
 980                    os.remove(f)
 981                return
 982            elif response == "a":
 983                os.system(applyPatchCmd)
 984                if len(filesToAdd) > 0:
 985                    print "You may also want to call p4 add on the following files:"
 986                    print " ".join(filesToAdd)
 987                if len(filesToDelete):
 988                    print "The following files should be scheduled for deletion with p4 delete:"
 989                    print " ".join(filesToDelete)
 990                die("Please resolve and submit the conflict manually and "
 991                    + "continue afterwards with git-p4 submit --continue")
 992            elif response == "w":
 993                system(diffcmd + " > patch.txt")
 994                print "Patch saved to patch.txt in %s !" % self.clientPath
 995                die("Please resolve and submit the conflict manually and "
 996                    "continue afterwards with git-p4 submit --continue")
 997
 998        system(applyPatchCmd)
 999
1000        for f in filesToAdd:
1001            p4_add(f)
1002        for f in filesToDelete:
1003            p4_revert(f)
1004            p4_delete(f)
1005
1006        # Set/clear executable bits
1007        for f in filesToChangeExecBit.keys():
1008            mode = filesToChangeExecBit[f]
1009            setP4ExecBit(f, mode)
1010
1011        logMessage = extractLogMessageFromGitCommit(id)
1012        logMessage = logMessage.strip()
1013
1014        template = self.prepareSubmitTemplate()
1015
1016        if self.interactive:
1017            submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019            if self.preserveUser:
1020               submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022            if os.environ.has_key("P4DIFF"):
1023                del(os.environ["P4DIFF"])
1024            diff = ""
1025            for editedFile in editedFiles:
1026                diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028            newdiff = ""
1029            for newFile in filesToAdd:
1030                newdiff += "==== new file ====\n"
1031                newdiff += "--- /dev/null\n"
1032                newdiff += "+++ %s\n" % newFile
1033                f = open(newFile, "r")
1034                for line in f.readlines():
1035                    newdiff += "+" + line
1036                f.close()
1037
1038            if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043            separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045            (handle, fileName) = tempfile.mkstemp()
1046            tmpFile = os.fdopen(handle, "w+")
1047            if self.isWindows:
1048                submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                separatorLine = separatorLine.replace("\n", "\r\n")
1050                newdiff = newdiff.replace("\n", "\r\n")
1051            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052            tmpFile.close()
1053
1054            if self.edit_template(fileName):
1055                # read the edited message and submit
1056                tmpFile = open(fileName, "rb")
1057                message = tmpFile.read()
1058                tmpFile.close()
1059                submitTemplate = message[:message.index(separatorLine)]
1060                if self.isWindows:
1061                    submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                if self.preserveUser:
1065                    if p4User:
1066                        # Get last changelist number. Cannot easily get it from
1067                        # the submit command output as the output is
1068                        # unmarshalled.
1069                        changelist = self.lastP4Changelist()
1070                        self.modifyChangelistUser(changelist, p4User)
1071            else:
1072                # skip this patch
1073                print "Submission cancelled, undoing p4 changes."
1074                for f in editedFiles:
1075                    p4_revert(f)
1076                for f in filesToAdd:
1077                    p4_revert(f)
1078                    os.remove(f)
1079
1080            os.remove(fileName)
1081        else:
1082            fileName = "submit.txt"
1083            file = open(fileName, "w+")
1084            file.write(self.prepareLogMessage(template, logMessage))
1085            file.close()
1086            print ("Perforce submit template written as %s. "
1087                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                   % (fileName, fileName))
1089
1090    def run(self, args):
1091        if len(args) == 0:
1092            self.master = currentGitBranch()
1093            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                die("Detecting current git branch failed!")
1095        elif len(args) == 1:
1096            self.master = args[0]
1097            if not branchExists(self.master):
1098                die("Branch %s does not exist" % self.master)
1099        else:
1100            return False
1101
1102        allowSubmit = gitConfig("git-p4.allowSubmit")
1103        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104            die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106        [upstream, settings] = findUpstreamBranchPoint()
1107        self.depotPath = settings['depot-paths'][0]
1108        if len(self.origin) == 0:
1109            self.origin = upstream
1110
1111        if self.preserveUser:
1112            if not self.canChangeChangelists():
1113                die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115        if self.verbose:
1116            print "Origin branch is " + self.origin
1117
1118        if len(self.depotPath) == 0:
1119            print "Internal error: cannot locate perforce depot path from existing branches"
1120            sys.exit(128)
1121
1122        self.clientPath = p4Where(self.depotPath)
1123
1124        if len(self.clientPath) == 0:
1125            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126            sys.exit(128)
1127
1128        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129        self.oldWorkingDirectory = os.getcwd()
1130
1131        # ensure the clientPath exists
1132        if not os.path.exists(self.clientPath):
1133            os.makedirs(self.clientPath)
1134
1135        chdir(self.clientPath)
1136        print "Synchronizing p4 checkout..."
1137        p4_sync("...")
1138        self.check()
1139
1140        commits = []
1141        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142            commits.append(line.strip())
1143        commits.reverse()
1144
1145        if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146            self.checkAuthorship = False
1147        else:
1148            self.checkAuthorship = True
1149
1150        if self.preserveUser:
1151            self.checkValidP4Users(commits)
1152
1153        while len(commits) > 0:
1154            commit = commits[0]
1155            commits = commits[1:]
1156            self.applyCommit(commit)
1157            if not self.interactive:
1158                break
1159
1160        if len(commits) == 0:
1161            print "All changes applied!"
1162            chdir(self.oldWorkingDirectory)
1163
1164            sync = P4Sync()
1165            sync.run([])
1166
1167            rebase = P4Rebase()
1168            rebase.rebase()
1169
1170        return True
1171
1172class View(object):
1173    """Represent a p4 view ("p4 help views"), and map files in a
1174       repo according to the view."""
1175
1176    class Path(object):
1177        """A depot or client path, possibly containing wildcards.
1178           The only one supported is ... at the end, currently.
1179           Initialize with the full path, with //depot or //client."""
1180
1181        def __init__(self, path, is_depot):
1182            self.path = path
1183            self.is_depot = is_depot
1184            self.find_wildcards()
1185            # remember the prefix bit, useful for relative mappings
1186            m = re.match("(//[^/]+/)", self.path)
1187            if not m:
1188                die("Path %s does not start with //prefix/" % self.path)
1189            prefix = m.group(1)
1190            if not self.is_depot:
1191                # strip //client/ on client paths
1192                self.path = self.path[len(prefix):]
1193
1194        def find_wildcards(self):
1195            """Make sure wildcards are valid, and set up internal
1196               variables."""
1197
1198            self.ends_triple_dot = False
1199            # There are three wildcards allowed in p4 views
1200            # (see "p4 help views").  This code knows how to
1201            # handle "..." (only at the end), but cannot deal with
1202            # "%%n" or "*".  Only check the depot_side, as p4 should
1203            # validate that the client_side matches too.
1204            if re.search(r'%%[1-9]', self.path):
1205                die("Can't handle %%n wildcards in view: %s" % self.path)
1206            if self.path.find("*") >= 0:
1207                die("Can't handle * wildcards in view: %s" % self.path)
1208            triple_dot_index = self.path.find("...")
1209            if triple_dot_index >= 0:
1210                if triple_dot_index != len(self.path) - 3:
1211                    die("Can handle only single ... wildcard, at end: %s" %
1212                        self.path)
1213                self.ends_triple_dot = True
1214
1215        def ensure_compatible(self, other_path):
1216            """Make sure the wildcards agree."""
1217            if self.ends_triple_dot != other_path.ends_triple_dot:
1218                 die("Both paths must end with ... if either does;\n" +
1219                     "paths: %s %s" % (self.path, other_path.path))
1220
1221        def match_wildcards(self, test_path):
1222            """See if this test_path matches us, and fill in the value
1223               of the wildcards if so.  Returns a tuple of
1224               (True|False, wildcards[]).  For now, only the ... at end
1225               is supported, so at most one wildcard."""
1226            if self.ends_triple_dot:
1227                dotless = self.path[:-3]
1228                if test_path.startswith(dotless):
1229                    wildcard = test_path[len(dotless):]
1230                    return (True, [ wildcard ])
1231            else:
1232                if test_path == self.path:
1233                    return (True, [])
1234            return (False, [])
1235
1236        def match(self, test_path):
1237            """Just return if it matches; don't bother with the wildcards."""
1238            b, _ = self.match_wildcards(test_path)
1239            return b
1240
1241        def fill_in_wildcards(self, wildcards):
1242            """Return the relative path, with the wildcards filled in
1243               if there are any."""
1244            if self.ends_triple_dot:
1245                return self.path[:-3] + wildcards[0]
1246            else:
1247                return self.path
1248
1249    class Mapping(object):
1250        def __init__(self, depot_side, client_side, overlay, exclude):
1251            # depot_side is without the trailing /... if it had one
1252            self.depot_side = View.Path(depot_side, is_depot=True)
1253            self.client_side = View.Path(client_side, is_depot=False)
1254            self.overlay = overlay  # started with "+"
1255            self.exclude = exclude  # started with "-"
1256            assert not (self.overlay and self.exclude)
1257            self.depot_side.ensure_compatible(self.client_side)
1258
1259        def __str__(self):
1260            c = " "
1261            if self.overlay:
1262                c = "+"
1263            if self.exclude:
1264                c = "-"
1265            return "View.Mapping: %s%s -> %s" % \
1266                   (c, self.depot_side.path, self.client_side.path)
1267
1268        def map_depot_to_client(self, depot_path):
1269            """Calculate the client path if using this mapping on the
1270               given depot path; does not consider the effect of other
1271               mappings in a view.  Even excluded mappings are returned."""
1272            matches, wildcards = self.depot_side.match_wildcards(depot_path)
1273            if not matches:
1274                return ""
1275            client_path = self.client_side.fill_in_wildcards(wildcards)
1276            return client_path
1277
1278    #
1279    # View methods
1280    #
1281    def __init__(self):
1282        self.mappings = []
1283
1284    def append(self, view_line):
1285        """Parse a view line, splitting it into depot and client
1286           sides.  Append to self.mappings, preserving order."""
1287
1288        # Split the view line into exactly two words.  P4 enforces
1289        # structure on these lines that simplifies this quite a bit.
1290        #
1291        # Either or both words may be double-quoted.
1292        # Single quotes do not matter.
1293        # Double-quote marks cannot occur inside the words.
1294        # A + or - prefix is also inside the quotes.
1295        # There are no quotes unless they contain a space.
1296        # The line is already white-space stripped.
1297        # The two words are separated by a single space.
1298        #
1299        if view_line[0] == '"':
1300            # First word is double quoted.  Find its end.
1301            close_quote_index = view_line.find('"', 1)
1302            if close_quote_index <= 0:
1303                die("No first-word closing quote found: %s" % view_line)
1304            depot_side = view_line[1:close_quote_index]
1305            # skip closing quote and space
1306            rhs_index = close_quote_index + 1 + 1
1307        else:
1308            space_index = view_line.find(" ")
1309            if space_index <= 0:
1310                die("No word-splitting space found: %s" % view_line)
1311            depot_side = view_line[0:space_index]
1312            rhs_index = space_index + 1
1313
1314        if view_line[rhs_index] == '"':
1315            # Second word is double quoted.  Make sure there is a
1316            # double quote at the end too.
1317            if not view_line.endswith('"'):
1318                die("View line with rhs quote should end with one: %s" %
1319                    view_line)
1320            # skip the quotes
1321            client_side = view_line[rhs_index+1:-1]
1322        else:
1323            client_side = view_line[rhs_index:]
1324
1325        # prefix + means overlay on previous mapping
1326        overlay = False
1327        if depot_side.startswith("+"):
1328            overlay = True
1329            depot_side = depot_side[1:]
1330
1331        # prefix - means exclude this path
1332        exclude = False
1333        if depot_side.startswith("-"):
1334            exclude = True
1335            depot_side = depot_side[1:]
1336
1337        m = View.Mapping(depot_side, client_side, overlay, exclude)
1338        self.mappings.append(m)
1339
1340    def map_in_client(self, depot_path):
1341        """Return the relative location in the client where this
1342           depot file should live.  Returns "" if the file should
1343           not be mapped in the client."""
1344
1345        paths_filled = []
1346        client_path = ""
1347
1348        # look at later entries first
1349        for m in self.mappings[::-1]:
1350
1351            # see where will this path end up in the client
1352            p = m.map_depot_to_client(depot_path)
1353
1354            if p == "":
1355                # Depot path does not belong in client.  Must remember
1356                # this, as previous items should not cause files to
1357                # exist in this path either.  Remember that the list is
1358                # being walked from the end, which has higher precedence.
1359                # Overlap mappings do not exclude previous mappings.
1360                if not m.overlay:
1361                    paths_filled.append(m.client_side)
1362
1363            else:
1364                # This mapping matched; no need to search any further.
1365                # But, the mapping could be rejected if the client path
1366                # has already been claimed by an earlier mapping (i.e.
1367                # one later in the list, which we are walking backwards).
1368                already_mapped_in_client = False
1369                for f in paths_filled:
1370                    # this is View.Path.match
1371                    if f.match(p):
1372                        already_mapped_in_client = True
1373                        break
1374                if not already_mapped_in_client:
1375                    # Include this file, unless it is from a line that
1376                    # explicitly said to exclude it.
1377                    if not m.exclude:
1378                        client_path = p
1379
1380                # a match, even if rejected, always stops the search
1381                break
1382
1383        return client_path
1384
1385class P4Sync(Command, P4UserMap):
1386    delete_actions = ( "delete", "move/delete", "purge" )
1387
1388    def __init__(self):
1389        Command.__init__(self)
1390        P4UserMap.__init__(self)
1391        self.options = [
1392                optparse.make_option("--branch", dest="branch"),
1393                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1394                optparse.make_option("--changesfile", dest="changesFile"),
1395                optparse.make_option("--silent", dest="silent", action="store_true"),
1396                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1397                optparse.make_option("--verbose", dest="verbose", action="store_true"),
1398                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1399                                     help="Import into refs/heads/ , not refs/remotes"),
1400                optparse.make_option("--max-changes", dest="maxChanges"),
1401                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1402                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1403                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1404                                     help="Only sync files that are included in the Perforce Client Spec")
1405        ]
1406        self.description = """Imports from Perforce into a git repository.\n
1407    example:
1408    //depot/my/project/ -- to import the current head
1409    //depot/my/project/@all -- to import everything
1410    //depot/my/project/@1,6 -- to import only from revision 1 to 6
1411
1412    (a ... is not needed in the path p4 specification, it's added implicitly)"""
1413
1414        self.usage += " //depot/path[@revRange]"
1415        self.silent = False
1416        self.createdBranches = set()
1417        self.committedChanges = set()
1418        self.branch = ""
1419        self.detectBranches = False
1420        self.detectLabels = False
1421        self.changesFile = ""
1422        self.syncWithOrigin = True
1423        self.verbose = False
1424        self.importIntoRemotes = True
1425        self.maxChanges = ""
1426        self.isWindows = (platform.system() == "Windows")
1427        self.keepRepoPath = False
1428        self.depotPaths = None
1429        self.p4BranchesInGit = []
1430        self.cloneExclude = []
1431        self.useClientSpec = False
1432        self.clientSpecDirs = None
1433        self.tempBranches = []
1434        self.tempBranchLocation = "git-p4-tmp"
1435
1436        if gitConfig("git-p4.syncFromOrigin") == "false":
1437            self.syncWithOrigin = False
1438
1439    #
1440    # P4 wildcards are not allowed in filenames.  P4 complains
1441    # if you simply add them, but you can force it with "-f", in
1442    # which case it translates them into %xx encoding internally.
1443    # Search for and fix just these four characters.  Do % last so
1444    # that fixing it does not inadvertently create new %-escapes.
1445    #
1446    def wildcard_decode(self, path):
1447        # Cannot have * in a filename in windows; untested as to
1448        # what p4 would do in such a case.
1449        if not self.isWindows:
1450            path = path.replace("%2A", "*")
1451        path = path.replace("%23", "#") \
1452                   .replace("%40", "@") \
1453                   .replace("%25", "%")
1454        return path
1455
1456    # Force a checkpoint in fast-import and wait for it to finish
1457    def checkpoint(self):
1458        self.gitStream.write("checkpoint\n\n")
1459        self.gitStream.write("progress checkpoint\n\n")
1460        out = self.gitOutput.readline()
1461        if self.verbose:
1462            print "checkpoint finished: " + out
1463
1464    def extractFilesFromCommit(self, commit):
1465        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1466                             for path in self.cloneExclude]
1467        files = []
1468        fnum = 0
1469        while commit.has_key("depotFile%s" % fnum):
1470            path =  commit["depotFile%s" % fnum]
1471
1472            if [p for p in self.cloneExclude
1473                if p4PathStartsWith(path, p)]:
1474                found = False
1475            else:
1476                found = [p for p in self.depotPaths
1477                         if p4PathStartsWith(path, p)]
1478            if not found:
1479                fnum = fnum + 1
1480                continue
1481
1482            file = {}
1483            file["path"] = path
1484            file["rev"] = commit["rev%s" % fnum]
1485            file["action"] = commit["action%s" % fnum]
1486            file["type"] = commit["type%s" % fnum]
1487            files.append(file)
1488            fnum = fnum + 1
1489        return files
1490
1491    def stripRepoPath(self, path, prefixes):
1492        if self.useClientSpec:
1493            return self.clientSpecDirs.map_in_client(path)
1494
1495        if self.keepRepoPath:
1496            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1497
1498        for p in prefixes:
1499            if p4PathStartsWith(path, p):
1500                path = path[len(p):]
1501
1502        return path
1503
1504    def splitFilesIntoBranches(self, commit):
1505        branches = {}
1506        fnum = 0
1507        while commit.has_key("depotFile%s" % fnum):
1508            path =  commit["depotFile%s" % fnum]
1509            found = [p for p in self.depotPaths
1510                     if p4PathStartsWith(path, p)]
1511            if not found:
1512                fnum = fnum + 1
1513                continue
1514
1515            file = {}
1516            file["path"] = path
1517            file["rev"] = commit["rev%s" % fnum]
1518            file["action"] = commit["action%s" % fnum]
1519            file["type"] = commit["type%s" % fnum]
1520            fnum = fnum + 1
1521
1522            relPath = self.stripRepoPath(path, self.depotPaths)
1523
1524            for branch in self.knownBranches.keys():
1525
1526                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1527                if relPath.startswith(branch + "/"):
1528                    if branch not in branches:
1529                        branches[branch] = []
1530                    branches[branch].append(file)
1531                    break
1532
1533        return branches
1534
1535    # output one file from the P4 stream
1536    # - helper for streamP4Files
1537
1538    def streamOneP4File(self, file, contents):
1539        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1540        relPath = self.wildcard_decode(relPath)
1541        if verbose:
1542            sys.stderr.write("%s\n" % relPath)
1543
1544        (type_base, type_mods) = split_p4_type(file["type"])
1545
1546        git_mode = "100644"
1547        if "x" in type_mods:
1548            git_mode = "100755"
1549        if type_base == "symlink":
1550            git_mode = "120000"
1551            # p4 print on a symlink contains "target\n"; remove the newline
1552            data = ''.join(contents)
1553            contents = [data[:-1]]
1554
1555        if type_base == "utf16":
1556            # p4 delivers different text in the python output to -G
1557            # than it does when using "print -o", or normal p4 client
1558            # operations.  utf16 is converted to ascii or utf8, perhaps.
1559            # But ascii text saved as -t utf16 is completely mangled.
1560            # Invoke print -o to get the real contents.
1561            text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1562            contents = [ text ]
1563
1564        if type_base == "apple":
1565            # Apple filetype files will be streamed as a concatenation of
1566            # its appledouble header and the contents.  This is useless
1567            # on both macs and non-macs.  If using "print -q -o xx", it
1568            # will create "xx" with the data, and "%xx" with the header.
1569            # This is also not very useful.
1570            #
1571            # Ideally, someday, this script can learn how to generate
1572            # appledouble files directly and import those to git, but
1573            # non-mac machines can never find a use for apple filetype.
1574            print "\nIgnoring apple filetype file %s" % file['depotFile']
1575            return
1576
1577        # Perhaps windows wants unicode, utf16 newlines translated too;
1578        # but this is not doing it.
1579        if self.isWindows and type_base == "text":
1580            mangled = []
1581            for data in contents:
1582                data = data.replace("\r\n", "\n")
1583                mangled.append(data)
1584            contents = mangled
1585
1586        # Note that we do not try to de-mangle keywords on utf16 files,
1587        # even though in theory somebody may want that.
1588        if type_base in ("text", "unicode", "binary"):
1589            if "ko" in type_mods:
1590                text = ''.join(contents)
1591                text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1592                contents = [ text ]
1593            elif "k" in type_mods:
1594                text = ''.join(contents)
1595                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1596                contents = [ text ]
1597
1598        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1599
1600        # total length...
1601        length = 0
1602        for d in contents:
1603            length = length + len(d)
1604
1605        self.gitStream.write("data %d\n" % length)
1606        for d in contents:
1607            self.gitStream.write(d)
1608        self.gitStream.write("\n")
1609
1610    def streamOneP4Deletion(self, file):
1611        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1612        if verbose:
1613            sys.stderr.write("delete %s\n" % relPath)
1614        self.gitStream.write("D %s\n" % relPath)
1615
1616    # handle another chunk of streaming data
1617    def streamP4FilesCb(self, marshalled):
1618
1619        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1620            # start of a new file - output the old one first
1621            self.streamOneP4File(self.stream_file, self.stream_contents)
1622            self.stream_file = {}
1623            self.stream_contents = []
1624            self.stream_have_file_info = False
1625
1626        # pick up the new file information... for the
1627        # 'data' field we need to append to our array
1628        for k in marshalled.keys():
1629            if k == 'data':
1630                self.stream_contents.append(marshalled['data'])
1631            else:
1632                self.stream_file[k] = marshalled[k]
1633
1634        self.stream_have_file_info = True
1635
1636    # Stream directly from "p4 files" into "git fast-import"
1637    def streamP4Files(self, files):
1638        filesForCommit = []
1639        filesToRead = []
1640        filesToDelete = []
1641
1642        for f in files:
1643            # if using a client spec, only add the files that have
1644            # a path in the client
1645            if self.clientSpecDirs:
1646                if self.clientSpecDirs.map_in_client(f['path']) == "":
1647                    continue
1648
1649            filesForCommit.append(f)
1650            if f['action'] in self.delete_actions:
1651                filesToDelete.append(f)
1652            else:
1653                filesToRead.append(f)
1654
1655        # deleted files...
1656        for f in filesToDelete:
1657            self.streamOneP4Deletion(f)
1658
1659        if len(filesToRead) > 0:
1660            self.stream_file = {}
1661            self.stream_contents = []
1662            self.stream_have_file_info = False
1663
1664            # curry self argument
1665            def streamP4FilesCbSelf(entry):
1666                self.streamP4FilesCb(entry)
1667
1668            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1669
1670            p4CmdList(["-x", "-", "print"],
1671                      stdin=fileArgs,
1672                      cb=streamP4FilesCbSelf)
1673
1674            # do the last chunk
1675            if self.stream_file.has_key('depotFile'):
1676                self.streamOneP4File(self.stream_file, self.stream_contents)
1677
1678    def make_email(self, userid):
1679        if userid in self.users:
1680            return self.users[userid]
1681        else:
1682            return "%s <a@b>" % userid
1683
1684    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1685        epoch = details["time"]
1686        author = details["user"]
1687        self.branchPrefixes = branchPrefixes
1688
1689        if self.verbose:
1690            print "commit into %s" % branch
1691
1692        # start with reading files; if that fails, we should not
1693        # create a commit.
1694        new_files = []
1695        for f in files:
1696            if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1697                new_files.append (f)
1698            else:
1699                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1700
1701        self.gitStream.write("commit %s\n" % branch)
1702#        gitStream.write("mark :%s\n" % details["change"])
1703        self.committedChanges.add(int(details["change"]))
1704        committer = ""
1705        if author not in self.users:
1706            self.getUserMapFromPerforceServer()
1707        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
1708
1709        self.gitStream.write("committer %s\n" % committer)
1710
1711        self.gitStream.write("data <<EOT\n")
1712        self.gitStream.write(details["desc"])
1713        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1714                             % (','.join (branchPrefixes), details["change"]))
1715        if len(details['options']) > 0:
1716            self.gitStream.write(": options = %s" % details['options'])
1717        self.gitStream.write("]\nEOT\n\n")
1718
1719        if len(parent) > 0:
1720            if self.verbose:
1721                print "parent %s" % parent
1722            self.gitStream.write("from %s\n" % parent)
1723
1724        self.streamP4Files(new_files)
1725        self.gitStream.write("\n")
1726
1727        change = int(details["change"])
1728
1729        if self.labels.has_key(change):
1730            label = self.labels[change]
1731            labelDetails = label[0]
1732            labelRevisions = label[1]
1733            if self.verbose:
1734                print "Change %s is labelled %s" % (change, labelDetails)
1735
1736            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1737                                                    for p in branchPrefixes])
1738
1739            if len(files) == len(labelRevisions):
1740
1741                cleanedFiles = {}
1742                for info in files:
1743                    if info["action"] in self.delete_actions:
1744                        continue
1745                    cleanedFiles[info["depotFile"]] = info["rev"]
1746
1747                if cleanedFiles == labelRevisions:
1748                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1749                    self.gitStream.write("from %s\n" % branch)
1750
1751                    owner = labelDetails["Owner"]
1752
1753                    # Try to use the owner of the p4 label, or failing that,
1754                    # the current p4 user id.
1755                    if owner:
1756                        email = self.make_email(owner)
1757                    else:
1758                        email = self.make_email(self.p4UserId())
1759                    tagger = "%s %s %s" % (email, epoch, self.tz)
1760
1761                    self.gitStream.write("tagger %s\n" % tagger)
1762
1763                    description = labelDetails["Description"]
1764                    self.gitStream.write("data %d\n" % len(description))
1765                    self.gitStream.write(description)
1766                    self.gitStream.write("\n")
1767
1768                else:
1769                    if not self.silent:
1770                        print ("Tag %s does not match with change %s: files do not match."
1771                               % (labelDetails["label"], change))
1772
1773            else:
1774                if not self.silent:
1775                    print ("Tag %s does not match with change %s: file count is different."
1776                           % (labelDetails["label"], change))
1777
1778    def getLabels(self):
1779        self.labels = {}
1780
1781        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
1782        if len(l) > 0 and not self.silent:
1783            print "Finding files belonging to labels in %s" % `self.depotPaths`
1784
1785        for output in l:
1786            label = output["label"]
1787            revisions = {}
1788            newestChange = 0
1789            if self.verbose:
1790                print "Querying files for label %s" % label
1791            for file in p4CmdList(["files"] +
1792                                      ["%s...@%s" % (p, label)
1793                                          for p in self.depotPaths]):
1794                revisions[file["depotFile"]] = file["rev"]
1795                change = int(file["change"])
1796                if change > newestChange:
1797                    newestChange = change
1798
1799            self.labels[newestChange] = [output, revisions]
1800
1801        if self.verbose:
1802            print "Label changes: %s" % self.labels.keys()
1803
1804    def guessProjectName(self):
1805        for p in self.depotPaths:
1806            if p.endswith("/"):
1807                p = p[:-1]
1808            p = p[p.strip().rfind("/") + 1:]
1809            if not p.endswith("/"):
1810               p += "/"
1811            return p
1812
1813    def getBranchMapping(self):
1814        lostAndFoundBranches = set()
1815
1816        user = gitConfig("git-p4.branchUser")
1817        if len(user) > 0:
1818            command = "branches -u %s" % user
1819        else:
1820            command = "branches"
1821
1822        for info in p4CmdList(command):
1823            details = p4Cmd(["branch", "-o", info["branch"]])
1824            viewIdx = 0
1825            while details.has_key("View%s" % viewIdx):
1826                paths = details["View%s" % viewIdx].split(" ")
1827                viewIdx = viewIdx + 1
1828                # require standard //depot/foo/... //depot/bar/... mapping
1829                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1830                    continue
1831                source = paths[0]
1832                destination = paths[1]
1833                ## HACK
1834                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1835                    source = source[len(self.depotPaths[0]):-4]
1836                    destination = destination[len(self.depotPaths[0]):-4]
1837
1838                    if destination in self.knownBranches:
1839                        if not self.silent:
1840                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1841                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1842                        continue
1843
1844                    self.knownBranches[destination] = source
1845
1846                    lostAndFoundBranches.discard(destination)
1847
1848                    if source not in self.knownBranches:
1849                        lostAndFoundBranches.add(source)
1850
1851        # Perforce does not strictly require branches to be defined, so we also
1852        # check git config for a branch list.
1853        #
1854        # Example of branch definition in git config file:
1855        # [git-p4]
1856        #   branchList=main:branchA
1857        #   branchList=main:branchB
1858        #   branchList=branchA:branchC
1859        configBranches = gitConfigList("git-p4.branchList")
1860        for branch in configBranches:
1861            if branch:
1862                (source, destination) = branch.split(":")
1863                self.knownBranches[destination] = source
1864
1865                lostAndFoundBranches.discard(destination)
1866
1867                if source not in self.knownBranches:
1868                    lostAndFoundBranches.add(source)
1869
1870
1871        for branch in lostAndFoundBranches:
1872            self.knownBranches[branch] = branch
1873
1874    def getBranchMappingFromGitBranches(self):
1875        branches = p4BranchesInGit(self.importIntoRemotes)
1876        for branch in branches.keys():
1877            if branch == "master":
1878                branch = "main"
1879            else:
1880                branch = branch[len(self.projectName):]
1881            self.knownBranches[branch] = branch
1882
1883    def listExistingP4GitBranches(self):
1884        # branches holds mapping from name to commit
1885        branches = p4BranchesInGit(self.importIntoRemotes)
1886        self.p4BranchesInGit = branches.keys()
1887        for branch in branches.keys():
1888            self.initialParents[self.refPrefix + branch] = branches[branch]
1889
1890    def updateOptionDict(self, d):
1891        option_keys = {}
1892        if self.keepRepoPath:
1893            option_keys['keepRepoPath'] = 1
1894
1895        d["options"] = ' '.join(sorted(option_keys.keys()))
1896
1897    def readOptions(self, d):
1898        self.keepRepoPath = (d.has_key('options')
1899                             and ('keepRepoPath' in d['options']))
1900
1901    def gitRefForBranch(self, branch):
1902        if branch == "main":
1903            return self.refPrefix + "master"
1904
1905        if len(branch) <= 0:
1906            return branch
1907
1908        return self.refPrefix + self.projectName + branch
1909
1910    def gitCommitByP4Change(self, ref, change):
1911        if self.verbose:
1912            print "looking in ref " + ref + " for change %s using bisect..." % change
1913
1914        earliestCommit = ""
1915        latestCommit = parseRevision(ref)
1916
1917        while True:
1918            if self.verbose:
1919                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1920            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1921            if len(next) == 0:
1922                if self.verbose:
1923                    print "argh"
1924                return ""
1925            log = extractLogMessageFromGitCommit(next)
1926            settings = extractSettingsGitLog(log)
1927            currentChange = int(settings['change'])
1928            if self.verbose:
1929                print "current change %s" % currentChange
1930
1931            if currentChange == change:
1932                if self.verbose:
1933                    print "found %s" % next
1934                return next
1935
1936            if currentChange < change:
1937                earliestCommit = "^%s" % next
1938            else:
1939                latestCommit = "%s" % next
1940
1941        return ""
1942
1943    def importNewBranch(self, branch, maxChange):
1944        # make fast-import flush all changes to disk and update the refs using the checkpoint
1945        # command so that we can try to find the branch parent in the git history
1946        self.gitStream.write("checkpoint\n\n");
1947        self.gitStream.flush();
1948        branchPrefix = self.depotPaths[0] + branch + "/"
1949        range = "@1,%s" % maxChange
1950        #print "prefix" + branchPrefix
1951        changes = p4ChangesForPaths([branchPrefix], range)
1952        if len(changes) <= 0:
1953            return False
1954        firstChange = changes[0]
1955        #print "first change in branch: %s" % firstChange
1956        sourceBranch = self.knownBranches[branch]
1957        sourceDepotPath = self.depotPaths[0] + sourceBranch
1958        sourceRef = self.gitRefForBranch(sourceBranch)
1959        #print "source " + sourceBranch
1960
1961        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
1962        #print "branch parent: %s" % branchParentChange
1963        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1964        if len(gitParent) > 0:
1965            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1966            #print "parent git commit: %s" % gitParent
1967
1968        self.importChanges(changes)
1969        return True
1970
1971    def searchParent(self, parent, branch, target):
1972        parentFound = False
1973        for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
1974            blob = blob.strip()
1975            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
1976                parentFound = True
1977                if self.verbose:
1978                    print "Found parent of %s in commit %s" % (branch, blob)
1979                break
1980        if parentFound:
1981            return blob
1982        else:
1983            return None
1984
1985    def importChanges(self, changes):
1986        cnt = 1
1987        for change in changes:
1988            description = p4Cmd(["describe", str(change)])
1989            self.updateOptionDict(description)
1990
1991            if not self.silent:
1992                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1993                sys.stdout.flush()
1994            cnt = cnt + 1
1995
1996            try:
1997                if self.detectBranches:
1998                    branches = self.splitFilesIntoBranches(description)
1999                    for branch in branches.keys():
2000                        ## HACK  --hwn
2001                        branchPrefix = self.depotPaths[0] + branch + "/"
2002
2003                        parent = ""
2004
2005                        filesForCommit = branches[branch]
2006
2007                        if self.verbose:
2008                            print "branch is %s" % branch
2009
2010                        self.updatedBranches.add(branch)
2011
2012                        if branch not in self.createdBranches:
2013                            self.createdBranches.add(branch)
2014                            parent = self.knownBranches[branch]
2015                            if parent == branch:
2016                                parent = ""
2017                            else:
2018                                fullBranch = self.projectName + branch
2019                                if fullBranch not in self.p4BranchesInGit:
2020                                    if not self.silent:
2021                                        print("\n    Importing new branch %s" % fullBranch);
2022                                    if self.importNewBranch(branch, change - 1):
2023                                        parent = ""
2024                                        self.p4BranchesInGit.append(fullBranch)
2025                                    if not self.silent:
2026                                        print("\n    Resuming with change %s" % change);
2027
2028                                if self.verbose:
2029                                    print "parent determined through known branches: %s" % parent
2030
2031                        branch = self.gitRefForBranch(branch)
2032                        parent = self.gitRefForBranch(parent)
2033
2034                        if self.verbose:
2035                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2036
2037                        if len(parent) == 0 and branch in self.initialParents:
2038                            parent = self.initialParents[branch]
2039                            del self.initialParents[branch]
2040
2041                        blob = None
2042                        if len(parent) > 0:
2043                            tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2044                            if self.verbose:
2045                                print "Creating temporary branch: " + tempBranch
2046                            self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2047                            self.tempBranches.append(tempBranch)
2048                            self.checkpoint()
2049                            blob = self.searchParent(parent, branch, tempBranch)
2050                        if blob:
2051                            self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2052                        else:
2053                            if self.verbose:
2054                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2055                            self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2056                else:
2057                    files = self.extractFilesFromCommit(description)
2058                    self.commit(description, files, self.branch, self.depotPaths,
2059                                self.initialParent)
2060                    self.initialParent = ""
2061            except IOError:
2062                print self.gitError.read()
2063                sys.exit(1)
2064
2065    def importHeadRevision(self, revision):
2066        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2067
2068        details = {}
2069        details["user"] = "git perforce import user"
2070        details["desc"] = ("Initial import of %s from the state at revision %s\n"
2071                           % (' '.join(self.depotPaths), revision))
2072        details["change"] = revision
2073        newestRevision = 0
2074
2075        fileCnt = 0
2076        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2077
2078        for info in p4CmdList(["files"] + fileArgs):
2079
2080            if 'code' in info and info['code'] == 'error':
2081                sys.stderr.write("p4 returned an error: %s\n"
2082                                 % info['data'])
2083                if info['data'].find("must refer to client") >= 0:
2084                    sys.stderr.write("This particular p4 error is misleading.\n")
2085                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
2086                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2087                sys.exit(1)
2088            if 'p4ExitCode' in info:
2089                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2090                sys.exit(1)
2091
2092
2093            change = int(info["change"])
2094            if change > newestRevision:
2095                newestRevision = change
2096
2097            if info["action"] in self.delete_actions:
2098                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2099                #fileCnt = fileCnt + 1
2100                continue
2101
2102            for prop in ["depotFile", "rev", "action", "type" ]:
2103                details["%s%s" % (prop, fileCnt)] = info[prop]
2104
2105            fileCnt = fileCnt + 1
2106
2107        details["change"] = newestRevision
2108
2109        # Use time from top-most change so that all git-p4 clones of
2110        # the same p4 repo have the same commit SHA1s.
2111        res = p4CmdList("describe -s %d" % newestRevision)
2112        newestTime = None
2113        for r in res:
2114            if r.has_key('time'):
2115                newestTime = int(r['time'])
2116        if newestTime is None:
2117            die("\"describe -s\" on newest change %d did not give a time")
2118        details["time"] = newestTime
2119
2120        self.updateOptionDict(details)
2121        try:
2122            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2123        except IOError:
2124            print "IO error with git fast-import. Is your git version recent enough?"
2125            print self.gitError.read()
2126
2127
2128    def getClientSpec(self):
2129        specList = p4CmdList("client -o")
2130        if len(specList) != 1:
2131            die('Output from "client -o" is %d lines, expecting 1' %
2132                len(specList))
2133
2134        # dictionary of all client parameters
2135        entry = specList[0]
2136
2137        # just the keys that start with "View"
2138        view_keys = [ k for k in entry.keys() if k.startswith("View") ]
2139
2140        # hold this new View
2141        view = View()
2142
2143        # append the lines, in order, to the view
2144        for view_num in range(len(view_keys)):
2145            k = "View%d" % view_num
2146            if k not in view_keys:
2147                die("Expected view key %s missing" % k)
2148            view.append(entry[k])
2149
2150        self.clientSpecDirs = view
2151        if self.verbose:
2152            for i, m in enumerate(self.clientSpecDirs.mappings):
2153                    print "clientSpecDirs %d: %s" % (i, str(m))
2154
2155    def run(self, args):
2156        self.depotPaths = []
2157        self.changeRange = ""
2158        self.initialParent = ""
2159        self.previousDepotPaths = []
2160
2161        # map from branch depot path to parent branch
2162        self.knownBranches = {}
2163        self.initialParents = {}
2164        self.hasOrigin = originP4BranchesExist()
2165        if not self.syncWithOrigin:
2166            self.hasOrigin = False
2167
2168        if self.importIntoRemotes:
2169            self.refPrefix = "refs/remotes/p4/"
2170        else:
2171            self.refPrefix = "refs/heads/p4/"
2172
2173        if self.syncWithOrigin and self.hasOrigin:
2174            if not self.silent:
2175                print "Syncing with origin first by calling git fetch origin"
2176            system("git fetch origin")
2177
2178        if len(self.branch) == 0:
2179            self.branch = self.refPrefix + "master"
2180            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2181                system("git update-ref %s refs/heads/p4" % self.branch)
2182                system("git branch -D p4");
2183            # create it /after/ importing, when master exists
2184            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2185                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2186
2187        if not self.useClientSpec:
2188            if gitConfig("git-p4.useclientspec", "--bool") == "true":
2189                self.useClientSpec = True
2190        if self.useClientSpec:
2191            self.getClientSpec()
2192
2193        # TODO: should always look at previous commits,
2194        # merge with previous imports, if possible.
2195        if args == []:
2196            if self.hasOrigin:
2197                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2198            self.listExistingP4GitBranches()
2199
2200            if len(self.p4BranchesInGit) > 1:
2201                if not self.silent:
2202                    print "Importing from/into multiple branches"
2203                self.detectBranches = True
2204
2205            if self.verbose:
2206                print "branches: %s" % self.p4BranchesInGit
2207
2208            p4Change = 0
2209            for branch in self.p4BranchesInGit:
2210                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2211
2212                settings = extractSettingsGitLog(logMsg)
2213
2214                self.readOptions(settings)
2215                if (settings.has_key('depot-paths')
2216                    and settings.has_key ('change')):
2217                    change = int(settings['change']) + 1
2218                    p4Change = max(p4Change, change)
2219
2220                    depotPaths = sorted(settings['depot-paths'])
2221                    if self.previousDepotPaths == []:
2222                        self.previousDepotPaths = depotPaths
2223                    else:
2224                        paths = []
2225                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2226                            prev_list = prev.split("/")
2227                            cur_list = cur.split("/")
2228                            for i in range(0, min(len(cur_list), len(prev_list))):
2229                                if cur_list[i] <> prev_list[i]:
2230                                    i = i - 1
2231                                    break
2232
2233                            paths.append ("/".join(cur_list[:i + 1]))
2234
2235                        self.previousDepotPaths = paths
2236
2237            if p4Change > 0:
2238                self.depotPaths = sorted(self.previousDepotPaths)
2239                self.changeRange = "@%s,#head" % p4Change
2240                if not self.detectBranches:
2241                    self.initialParent = parseRevision(self.branch)
2242                if not self.silent and not self.detectBranches:
2243                    print "Performing incremental import into %s git branch" % self.branch
2244
2245        if not self.branch.startswith("refs/"):
2246            self.branch = "refs/heads/" + self.branch
2247
2248        if len(args) == 0 and self.depotPaths:
2249            if not self.silent:
2250                print "Depot paths: %s" % ' '.join(self.depotPaths)
2251        else:
2252            if self.depotPaths and self.depotPaths != args:
2253                print ("previous import used depot path %s and now %s was specified. "
2254                       "This doesn't work!" % (' '.join (self.depotPaths),
2255                                               ' '.join (args)))
2256                sys.exit(1)
2257
2258            self.depotPaths = sorted(args)
2259
2260        revision = ""
2261        self.users = {}
2262
2263        # Make sure no revision specifiers are used when --changesfile
2264        # is specified.
2265        bad_changesfile = False
2266        if len(self.changesFile) > 0:
2267            for p in self.depotPaths:
2268                if p.find("@") >= 0 or p.find("#") >= 0:
2269                    bad_changesfile = True
2270                    break
2271        if bad_changesfile:
2272            die("Option --changesfile is incompatible with revision specifiers")
2273
2274        newPaths = []
2275        for p in self.depotPaths:
2276            if p.find("@") != -1:
2277                atIdx = p.index("@")
2278                self.changeRange = p[atIdx:]
2279                if self.changeRange == "@all":
2280                    self.changeRange = ""
2281                elif ',' not in self.changeRange:
2282                    revision = self.changeRange
2283                    self.changeRange = ""
2284                p = p[:atIdx]
2285            elif p.find("#") != -1:
2286                hashIdx = p.index("#")
2287                revision = p[hashIdx:]
2288                p = p[:hashIdx]
2289            elif self.previousDepotPaths == []:
2290                # pay attention to changesfile, if given, else import
2291                # the entire p4 tree at the head revision
2292                if len(self.changesFile) == 0:
2293                    revision = "#head"
2294
2295            p = re.sub ("\.\.\.$", "", p)
2296            if not p.endswith("/"):
2297                p += "/"
2298
2299            newPaths.append(p)
2300
2301        self.depotPaths = newPaths
2302
2303
2304        self.loadUserMapFromCache()
2305        self.labels = {}
2306        if self.detectLabels:
2307            self.getLabels();
2308
2309        if self.detectBranches:
2310            ## FIXME - what's a P4 projectName ?
2311            self.projectName = self.guessProjectName()
2312
2313            if self.hasOrigin:
2314                self.getBranchMappingFromGitBranches()
2315            else:
2316                self.getBranchMapping()
2317            if self.verbose:
2318                print "p4-git branches: %s" % self.p4BranchesInGit
2319                print "initial parents: %s" % self.initialParents
2320            for b in self.p4BranchesInGit:
2321                if b != "master":
2322
2323                    ## FIXME
2324                    b = b[len(self.projectName):]
2325                self.createdBranches.add(b)
2326
2327        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2328
2329        importProcess = subprocess.Popen(["git", "fast-import"],
2330                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2331                                         stderr=subprocess.PIPE);
2332        self.gitOutput = importProcess.stdout
2333        self.gitStream = importProcess.stdin
2334        self.gitError = importProcess.stderr
2335
2336        if revision:
2337            self.importHeadRevision(revision)
2338        else:
2339            changes = []
2340
2341            if len(self.changesFile) > 0:
2342                output = open(self.changesFile).readlines()
2343                changeSet = set()
2344                for line in output:
2345                    changeSet.add(int(line))
2346
2347                for change in changeSet:
2348                    changes.append(change)
2349
2350                changes.sort()
2351            else:
2352                # catch "git-p4 sync" with no new branches, in a repo that
2353                # does not have any existing git-p4 branches
2354                if len(args) == 0 and not self.p4BranchesInGit:
2355                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2356                if self.verbose:
2357                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2358                                                              self.changeRange)
2359                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2360
2361                if len(self.maxChanges) > 0:
2362                    changes = changes[:min(int(self.maxChanges), len(changes))]
2363
2364            if len(changes) == 0:
2365                if not self.silent:
2366                    print "No changes to import!"
2367                return True
2368
2369            if not self.silent and not self.detectBranches:
2370                print "Import destination: %s" % self.branch
2371
2372            self.updatedBranches = set()
2373
2374            self.importChanges(changes)
2375
2376            if not self.silent:
2377                print ""
2378                if len(self.updatedBranches) > 0:
2379                    sys.stdout.write("Updated branches: ")
2380                    for b in self.updatedBranches:
2381                        sys.stdout.write("%s " % b)
2382                    sys.stdout.write("\n")
2383
2384        self.gitStream.close()
2385        if importProcess.wait() != 0:
2386            die("fast-import failed: %s" % self.gitError.read())
2387        self.gitOutput.close()
2388        self.gitError.close()
2389
2390        # Cleanup temporary branches created during import
2391        if self.tempBranches != []:
2392            for branch in self.tempBranches:
2393                read_pipe("git update-ref -d %s" % branch)
2394            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2395
2396        return True
2397
2398class P4Rebase(Command):
2399    def __init__(self):
2400        Command.__init__(self)
2401        self.options = [ ]
2402        self.description = ("Fetches the latest revision from perforce and "
2403                            + "rebases the current work (branch) against it")
2404        self.verbose = False
2405
2406    def run(self, args):
2407        sync = P4Sync()
2408        sync.run([])
2409
2410        return self.rebase()
2411
2412    def rebase(self):
2413        if os.system("git update-index --refresh") != 0:
2414            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2415        if len(read_pipe("git diff-index HEAD --")) > 0:
2416            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2417
2418        [upstream, settings] = findUpstreamBranchPoint()
2419        if len(upstream) == 0:
2420            die("Cannot find upstream branchpoint for rebase")
2421
2422        # the branchpoint may be p4/foo~3, so strip off the parent
2423        upstream = re.sub("~[0-9]+$", "", upstream)
2424
2425        print "Rebasing the current branch onto %s" % upstream
2426        oldHead = read_pipe("git rev-parse HEAD").strip()
2427        system("git rebase %s" % upstream)
2428        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2429        return True
2430
2431class P4Clone(P4Sync):
2432    def __init__(self):
2433        P4Sync.__init__(self)
2434        self.description = "Creates a new git repository and imports from Perforce into it"
2435        self.usage = "usage: %prog [options] //depot/path[@revRange]"
2436        self.options += [
2437            optparse.make_option("--destination", dest="cloneDestination",
2438                                 action='store', default=None,
2439                                 help="where to leave result of the clone"),
2440            optparse.make_option("-/", dest="cloneExclude",
2441                                 action="append", type="string",
2442                                 help="exclude depot path"),
2443            optparse.make_option("--bare", dest="cloneBare",
2444                                 action="store_true", default=False),
2445        ]
2446        self.cloneDestination = None
2447        self.needsGit = False
2448        self.cloneBare = False
2449
2450    # This is required for the "append" cloneExclude action
2451    def ensure_value(self, attr, value):
2452        if not hasattr(self, attr) or getattr(self, attr) is None:
2453            setattr(self, attr, value)
2454        return getattr(self, attr)
2455
2456    def defaultDestination(self, args):
2457        ## TODO: use common prefix of args?
2458        depotPath = args[0]
2459        depotDir = re.sub("(@[^@]*)$", "", depotPath)
2460        depotDir = re.sub("(#[^#]*)$", "", depotDir)
2461        depotDir = re.sub(r"\.\.\.$", "", depotDir)
2462        depotDir = re.sub(r"/$", "", depotDir)
2463        return os.path.split(depotDir)[1]
2464
2465    def run(self, args):
2466        if len(args) < 1:
2467            return False
2468
2469        if self.keepRepoPath and not self.cloneDestination:
2470            sys.stderr.write("Must specify destination for --keep-path\n")
2471            sys.exit(1)
2472
2473        depotPaths = args
2474
2475        if not self.cloneDestination and len(depotPaths) > 1:
2476            self.cloneDestination = depotPaths[-1]
2477            depotPaths = depotPaths[:-1]
2478
2479        self.cloneExclude = ["/"+p for p in self.cloneExclude]
2480        for p in depotPaths:
2481            if not p.startswith("//"):
2482                return False
2483
2484        if not self.cloneDestination:
2485            self.cloneDestination = self.defaultDestination(args)
2486
2487        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2488
2489        if not os.path.exists(self.cloneDestination):
2490            os.makedirs(self.cloneDestination)
2491        chdir(self.cloneDestination)
2492
2493        init_cmd = [ "git", "init" ]
2494        if self.cloneBare:
2495            init_cmd.append("--bare")
2496        subprocess.check_call(init_cmd)
2497
2498        if not P4Sync.run(self, depotPaths):
2499            return False
2500        if self.branch != "master":
2501            if self.importIntoRemotes:
2502                masterbranch = "refs/remotes/p4/master"
2503            else:
2504                masterbranch = "refs/heads/p4/master"
2505            if gitBranchExists(masterbranch):
2506                system("git branch master %s" % masterbranch)
2507                if not self.cloneBare:
2508                    system("git checkout -f")
2509            else:
2510                print "Could not detect main branch. No checkout/master branch created."
2511
2512        return True
2513
2514class P4Branches(Command):
2515    def __init__(self):
2516        Command.__init__(self)
2517        self.options = [ ]
2518        self.description = ("Shows the git branches that hold imports and their "
2519                            + "corresponding perforce depot paths")
2520        self.verbose = False
2521
2522    def run(self, args):
2523        if originP4BranchesExist():
2524            createOrUpdateBranchesFromOrigin()
2525
2526        cmdline = "git rev-parse --symbolic "
2527        cmdline += " --remotes"
2528
2529        for line in read_pipe_lines(cmdline):
2530            line = line.strip()
2531
2532            if not line.startswith('p4/') or line == "p4/HEAD":
2533                continue
2534            branch = line
2535
2536            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2537            settings = extractSettingsGitLog(log)
2538
2539            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2540        return True
2541
2542class HelpFormatter(optparse.IndentedHelpFormatter):
2543    def __init__(self):
2544        optparse.IndentedHelpFormatter.__init__(self)
2545
2546    def format_description(self, description):
2547        if description:
2548            return description + "\n"
2549        else:
2550            return ""
2551
2552def printUsage(commands):
2553    print "usage: %s <command> [options]" % sys.argv[0]
2554    print ""
2555    print "valid commands: %s" % ", ".join(commands)
2556    print ""
2557    print "Try %s <command> --help for command specific help." % sys.argv[0]
2558    print ""
2559
2560commands = {
2561    "debug" : P4Debug,
2562    "submit" : P4Submit,
2563    "commit" : P4Submit,
2564    "sync" : P4Sync,
2565    "rebase" : P4Rebase,
2566    "clone" : P4Clone,
2567    "rollback" : P4RollBack,
2568    "branches" : P4Branches
2569}
2570
2571
2572def main():
2573    if len(sys.argv[1:]) == 0:
2574        printUsage(commands.keys())
2575        sys.exit(2)
2576
2577    cmd = ""
2578    cmdName = sys.argv[1]
2579    try:
2580        klass = commands[cmdName]
2581        cmd = klass()
2582    except KeyError:
2583        print "unknown command %s" % cmdName
2584        print ""
2585        printUsage(commands.keys())
2586        sys.exit(2)
2587
2588    options = cmd.options
2589    cmd.gitdir = os.environ.get("GIT_DIR", None)
2590
2591    args = sys.argv[2:]
2592
2593    if len(options) > 0:
2594        if cmd.needsGit:
2595            options.append(optparse.make_option("--git-dir", dest="gitdir"))
2596
2597        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2598                                       options,
2599                                       description = cmd.description,
2600                                       formatter = HelpFormatter())
2601
2602        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2603    global verbose
2604    verbose = cmd.verbose
2605    if cmd.needsGit:
2606        if cmd.gitdir == None:
2607            cmd.gitdir = os.path.abspath(".git")
2608            if not isValidGitDir(cmd.gitdir):
2609                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2610                if os.path.exists(cmd.gitdir):
2611                    cdup = read_pipe("git rev-parse --show-cdup").strip()
2612                    if len(cdup) > 0:
2613                        chdir(cdup);
2614
2615        if not isValidGitDir(cmd.gitdir):
2616            if isValidGitDir(cmd.gitdir + "/.git"):
2617                cmd.gitdir += "/.git"
2618            else:
2619                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2620
2621        os.environ["GIT_DIR"] = cmd.gitdir
2622
2623    if not cmd.run(args):
2624        parser.print_help()
2625        sys.exit(2)
2626
2627
2628if __name__ == '__main__':
2629    main()