contrib / fast-import / git-p4on commit Merge branch 'jl/submodule-re-add' into maint (5febbda)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = ["p4"]
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += ["-u",user]
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += ["-P", password]
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += ["-p", port]
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += ["-h", host]
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += ["-c", client]
  46
  47
  48    if isinstance(cmd,basestring):
  49        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50    else:
  51        real_cmd += cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56    # not using the shell, we have to set it ourselves.  This path could
  57    # be relative, so go there first, then figure out where we ended up.
  58    os.chdir(dir)
  59    os.environ['PWD'] = os.getcwd()
  60
  61def die(msg):
  62    if verbose:
  63        raise Exception(msg)
  64    else:
  65        sys.stderr.write(msg + "\n")
  66        sys.exit(1)
  67
  68def write_pipe(c, stdin):
  69    if verbose:
  70        sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72    expand = isinstance(c,basestring)
  73    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74    pipe = p.stdin
  75    val = pipe.write(stdin)
  76    pipe.close()
  77    if p.wait():
  78        die('Command failed: %s' % str(c))
  79
  80    return val
  81
  82def p4_write_pipe(c, stdin):
  83    real_cmd = p4_build_cmd(c)
  84    return write_pipe(real_cmd, stdin)
  85
  86def read_pipe(c, ignore_error=False):
  87    if verbose:
  88        sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90    expand = isinstance(c,basestring)
  91    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92    pipe = p.stdout
  93    val = pipe.read()
  94    if p.wait() and not ignore_error:
  95        die('Command failed: %s' % str(c))
  96
  97    return val
  98
  99def p4_read_pipe(c, ignore_error=False):
 100    real_cmd = p4_build_cmd(c)
 101    return read_pipe(real_cmd, ignore_error)
 102
 103def read_pipe_lines(c):
 104    if verbose:
 105        sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107    expand = isinstance(c, basestring)
 108    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109    pipe = p.stdout
 110    val = pipe.readlines()
 111    if pipe.close() or p.wait():
 112        die('Command failed: %s' % str(c))
 113
 114    return val
 115
 116def p4_read_pipe_lines(c):
 117    """Specifically invoke p4 on the command supplied. """
 118    real_cmd = p4_build_cmd(c)
 119    return read_pipe_lines(real_cmd)
 120
 121def system(cmd):
 122    expand = isinstance(cmd,basestring)
 123    if verbose:
 124        sys.stderr.write("executing %s\n" % str(cmd))
 125    subprocess.check_call(cmd, shell=expand)
 126
 127def p4_system(cmd):
 128    """Specifically invoke p4 as the system command. """
 129    real_cmd = p4_build_cmd(cmd)
 130    expand = isinstance(real_cmd, basestring)
 131    subprocess.check_call(real_cmd, shell=expand)
 132
 133def p4_integrate(src, dest):
 134    p4_system(["integrate", "-Dt", src, dest])
 135
 136def p4_sync(path):
 137    p4_system(["sync", path])
 138
 139def p4_add(f):
 140    p4_system(["add", f])
 141
 142def p4_delete(f):
 143    p4_system(["delete", f])
 144
 145def p4_edit(f):
 146    p4_system(["edit", f])
 147
 148def p4_revert(f):
 149    p4_system(["revert", f])
 150
 151def p4_reopen(type, file):
 152    p4_system(["reopen", "-t", type, file])
 153
 154#
 155# Canonicalize the p4 type and return a tuple of the
 156# base type, plus any modifiers.  See "p4 help filetypes"
 157# for a list and explanation.
 158#
 159def split_p4_type(p4type):
 160
 161    p4_filetypes_historical = {
 162        "ctempobj": "binary+Sw",
 163        "ctext": "text+C",
 164        "cxtext": "text+Cx",
 165        "ktext": "text+k",
 166        "kxtext": "text+kx",
 167        "ltext": "text+F",
 168        "tempobj": "binary+FSw",
 169        "ubinary": "binary+F",
 170        "uresource": "resource+F",
 171        "uxbinary": "binary+Fx",
 172        "xbinary": "binary+x",
 173        "xltext": "text+Fx",
 174        "xtempobj": "binary+Swx",
 175        "xtext": "text+x",
 176        "xunicode": "unicode+x",
 177        "xutf16": "utf16+x",
 178    }
 179    if p4type in p4_filetypes_historical:
 180        p4type = p4_filetypes_historical[p4type]
 181    mods = ""
 182    s = p4type.split("+")
 183    base = s[0]
 184    mods = ""
 185    if len(s) > 1:
 186        mods = s[1]
 187    return (base, mods)
 188
 189
 190def setP4ExecBit(file, mode):
 191    # Reopens an already open file and changes the execute bit to match
 192    # the execute bit setting in the passed in mode.
 193
 194    p4Type = "+x"
 195
 196    if not isModeExec(mode):
 197        p4Type = getP4OpenedType(file)
 198        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200        if p4Type[-1] == "+":
 201            p4Type = p4Type[0:-1]
 202
 203    p4_reopen(p4Type, file)
 204
 205def getP4OpenedType(file):
 206    # Returns the perforce file type for the given file.
 207
 208    result = p4_read_pipe(["opened", file])
 209    match = re.match(".*\((.+)\)\r?$", result)
 210    if match:
 211        return match.group(1)
 212    else:
 213        die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215def diffTreePattern():
 216    # This is a simple generator for the diff tree regex pattern. This could be
 217    # a class variable if this and parseDiffTreeEntry were a part of a class.
 218    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219    while True:
 220        yield pattern
 221
 222def parseDiffTreeEntry(entry):
 223    """Parses a single diff tree entry into its component elements.
 224
 225    See git-diff-tree(1) manpage for details about the format of the diff
 226    output. This method returns a dictionary with the following elements:
 227
 228    src_mode - The mode of the source file
 229    dst_mode - The mode of the destination file
 230    src_sha1 - The sha1 for the source file
 231    dst_sha1 - The sha1 fr the destination file
 232    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233    status_score - The score for the status (applicable for 'C' and 'R'
 234                   statuses). This is None if there is no score.
 235    src - The path for the source file.
 236    dst - The path for the destination file. This is only present for
 237          copy or renames. If it is not present, this is None.
 238
 239    If the pattern is not matched, None is returned."""
 240
 241    match = diffTreePattern().next().match(entry)
 242    if match:
 243        return {
 244            'src_mode': match.group(1),
 245            'dst_mode': match.group(2),
 246            'src_sha1': match.group(3),
 247            'dst_sha1': match.group(4),
 248            'status': match.group(5),
 249            'status_score': match.group(6),
 250            'src': match.group(7),
 251            'dst': match.group(10)
 252        }
 253    return None
 254
 255def isModeExec(mode):
 256    # Returns True if the given git mode represents an executable file,
 257    # otherwise False.
 258    return mode[-3:] == "755"
 259
 260def isModeExecChanged(src_mode, dst_mode):
 261    return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265    if isinstance(cmd,basestring):
 266        cmd = "-G " + cmd
 267        expand = True
 268    else:
 269        cmd = ["-G"] + cmd
 270        expand = False
 271
 272    cmd = p4_build_cmd(cmd)
 273    if verbose:
 274        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276    # Use a temporary file to avoid deadlocks without
 277    # subprocess.communicate(), which would put another copy
 278    # of stdout into memory.
 279    stdin_file = None
 280    if stdin is not None:
 281        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282        if isinstance(stdin,basestring):
 283            stdin_file.write(stdin)
 284        else:
 285            for i in stdin:
 286                stdin_file.write(i + '\n')
 287        stdin_file.flush()
 288        stdin_file.seek(0)
 289
 290    p4 = subprocess.Popen(cmd,
 291                          shell=expand,
 292                          stdin=stdin_file,
 293                          stdout=subprocess.PIPE)
 294
 295    result = []
 296    try:
 297        while True:
 298            entry = marshal.load(p4.stdout)
 299            if cb is not None:
 300                cb(entry)
 301            else:
 302                result.append(entry)
 303    except EOFError:
 304        pass
 305    exitCode = p4.wait()
 306    if exitCode != 0:
 307        entry = {}
 308        entry["p4ExitCode"] = exitCode
 309        result.append(entry)
 310
 311    return result
 312
 313def p4Cmd(cmd):
 314    list = p4CmdList(cmd)
 315    result = {}
 316    for entry in list:
 317        result.update(entry)
 318    return result;
 319
 320def p4Where(depotPath):
 321    if not depotPath.endswith("/"):
 322        depotPath += "/"
 323    depotPath = depotPath + "..."
 324    outputList = p4CmdList(["where", depotPath])
 325    output = None
 326    for entry in outputList:
 327        if "depotFile" in entry:
 328            if entry["depotFile"] == depotPath:
 329                output = entry
 330                break
 331        elif "data" in entry:
 332            data = entry.get("data")
 333            space = data.find(" ")
 334            if data[:space] == depotPath:
 335                output = entry
 336                break
 337    if output == None:
 338        return ""
 339    if output["code"] == "error":
 340        return ""
 341    clientPath = ""
 342    if "path" in output:
 343        clientPath = output.get("path")
 344    elif "data" in output:
 345        data = output.get("data")
 346        lastSpace = data.rfind(" ")
 347        clientPath = data[lastSpace + 1:]
 348
 349    if clientPath.endswith("..."):
 350        clientPath = clientPath[:-3]
 351    return clientPath
 352
 353def currentGitBranch():
 354    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356def isValidGitDir(path):
 357    if (os.path.exists(path + "/HEAD")
 358        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359        return True;
 360    return False
 361
 362def parseRevision(ref):
 363    return read_pipe("git rev-parse %s" % ref).strip()
 364
 365def branchExists(ref):
 366    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                     ignore_error=True)
 368    return len(rev) > 0
 369
 370def extractLogMessageFromGitCommit(commit):
 371    logMessage = ""
 372
 373    ## fixme: title is first line of commit, not 1st paragraph.
 374    foundTitle = False
 375    for log in read_pipe_lines("git cat-file commit %s" % commit):
 376       if not foundTitle:
 377           if len(log) == 1:
 378               foundTitle = True
 379           continue
 380
 381       logMessage += log
 382    return logMessage
 383
 384def extractSettingsGitLog(log):
 385    values = {}
 386    for line in log.split("\n"):
 387        line = line.strip()
 388        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389        if not m:
 390            continue
 391
 392        assignments = m.group(1).split (':')
 393        for a in assignments:
 394            vals = a.split ('=')
 395            key = vals[0].strip()
 396            val = ('='.join (vals[1:])).strip()
 397            if val.endswith ('\"') and val.startswith('"'):
 398                val = val[1:-1]
 399
 400            values[key] = val
 401
 402    paths = values.get("depot-paths")
 403    if not paths:
 404        paths = values.get("depot-path")
 405    if paths:
 406        values['depot-paths'] = paths.split(',')
 407    return values
 408
 409def gitBranchExists(branch):
 410    proc = subprocess.Popen(["git", "rev-parse", branch],
 411                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412    return proc.wait() == 0;
 413
 414_gitConfig = {}
 415def gitConfig(key, args = None): # set args to "--bool", for instance
 416    if not _gitConfig.has_key(key):
 417        argsFilter = ""
 418        if args != None:
 419            argsFilter = "%s " % args
 420        cmd = "git config %s%s" % (argsFilter, key)
 421        _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422    return _gitConfig[key]
 423
 424def gitConfigList(key):
 425    if not _gitConfig.has_key(key):
 426        _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427    return _gitConfig[key]
 428
 429def p4BranchesInGit(branchesAreInRemotes = True):
 430    branches = {}
 431
 432    cmdline = "git rev-parse --symbolic "
 433    if branchesAreInRemotes:
 434        cmdline += " --remotes"
 435    else:
 436        cmdline += " --branches"
 437
 438    for line in read_pipe_lines(cmdline):
 439        line = line.strip()
 440
 441        ## only import to p4/
 442        if not line.startswith('p4/') or line == "p4/HEAD":
 443            continue
 444        branch = line
 445
 446        # strip off p4
 447        branch = re.sub ("^p4/", "", line)
 448
 449        branches[branch] = parseRevision(line)
 450    return branches
 451
 452def findUpstreamBranchPoint(head = "HEAD"):
 453    branches = p4BranchesInGit()
 454    # map from depot-path to branch name
 455    branchByDepotPath = {}
 456    for branch in branches.keys():
 457        tip = branches[branch]
 458        log = extractLogMessageFromGitCommit(tip)
 459        settings = extractSettingsGitLog(log)
 460        if settings.has_key("depot-paths"):
 461            paths = ",".join(settings["depot-paths"])
 462            branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464    settings = None
 465    parent = 0
 466    while parent < 65535:
 467        commit = head + "~%s" % parent
 468        log = extractLogMessageFromGitCommit(commit)
 469        settings = extractSettingsGitLog(log)
 470        if settings.has_key("depot-paths"):
 471            paths = ",".join(settings["depot-paths"])
 472            if branchByDepotPath.has_key(paths):
 473                return [branchByDepotPath[paths], settings]
 474
 475        parent = parent + 1
 476
 477    return ["", settings]
 478
 479def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480    if not silent:
 481        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482               % localRefPrefix)
 483
 484    originPrefix = "origin/p4/"
 485
 486    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487        line = line.strip()
 488        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489            continue
 490
 491        headName = line[len(originPrefix):]
 492        remoteHead = localRefPrefix + headName
 493        originHead = line
 494
 495        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496        if (not original.has_key('depot-paths')
 497            or not original.has_key('change')):
 498            continue
 499
 500        update = False
 501        if not gitBranchExists(remoteHead):
 502            if verbose:
 503                print "creating %s" % remoteHead
 504            update = True
 505        else:
 506            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507            if settings.has_key('change') > 0:
 508                if settings['depot-paths'] == original['depot-paths']:
 509                    originP4Change = int(original['change'])
 510                    p4Change = int(settings['change'])
 511                    if originP4Change > p4Change:
 512                        print ("%s (%s) is newer than %s (%s). "
 513                               "Updating p4 branch from origin."
 514                               % (originHead, originP4Change,
 515                                  remoteHead, p4Change))
 516                        update = True
 517                else:
 518                    print ("Ignoring: %s was imported from %s while "
 519                           "%s was imported from %s"
 520                           % (originHead, ','.join(original['depot-paths']),
 521                              remoteHead, ','.join(settings['depot-paths'])))
 522
 523        if update:
 524            system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526def originP4BranchesExist():
 527        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529def p4ChangesForPaths(depotPaths, changeRange):
 530    assert depotPaths
 531    cmd = ['changes']
 532    for p in depotPaths:
 533        cmd += ["%s...%s" % (p, changeRange)]
 534    output = p4_read_pipe_lines(cmd)
 535
 536    changes = {}
 537    for line in output:
 538        changeNum = int(line.split(" ")[1])
 539        changes[changeNum] = True
 540
 541    changelist = changes.keys()
 542    changelist.sort()
 543    return changelist
 544
 545def p4PathStartsWith(path, prefix):
 546    # This method tries to remedy a potential mixed-case issue:
 547    #
 548    # If UserA adds  //depot/DirA/file1
 549    # and UserB adds //depot/dira/file2
 550    #
 551    # we may or may not have a problem. If you have core.ignorecase=true,
 552    # we treat DirA and dira as the same directory
 553    ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554    if ignorecase:
 555        return path.lower().startswith(prefix.lower())
 556    return path.startswith(prefix)
 557
 558class Command:
 559    def __init__(self):
 560        self.usage = "usage: %prog [options]"
 561        self.needsGit = True
 562
 563class P4UserMap:
 564    def __init__(self):
 565        self.userMapFromPerforceServer = False
 566
 567    def getUserCacheFilename(self):
 568        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 569        return home + "/.gitp4-usercache.txt"
 570
 571    def getUserMapFromPerforceServer(self):
 572        if self.userMapFromPerforceServer:
 573            return
 574        self.users = {}
 575        self.emails = {}
 576
 577        for output in p4CmdList("users"):
 578            if not output.has_key("User"):
 579                continue
 580            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 581            self.emails[output["Email"]] = output["User"]
 582
 583
 584        s = ''
 585        for (key, val) in self.users.items():
 586            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 587
 588        open(self.getUserCacheFilename(), "wb").write(s)
 589        self.userMapFromPerforceServer = True
 590
 591    def loadUserMapFromCache(self):
 592        self.users = {}
 593        self.userMapFromPerforceServer = False
 594        try:
 595            cache = open(self.getUserCacheFilename(), "rb")
 596            lines = cache.readlines()
 597            cache.close()
 598            for line in lines:
 599                entry = line.strip().split("\t")
 600                self.users[entry[0]] = entry[1]
 601        except IOError:
 602            self.getUserMapFromPerforceServer()
 603
 604class P4Debug(Command):
 605    def __init__(self):
 606        Command.__init__(self)
 607        self.options = [
 608            optparse.make_option("--verbose", dest="verbose", action="store_true",
 609                                 default=False),
 610            ]
 611        self.description = "A tool to debug the output of p4 -G."
 612        self.needsGit = False
 613        self.verbose = False
 614
 615    def run(self, args):
 616        j = 0
 617        for output in p4CmdList(args):
 618            print 'Element: %d' % j
 619            j += 1
 620            print output
 621        return True
 622
 623class P4RollBack(Command):
 624    def __init__(self):
 625        Command.__init__(self)
 626        self.options = [
 627            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 628            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 629        ]
 630        self.description = "A tool to debug the multi-branch import. Don't use :)"
 631        self.verbose = False
 632        self.rollbackLocalBranches = False
 633
 634    def run(self, args):
 635        if len(args) != 1:
 636            return False
 637        maxChange = int(args[0])
 638
 639        if "p4ExitCode" in p4Cmd("changes -m 1"):
 640            die("Problems executing p4");
 641
 642        if self.rollbackLocalBranches:
 643            refPrefix = "refs/heads/"
 644            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 645        else:
 646            refPrefix = "refs/remotes/"
 647            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 648
 649        for line in lines:
 650            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 651                line = line.strip()
 652                ref = refPrefix + line
 653                log = extractLogMessageFromGitCommit(ref)
 654                settings = extractSettingsGitLog(log)
 655
 656                depotPaths = settings['depot-paths']
 657                change = settings['change']
 658
 659                changed = False
 660
 661                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 662                                                           for p in depotPaths]))) == 0:
 663                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 664                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 665                    continue
 666
 667                while change and int(change) > maxChange:
 668                    changed = True
 669                    if self.verbose:
 670                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 671                    system("git update-ref %s \"%s^\"" % (ref, ref))
 672                    log = extractLogMessageFromGitCommit(ref)
 673                    settings =  extractSettingsGitLog(log)
 674
 675
 676                    depotPaths = settings['depot-paths']
 677                    change = settings['change']
 678
 679                if changed:
 680                    print "%s rewound to %s" % (ref, change)
 681
 682        return True
 683
 684class P4Submit(Command, P4UserMap):
 685    def __init__(self):
 686        Command.__init__(self)
 687        P4UserMap.__init__(self)
 688        self.options = [
 689                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 690                optparse.make_option("--origin", dest="origin"),
 691                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 692                # preserve the user, requires relevant p4 permissions
 693                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 694        ]
 695        self.description = "Submit changes from git to the perforce depot."
 696        self.usage += " [name of git branch to submit into perforce depot]"
 697        self.interactive = True
 698        self.origin = ""
 699        self.detectRenames = False
 700        self.verbose = False
 701        self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 702        self.isWindows = (platform.system() == "Windows")
 703        self.myP4UserId = None
 704
 705    def check(self):
 706        if len(p4CmdList("opened ...")) > 0:
 707            die("You have files opened with perforce! Close them before starting the sync.")
 708
 709    # replaces everything between 'Description:' and the next P4 submit template field with the
 710    # commit message
 711    def prepareLogMessage(self, template, message):
 712        result = ""
 713
 714        inDescriptionSection = False
 715
 716        for line in template.split("\n"):
 717            if line.startswith("#"):
 718                result += line + "\n"
 719                continue
 720
 721            if inDescriptionSection:
 722                if line.startswith("Files:") or line.startswith("Jobs:"):
 723                    inDescriptionSection = False
 724                else:
 725                    continue
 726            else:
 727                if line.startswith("Description:"):
 728                    inDescriptionSection = True
 729                    line += "\n"
 730                    for messageLine in message.split("\n"):
 731                        line += "\t" + messageLine + "\n"
 732
 733            result += line + "\n"
 734
 735        return result
 736
 737    def p4UserForCommit(self,id):
 738        # Return the tuple (perforce user,git email) for a given git commit id
 739        self.getUserMapFromPerforceServer()
 740        gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 741        gitEmail = gitEmail.strip()
 742        if not self.emails.has_key(gitEmail):
 743            return (None,gitEmail)
 744        else:
 745            return (self.emails[gitEmail],gitEmail)
 746
 747    def checkValidP4Users(self,commits):
 748        # check if any git authors cannot be mapped to p4 users
 749        for id in commits:
 750            (user,email) = self.p4UserForCommit(id)
 751            if not user:
 752                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 753                if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 754                    print "%s" % msg
 755                else:
 756                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 757
 758    def lastP4Changelist(self):
 759        # Get back the last changelist number submitted in this client spec. This
 760        # then gets used to patch up the username in the change. If the same
 761        # client spec is being used by multiple processes then this might go
 762        # wrong.
 763        results = p4CmdList("client -o")        # find the current client
 764        client = None
 765        for r in results:
 766            if r.has_key('Client'):
 767                client = r['Client']
 768                break
 769        if not client:
 770            die("could not get client spec")
 771        results = p4CmdList(["changes", "-c", client, "-m", "1"])
 772        for r in results:
 773            if r.has_key('change'):
 774                return r['change']
 775        die("Could not get changelist number for last submit - cannot patch up user details")
 776
 777    def modifyChangelistUser(self, changelist, newUser):
 778        # fixup the user field of a changelist after it has been submitted.
 779        changes = p4CmdList("change -o %s" % changelist)
 780        if len(changes) != 1:
 781            die("Bad output from p4 change modifying %s to user %s" %
 782                (changelist, newUser))
 783
 784        c = changes[0]
 785        if c['User'] == newUser: return   # nothing to do
 786        c['User'] = newUser
 787        input = marshal.dumps(c)
 788
 789        result = p4CmdList("change -f -i", stdin=input)
 790        for r in result:
 791            if r.has_key('code'):
 792                if r['code'] == 'error':
 793                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 794            if r.has_key('data'):
 795                print("Updated user field for changelist %s to %s" % (changelist, newUser))
 796                return
 797        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 798
 799    def canChangeChangelists(self):
 800        # check to see if we have p4 admin or super-user permissions, either of
 801        # which are required to modify changelists.
 802        results = p4CmdList("protects %s" % self.depotPath)
 803        for r in results:
 804            if r.has_key('perm'):
 805                if r['perm'] == 'admin':
 806                    return 1
 807                if r['perm'] == 'super':
 808                    return 1
 809        return 0
 810
 811    def p4UserId(self):
 812        if self.myP4UserId:
 813            return self.myP4UserId
 814
 815        results = p4CmdList("user -o")
 816        for r in results:
 817            if r.has_key('User'):
 818                self.myP4UserId = r['User']
 819                return r['User']
 820        die("Could not find your p4 user id")
 821
 822    def p4UserIsMe(self, p4User):
 823        # return True if the given p4 user is actually me
 824        me = self.p4UserId()
 825        if not p4User or p4User != me:
 826            return False
 827        else:
 828            return True
 829
 830    def prepareSubmitTemplate(self):
 831        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832        template = ""
 833        inFilesSection = False
 834        for line in p4_read_pipe_lines(['change', '-o']):
 835            if line.endswith("\r\n"):
 836                line = line[:-2] + "\n"
 837            if inFilesSection:
 838                if line.startswith("\t"):
 839                    # path starts and ends with a tab
 840                    path = line[1:]
 841                    lastTab = path.rfind("\t")
 842                    if lastTab != -1:
 843                        path = path[:lastTab]
 844                        if not p4PathStartsWith(path, self.depotPath):
 845                            continue
 846                else:
 847                    inFilesSection = False
 848            else:
 849                if line.startswith("Files:"):
 850                    inFilesSection = True
 851
 852            template += line
 853
 854        return template
 855
 856    def edit_template(self, template_file):
 857        """Invoke the editor to let the user change the submission
 858           message.  Return true if okay to continue with the submit."""
 859
 860        # if configured to skip the editing part, just submit
 861        if gitConfig("git-p4.skipSubmitEdit") == "true":
 862            return True
 863
 864        # look at the modification time, to check later if the user saved
 865        # the file
 866        mtime = os.stat(template_file).st_mtime
 867
 868        # invoke the editor
 869        if os.environ.has_key("P4EDITOR"):
 870            editor = os.environ.get("P4EDITOR")
 871        else:
 872            editor = read_pipe("git var GIT_EDITOR").strip()
 873        system(editor + " " + template_file)
 874
 875        # If the file was not saved, prompt to see if this patch should
 876        # be skipped.  But skip this verification step if configured so.
 877        if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878            return True
 879
 880        # modification time updated means user saved the file
 881        if os.stat(template_file).st_mtime > mtime:
 882            return True
 883
 884        while True:
 885            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886            if response == 'y':
 887                return True
 888            if response == 'n':
 889                return False
 890
 891    def applyCommit(self, id):
 892        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894        (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896        if not self.detectRenames:
 897            # If not explicitly set check the config variable
 898            self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900        if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901            diffOpts = ""
 902        elif self.detectRenames.lower() == "true":
 903            diffOpts = "-M"
 904        else:
 905            diffOpts = "-M%s" % self.detectRenames
 906
 907        detectCopies = gitConfig("git-p4.detectCopies")
 908        if detectCopies.lower() == "true":
 909            diffOpts += " -C"
 910        elif detectCopies != "" and detectCopies.lower() != "false":
 911            diffOpts += " -C%s" % detectCopies
 912
 913        if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914            diffOpts += " --find-copies-harder"
 915
 916        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917        filesToAdd = set()
 918        filesToDelete = set()
 919        editedFiles = set()
 920        filesToChangeExecBit = {}
 921        for line in diff:
 922            diff = parseDiffTreeEntry(line)
 923            modifier = diff['status']
 924            path = diff['src']
 925            if modifier == "M":
 926                p4_edit(path)
 927                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                    filesToChangeExecBit[path] = diff['dst_mode']
 929                editedFiles.add(path)
 930            elif modifier == "A":
 931                filesToAdd.add(path)
 932                filesToChangeExecBit[path] = diff['dst_mode']
 933                if path in filesToDelete:
 934                    filesToDelete.remove(path)
 935            elif modifier == "D":
 936                filesToDelete.add(path)
 937                if path in filesToAdd:
 938                    filesToAdd.remove(path)
 939            elif modifier == "C":
 940                src, dest = diff['src'], diff['dst']
 941                p4_integrate(src, dest)
 942                if diff['src_sha1'] != diff['dst_sha1']:
 943                    p4_edit(dest)
 944                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                    p4_edit(dest)
 946                    filesToChangeExecBit[dest] = diff['dst_mode']
 947                os.unlink(dest)
 948                editedFiles.add(dest)
 949            elif modifier == "R":
 950                src, dest = diff['src'], diff['dst']
 951                p4_integrate(src, dest)
 952                if diff['src_sha1'] != diff['dst_sha1']:
 953                    p4_edit(dest)
 954                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                    p4_edit(dest)
 956                    filesToChangeExecBit[dest] = diff['dst_mode']
 957                os.unlink(dest)
 958                editedFiles.add(dest)
 959                filesToDelete.add(src)
 960            else:
 961                die("unknown modifier %s for %s" % (modifier, path))
 962
 963        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964        patchcmd = diffcmd + " | git apply "
 965        tryPatchCmd = patchcmd + "--check -"
 966        applyPatchCmd = patchcmd + "--check --apply -"
 967
 968        if os.system(tryPatchCmd) != 0:
 969            print "Unfortunately applying the change failed!"
 970            print "What do you want to do?"
 971            response = "x"
 972            while response != "s" and response != "a" and response != "w":
 973                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975            if response == "s":
 976                print "Skipping! Good luck with the next patches..."
 977                for f in editedFiles:
 978                    p4_revert(f)
 979                for f in filesToAdd:
 980                    os.remove(f)
 981                return
 982            elif response == "a":
 983                os.system(applyPatchCmd)
 984                if len(filesToAdd) > 0:
 985                    print "You may also want to call p4 add on the following files:"
 986                    print " ".join(filesToAdd)
 987                if len(filesToDelete):
 988                    print "The following files should be scheduled for deletion with p4 delete:"
 989                    print " ".join(filesToDelete)
 990                die("Please resolve and submit the conflict manually and "
 991                    + "continue afterwards with git-p4 submit --continue")
 992            elif response == "w":
 993                system(diffcmd + " > patch.txt")
 994                print "Patch saved to patch.txt in %s !" % self.clientPath
 995                die("Please resolve and submit the conflict manually and "
 996                    "continue afterwards with git-p4 submit --continue")
 997
 998        system(applyPatchCmd)
 999
1000        for f in filesToAdd:
1001            p4_add(f)
1002        for f in filesToDelete:
1003            p4_revert(f)
1004            p4_delete(f)
1005
1006        # Set/clear executable bits
1007        for f in filesToChangeExecBit.keys():
1008            mode = filesToChangeExecBit[f]
1009            setP4ExecBit(f, mode)
1010
1011        logMessage = extractLogMessageFromGitCommit(id)
1012        logMessage = logMessage.strip()
1013
1014        template = self.prepareSubmitTemplate()
1015
1016        if self.interactive:
1017            submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019            if self.preserveUser:
1020               submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022            if os.environ.has_key("P4DIFF"):
1023                del(os.environ["P4DIFF"])
1024            diff = ""
1025            for editedFile in editedFiles:
1026                diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028            newdiff = ""
1029            for newFile in filesToAdd:
1030                newdiff += "==== new file ====\n"
1031                newdiff += "--- /dev/null\n"
1032                newdiff += "+++ %s\n" % newFile
1033                f = open(newFile, "r")
1034                for line in f.readlines():
1035                    newdiff += "+" + line
1036                f.close()
1037
1038            if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043            separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045            (handle, fileName) = tempfile.mkstemp()
1046            tmpFile = os.fdopen(handle, "w+")
1047            if self.isWindows:
1048                submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                separatorLine = separatorLine.replace("\n", "\r\n")
1050                newdiff = newdiff.replace("\n", "\r\n")
1051            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052            tmpFile.close()
1053
1054            if self.edit_template(fileName):
1055                # read the edited message and submit
1056                tmpFile = open(fileName, "rb")
1057                message = tmpFile.read()
1058                tmpFile.close()
1059                submitTemplate = message[:message.index(separatorLine)]
1060                if self.isWindows:
1061                    submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                if self.preserveUser:
1065                    if p4User:
1066                        # Get last changelist number. Cannot easily get it from
1067                        # the submit command output as the output is
1068                        # unmarshalled.
1069                        changelist = self.lastP4Changelist()
1070                        self.modifyChangelistUser(changelist, p4User)
1071            else:
1072                # skip this patch
1073                print "Submission cancelled, undoing p4 changes."
1074                for f in editedFiles:
1075                    p4_revert(f)
1076                for f in filesToAdd:
1077                    p4_revert(f)
1078                    os.remove(f)
1079
1080            os.remove(fileName)
1081        else:
1082            fileName = "submit.txt"
1083            file = open(fileName, "w+")
1084            file.write(self.prepareLogMessage(template, logMessage))
1085            file.close()
1086            print ("Perforce submit template written as %s. "
1087                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                   % (fileName, fileName))
1089
1090    def run(self, args):
1091        if len(args) == 0:
1092            self.master = currentGitBranch()
1093            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                die("Detecting current git branch failed!")
1095        elif len(args) == 1:
1096            self.master = args[0]
1097            if not branchExists(self.master):
1098                die("Branch %s does not exist" % self.master)
1099        else:
1100            return False
1101
1102        allowSubmit = gitConfig("git-p4.allowSubmit")
1103        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104            die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106        [upstream, settings] = findUpstreamBranchPoint()
1107        self.depotPath = settings['depot-paths'][0]
1108        if len(self.origin) == 0:
1109            self.origin = upstream
1110
1111        if self.preserveUser:
1112            if not self.canChangeChangelists():
1113                die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115        if self.verbose:
1116            print "Origin branch is " + self.origin
1117
1118        if len(self.depotPath) == 0:
1119            print "Internal error: cannot locate perforce depot path from existing branches"
1120            sys.exit(128)
1121
1122        self.clientPath = p4Where(self.depotPath)
1123
1124        if len(self.clientPath) == 0:
1125            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126            sys.exit(128)
1127
1128        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129        self.oldWorkingDirectory = os.getcwd()
1130
1131        # ensure the clientPath exists
1132        if not os.path.exists(self.clientPath):
1133            os.makedirs(self.clientPath)
1134
1135        chdir(self.clientPath)
1136        print "Synchronizing p4 checkout..."
1137        p4_sync("...")
1138        self.check()
1139
1140        commits = []
1141        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142            commits.append(line.strip())
1143        commits.reverse()
1144
1145        if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146            self.checkAuthorship = False
1147        else:
1148            self.checkAuthorship = True
1149
1150        if self.preserveUser:
1151            self.checkValidP4Users(commits)
1152
1153        while len(commits) > 0:
1154            commit = commits[0]
1155            commits = commits[1:]
1156            self.applyCommit(commit)
1157            if not self.interactive:
1158                break
1159
1160        if len(commits) == 0:
1161            print "All changes applied!"
1162            chdir(self.oldWorkingDirectory)
1163
1164            sync = P4Sync()
1165            sync.run([])
1166
1167            rebase = P4Rebase()
1168            rebase.rebase()
1169
1170        return True
1171
1172class View(object):
1173    """Represent a p4 view ("p4 help views"), and map files in a
1174       repo according to the view."""
1175
1176    class Path(object):
1177        """A depot or client path, possibly containing wildcards.
1178           The only one supported is ... at the end, currently.
1179           Initialize with the full path, with //depot or //client."""
1180
1181        def __init__(self, path, is_depot):
1182            self.path = path
1183            self.is_depot = is_depot
1184            self.find_wildcards()
1185            # remember the prefix bit, useful for relative mappings
1186            m = re.match("(//[^/]+/)", self.path)
1187            if not m:
1188                die("Path %s does not start with //prefix/" % self.path)
1189            prefix = m.group(1)
1190            if not self.is_depot:
1191                # strip //client/ on client paths
1192                self.path = self.path[len(prefix):]
1193
1194        def find_wildcards(self):
1195            """Make sure wildcards are valid, and set up internal
1196               variables."""
1197
1198            self.ends_triple_dot = False
1199            # There are three wildcards allowed in p4 views
1200            # (see "p4 help views").  This code knows how to
1201            # handle "..." (only at the end), but cannot deal with
1202            # "%%n" or "*".  Only check the depot_side, as p4 should
1203            # validate that the client_side matches too.
1204            if re.search(r'%%[1-9]', self.path):
1205                die("Can't handle %%n wildcards in view: %s" % self.path)
1206            if self.path.find("*") >= 0:
1207                die("Can't handle * wildcards in view: %s" % self.path)
1208            triple_dot_index = self.path.find("...")
1209            if triple_dot_index >= 0:
1210                if not self.path.endswith("..."):
1211                    die("Can handle ... wildcard only at end of path: %s" %
1212                        self.path)
1213                self.ends_triple_dot = True
1214
1215        def ensure_compatible(self, other_path):
1216            """Make sure the wildcards agree."""
1217            if self.ends_triple_dot != other_path.ends_triple_dot:
1218                 die("Both paths must end with ... if either does;\n" +
1219                     "paths: %s %s" % (self.path, other_path.path))
1220
1221        def match_wildcards(self, test_path):
1222            """See if this test_path matches us, and fill in the value
1223               of the wildcards if so.  Returns a tuple of
1224               (True|False, wildcards[]).  For now, only the ... at end
1225               is supported, so at most one wildcard."""
1226            if self.ends_triple_dot:
1227                dotless = self.path[:-3]
1228                if test_path.startswith(dotless):
1229                    wildcard = test_path[len(dotless):]
1230                    return (True, [ wildcard ])
1231            else:
1232                if test_path == self.path:
1233                    return (True, [])
1234            return (False, [])
1235
1236        def match(self, test_path):
1237            """Just return if it matches; don't bother with the wildcards."""
1238            b, _ = self.match_wildcards(test_path)
1239            return b
1240
1241        def fill_in_wildcards(self, wildcards):
1242            """Return the relative path, with the wildcards filled in
1243               if there are any."""
1244            if self.ends_triple_dot:
1245                return self.path[:-3] + wildcards[0]
1246            else:
1247                return self.path
1248
1249    class Mapping(object):
1250        def __init__(self, depot_side, client_side, overlay, exclude):
1251            # depot_side is without the trailing /... if it had one
1252            self.depot_side = View.Path(depot_side, is_depot=True)
1253            self.client_side = View.Path(client_side, is_depot=False)
1254            self.overlay = overlay  # started with "+"
1255            self.exclude = exclude  # started with "-"
1256            assert not (self.overlay and self.exclude)
1257            self.depot_side.ensure_compatible(self.client_side)
1258
1259        def __str__(self):
1260            c = " "
1261            if self.overlay:
1262                c = "+"
1263            if self.exclude:
1264                c = "-"
1265            return "View.Mapping: %s%s -> %s" % \
1266                   (c, self.depot_side, self.client_side)
1267
1268        def map_depot_to_client(self, depot_path):
1269            """Calculate the client path if using this mapping on the
1270               given depot path; does not consider the effect of other
1271               mappings in a view.  Even excluded mappings are returned."""
1272            matches, wildcards = self.depot_side.match_wildcards(depot_path)
1273            if not matches:
1274                return ""
1275            client_path = self.client_side.fill_in_wildcards(wildcards)
1276            return client_path
1277
1278    #
1279    # View methods
1280    #
1281    def __init__(self):
1282        self.mappings = []
1283
1284    def append(self, view_line):
1285        """Parse a view line, splitting it into depot and client
1286           sides.  Append to self.mappings, preserving order."""
1287
1288        # Split the view line into exactly two words.  P4 enforces
1289        # structure on these lines that simplifies this quite a bit.
1290        #
1291        # Either or both words may be double-quoted.
1292        # Single quotes do not matter.
1293        # Double-quote marks cannot occur inside the words.
1294        # A + or - prefix is also inside the quotes.
1295        # There are no quotes unless they contain a space.
1296        # The line is already white-space stripped.
1297        # The two words are separated by a single space.
1298        #
1299        if view_line[0] == '"':
1300            # First word is double quoted.  Find its end.
1301            close_quote_index = view_line.find('"', 1)
1302            if close_quote_index <= 0:
1303                die("No first-word closing quote found: %s" % view_line)
1304            depot_side = view_line[1:close_quote_index]
1305            # skip closing quote and space
1306            rhs_index = close_quote_index + 1 + 1
1307        else:
1308            space_index = view_line.find(" ")
1309            if space_index <= 0:
1310                die("No word-splitting space found: %s" % view_line)
1311            depot_side = view_line[0:space_index]
1312            rhs_index = space_index + 1
1313
1314        if view_line[rhs_index] == '"':
1315            # Second word is double quoted.  Make sure there is a
1316            # double quote at the end too.
1317            if not view_line.endswith('"'):
1318                die("View line with rhs quote should end with one: %s" %
1319                    view_line)
1320            # skip the quotes
1321            client_side = view_line[rhs_index+1:-1]
1322        else:
1323            client_side = view_line[rhs_index:]
1324
1325        # prefix + means overlay on previous mapping
1326        overlay = False
1327        if depot_side.startswith("+"):
1328            overlay = True
1329            depot_side = depot_side[1:]
1330
1331        # prefix - means exclude this path
1332        exclude = False
1333        if depot_side.startswith("-"):
1334            exclude = True
1335            depot_side = depot_side[1:]
1336
1337        m = View.Mapping(depot_side, client_side, overlay, exclude)
1338        self.mappings.append(m)
1339
1340    def map_in_client(self, depot_path):
1341        """Return the relative location in the client where this
1342           depot file should live.  Returns "" if the file should
1343           not be mapped in the client."""
1344
1345        paths_filled = []
1346        client_path = ""
1347
1348        # look at later entries first
1349        for m in self.mappings[::-1]:
1350
1351            # see where will this path end up in the client
1352            p = m.map_depot_to_client(depot_path)
1353
1354            if p == "":
1355                # Depot path does not belong in client.  Must remember
1356                # this, as previous items should not cause files to
1357                # exist in this path either.  Remember that the list is
1358                # being walked from the end, which has higher precedence.
1359                # Overlap mappings do not exclude previous mappings.
1360                if not m.overlay:
1361                    paths_filled.append(m.client_side)
1362
1363            else:
1364                # This mapping matched; no need to search any further.
1365                # But, the mapping could be rejected if the client path
1366                # has already been claimed by an earlier mapping.
1367                already_mapped_in_client = False
1368                for f in paths_filled:
1369                    # this is View.Path.match
1370                    if f.match(p):
1371                        already_mapped_in_client = True
1372                        break
1373                if not already_mapped_in_client:
1374                    # Include this file, unless it is from a line that
1375                    # explicitly said to exclude it.
1376                    if not m.exclude:
1377                        client_path = p
1378
1379                # a match, even if rejected, always stops the search
1380                break
1381
1382        return client_path
1383
1384class P4Sync(Command, P4UserMap):
1385    delete_actions = ( "delete", "move/delete", "purge" )
1386
1387    def __init__(self):
1388        Command.__init__(self)
1389        P4UserMap.__init__(self)
1390        self.options = [
1391                optparse.make_option("--branch", dest="branch"),
1392                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1393                optparse.make_option("--changesfile", dest="changesFile"),
1394                optparse.make_option("--silent", dest="silent", action="store_true"),
1395                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1396                optparse.make_option("--verbose", dest="verbose", action="store_true"),
1397                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1398                                     help="Import into refs/heads/ , not refs/remotes"),
1399                optparse.make_option("--max-changes", dest="maxChanges"),
1400                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1401                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1402                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1403                                     help="Only sync files that are included in the Perforce Client Spec")
1404        ]
1405        self.description = """Imports from Perforce into a git repository.\n
1406    example:
1407    //depot/my/project/ -- to import the current head
1408    //depot/my/project/@all -- to import everything
1409    //depot/my/project/@1,6 -- to import only from revision 1 to 6
1410
1411    (a ... is not needed in the path p4 specification, it's added implicitly)"""
1412
1413        self.usage += " //depot/path[@revRange]"
1414        self.silent = False
1415        self.createdBranches = set()
1416        self.committedChanges = set()
1417        self.branch = ""
1418        self.detectBranches = False
1419        self.detectLabels = False
1420        self.changesFile = ""
1421        self.syncWithOrigin = True
1422        self.verbose = False
1423        self.importIntoRemotes = True
1424        self.maxChanges = ""
1425        self.isWindows = (platform.system() == "Windows")
1426        self.keepRepoPath = False
1427        self.depotPaths = None
1428        self.p4BranchesInGit = []
1429        self.cloneExclude = []
1430        self.useClientSpec = False
1431        self.clientSpecDirs = None
1432
1433        if gitConfig("git-p4.syncFromOrigin") == "false":
1434            self.syncWithOrigin = False
1435
1436    #
1437    # P4 wildcards are not allowed in filenames.  P4 complains
1438    # if you simply add them, but you can force it with "-f", in
1439    # which case it translates them into %xx encoding internally.
1440    # Search for and fix just these four characters.  Do % last so
1441    # that fixing it does not inadvertently create new %-escapes.
1442    #
1443    def wildcard_decode(self, path):
1444        # Cannot have * in a filename in windows; untested as to
1445        # what p4 would do in such a case.
1446        if not self.isWindows:
1447            path = path.replace("%2A", "*")
1448        path = path.replace("%23", "#") \
1449                   .replace("%40", "@") \
1450                   .replace("%25", "%")
1451        return path
1452
1453    def extractFilesFromCommit(self, commit):
1454        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1455                             for path in self.cloneExclude]
1456        files = []
1457        fnum = 0
1458        while commit.has_key("depotFile%s" % fnum):
1459            path =  commit["depotFile%s" % fnum]
1460
1461            if [p for p in self.cloneExclude
1462                if p4PathStartsWith(path, p)]:
1463                found = False
1464            else:
1465                found = [p for p in self.depotPaths
1466                         if p4PathStartsWith(path, p)]
1467            if not found:
1468                fnum = fnum + 1
1469                continue
1470
1471            file = {}
1472            file["path"] = path
1473            file["rev"] = commit["rev%s" % fnum]
1474            file["action"] = commit["action%s" % fnum]
1475            file["type"] = commit["type%s" % fnum]
1476            files.append(file)
1477            fnum = fnum + 1
1478        return files
1479
1480    def stripRepoPath(self, path, prefixes):
1481        if self.useClientSpec:
1482            return self.clientSpecDirs.map_in_client(path)
1483
1484        if self.keepRepoPath:
1485            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1486
1487        for p in prefixes:
1488            if p4PathStartsWith(path, p):
1489                path = path[len(p):]
1490
1491        return path
1492
1493    def splitFilesIntoBranches(self, commit):
1494        branches = {}
1495        fnum = 0
1496        while commit.has_key("depotFile%s" % fnum):
1497            path =  commit["depotFile%s" % fnum]
1498            found = [p for p in self.depotPaths
1499                     if p4PathStartsWith(path, p)]
1500            if not found:
1501                fnum = fnum + 1
1502                continue
1503
1504            file = {}
1505            file["path"] = path
1506            file["rev"] = commit["rev%s" % fnum]
1507            file["action"] = commit["action%s" % fnum]
1508            file["type"] = commit["type%s" % fnum]
1509            fnum = fnum + 1
1510
1511            relPath = self.stripRepoPath(path, self.depotPaths)
1512
1513            for branch in self.knownBranches.keys():
1514
1515                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1516                if relPath.startswith(branch + "/"):
1517                    if branch not in branches:
1518                        branches[branch] = []
1519                    branches[branch].append(file)
1520                    break
1521
1522        return branches
1523
1524    # output one file from the P4 stream
1525    # - helper for streamP4Files
1526
1527    def streamOneP4File(self, file, contents):
1528        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1529        relPath = self.wildcard_decode(relPath)
1530        if verbose:
1531            sys.stderr.write("%s\n" % relPath)
1532
1533        (type_base, type_mods) = split_p4_type(file["type"])
1534
1535        git_mode = "100644"
1536        if "x" in type_mods:
1537            git_mode = "100755"
1538        if type_base == "symlink":
1539            git_mode = "120000"
1540            # p4 print on a symlink contains "target\n"; remove the newline
1541            data = ''.join(contents)
1542            contents = [data[:-1]]
1543
1544        if type_base == "utf16":
1545            # p4 delivers different text in the python output to -G
1546            # than it does when using "print -o", or normal p4 client
1547            # operations.  utf16 is converted to ascii or utf8, perhaps.
1548            # But ascii text saved as -t utf16 is completely mangled.
1549            # Invoke print -o to get the real contents.
1550            text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1551            contents = [ text ]
1552
1553        if type_base == "apple":
1554            # Apple filetype files will be streamed as a concatenation of
1555            # its appledouble header and the contents.  This is useless
1556            # on both macs and non-macs.  If using "print -q -o xx", it
1557            # will create "xx" with the data, and "%xx" with the header.
1558            # This is also not very useful.
1559            #
1560            # Ideally, someday, this script can learn how to generate
1561            # appledouble files directly and import those to git, but
1562            # non-mac machines can never find a use for apple filetype.
1563            print "\nIgnoring apple filetype file %s" % file['depotFile']
1564            return
1565
1566        # Perhaps windows wants unicode, utf16 newlines translated too;
1567        # but this is not doing it.
1568        if self.isWindows and type_base == "text":
1569            mangled = []
1570            for data in contents:
1571                data = data.replace("\r\n", "\n")
1572                mangled.append(data)
1573            contents = mangled
1574
1575        # Note that we do not try to de-mangle keywords on utf16 files,
1576        # even though in theory somebody may want that.
1577        if type_base in ("text", "unicode", "binary"):
1578            if "ko" in type_mods:
1579                text = ''.join(contents)
1580                text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1581                contents = [ text ]
1582            elif "k" in type_mods:
1583                text = ''.join(contents)
1584                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1585                contents = [ text ]
1586
1587        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1588
1589        # total length...
1590        length = 0
1591        for d in contents:
1592            length = length + len(d)
1593
1594        self.gitStream.write("data %d\n" % length)
1595        for d in contents:
1596            self.gitStream.write(d)
1597        self.gitStream.write("\n")
1598
1599    def streamOneP4Deletion(self, file):
1600        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1601        if verbose:
1602            sys.stderr.write("delete %s\n" % relPath)
1603        self.gitStream.write("D %s\n" % relPath)
1604
1605    # handle another chunk of streaming data
1606    def streamP4FilesCb(self, marshalled):
1607
1608        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1609            # start of a new file - output the old one first
1610            self.streamOneP4File(self.stream_file, self.stream_contents)
1611            self.stream_file = {}
1612            self.stream_contents = []
1613            self.stream_have_file_info = False
1614
1615        # pick up the new file information... for the
1616        # 'data' field we need to append to our array
1617        for k in marshalled.keys():
1618            if k == 'data':
1619                self.stream_contents.append(marshalled['data'])
1620            else:
1621                self.stream_file[k] = marshalled[k]
1622
1623        self.stream_have_file_info = True
1624
1625    # Stream directly from "p4 files" into "git fast-import"
1626    def streamP4Files(self, files):
1627        filesForCommit = []
1628        filesToRead = []
1629        filesToDelete = []
1630
1631        for f in files:
1632            # if using a client spec, only add the files that have
1633            # a path in the client
1634            if self.clientSpecDirs:
1635                if self.clientSpecDirs.map_in_client(f['path']) == "":
1636                    continue
1637
1638            filesForCommit.append(f)
1639            if f['action'] in self.delete_actions:
1640                filesToDelete.append(f)
1641            else:
1642                filesToRead.append(f)
1643
1644        # deleted files...
1645        for f in filesToDelete:
1646            self.streamOneP4Deletion(f)
1647
1648        if len(filesToRead) > 0:
1649            self.stream_file = {}
1650            self.stream_contents = []
1651            self.stream_have_file_info = False
1652
1653            # curry self argument
1654            def streamP4FilesCbSelf(entry):
1655                self.streamP4FilesCb(entry)
1656
1657            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1658
1659            p4CmdList(["-x", "-", "print"],
1660                      stdin=fileArgs,
1661                      cb=streamP4FilesCbSelf)
1662
1663            # do the last chunk
1664            if self.stream_file.has_key('depotFile'):
1665                self.streamOneP4File(self.stream_file, self.stream_contents)
1666
1667    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1668        epoch = details["time"]
1669        author = details["user"]
1670        self.branchPrefixes = branchPrefixes
1671
1672        if self.verbose:
1673            print "commit into %s" % branch
1674
1675        # start with reading files; if that fails, we should not
1676        # create a commit.
1677        new_files = []
1678        for f in files:
1679            if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1680                new_files.append (f)
1681            else:
1682                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1683
1684        self.gitStream.write("commit %s\n" % branch)
1685#        gitStream.write("mark :%s\n" % details["change"])
1686        self.committedChanges.add(int(details["change"]))
1687        committer = ""
1688        if author not in self.users:
1689            self.getUserMapFromPerforceServer()
1690        if author in self.users:
1691            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1692        else:
1693            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1694
1695        self.gitStream.write("committer %s\n" % committer)
1696
1697        self.gitStream.write("data <<EOT\n")
1698        self.gitStream.write(details["desc"])
1699        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1700                             % (','.join (branchPrefixes), details["change"]))
1701        if len(details['options']) > 0:
1702            self.gitStream.write(": options = %s" % details['options'])
1703        self.gitStream.write("]\nEOT\n\n")
1704
1705        if len(parent) > 0:
1706            if self.verbose:
1707                print "parent %s" % parent
1708            self.gitStream.write("from %s\n" % parent)
1709
1710        self.streamP4Files(new_files)
1711        self.gitStream.write("\n")
1712
1713        change = int(details["change"])
1714
1715        if self.labels.has_key(change):
1716            label = self.labels[change]
1717            labelDetails = label[0]
1718            labelRevisions = label[1]
1719            if self.verbose:
1720                print "Change %s is labelled %s" % (change, labelDetails)
1721
1722            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1723                                                    for p in branchPrefixes])
1724
1725            if len(files) == len(labelRevisions):
1726
1727                cleanedFiles = {}
1728                for info in files:
1729                    if info["action"] in self.delete_actions:
1730                        continue
1731                    cleanedFiles[info["depotFile"]] = info["rev"]
1732
1733                if cleanedFiles == labelRevisions:
1734                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1735                    self.gitStream.write("from %s\n" % branch)
1736
1737                    owner = labelDetails["Owner"]
1738                    tagger = ""
1739                    if author in self.users:
1740                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1741                    else:
1742                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1743                    self.gitStream.write("tagger %s\n" % tagger)
1744                    self.gitStream.write("data <<EOT\n")
1745                    self.gitStream.write(labelDetails["Description"])
1746                    self.gitStream.write("EOT\n\n")
1747
1748                else:
1749                    if not self.silent:
1750                        print ("Tag %s does not match with change %s: files do not match."
1751                               % (labelDetails["label"], change))
1752
1753            else:
1754                if not self.silent:
1755                    print ("Tag %s does not match with change %s: file count is different."
1756                           % (labelDetails["label"], change))
1757
1758    def getLabels(self):
1759        self.labels = {}
1760
1761        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1762        if len(l) > 0 and not self.silent:
1763            print "Finding files belonging to labels in %s" % `self.depotPaths`
1764
1765        for output in l:
1766            label = output["label"]
1767            revisions = {}
1768            newestChange = 0
1769            if self.verbose:
1770                print "Querying files for label %s" % label
1771            for file in p4CmdList(["files"] +
1772                                      ["%s...@%s" % (p, label)
1773                                          for p in self.depotPaths]):
1774                revisions[file["depotFile"]] = file["rev"]
1775                change = int(file["change"])
1776                if change > newestChange:
1777                    newestChange = change
1778
1779            self.labels[newestChange] = [output, revisions]
1780
1781        if self.verbose:
1782            print "Label changes: %s" % self.labels.keys()
1783
1784    def guessProjectName(self):
1785        for p in self.depotPaths:
1786            if p.endswith("/"):
1787                p = p[:-1]
1788            p = p[p.strip().rfind("/") + 1:]
1789            if not p.endswith("/"):
1790               p += "/"
1791            return p
1792
1793    def getBranchMapping(self):
1794        lostAndFoundBranches = set()
1795
1796        user = gitConfig("git-p4.branchUser")
1797        if len(user) > 0:
1798            command = "branches -u %s" % user
1799        else:
1800            command = "branches"
1801
1802        for info in p4CmdList(command):
1803            details = p4Cmd("branch -o %s" % info["branch"])
1804            viewIdx = 0
1805            while details.has_key("View%s" % viewIdx):
1806                paths = details["View%s" % viewIdx].split(" ")
1807                viewIdx = viewIdx + 1
1808                # require standard //depot/foo/... //depot/bar/... mapping
1809                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1810                    continue
1811                source = paths[0]
1812                destination = paths[1]
1813                ## HACK
1814                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1815                    source = source[len(self.depotPaths[0]):-4]
1816                    destination = destination[len(self.depotPaths[0]):-4]
1817
1818                    if destination in self.knownBranches:
1819                        if not self.silent:
1820                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1821                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1822                        continue
1823
1824                    self.knownBranches[destination] = source
1825
1826                    lostAndFoundBranches.discard(destination)
1827
1828                    if source not in self.knownBranches:
1829                        lostAndFoundBranches.add(source)
1830
1831        # Perforce does not strictly require branches to be defined, so we also
1832        # check git config for a branch list.
1833        #
1834        # Example of branch definition in git config file:
1835        # [git-p4]
1836        #   branchList=main:branchA
1837        #   branchList=main:branchB
1838        #   branchList=branchA:branchC
1839        configBranches = gitConfigList("git-p4.branchList")
1840        for branch in configBranches:
1841            if branch:
1842                (source, destination) = branch.split(":")
1843                self.knownBranches[destination] = source
1844
1845                lostAndFoundBranches.discard(destination)
1846
1847                if source not in self.knownBranches:
1848                    lostAndFoundBranches.add(source)
1849
1850
1851        for branch in lostAndFoundBranches:
1852            self.knownBranches[branch] = branch
1853
1854    def getBranchMappingFromGitBranches(self):
1855        branches = p4BranchesInGit(self.importIntoRemotes)
1856        for branch in branches.keys():
1857            if branch == "master":
1858                branch = "main"
1859            else:
1860                branch = branch[len(self.projectName):]
1861            self.knownBranches[branch] = branch
1862
1863    def listExistingP4GitBranches(self):
1864        # branches holds mapping from name to commit
1865        branches = p4BranchesInGit(self.importIntoRemotes)
1866        self.p4BranchesInGit = branches.keys()
1867        for branch in branches.keys():
1868            self.initialParents[self.refPrefix + branch] = branches[branch]
1869
1870    def updateOptionDict(self, d):
1871        option_keys = {}
1872        if self.keepRepoPath:
1873            option_keys['keepRepoPath'] = 1
1874
1875        d["options"] = ' '.join(sorted(option_keys.keys()))
1876
1877    def readOptions(self, d):
1878        self.keepRepoPath = (d.has_key('options')
1879                             and ('keepRepoPath' in d['options']))
1880
1881    def gitRefForBranch(self, branch):
1882        if branch == "main":
1883            return self.refPrefix + "master"
1884
1885        if len(branch) <= 0:
1886            return branch
1887
1888        return self.refPrefix + self.projectName + branch
1889
1890    def gitCommitByP4Change(self, ref, change):
1891        if self.verbose:
1892            print "looking in ref " + ref + " for change %s using bisect..." % change
1893
1894        earliestCommit = ""
1895        latestCommit = parseRevision(ref)
1896
1897        while True:
1898            if self.verbose:
1899                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1900            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1901            if len(next) == 0:
1902                if self.verbose:
1903                    print "argh"
1904                return ""
1905            log = extractLogMessageFromGitCommit(next)
1906            settings = extractSettingsGitLog(log)
1907            currentChange = int(settings['change'])
1908            if self.verbose:
1909                print "current change %s" % currentChange
1910
1911            if currentChange == change:
1912                if self.verbose:
1913                    print "found %s" % next
1914                return next
1915
1916            if currentChange < change:
1917                earliestCommit = "^%s" % next
1918            else:
1919                latestCommit = "%s" % next
1920
1921        return ""
1922
1923    def importNewBranch(self, branch, maxChange):
1924        # make fast-import flush all changes to disk and update the refs using the checkpoint
1925        # command so that we can try to find the branch parent in the git history
1926        self.gitStream.write("checkpoint\n\n");
1927        self.gitStream.flush();
1928        branchPrefix = self.depotPaths[0] + branch + "/"
1929        range = "@1,%s" % maxChange
1930        #print "prefix" + branchPrefix
1931        changes = p4ChangesForPaths([branchPrefix], range)
1932        if len(changes) <= 0:
1933            return False
1934        firstChange = changes[0]
1935        #print "first change in branch: %s" % firstChange
1936        sourceBranch = self.knownBranches[branch]
1937        sourceDepotPath = self.depotPaths[0] + sourceBranch
1938        sourceRef = self.gitRefForBranch(sourceBranch)
1939        #print "source " + sourceBranch
1940
1941        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1942        #print "branch parent: %s" % branchParentChange
1943        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1944        if len(gitParent) > 0:
1945            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1946            #print "parent git commit: %s" % gitParent
1947
1948        self.importChanges(changes)
1949        return True
1950
1951    def importChanges(self, changes):
1952        cnt = 1
1953        for change in changes:
1954            description = p4Cmd("describe %s" % change)
1955            self.updateOptionDict(description)
1956
1957            if not self.silent:
1958                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1959                sys.stdout.flush()
1960            cnt = cnt + 1
1961
1962            try:
1963                if self.detectBranches:
1964                    branches = self.splitFilesIntoBranches(description)
1965                    for branch in branches.keys():
1966                        ## HACK  --hwn
1967                        branchPrefix = self.depotPaths[0] + branch + "/"
1968
1969                        parent = ""
1970
1971                        filesForCommit = branches[branch]
1972
1973                        if self.verbose:
1974                            print "branch is %s" % branch
1975
1976                        self.updatedBranches.add(branch)
1977
1978                        if branch not in self.createdBranches:
1979                            self.createdBranches.add(branch)
1980                            parent = self.knownBranches[branch]
1981                            if parent == branch:
1982                                parent = ""
1983                            else:
1984                                fullBranch = self.projectName + branch
1985                                if fullBranch not in self.p4BranchesInGit:
1986                                    if not self.silent:
1987                                        print("\n    Importing new branch %s" % fullBranch);
1988                                    if self.importNewBranch(branch, change - 1):
1989                                        parent = ""
1990                                        self.p4BranchesInGit.append(fullBranch)
1991                                    if not self.silent:
1992                                        print("\n    Resuming with change %s" % change);
1993
1994                                if self.verbose:
1995                                    print "parent determined through known branches: %s" % parent
1996
1997                        branch = self.gitRefForBranch(branch)
1998                        parent = self.gitRefForBranch(parent)
1999
2000                        if self.verbose:
2001                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2002
2003                        if len(parent) == 0 and branch in self.initialParents:
2004                            parent = self.initialParents[branch]
2005                            del self.initialParents[branch]
2006
2007                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2008                else:
2009                    files = self.extractFilesFromCommit(description)
2010                    self.commit(description, files, self.branch, self.depotPaths,
2011                                self.initialParent)
2012                    self.initialParent = ""
2013            except IOError:
2014                print self.gitError.read()
2015                sys.exit(1)
2016
2017    def importHeadRevision(self, revision):
2018        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2019
2020        details = {}
2021        details["user"] = "git perforce import user"
2022        details["desc"] = ("Initial import of %s from the state at revision %s\n"
2023                           % (' '.join(self.depotPaths), revision))
2024        details["change"] = revision
2025        newestRevision = 0
2026
2027        fileCnt = 0
2028        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2029
2030        for info in p4CmdList(["files"] + fileArgs):
2031
2032            if 'code' in info and info['code'] == 'error':
2033                sys.stderr.write("p4 returned an error: %s\n"
2034                                 % info['data'])
2035                if info['data'].find("must refer to client") >= 0:
2036                    sys.stderr.write("This particular p4 error is misleading.\n")
2037                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
2038                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2039                sys.exit(1)
2040            if 'p4ExitCode' in info:
2041                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2042                sys.exit(1)
2043
2044
2045            change = int(info["change"])
2046            if change > newestRevision:
2047                newestRevision = change
2048
2049            if info["action"] in self.delete_actions:
2050                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2051                #fileCnt = fileCnt + 1
2052                continue
2053
2054            for prop in ["depotFile", "rev", "action", "type" ]:
2055                details["%s%s" % (prop, fileCnt)] = info[prop]
2056
2057            fileCnt = fileCnt + 1
2058
2059        details["change"] = newestRevision
2060
2061        # Use time from top-most change so that all git-p4 clones of
2062        # the same p4 repo have the same commit SHA1s.
2063        res = p4CmdList("describe -s %d" % newestRevision)
2064        newestTime = None
2065        for r in res:
2066            if r.has_key('time'):
2067                newestTime = int(r['time'])
2068        if newestTime is None:
2069            die("\"describe -s\" on newest change %d did not give a time")
2070        details["time"] = newestTime
2071
2072        self.updateOptionDict(details)
2073        try:
2074            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2075        except IOError:
2076            print "IO error with git fast-import. Is your git version recent enough?"
2077            print self.gitError.read()
2078
2079
2080    def getClientSpec(self):
2081        specList = p4CmdList("client -o")
2082        if len(specList) != 1:
2083            die('Output from "client -o" is %d lines, expecting 1' %
2084                len(specList))
2085
2086        # dictionary of all client parameters
2087        entry = specList[0]
2088
2089        # just the keys that start with "View"
2090        view_keys = [ k for k in entry.keys() if k.startswith("View") ]
2091
2092        # hold this new View
2093        view = View()
2094
2095        # append the lines, in order, to the view
2096        for view_num in range(len(view_keys)):
2097            k = "View%d" % view_num
2098            if k not in view_keys:
2099                die("Expected view key %s missing" % k)
2100            view.append(entry[k])
2101
2102        self.clientSpecDirs = view
2103        if self.verbose:
2104            for i, m in enumerate(self.clientSpecDirs.mappings):
2105                    print "clientSpecDirs %d: %s" % (i, str(m))
2106
2107    def run(self, args):
2108        self.depotPaths = []
2109        self.changeRange = ""
2110        self.initialParent = ""
2111        self.previousDepotPaths = []
2112
2113        # map from branch depot path to parent branch
2114        self.knownBranches = {}
2115        self.initialParents = {}
2116        self.hasOrigin = originP4BranchesExist()
2117        if not self.syncWithOrigin:
2118            self.hasOrigin = False
2119
2120        if self.importIntoRemotes:
2121            self.refPrefix = "refs/remotes/p4/"
2122        else:
2123            self.refPrefix = "refs/heads/p4/"
2124
2125        if self.syncWithOrigin and self.hasOrigin:
2126            if not self.silent:
2127                print "Syncing with origin first by calling git fetch origin"
2128            system("git fetch origin")
2129
2130        if len(self.branch) == 0:
2131            self.branch = self.refPrefix + "master"
2132            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2133                system("git update-ref %s refs/heads/p4" % self.branch)
2134                system("git branch -D p4");
2135            # create it /after/ importing, when master exists
2136            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2137                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2138
2139        if not self.useClientSpec:
2140            if gitConfig("git-p4.useclientspec", "--bool") == "true":
2141                self.useClientSpec = True
2142        if self.useClientSpec:
2143            self.getClientSpec()
2144
2145        # TODO: should always look at previous commits,
2146        # merge with previous imports, if possible.
2147        if args == []:
2148            if self.hasOrigin:
2149                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2150            self.listExistingP4GitBranches()
2151
2152            if len(self.p4BranchesInGit) > 1:
2153                if not self.silent:
2154                    print "Importing from/into multiple branches"
2155                self.detectBranches = True
2156
2157            if self.verbose:
2158                print "branches: %s" % self.p4BranchesInGit
2159
2160            p4Change = 0
2161            for branch in self.p4BranchesInGit:
2162                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2163
2164                settings = extractSettingsGitLog(logMsg)
2165
2166                self.readOptions(settings)
2167                if (settings.has_key('depot-paths')
2168                    and settings.has_key ('change')):
2169                    change = int(settings['change']) + 1
2170                    p4Change = max(p4Change, change)
2171
2172                    depotPaths = sorted(settings['depot-paths'])
2173                    if self.previousDepotPaths == []:
2174                        self.previousDepotPaths = depotPaths
2175                    else:
2176                        paths = []
2177                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2178                            prev_list = prev.split("/")
2179                            cur_list = cur.split("/")
2180                            for i in range(0, min(len(cur_list), len(prev_list))):
2181                                if cur_list[i] <> prev_list[i]:
2182                                    i = i - 1
2183                                    break
2184
2185                            paths.append ("/".join(cur_list[:i + 1]))
2186
2187                        self.previousDepotPaths = paths
2188
2189            if p4Change > 0:
2190                self.depotPaths = sorted(self.previousDepotPaths)
2191                self.changeRange = "@%s,#head" % p4Change
2192                if not self.detectBranches:
2193                    self.initialParent = parseRevision(self.branch)
2194                if not self.silent and not self.detectBranches:
2195                    print "Performing incremental import into %s git branch" % self.branch
2196
2197        if not self.branch.startswith("refs/"):
2198            self.branch = "refs/heads/" + self.branch
2199
2200        if len(args) == 0 and self.depotPaths:
2201            if not self.silent:
2202                print "Depot paths: %s" % ' '.join(self.depotPaths)
2203        else:
2204            if self.depotPaths and self.depotPaths != args:
2205                print ("previous import used depot path %s and now %s was specified. "
2206                       "This doesn't work!" % (' '.join (self.depotPaths),
2207                                               ' '.join (args)))
2208                sys.exit(1)
2209
2210            self.depotPaths = sorted(args)
2211
2212        revision = ""
2213        self.users = {}
2214
2215        # Make sure no revision specifiers are used when --changesfile
2216        # is specified.
2217        bad_changesfile = False
2218        if len(self.changesFile) > 0:
2219            for p in self.depotPaths:
2220                if p.find("@") >= 0 or p.find("#") >= 0:
2221                    bad_changesfile = True
2222                    break
2223        if bad_changesfile:
2224            die("Option --changesfile is incompatible with revision specifiers")
2225
2226        newPaths = []
2227        for p in self.depotPaths:
2228            if p.find("@") != -1:
2229                atIdx = p.index("@")
2230                self.changeRange = p[atIdx:]
2231                if self.changeRange == "@all":
2232                    self.changeRange = ""
2233                elif ',' not in self.changeRange:
2234                    revision = self.changeRange
2235                    self.changeRange = ""
2236                p = p[:atIdx]
2237            elif p.find("#") != -1:
2238                hashIdx = p.index("#")
2239                revision = p[hashIdx:]
2240                p = p[:hashIdx]
2241            elif self.previousDepotPaths == []:
2242                # pay attention to changesfile, if given, else import
2243                # the entire p4 tree at the head revision
2244                if len(self.changesFile) == 0:
2245                    revision = "#head"
2246
2247            p = re.sub ("\.\.\.$", "", p)
2248            if not p.endswith("/"):
2249                p += "/"
2250
2251            newPaths.append(p)
2252
2253        self.depotPaths = newPaths
2254
2255
2256        self.loadUserMapFromCache()
2257        self.labels = {}
2258        if self.detectLabels:
2259            self.getLabels();
2260
2261        if self.detectBranches:
2262            ## FIXME - what's a P4 projectName ?
2263            self.projectName = self.guessProjectName()
2264
2265            if self.hasOrigin:
2266                self.getBranchMappingFromGitBranches()
2267            else:
2268                self.getBranchMapping()
2269            if self.verbose:
2270                print "p4-git branches: %s" % self.p4BranchesInGit
2271                print "initial parents: %s" % self.initialParents
2272            for b in self.p4BranchesInGit:
2273                if b != "master":
2274
2275                    ## FIXME
2276                    b = b[len(self.projectName):]
2277                self.createdBranches.add(b)
2278
2279        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2280
2281        importProcess = subprocess.Popen(["git", "fast-import"],
2282                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2283                                         stderr=subprocess.PIPE);
2284        self.gitOutput = importProcess.stdout
2285        self.gitStream = importProcess.stdin
2286        self.gitError = importProcess.stderr
2287
2288        if revision:
2289            self.importHeadRevision(revision)
2290        else:
2291            changes = []
2292
2293            if len(self.changesFile) > 0:
2294                output = open(self.changesFile).readlines()
2295                changeSet = set()
2296                for line in output:
2297                    changeSet.add(int(line))
2298
2299                for change in changeSet:
2300                    changes.append(change)
2301
2302                changes.sort()
2303            else:
2304                # catch "git-p4 sync" with no new branches, in a repo that
2305                # does not have any existing git-p4 branches
2306                if len(args) == 0 and not self.p4BranchesInGit:
2307                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2308                if self.verbose:
2309                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2310                                                              self.changeRange)
2311                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2312
2313                if len(self.maxChanges) > 0:
2314                    changes = changes[:min(int(self.maxChanges), len(changes))]
2315
2316            if len(changes) == 0:
2317                if not self.silent:
2318                    print "No changes to import!"
2319                return True
2320
2321            if not self.silent and not self.detectBranches:
2322                print "Import destination: %s" % self.branch
2323
2324            self.updatedBranches = set()
2325
2326            self.importChanges(changes)
2327
2328            if not self.silent:
2329                print ""
2330                if len(self.updatedBranches) > 0:
2331                    sys.stdout.write("Updated branches: ")
2332                    for b in self.updatedBranches:
2333                        sys.stdout.write("%s " % b)
2334                    sys.stdout.write("\n")
2335
2336        self.gitStream.close()
2337        if importProcess.wait() != 0:
2338            die("fast-import failed: %s" % self.gitError.read())
2339        self.gitOutput.close()
2340        self.gitError.close()
2341
2342        return True
2343
2344class P4Rebase(Command):
2345    def __init__(self):
2346        Command.__init__(self)
2347        self.options = [ ]
2348        self.description = ("Fetches the latest revision from perforce and "
2349                            + "rebases the current work (branch) against it")
2350        self.verbose = False
2351
2352    def run(self, args):
2353        sync = P4Sync()
2354        sync.run([])
2355
2356        return self.rebase()
2357
2358    def rebase(self):
2359        if os.system("git update-index --refresh") != 0:
2360            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2361        if len(read_pipe("git diff-index HEAD --")) > 0:
2362            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2363
2364        [upstream, settings] = findUpstreamBranchPoint()
2365        if len(upstream) == 0:
2366            die("Cannot find upstream branchpoint for rebase")
2367
2368        # the branchpoint may be p4/foo~3, so strip off the parent
2369        upstream = re.sub("~[0-9]+$", "", upstream)
2370
2371        print "Rebasing the current branch onto %s" % upstream
2372        oldHead = read_pipe("git rev-parse HEAD").strip()
2373        system("git rebase %s" % upstream)
2374        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2375        return True
2376
2377class P4Clone(P4Sync):
2378    def __init__(self):
2379        P4Sync.__init__(self)
2380        self.description = "Creates a new git repository and imports from Perforce into it"
2381        self.usage = "usage: %prog [options] //depot/path[@revRange]"
2382        self.options += [
2383            optparse.make_option("--destination", dest="cloneDestination",
2384                                 action='store', default=None,
2385                                 help="where to leave result of the clone"),
2386            optparse.make_option("-/", dest="cloneExclude",
2387                                 action="append", type="string",
2388                                 help="exclude depot path"),
2389            optparse.make_option("--bare", dest="cloneBare",
2390                                 action="store_true", default=False),
2391        ]
2392        self.cloneDestination = None
2393        self.needsGit = False
2394        self.cloneBare = False
2395
2396    # This is required for the "append" cloneExclude action
2397    def ensure_value(self, attr, value):
2398        if not hasattr(self, attr) or getattr(self, attr) is None:
2399            setattr(self, attr, value)
2400        return getattr(self, attr)
2401
2402    def defaultDestination(self, args):
2403        ## TODO: use common prefix of args?
2404        depotPath = args[0]
2405        depotDir = re.sub("(@[^@]*)$", "", depotPath)
2406        depotDir = re.sub("(#[^#]*)$", "", depotDir)
2407        depotDir = re.sub(r"\.\.\.$", "", depotDir)
2408        depotDir = re.sub(r"/$", "", depotDir)
2409        return os.path.split(depotDir)[1]
2410
2411    def run(self, args):
2412        if len(args) < 1:
2413            return False
2414
2415        if self.keepRepoPath and not self.cloneDestination:
2416            sys.stderr.write("Must specify destination for --keep-path\n")
2417            sys.exit(1)
2418
2419        depotPaths = args
2420
2421        if not self.cloneDestination and len(depotPaths) > 1:
2422            self.cloneDestination = depotPaths[-1]
2423            depotPaths = depotPaths[:-1]
2424
2425        self.cloneExclude = ["/"+p for p in self.cloneExclude]
2426        for p in depotPaths:
2427            if not p.startswith("//"):
2428                return False
2429
2430        if not self.cloneDestination:
2431            self.cloneDestination = self.defaultDestination(args)
2432
2433        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2434
2435        if not os.path.exists(self.cloneDestination):
2436            os.makedirs(self.cloneDestination)
2437        chdir(self.cloneDestination)
2438
2439        init_cmd = [ "git", "init" ]
2440        if self.cloneBare:
2441            init_cmd.append("--bare")
2442        subprocess.check_call(init_cmd)
2443
2444        if not P4Sync.run(self, depotPaths):
2445            return False
2446        if self.branch != "master":
2447            if self.importIntoRemotes:
2448                masterbranch = "refs/remotes/p4/master"
2449            else:
2450                masterbranch = "refs/heads/p4/master"
2451            if gitBranchExists(masterbranch):
2452                system("git branch master %s" % masterbranch)
2453                if not self.cloneBare:
2454                    system("git checkout -f")
2455            else:
2456                print "Could not detect main branch. No checkout/master branch created."
2457
2458        return True
2459
2460class P4Branches(Command):
2461    def __init__(self):
2462        Command.__init__(self)
2463        self.options = [ ]
2464        self.description = ("Shows the git branches that hold imports and their "
2465                            + "corresponding perforce depot paths")
2466        self.verbose = False
2467
2468    def run(self, args):
2469        if originP4BranchesExist():
2470            createOrUpdateBranchesFromOrigin()
2471
2472        cmdline = "git rev-parse --symbolic "
2473        cmdline += " --remotes"
2474
2475        for line in read_pipe_lines(cmdline):
2476            line = line.strip()
2477
2478            if not line.startswith('p4/') or line == "p4/HEAD":
2479                continue
2480            branch = line
2481
2482            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2483            settings = extractSettingsGitLog(log)
2484
2485            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2486        return True
2487
2488class HelpFormatter(optparse.IndentedHelpFormatter):
2489    def __init__(self):
2490        optparse.IndentedHelpFormatter.__init__(self)
2491
2492    def format_description(self, description):
2493        if description:
2494            return description + "\n"
2495        else:
2496            return ""
2497
2498def printUsage(commands):
2499    print "usage: %s <command> [options]" % sys.argv[0]
2500    print ""
2501    print "valid commands: %s" % ", ".join(commands)
2502    print ""
2503    print "Try %s <command> --help for command specific help." % sys.argv[0]
2504    print ""
2505
2506commands = {
2507    "debug" : P4Debug,
2508    "submit" : P4Submit,
2509    "commit" : P4Submit,
2510    "sync" : P4Sync,
2511    "rebase" : P4Rebase,
2512    "clone" : P4Clone,
2513    "rollback" : P4RollBack,
2514    "branches" : P4Branches
2515}
2516
2517
2518def main():
2519    if len(sys.argv[1:]) == 0:
2520        printUsage(commands.keys())
2521        sys.exit(2)
2522
2523    cmd = ""
2524    cmdName = sys.argv[1]
2525    try:
2526        klass = commands[cmdName]
2527        cmd = klass()
2528    except KeyError:
2529        print "unknown command %s" % cmdName
2530        print ""
2531        printUsage(commands.keys())
2532        sys.exit(2)
2533
2534    options = cmd.options
2535    cmd.gitdir = os.environ.get("GIT_DIR", None)
2536
2537    args = sys.argv[2:]
2538
2539    if len(options) > 0:
2540        if cmd.needsGit:
2541            options.append(optparse.make_option("--git-dir", dest="gitdir"))
2542
2543        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2544                                       options,
2545                                       description = cmd.description,
2546                                       formatter = HelpFormatter())
2547
2548        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2549    global verbose
2550    verbose = cmd.verbose
2551    if cmd.needsGit:
2552        if cmd.gitdir == None:
2553            cmd.gitdir = os.path.abspath(".git")
2554            if not isValidGitDir(cmd.gitdir):
2555                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2556                if os.path.exists(cmd.gitdir):
2557                    cdup = read_pipe("git rev-parse --show-cdup").strip()
2558                    if len(cdup) > 0:
2559                        chdir(cdup);
2560
2561        if not isValidGitDir(cmd.gitdir):
2562            if isValidGitDir(cmd.gitdir + "/.git"):
2563                cmd.gitdir += "/.git"
2564            else:
2565                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2566
2567        os.environ["GIT_DIR"] = cmd.gitdir
2568
2569    if not cmd.run(args):
2570        parser.print_help()
2571        sys.exit(2)
2572
2573
2574if __name__ == '__main__':
2575    main()