8f28e0a6fbe8924a78ccb5bc180d445b464da354
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = ["p4"]
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += ["-u",user]
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += ["-P", password]
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += ["-p", port]
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += ["-h", host]
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += ["-c", client]
  46
  47
  48    if isinstance(cmd,basestring):
  49        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50    else:
  51        real_cmd += cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56    # not using the shell, we have to set it ourselves.  This path could
  57    # be relative, so go there first, then figure out where we ended up.
  58    os.chdir(dir)
  59    os.environ['PWD'] = os.getcwd()
  60
  61def die(msg):
  62    if verbose:
  63        raise Exception(msg)
  64    else:
  65        sys.stderr.write(msg + "\n")
  66        sys.exit(1)
  67
  68def write_pipe(c, stdin):
  69    if verbose:
  70        sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72    expand = isinstance(c,basestring)
  73    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74    pipe = p.stdin
  75    val = pipe.write(stdin)
  76    pipe.close()
  77    if p.wait():
  78        die('Command failed: %s' % str(c))
  79
  80    return val
  81
  82def p4_write_pipe(c, stdin):
  83    real_cmd = p4_build_cmd(c)
  84    return write_pipe(real_cmd, stdin)
  85
  86def read_pipe(c, ignore_error=False):
  87    if verbose:
  88        sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90    expand = isinstance(c,basestring)
  91    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92    pipe = p.stdout
  93    val = pipe.read()
  94    if p.wait() and not ignore_error:
  95        die('Command failed: %s' % str(c))
  96
  97    return val
  98
  99def p4_read_pipe(c, ignore_error=False):
 100    real_cmd = p4_build_cmd(c)
 101    return read_pipe(real_cmd, ignore_error)
 102
 103def read_pipe_lines(c):
 104    if verbose:
 105        sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107    expand = isinstance(c, basestring)
 108    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109    pipe = p.stdout
 110    val = pipe.readlines()
 111    if pipe.close() or p.wait():
 112        die('Command failed: %s' % str(c))
 113
 114    return val
 115
 116def p4_read_pipe_lines(c):
 117    """Specifically invoke p4 on the command supplied. """
 118    real_cmd = p4_build_cmd(c)
 119    return read_pipe_lines(real_cmd)
 120
 121def system(cmd):
 122    expand = isinstance(cmd,basestring)
 123    if verbose:
 124        sys.stderr.write("executing %s\n" % str(cmd))
 125    subprocess.check_call(cmd, shell=expand)
 126
 127def p4_system(cmd):
 128    """Specifically invoke p4 as the system command. """
 129    real_cmd = p4_build_cmd(cmd)
 130    expand = isinstance(real_cmd, basestring)
 131    subprocess.check_call(real_cmd, shell=expand)
 132
 133def p4_integrate(src, dest):
 134    p4_system(["integrate", "-Dt", src, dest])
 135
 136def p4_sync(path):
 137    p4_system(["sync", path])
 138
 139def p4_add(f):
 140    p4_system(["add", f])
 141
 142def p4_delete(f):
 143    p4_system(["delete", f])
 144
 145def p4_edit(f):
 146    p4_system(["edit", f])
 147
 148def p4_revert(f):
 149    p4_system(["revert", f])
 150
 151def p4_reopen(type, file):
 152    p4_system(["reopen", "-t", type, file])
 153
 154#
 155# Canonicalize the p4 type and return a tuple of the
 156# base type, plus any modifiers.  See "p4 help filetypes"
 157# for a list and explanation.
 158#
 159def split_p4_type(p4type):
 160
 161    p4_filetypes_historical = {
 162        "ctempobj": "binary+Sw",
 163        "ctext": "text+C",
 164        "cxtext": "text+Cx",
 165        "ktext": "text+k",
 166        "kxtext": "text+kx",
 167        "ltext": "text+F",
 168        "tempobj": "binary+FSw",
 169        "ubinary": "binary+F",
 170        "uresource": "resource+F",
 171        "uxbinary": "binary+Fx",
 172        "xbinary": "binary+x",
 173        "xltext": "text+Fx",
 174        "xtempobj": "binary+Swx",
 175        "xtext": "text+x",
 176        "xunicode": "unicode+x",
 177        "xutf16": "utf16+x",
 178    }
 179    if p4type in p4_filetypes_historical:
 180        p4type = p4_filetypes_historical[p4type]
 181    mods = ""
 182    s = p4type.split("+")
 183    base = s[0]
 184    mods = ""
 185    if len(s) > 1:
 186        mods = s[1]
 187    return (base, mods)
 188
 189
 190def setP4ExecBit(file, mode):
 191    # Reopens an already open file and changes the execute bit to match
 192    # the execute bit setting in the passed in mode.
 193
 194    p4Type = "+x"
 195
 196    if not isModeExec(mode):
 197        p4Type = getP4OpenedType(file)
 198        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200        if p4Type[-1] == "+":
 201            p4Type = p4Type[0:-1]
 202
 203    p4_reopen(p4Type, file)
 204
 205def getP4OpenedType(file):
 206    # Returns the perforce file type for the given file.
 207
 208    result = p4_read_pipe(["opened", file])
 209    match = re.match(".*\((.+)\)\r?$", result)
 210    if match:
 211        return match.group(1)
 212    else:
 213        die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215def diffTreePattern():
 216    # This is a simple generator for the diff tree regex pattern. This could be
 217    # a class variable if this and parseDiffTreeEntry were a part of a class.
 218    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219    while True:
 220        yield pattern
 221
 222def parseDiffTreeEntry(entry):
 223    """Parses a single diff tree entry into its component elements.
 224
 225    See git-diff-tree(1) manpage for details about the format of the diff
 226    output. This method returns a dictionary with the following elements:
 227
 228    src_mode - The mode of the source file
 229    dst_mode - The mode of the destination file
 230    src_sha1 - The sha1 for the source file
 231    dst_sha1 - The sha1 fr the destination file
 232    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233    status_score - The score for the status (applicable for 'C' and 'R'
 234                   statuses). This is None if there is no score.
 235    src - The path for the source file.
 236    dst - The path for the destination file. This is only present for
 237          copy or renames. If it is not present, this is None.
 238
 239    If the pattern is not matched, None is returned."""
 240
 241    match = diffTreePattern().next().match(entry)
 242    if match:
 243        return {
 244            'src_mode': match.group(1),
 245            'dst_mode': match.group(2),
 246            'src_sha1': match.group(3),
 247            'dst_sha1': match.group(4),
 248            'status': match.group(5),
 249            'status_score': match.group(6),
 250            'src': match.group(7),
 251            'dst': match.group(10)
 252        }
 253    return None
 254
 255def isModeExec(mode):
 256    # Returns True if the given git mode represents an executable file,
 257    # otherwise False.
 258    return mode[-3:] == "755"
 259
 260def isModeExecChanged(src_mode, dst_mode):
 261    return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265    if isinstance(cmd,basestring):
 266        cmd = "-G " + cmd
 267        expand = True
 268    else:
 269        cmd = ["-G"] + cmd
 270        expand = False
 271
 272    cmd = p4_build_cmd(cmd)
 273    if verbose:
 274        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276    # Use a temporary file to avoid deadlocks without
 277    # subprocess.communicate(), which would put another copy
 278    # of stdout into memory.
 279    stdin_file = None
 280    if stdin is not None:
 281        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282        if isinstance(stdin,basestring):
 283            stdin_file.write(stdin)
 284        else:
 285            for i in stdin:
 286                stdin_file.write(i + '\n')
 287        stdin_file.flush()
 288        stdin_file.seek(0)
 289
 290    p4 = subprocess.Popen(cmd,
 291                          shell=expand,
 292                          stdin=stdin_file,
 293                          stdout=subprocess.PIPE)
 294
 295    result = []
 296    try:
 297        while True:
 298            entry = marshal.load(p4.stdout)
 299            if cb is not None:
 300                cb(entry)
 301            else:
 302                result.append(entry)
 303    except EOFError:
 304        pass
 305    exitCode = p4.wait()
 306    if exitCode != 0:
 307        entry = {}
 308        entry["p4ExitCode"] = exitCode
 309        result.append(entry)
 310
 311    return result
 312
 313def p4Cmd(cmd):
 314    list = p4CmdList(cmd)
 315    result = {}
 316    for entry in list:
 317        result.update(entry)
 318    return result;
 319
 320def p4Where(depotPath):
 321    if not depotPath.endswith("/"):
 322        depotPath += "/"
 323    depotPath = depotPath + "..."
 324    outputList = p4CmdList(["where", depotPath])
 325    output = None
 326    for entry in outputList:
 327        if "depotFile" in entry:
 328            if entry["depotFile"] == depotPath:
 329                output = entry
 330                break
 331        elif "data" in entry:
 332            data = entry.get("data")
 333            space = data.find(" ")
 334            if data[:space] == depotPath:
 335                output = entry
 336                break
 337    if output == None:
 338        return ""
 339    if output["code"] == "error":
 340        return ""
 341    clientPath = ""
 342    if "path" in output:
 343        clientPath = output.get("path")
 344    elif "data" in output:
 345        data = output.get("data")
 346        lastSpace = data.rfind(" ")
 347        clientPath = data[lastSpace + 1:]
 348
 349    if clientPath.endswith("..."):
 350        clientPath = clientPath[:-3]
 351    return clientPath
 352
 353def currentGitBranch():
 354    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356def isValidGitDir(path):
 357    if (os.path.exists(path + "/HEAD")
 358        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359        return True;
 360    return False
 361
 362def parseRevision(ref):
 363    return read_pipe("git rev-parse %s" % ref).strip()
 364
 365def branchExists(ref):
 366    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                     ignore_error=True)
 368    return len(rev) > 0
 369
 370def extractLogMessageFromGitCommit(commit):
 371    logMessage = ""
 372
 373    ## fixme: title is first line of commit, not 1st paragraph.
 374    foundTitle = False
 375    for log in read_pipe_lines("git cat-file commit %s" % commit):
 376       if not foundTitle:
 377           if len(log) == 1:
 378               foundTitle = True
 379           continue
 380
 381       logMessage += log
 382    return logMessage
 383
 384def extractSettingsGitLog(log):
 385    values = {}
 386    for line in log.split("\n"):
 387        line = line.strip()
 388        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389        if not m:
 390            continue
 391
 392        assignments = m.group(1).split (':')
 393        for a in assignments:
 394            vals = a.split ('=')
 395            key = vals[0].strip()
 396            val = ('='.join (vals[1:])).strip()
 397            if val.endswith ('\"') and val.startswith('"'):
 398                val = val[1:-1]
 399
 400            values[key] = val
 401
 402    paths = values.get("depot-paths")
 403    if not paths:
 404        paths = values.get("depot-path")
 405    if paths:
 406        values['depot-paths'] = paths.split(',')
 407    return values
 408
 409def gitBranchExists(branch):
 410    proc = subprocess.Popen(["git", "rev-parse", branch],
 411                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412    return proc.wait() == 0;
 413
 414_gitConfig = {}
 415def gitConfig(key, args = None): # set args to "--bool", for instance
 416    if not _gitConfig.has_key(key):
 417        argsFilter = ""
 418        if args != None:
 419            argsFilter = "%s " % args
 420        cmd = "git config %s%s" % (argsFilter, key)
 421        _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422    return _gitConfig[key]
 423
 424def gitConfigList(key):
 425    if not _gitConfig.has_key(key):
 426        _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427    return _gitConfig[key]
 428
 429def p4BranchesInGit(branchesAreInRemotes = True):
 430    branches = {}
 431
 432    cmdline = "git rev-parse --symbolic "
 433    if branchesAreInRemotes:
 434        cmdline += " --remotes"
 435    else:
 436        cmdline += " --branches"
 437
 438    for line in read_pipe_lines(cmdline):
 439        line = line.strip()
 440
 441        ## only import to p4/
 442        if not line.startswith('p4/') or line == "p4/HEAD":
 443            continue
 444        branch = line
 445
 446        # strip off p4
 447        branch = re.sub ("^p4/", "", line)
 448
 449        branches[branch] = parseRevision(line)
 450    return branches
 451
 452def findUpstreamBranchPoint(head = "HEAD"):
 453    branches = p4BranchesInGit()
 454    # map from depot-path to branch name
 455    branchByDepotPath = {}
 456    for branch in branches.keys():
 457        tip = branches[branch]
 458        log = extractLogMessageFromGitCommit(tip)
 459        settings = extractSettingsGitLog(log)
 460        if settings.has_key("depot-paths"):
 461            paths = ",".join(settings["depot-paths"])
 462            branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464    settings = None
 465    parent = 0
 466    while parent < 65535:
 467        commit = head + "~%s" % parent
 468        log = extractLogMessageFromGitCommit(commit)
 469        settings = extractSettingsGitLog(log)
 470        if settings.has_key("depot-paths"):
 471            paths = ",".join(settings["depot-paths"])
 472            if branchByDepotPath.has_key(paths):
 473                return [branchByDepotPath[paths], settings]
 474
 475        parent = parent + 1
 476
 477    return ["", settings]
 478
 479def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480    if not silent:
 481        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482               % localRefPrefix)
 483
 484    originPrefix = "origin/p4/"
 485
 486    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487        line = line.strip()
 488        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489            continue
 490
 491        headName = line[len(originPrefix):]
 492        remoteHead = localRefPrefix + headName
 493        originHead = line
 494
 495        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496        if (not original.has_key('depot-paths')
 497            or not original.has_key('change')):
 498            continue
 499
 500        update = False
 501        if not gitBranchExists(remoteHead):
 502            if verbose:
 503                print "creating %s" % remoteHead
 504            update = True
 505        else:
 506            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507            if settings.has_key('change') > 0:
 508                if settings['depot-paths'] == original['depot-paths']:
 509                    originP4Change = int(original['change'])
 510                    p4Change = int(settings['change'])
 511                    if originP4Change > p4Change:
 512                        print ("%s (%s) is newer than %s (%s). "
 513                               "Updating p4 branch from origin."
 514                               % (originHead, originP4Change,
 515                                  remoteHead, p4Change))
 516                        update = True
 517                else:
 518                    print ("Ignoring: %s was imported from %s while "
 519                           "%s was imported from %s"
 520                           % (originHead, ','.join(original['depot-paths']),
 521                              remoteHead, ','.join(settings['depot-paths'])))
 522
 523        if update:
 524            system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526def originP4BranchesExist():
 527        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529def p4ChangesForPaths(depotPaths, changeRange):
 530    assert depotPaths
 531    cmd = ['changes']
 532    for p in depotPaths:
 533        cmd += ["%s...%s" % (p, changeRange)]
 534    output = p4_read_pipe_lines(cmd)
 535
 536    changes = {}
 537    for line in output:
 538        changeNum = int(line.split(" ")[1])
 539        changes[changeNum] = True
 540
 541    changelist = changes.keys()
 542    changelist.sort()
 543    return changelist
 544
 545def p4PathStartsWith(path, prefix):
 546    # This method tries to remedy a potential mixed-case issue:
 547    #
 548    # If UserA adds  //depot/DirA/file1
 549    # and UserB adds //depot/dira/file2
 550    #
 551    # we may or may not have a problem. If you have core.ignorecase=true,
 552    # we treat DirA and dira as the same directory
 553    ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554    if ignorecase:
 555        return path.lower().startswith(prefix.lower())
 556    return path.startswith(prefix)
 557
 558class Command:
 559    def __init__(self):
 560        self.usage = "usage: %prog [options]"
 561        self.needsGit = True
 562
 563class P4UserMap:
 564    def __init__(self):
 565        self.userMapFromPerforceServer = False
 566
 567    def getUserCacheFilename(self):
 568        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 569        return home + "/.gitp4-usercache.txt"
 570
 571    def getUserMapFromPerforceServer(self):
 572        if self.userMapFromPerforceServer:
 573            return
 574        self.users = {}
 575        self.emails = {}
 576
 577        for output in p4CmdList("users"):
 578            if not output.has_key("User"):
 579                continue
 580            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 581            self.emails[output["Email"]] = output["User"]
 582
 583
 584        s = ''
 585        for (key, val) in self.users.items():
 586            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 587
 588        open(self.getUserCacheFilename(), "wb").write(s)
 589        self.userMapFromPerforceServer = True
 590
 591    def loadUserMapFromCache(self):
 592        self.users = {}
 593        self.userMapFromPerforceServer = False
 594        try:
 595            cache = open(self.getUserCacheFilename(), "rb")
 596            lines = cache.readlines()
 597            cache.close()
 598            for line in lines:
 599                entry = line.strip().split("\t")
 600                self.users[entry[0]] = entry[1]
 601        except IOError:
 602            self.getUserMapFromPerforceServer()
 603
 604class P4Debug(Command):
 605    def __init__(self):
 606        Command.__init__(self)
 607        self.options = [
 608            optparse.make_option("--verbose", dest="verbose", action="store_true",
 609                                 default=False),
 610            ]
 611        self.description = "A tool to debug the output of p4 -G."
 612        self.needsGit = False
 613        self.verbose = False
 614
 615    def run(self, args):
 616        j = 0
 617        for output in p4CmdList(args):
 618            print 'Element: %d' % j
 619            j += 1
 620            print output
 621        return True
 622
 623class P4RollBack(Command):
 624    def __init__(self):
 625        Command.__init__(self)
 626        self.options = [
 627            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 628            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 629        ]
 630        self.description = "A tool to debug the multi-branch import. Don't use :)"
 631        self.verbose = False
 632        self.rollbackLocalBranches = False
 633
 634    def run(self, args):
 635        if len(args) != 1:
 636            return False
 637        maxChange = int(args[0])
 638
 639        if "p4ExitCode" in p4Cmd("changes -m 1"):
 640            die("Problems executing p4");
 641
 642        if self.rollbackLocalBranches:
 643            refPrefix = "refs/heads/"
 644            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 645        else:
 646            refPrefix = "refs/remotes/"
 647            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 648
 649        for line in lines:
 650            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 651                line = line.strip()
 652                ref = refPrefix + line
 653                log = extractLogMessageFromGitCommit(ref)
 654                settings = extractSettingsGitLog(log)
 655
 656                depotPaths = settings['depot-paths']
 657                change = settings['change']
 658
 659                changed = False
 660
 661                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 662                                                           for p in depotPaths]))) == 0:
 663                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 664                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 665                    continue
 666
 667                while change and int(change) > maxChange:
 668                    changed = True
 669                    if self.verbose:
 670                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 671                    system("git update-ref %s \"%s^\"" % (ref, ref))
 672                    log = extractLogMessageFromGitCommit(ref)
 673                    settings =  extractSettingsGitLog(log)
 674
 675
 676                    depotPaths = settings['depot-paths']
 677                    change = settings['change']
 678
 679                if changed:
 680                    print "%s rewound to %s" % (ref, change)
 681
 682        return True
 683
 684class P4Submit(Command, P4UserMap):
 685    def __init__(self):
 686        Command.__init__(self)
 687        P4UserMap.__init__(self)
 688        self.options = [
 689                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 690                optparse.make_option("--origin", dest="origin"),
 691                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 692                # preserve the user, requires relevant p4 permissions
 693                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 694        ]
 695        self.description = "Submit changes from git to the perforce depot."
 696        self.usage += " [name of git branch to submit into perforce depot]"
 697        self.interactive = True
 698        self.origin = ""
 699        self.detectRenames = False
 700        self.verbose = False
 701        self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 702        self.isWindows = (platform.system() == "Windows")
 703        self.myP4UserId = None
 704
 705    def check(self):
 706        if len(p4CmdList("opened ...")) > 0:
 707            die("You have files opened with perforce! Close them before starting the sync.")
 708
 709    # replaces everything between 'Description:' and the next P4 submit template field with the
 710    # commit message
 711    def prepareLogMessage(self, template, message):
 712        result = ""
 713
 714        inDescriptionSection = False
 715
 716        for line in template.split("\n"):
 717            if line.startswith("#"):
 718                result += line + "\n"
 719                continue
 720
 721            if inDescriptionSection:
 722                if line.startswith("Files:") or line.startswith("Jobs:"):
 723                    inDescriptionSection = False
 724                else:
 725                    continue
 726            else:
 727                if line.startswith("Description:"):
 728                    inDescriptionSection = True
 729                    line += "\n"
 730                    for messageLine in message.split("\n"):
 731                        line += "\t" + messageLine + "\n"
 732
 733            result += line + "\n"
 734
 735        return result
 736
 737    def p4UserForCommit(self,id):
 738        # Return the tuple (perforce user,git email) for a given git commit id
 739        self.getUserMapFromPerforceServer()
 740        gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 741        gitEmail = gitEmail.strip()
 742        if not self.emails.has_key(gitEmail):
 743            return (None,gitEmail)
 744        else:
 745            return (self.emails[gitEmail],gitEmail)
 746
 747    def checkValidP4Users(self,commits):
 748        # check if any git authors cannot be mapped to p4 users
 749        for id in commits:
 750            (user,email) = self.p4UserForCommit(id)
 751            if not user:
 752                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 753                if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 754                    print "%s" % msg
 755                else:
 756                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 757
 758    def lastP4Changelist(self):
 759        # Get back the last changelist number submitted in this client spec. This
 760        # then gets used to patch up the username in the change. If the same
 761        # client spec is being used by multiple processes then this might go
 762        # wrong.
 763        results = p4CmdList("client -o")        # find the current client
 764        client = None
 765        for r in results:
 766            if r.has_key('Client'):
 767                client = r['Client']
 768                break
 769        if not client:
 770            die("could not get client spec")
 771        results = p4CmdList(["changes", "-c", client, "-m", "1"])
 772        for r in results:
 773            if r.has_key('change'):
 774                return r['change']
 775        die("Could not get changelist number for last submit - cannot patch up user details")
 776
 777    def modifyChangelistUser(self, changelist, newUser):
 778        # fixup the user field of a changelist after it has been submitted.
 779        changes = p4CmdList("change -o %s" % changelist)
 780        if len(changes) != 1:
 781            die("Bad output from p4 change modifying %s to user %s" %
 782                (changelist, newUser))
 783
 784        c = changes[0]
 785        if c['User'] == newUser: return   # nothing to do
 786        c['User'] = newUser
 787        input = marshal.dumps(c)
 788
 789        result = p4CmdList("change -f -i", stdin=input)
 790        for r in result:
 791            if r.has_key('code'):
 792                if r['code'] == 'error':
 793                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 794            if r.has_key('data'):
 795                print("Updated user field for changelist %s to %s" % (changelist, newUser))
 796                return
 797        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 798
 799    def canChangeChangelists(self):
 800        # check to see if we have p4 admin or super-user permissions, either of
 801        # which are required to modify changelists.
 802        results = p4CmdList("protects %s" % self.depotPath)
 803        for r in results:
 804            if r.has_key('perm'):
 805                if r['perm'] == 'admin':
 806                    return 1
 807                if r['perm'] == 'super':
 808                    return 1
 809        return 0
 810
 811    def p4UserId(self):
 812        if self.myP4UserId:
 813            return self.myP4UserId
 814
 815        results = p4CmdList("user -o")
 816        for r in results:
 817            if r.has_key('User'):
 818                self.myP4UserId = r['User']
 819                return r['User']
 820        die("Could not find your p4 user id")
 821
 822    def p4UserIsMe(self, p4User):
 823        # return True if the given p4 user is actually me
 824        me = self.p4UserId()
 825        if not p4User or p4User != me:
 826            return False
 827        else:
 828            return True
 829
 830    def prepareSubmitTemplate(self):
 831        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832        template = ""
 833        inFilesSection = False
 834        for line in p4_read_pipe_lines(['change', '-o']):
 835            if line.endswith("\r\n"):
 836                line = line[:-2] + "\n"
 837            if inFilesSection:
 838                if line.startswith("\t"):
 839                    # path starts and ends with a tab
 840                    path = line[1:]
 841                    lastTab = path.rfind("\t")
 842                    if lastTab != -1:
 843                        path = path[:lastTab]
 844                        if not p4PathStartsWith(path, self.depotPath):
 845                            continue
 846                else:
 847                    inFilesSection = False
 848            else:
 849                if line.startswith("Files:"):
 850                    inFilesSection = True
 851
 852            template += line
 853
 854        return template
 855
 856    def edit_template(self, template_file):
 857        """Invoke the editor to let the user change the submission
 858           message.  Return true if okay to continue with the submit."""
 859
 860        # if configured to skip the editing part, just submit
 861        if gitConfig("git-p4.skipSubmitEdit") == "true":
 862            return True
 863
 864        # look at the modification time, to check later if the user saved
 865        # the file
 866        mtime = os.stat(template_file).st_mtime
 867
 868        # invoke the editor
 869        if os.environ.has_key("P4EDITOR"):
 870            editor = os.environ.get("P4EDITOR")
 871        else:
 872            editor = read_pipe("git var GIT_EDITOR").strip()
 873        system(editor + " " + template_file)
 874
 875        # If the file was not saved, prompt to see if this patch should
 876        # be skipped.  But skip this verification step if configured so.
 877        if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878            return True
 879
 880        # modification time updated means user saved the file
 881        if os.stat(template_file).st_mtime > mtime:
 882            return True
 883
 884        while True:
 885            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886            if response == 'y':
 887                return True
 888            if response == 'n':
 889                return False
 890
 891    def applyCommit(self, id):
 892        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894        (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896        if not self.detectRenames:
 897            # If not explicitly set check the config variable
 898            self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900        if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901            diffOpts = ""
 902        elif self.detectRenames.lower() == "true":
 903            diffOpts = "-M"
 904        else:
 905            diffOpts = "-M%s" % self.detectRenames
 906
 907        detectCopies = gitConfig("git-p4.detectCopies")
 908        if detectCopies.lower() == "true":
 909            diffOpts += " -C"
 910        elif detectCopies != "" and detectCopies.lower() != "false":
 911            diffOpts += " -C%s" % detectCopies
 912
 913        if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914            diffOpts += " --find-copies-harder"
 915
 916        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917        filesToAdd = set()
 918        filesToDelete = set()
 919        editedFiles = set()
 920        filesToChangeExecBit = {}
 921        for line in diff:
 922            diff = parseDiffTreeEntry(line)
 923            modifier = diff['status']
 924            path = diff['src']
 925            if modifier == "M":
 926                p4_edit(path)
 927                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                    filesToChangeExecBit[path] = diff['dst_mode']
 929                editedFiles.add(path)
 930            elif modifier == "A":
 931                filesToAdd.add(path)
 932                filesToChangeExecBit[path] = diff['dst_mode']
 933                if path in filesToDelete:
 934                    filesToDelete.remove(path)
 935            elif modifier == "D":
 936                filesToDelete.add(path)
 937                if path in filesToAdd:
 938                    filesToAdd.remove(path)
 939            elif modifier == "C":
 940                src, dest = diff['src'], diff['dst']
 941                p4_integrate(src, dest)
 942                if diff['src_sha1'] != diff['dst_sha1']:
 943                    p4_edit(dest)
 944                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                    p4_edit(dest)
 946                    filesToChangeExecBit[dest] = diff['dst_mode']
 947                os.unlink(dest)
 948                editedFiles.add(dest)
 949            elif modifier == "R":
 950                src, dest = diff['src'], diff['dst']
 951                p4_integrate(src, dest)
 952                if diff['src_sha1'] != diff['dst_sha1']:
 953                    p4_edit(dest)
 954                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                    p4_edit(dest)
 956                    filesToChangeExecBit[dest] = diff['dst_mode']
 957                os.unlink(dest)
 958                editedFiles.add(dest)
 959                filesToDelete.add(src)
 960            else:
 961                die("unknown modifier %s for %s" % (modifier, path))
 962
 963        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964        patchcmd = diffcmd + " | git apply "
 965        tryPatchCmd = patchcmd + "--check -"
 966        applyPatchCmd = patchcmd + "--check --apply -"
 967
 968        if os.system(tryPatchCmd) != 0:
 969            print "Unfortunately applying the change failed!"
 970            print "What do you want to do?"
 971            response = "x"
 972            while response != "s" and response != "a" and response != "w":
 973                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975            if response == "s":
 976                print "Skipping! Good luck with the next patches..."
 977                for f in editedFiles:
 978                    p4_revert(f)
 979                for f in filesToAdd:
 980                    os.remove(f)
 981                return
 982            elif response == "a":
 983                os.system(applyPatchCmd)
 984                if len(filesToAdd) > 0:
 985                    print "You may also want to call p4 add on the following files:"
 986                    print " ".join(filesToAdd)
 987                if len(filesToDelete):
 988                    print "The following files should be scheduled for deletion with p4 delete:"
 989                    print " ".join(filesToDelete)
 990                die("Please resolve and submit the conflict manually and "
 991                    + "continue afterwards with git-p4 submit --continue")
 992            elif response == "w":
 993                system(diffcmd + " > patch.txt")
 994                print "Patch saved to patch.txt in %s !" % self.clientPath
 995                die("Please resolve and submit the conflict manually and "
 996                    "continue afterwards with git-p4 submit --continue")
 997
 998        system(applyPatchCmd)
 999
1000        for f in filesToAdd:
1001            p4_add(f)
1002        for f in filesToDelete:
1003            p4_revert(f)
1004            p4_delete(f)
1005
1006        # Set/clear executable bits
1007        for f in filesToChangeExecBit.keys():
1008            mode = filesToChangeExecBit[f]
1009            setP4ExecBit(f, mode)
1010
1011        logMessage = extractLogMessageFromGitCommit(id)
1012        logMessage = logMessage.strip()
1013
1014        template = self.prepareSubmitTemplate()
1015
1016        if self.interactive:
1017            submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019            if self.preserveUser:
1020               submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022            if os.environ.has_key("P4DIFF"):
1023                del(os.environ["P4DIFF"])
1024            diff = ""
1025            for editedFile in editedFiles:
1026                diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028            newdiff = ""
1029            for newFile in filesToAdd:
1030                newdiff += "==== new file ====\n"
1031                newdiff += "--- /dev/null\n"
1032                newdiff += "+++ %s\n" % newFile
1033                f = open(newFile, "r")
1034                for line in f.readlines():
1035                    newdiff += "+" + line
1036                f.close()
1037
1038            if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043            separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045            (handle, fileName) = tempfile.mkstemp()
1046            tmpFile = os.fdopen(handle, "w+")
1047            if self.isWindows:
1048                submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                separatorLine = separatorLine.replace("\n", "\r\n")
1050                newdiff = newdiff.replace("\n", "\r\n")
1051            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052            tmpFile.close()
1053
1054            if self.edit_template(fileName):
1055                # read the edited message and submit
1056                tmpFile = open(fileName, "rb")
1057                message = tmpFile.read()
1058                tmpFile.close()
1059                submitTemplate = message[:message.index(separatorLine)]
1060                if self.isWindows:
1061                    submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                if self.preserveUser:
1065                    if p4User:
1066                        # Get last changelist number. Cannot easily get it from
1067                        # the submit command output as the output is
1068                        # unmarshalled.
1069                        changelist = self.lastP4Changelist()
1070                        self.modifyChangelistUser(changelist, p4User)
1071            else:
1072                # skip this patch
1073                print "Submission cancelled, undoing p4 changes."
1074                for f in editedFiles:
1075                    p4_revert(f)
1076                for f in filesToAdd:
1077                    p4_revert(f)
1078                    os.remove(f)
1079
1080            os.remove(fileName)
1081        else:
1082            fileName = "submit.txt"
1083            file = open(fileName, "w+")
1084            file.write(self.prepareLogMessage(template, logMessage))
1085            file.close()
1086            print ("Perforce submit template written as %s. "
1087                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                   % (fileName, fileName))
1089
1090    def run(self, args):
1091        if len(args) == 0:
1092            self.master = currentGitBranch()
1093            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                die("Detecting current git branch failed!")
1095        elif len(args) == 1:
1096            self.master = args[0]
1097            if not branchExists(self.master):
1098                die("Branch %s does not exist" % self.master)
1099        else:
1100            return False
1101
1102        allowSubmit = gitConfig("git-p4.allowSubmit")
1103        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104            die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106        [upstream, settings] = findUpstreamBranchPoint()
1107        self.depotPath = settings['depot-paths'][0]
1108        if len(self.origin) == 0:
1109            self.origin = upstream
1110
1111        if self.preserveUser:
1112            if not self.canChangeChangelists():
1113                die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115        if self.verbose:
1116            print "Origin branch is " + self.origin
1117
1118        if len(self.depotPath) == 0:
1119            print "Internal error: cannot locate perforce depot path from existing branches"
1120            sys.exit(128)
1121
1122        self.clientPath = p4Where(self.depotPath)
1123
1124        if len(self.clientPath) == 0:
1125            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126            sys.exit(128)
1127
1128        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129        self.oldWorkingDirectory = os.getcwd()
1130
1131        # ensure the clientPath exists
1132        if not os.path.exists(self.clientPath):
1133            os.makedirs(self.clientPath)
1134
1135        chdir(self.clientPath)
1136        print "Synchronizing p4 checkout..."
1137        p4_sync("...")
1138        self.check()
1139
1140        commits = []
1141        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142            commits.append(line.strip())
1143        commits.reverse()
1144
1145        if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146            self.checkAuthorship = False
1147        else:
1148            self.checkAuthorship = True
1149
1150        if self.preserveUser:
1151            self.checkValidP4Users(commits)
1152
1153        while len(commits) > 0:
1154            commit = commits[0]
1155            commits = commits[1:]
1156            self.applyCommit(commit)
1157            if not self.interactive:
1158                break
1159
1160        if len(commits) == 0:
1161            print "All changes applied!"
1162            chdir(self.oldWorkingDirectory)
1163
1164            sync = P4Sync()
1165            sync.run([])
1166
1167            rebase = P4Rebase()
1168            rebase.rebase()
1169
1170        return True
1171
1172class P4Sync(Command, P4UserMap):
1173    delete_actions = ( "delete", "move/delete", "purge" )
1174
1175    def __init__(self):
1176        Command.__init__(self)
1177        P4UserMap.__init__(self)
1178        self.options = [
1179                optparse.make_option("--branch", dest="branch"),
1180                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1181                optparse.make_option("--changesfile", dest="changesFile"),
1182                optparse.make_option("--silent", dest="silent", action="store_true"),
1183                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1184                optparse.make_option("--verbose", dest="verbose", action="store_true"),
1185                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1186                                     help="Import into refs/heads/ , not refs/remotes"),
1187                optparse.make_option("--max-changes", dest="maxChanges"),
1188                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1189                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1190                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1191                                     help="Only sync files that are included in the Perforce Client Spec")
1192        ]
1193        self.description = """Imports from Perforce into a git repository.\n
1194    example:
1195    //depot/my/project/ -- to import the current head
1196    //depot/my/project/@all -- to import everything
1197    //depot/my/project/@1,6 -- to import only from revision 1 to 6
1198
1199    (a ... is not needed in the path p4 specification, it's added implicitly)"""
1200
1201        self.usage += " //depot/path[@revRange]"
1202        self.silent = False
1203        self.createdBranches = set()
1204        self.committedChanges = set()
1205        self.branch = ""
1206        self.detectBranches = False
1207        self.detectLabels = False
1208        self.changesFile = ""
1209        self.syncWithOrigin = True
1210        self.verbose = False
1211        self.importIntoRemotes = True
1212        self.maxChanges = ""
1213        self.isWindows = (platform.system() == "Windows")
1214        self.keepRepoPath = False
1215        self.depotPaths = None
1216        self.p4BranchesInGit = []
1217        self.cloneExclude = []
1218        self.useClientSpec = False
1219        self.clientSpecDirs = []
1220        self.haveSingleFileClientViews = False
1221
1222        if gitConfig("git-p4.syncFromOrigin") == "false":
1223            self.syncWithOrigin = False
1224
1225    #
1226    # P4 wildcards are not allowed in filenames.  P4 complains
1227    # if you simply add them, but you can force it with "-f", in
1228    # which case it translates them into %xx encoding internally.
1229    # Search for and fix just these four characters.  Do % last so
1230    # that fixing it does not inadvertently create new %-escapes.
1231    #
1232    def wildcard_decode(self, path):
1233        # Cannot have * in a filename in windows; untested as to
1234        # what p4 would do in such a case.
1235        if not self.isWindows:
1236            path = path.replace("%2A", "*")
1237        path = path.replace("%23", "#") \
1238                   .replace("%40", "@") \
1239                   .replace("%25", "%")
1240        return path
1241
1242    def extractFilesFromCommit(self, commit):
1243        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1244                             for path in self.cloneExclude]
1245        files = []
1246        fnum = 0
1247        while commit.has_key("depotFile%s" % fnum):
1248            path =  commit["depotFile%s" % fnum]
1249
1250            if [p for p in self.cloneExclude
1251                if p4PathStartsWith(path, p)]:
1252                found = False
1253            else:
1254                found = [p for p in self.depotPaths
1255                         if p4PathStartsWith(path, p)]
1256            if not found:
1257                fnum = fnum + 1
1258                continue
1259
1260            file = {}
1261            file["path"] = path
1262            file["rev"] = commit["rev%s" % fnum]
1263            file["action"] = commit["action%s" % fnum]
1264            file["type"] = commit["type%s" % fnum]
1265            files.append(file)
1266            fnum = fnum + 1
1267        return files
1268
1269    def stripRepoPath(self, path, prefixes):
1270        if self.useClientSpec:
1271
1272            # if using the client spec, we use the output directory
1273            # specified in the client.  For example, a view
1274            #   //depot/foo/branch/... //client/branch/foo/...
1275            # will end up putting all foo/branch files into
1276            #  branch/foo/
1277            for val in self.clientSpecDirs:
1278                if self.haveSingleFileClientViews and len(path) == abs(val[1][0]) and path == val[0]:
1279                    # since 'path' is a depot file path, if it matches the LeftMap,
1280                    # then the View is a single file mapping, so use the entire rightMap
1281                    # first two tests above avoid the last == test for common cases
1282                    path =  val[1][1]
1283                    # now strip out the client (//client/...)
1284                    path = re.sub("^(//[^/]+/)", '', path)
1285                    # the rest is local client path
1286                    return path
1287
1288                if path.startswith(val[0]):
1289                    # replace the depot path with the client path
1290                    path = path.replace(val[0], val[1][1])
1291                    # now strip out the client (//client/...)
1292                    path = re.sub("^(//[^/]+/)", '', path)
1293                    # the rest is all path
1294                    return path
1295
1296        if self.keepRepoPath:
1297            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1298
1299        for p in prefixes:
1300            if p4PathStartsWith(path, p):
1301                path = path[len(p):]
1302
1303        return path
1304
1305    def splitFilesIntoBranches(self, commit):
1306        branches = {}
1307        fnum = 0
1308        while commit.has_key("depotFile%s" % fnum):
1309            path =  commit["depotFile%s" % fnum]
1310            found = [p for p in self.depotPaths
1311                     if p4PathStartsWith(path, p)]
1312            if not found:
1313                fnum = fnum + 1
1314                continue
1315
1316            file = {}
1317            file["path"] = path
1318            file["rev"] = commit["rev%s" % fnum]
1319            file["action"] = commit["action%s" % fnum]
1320            file["type"] = commit["type%s" % fnum]
1321            fnum = fnum + 1
1322
1323            relPath = self.stripRepoPath(path, self.depotPaths)
1324
1325            for branch in self.knownBranches.keys():
1326
1327                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1328                if relPath.startswith(branch + "/"):
1329                    if branch not in branches:
1330                        branches[branch] = []
1331                    branches[branch].append(file)
1332                    break
1333
1334        return branches
1335
1336    # output one file from the P4 stream
1337    # - helper for streamP4Files
1338
1339    def streamOneP4File(self, file, contents):
1340        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1341        relPath = self.wildcard_decode(relPath)
1342        if verbose:
1343            sys.stderr.write("%s\n" % relPath)
1344
1345        (type_base, type_mods) = split_p4_type(file["type"])
1346
1347        git_mode = "100644"
1348        if "x" in type_mods:
1349            git_mode = "100755"
1350        if type_base == "symlink":
1351            git_mode = "120000"
1352            # p4 print on a symlink contains "target\n"; remove the newline
1353            data = ''.join(contents)
1354            contents = [data[:-1]]
1355
1356        if type_base == "utf16":
1357            # p4 delivers different text in the python output to -G
1358            # than it does when using "print -o", or normal p4 client
1359            # operations.  utf16 is converted to ascii or utf8, perhaps.
1360            # But ascii text saved as -t utf16 is completely mangled.
1361            # Invoke print -o to get the real contents.
1362            text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1363            contents = [ text ]
1364
1365        if type_base == "apple":
1366            # Apple filetype files will be streamed as a concatenation of
1367            # its appledouble header and the contents.  This is useless
1368            # on both macs and non-macs.  If using "print -q -o xx", it
1369            # will create "xx" with the data, and "%xx" with the header.
1370            # This is also not very useful.
1371            #
1372            # Ideally, someday, this script can learn how to generate
1373            # appledouble files directly and import those to git, but
1374            # non-mac machines can never find a use for apple filetype.
1375            print "\nIgnoring apple filetype file %s" % file['depotFile']
1376            return
1377
1378        # Perhaps windows wants unicode, utf16 newlines translated too;
1379        # but this is not doing it.
1380        if self.isWindows and type_base == "text":
1381            mangled = []
1382            for data in contents:
1383                data = data.replace("\r\n", "\n")
1384                mangled.append(data)
1385            contents = mangled
1386
1387        # Note that we do not try to de-mangle keywords on utf16 files,
1388        # even though in theory somebody may want that.
1389        if type_base in ("text", "unicode", "binary"):
1390            if "ko" in type_mods:
1391                text = ''.join(contents)
1392                text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1393                contents = [ text ]
1394            elif "k" in type_mods:
1395                text = ''.join(contents)
1396                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1397                contents = [ text ]
1398
1399        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1400
1401        # total length...
1402        length = 0
1403        for d in contents:
1404            length = length + len(d)
1405
1406        self.gitStream.write("data %d\n" % length)
1407        for d in contents:
1408            self.gitStream.write(d)
1409        self.gitStream.write("\n")
1410
1411    def streamOneP4Deletion(self, file):
1412        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1413        if verbose:
1414            sys.stderr.write("delete %s\n" % relPath)
1415        self.gitStream.write("D %s\n" % relPath)
1416
1417    # handle another chunk of streaming data
1418    def streamP4FilesCb(self, marshalled):
1419
1420        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1421            # start of a new file - output the old one first
1422            self.streamOneP4File(self.stream_file, self.stream_contents)
1423            self.stream_file = {}
1424            self.stream_contents = []
1425            self.stream_have_file_info = False
1426
1427        # pick up the new file information... for the
1428        # 'data' field we need to append to our array
1429        for k in marshalled.keys():
1430            if k == 'data':
1431                self.stream_contents.append(marshalled['data'])
1432            else:
1433                self.stream_file[k] = marshalled[k]
1434
1435        self.stream_have_file_info = True
1436
1437    # Stream directly from "p4 files" into "git fast-import"
1438    def streamP4Files(self, files):
1439        filesForCommit = []
1440        filesToRead = []
1441        filesToDelete = []
1442
1443        for f in files:
1444            includeFile = True
1445            for val in self.clientSpecDirs:
1446                if f['path'].startswith(val[0]):
1447                    if val[1][0] <= 0:
1448                        includeFile = False
1449                    break
1450
1451            if includeFile:
1452                filesForCommit.append(f)
1453                if f['action'] in self.delete_actions:
1454                    filesToDelete.append(f)
1455                else:
1456                    filesToRead.append(f)
1457
1458        # deleted files...
1459        for f in filesToDelete:
1460            self.streamOneP4Deletion(f)
1461
1462        if len(filesToRead) > 0:
1463            self.stream_file = {}
1464            self.stream_contents = []
1465            self.stream_have_file_info = False
1466
1467            # curry self argument
1468            def streamP4FilesCbSelf(entry):
1469                self.streamP4FilesCb(entry)
1470
1471            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1472
1473            p4CmdList(["-x", "-", "print"],
1474                      stdin=fileArgs,
1475                      cb=streamP4FilesCbSelf)
1476
1477            # do the last chunk
1478            if self.stream_file.has_key('depotFile'):
1479                self.streamOneP4File(self.stream_file, self.stream_contents)
1480
1481    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1482        epoch = details["time"]
1483        author = details["user"]
1484        self.branchPrefixes = branchPrefixes
1485
1486        if self.verbose:
1487            print "commit into %s" % branch
1488
1489        # start with reading files; if that fails, we should not
1490        # create a commit.
1491        new_files = []
1492        for f in files:
1493            if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1494                new_files.append (f)
1495            else:
1496                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1497
1498        self.gitStream.write("commit %s\n" % branch)
1499#        gitStream.write("mark :%s\n" % details["change"])
1500        self.committedChanges.add(int(details["change"]))
1501        committer = ""
1502        if author not in self.users:
1503            self.getUserMapFromPerforceServer()
1504        if author in self.users:
1505            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1506        else:
1507            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1508
1509        self.gitStream.write("committer %s\n" % committer)
1510
1511        self.gitStream.write("data <<EOT\n")
1512        self.gitStream.write(details["desc"])
1513        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1514                             % (','.join (branchPrefixes), details["change"]))
1515        if len(details['options']) > 0:
1516            self.gitStream.write(": options = %s" % details['options'])
1517        self.gitStream.write("]\nEOT\n\n")
1518
1519        if len(parent) > 0:
1520            if self.verbose:
1521                print "parent %s" % parent
1522            self.gitStream.write("from %s\n" % parent)
1523
1524        self.streamP4Files(new_files)
1525        self.gitStream.write("\n")
1526
1527        change = int(details["change"])
1528
1529        if self.labels.has_key(change):
1530            label = self.labels[change]
1531            labelDetails = label[0]
1532            labelRevisions = label[1]
1533            if self.verbose:
1534                print "Change %s is labelled %s" % (change, labelDetails)
1535
1536            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1537                                                    for p in branchPrefixes])
1538
1539            if len(files) == len(labelRevisions):
1540
1541                cleanedFiles = {}
1542                for info in files:
1543                    if info["action"] in self.delete_actions:
1544                        continue
1545                    cleanedFiles[info["depotFile"]] = info["rev"]
1546
1547                if cleanedFiles == labelRevisions:
1548                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1549                    self.gitStream.write("from %s\n" % branch)
1550
1551                    owner = labelDetails["Owner"]
1552                    tagger = ""
1553                    if author in self.users:
1554                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1555                    else:
1556                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1557                    self.gitStream.write("tagger %s\n" % tagger)
1558                    self.gitStream.write("data <<EOT\n")
1559                    self.gitStream.write(labelDetails["Description"])
1560                    self.gitStream.write("EOT\n\n")
1561
1562                else:
1563                    if not self.silent:
1564                        print ("Tag %s does not match with change %s: files do not match."
1565                               % (labelDetails["label"], change))
1566
1567            else:
1568                if not self.silent:
1569                    print ("Tag %s does not match with change %s: file count is different."
1570                           % (labelDetails["label"], change))
1571
1572    def getLabels(self):
1573        self.labels = {}
1574
1575        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1576        if len(l) > 0 and not self.silent:
1577            print "Finding files belonging to labels in %s" % `self.depotPaths`
1578
1579        for output in l:
1580            label = output["label"]
1581            revisions = {}
1582            newestChange = 0
1583            if self.verbose:
1584                print "Querying files for label %s" % label
1585            for file in p4CmdList(["files"] +
1586                                      ["%s...@%s" % (p, label)
1587                                          for p in self.depotPaths]):
1588                revisions[file["depotFile"]] = file["rev"]
1589                change = int(file["change"])
1590                if change > newestChange:
1591                    newestChange = change
1592
1593            self.labels[newestChange] = [output, revisions]
1594
1595        if self.verbose:
1596            print "Label changes: %s" % self.labels.keys()
1597
1598    def guessProjectName(self):
1599        for p in self.depotPaths:
1600            if p.endswith("/"):
1601                p = p[:-1]
1602            p = p[p.strip().rfind("/") + 1:]
1603            if not p.endswith("/"):
1604               p += "/"
1605            return p
1606
1607    def getBranchMapping(self):
1608        lostAndFoundBranches = set()
1609
1610        user = gitConfig("git-p4.branchUser")
1611        if len(user) > 0:
1612            command = "branches -u %s" % user
1613        else:
1614            command = "branches"
1615
1616        for info in p4CmdList(command):
1617            details = p4Cmd("branch -o %s" % info["branch"])
1618            viewIdx = 0
1619            while details.has_key("View%s" % viewIdx):
1620                paths = details["View%s" % viewIdx].split(" ")
1621                viewIdx = viewIdx + 1
1622                # require standard //depot/foo/... //depot/bar/... mapping
1623                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1624                    continue
1625                source = paths[0]
1626                destination = paths[1]
1627                ## HACK
1628                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1629                    source = source[len(self.depotPaths[0]):-4]
1630                    destination = destination[len(self.depotPaths[0]):-4]
1631
1632                    if destination in self.knownBranches:
1633                        if not self.silent:
1634                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1635                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1636                        continue
1637
1638                    self.knownBranches[destination] = source
1639
1640                    lostAndFoundBranches.discard(destination)
1641
1642                    if source not in self.knownBranches:
1643                        lostAndFoundBranches.add(source)
1644
1645        # Perforce does not strictly require branches to be defined, so we also
1646        # check git config for a branch list.
1647        #
1648        # Example of branch definition in git config file:
1649        # [git-p4]
1650        #   branchList=main:branchA
1651        #   branchList=main:branchB
1652        #   branchList=branchA:branchC
1653        configBranches = gitConfigList("git-p4.branchList")
1654        for branch in configBranches:
1655            if branch:
1656                (source, destination) = branch.split(":")
1657                self.knownBranches[destination] = source
1658
1659                lostAndFoundBranches.discard(destination)
1660
1661                if source not in self.knownBranches:
1662                    lostAndFoundBranches.add(source)
1663
1664
1665        for branch in lostAndFoundBranches:
1666            self.knownBranches[branch] = branch
1667
1668    def getBranchMappingFromGitBranches(self):
1669        branches = p4BranchesInGit(self.importIntoRemotes)
1670        for branch in branches.keys():
1671            if branch == "master":
1672                branch = "main"
1673            else:
1674                branch = branch[len(self.projectName):]
1675            self.knownBranches[branch] = branch
1676
1677    def listExistingP4GitBranches(self):
1678        # branches holds mapping from name to commit
1679        branches = p4BranchesInGit(self.importIntoRemotes)
1680        self.p4BranchesInGit = branches.keys()
1681        for branch in branches.keys():
1682            self.initialParents[self.refPrefix + branch] = branches[branch]
1683
1684    def updateOptionDict(self, d):
1685        option_keys = {}
1686        if self.keepRepoPath:
1687            option_keys['keepRepoPath'] = 1
1688
1689        d["options"] = ' '.join(sorted(option_keys.keys()))
1690
1691    def readOptions(self, d):
1692        self.keepRepoPath = (d.has_key('options')
1693                             and ('keepRepoPath' in d['options']))
1694
1695    def gitRefForBranch(self, branch):
1696        if branch == "main":
1697            return self.refPrefix + "master"
1698
1699        if len(branch) <= 0:
1700            return branch
1701
1702        return self.refPrefix + self.projectName + branch
1703
1704    def gitCommitByP4Change(self, ref, change):
1705        if self.verbose:
1706            print "looking in ref " + ref + " for change %s using bisect..." % change
1707
1708        earliestCommit = ""
1709        latestCommit = parseRevision(ref)
1710
1711        while True:
1712            if self.verbose:
1713                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1714            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1715            if len(next) == 0:
1716                if self.verbose:
1717                    print "argh"
1718                return ""
1719            log = extractLogMessageFromGitCommit(next)
1720            settings = extractSettingsGitLog(log)
1721            currentChange = int(settings['change'])
1722            if self.verbose:
1723                print "current change %s" % currentChange
1724
1725            if currentChange == change:
1726                if self.verbose:
1727                    print "found %s" % next
1728                return next
1729
1730            if currentChange < change:
1731                earliestCommit = "^%s" % next
1732            else:
1733                latestCommit = "%s" % next
1734
1735        return ""
1736
1737    def importNewBranch(self, branch, maxChange):
1738        # make fast-import flush all changes to disk and update the refs using the checkpoint
1739        # command so that we can try to find the branch parent in the git history
1740        self.gitStream.write("checkpoint\n\n");
1741        self.gitStream.flush();
1742        branchPrefix = self.depotPaths[0] + branch + "/"
1743        range = "@1,%s" % maxChange
1744        #print "prefix" + branchPrefix
1745        changes = p4ChangesForPaths([branchPrefix], range)
1746        if len(changes) <= 0:
1747            return False
1748        firstChange = changes[0]
1749        #print "first change in branch: %s" % firstChange
1750        sourceBranch = self.knownBranches[branch]
1751        sourceDepotPath = self.depotPaths[0] + sourceBranch
1752        sourceRef = self.gitRefForBranch(sourceBranch)
1753        #print "source " + sourceBranch
1754
1755        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1756        #print "branch parent: %s" % branchParentChange
1757        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1758        if len(gitParent) > 0:
1759            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1760            #print "parent git commit: %s" % gitParent
1761
1762        self.importChanges(changes)
1763        return True
1764
1765    def importChanges(self, changes):
1766        cnt = 1
1767        for change in changes:
1768            description = p4Cmd("describe %s" % change)
1769            self.updateOptionDict(description)
1770
1771            if not self.silent:
1772                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1773                sys.stdout.flush()
1774            cnt = cnt + 1
1775
1776            try:
1777                if self.detectBranches:
1778                    branches = self.splitFilesIntoBranches(description)
1779                    for branch in branches.keys():
1780                        ## HACK  --hwn
1781                        branchPrefix = self.depotPaths[0] + branch + "/"
1782
1783                        parent = ""
1784
1785                        filesForCommit = branches[branch]
1786
1787                        if self.verbose:
1788                            print "branch is %s" % branch
1789
1790                        self.updatedBranches.add(branch)
1791
1792                        if branch not in self.createdBranches:
1793                            self.createdBranches.add(branch)
1794                            parent = self.knownBranches[branch]
1795                            if parent == branch:
1796                                parent = ""
1797                            else:
1798                                fullBranch = self.projectName + branch
1799                                if fullBranch not in self.p4BranchesInGit:
1800                                    if not self.silent:
1801                                        print("\n    Importing new branch %s" % fullBranch);
1802                                    if self.importNewBranch(branch, change - 1):
1803                                        parent = ""
1804                                        self.p4BranchesInGit.append(fullBranch)
1805                                    if not self.silent:
1806                                        print("\n    Resuming with change %s" % change);
1807
1808                                if self.verbose:
1809                                    print "parent determined through known branches: %s" % parent
1810
1811                        branch = self.gitRefForBranch(branch)
1812                        parent = self.gitRefForBranch(parent)
1813
1814                        if self.verbose:
1815                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1816
1817                        if len(parent) == 0 and branch in self.initialParents:
1818                            parent = self.initialParents[branch]
1819                            del self.initialParents[branch]
1820
1821                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1822                else:
1823                    files = self.extractFilesFromCommit(description)
1824                    self.commit(description, files, self.branch, self.depotPaths,
1825                                self.initialParent)
1826                    self.initialParent = ""
1827            except IOError:
1828                print self.gitError.read()
1829                sys.exit(1)
1830
1831    def importHeadRevision(self, revision):
1832        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1833
1834        details = {}
1835        details["user"] = "git perforce import user"
1836        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1837                           % (' '.join(self.depotPaths), revision))
1838        details["change"] = revision
1839        newestRevision = 0
1840
1841        fileCnt = 0
1842        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1843
1844        for info in p4CmdList(["files"] + fileArgs):
1845
1846            if 'code' in info and info['code'] == 'error':
1847                sys.stderr.write("p4 returned an error: %s\n"
1848                                 % info['data'])
1849                if info['data'].find("must refer to client") >= 0:
1850                    sys.stderr.write("This particular p4 error is misleading.\n")
1851                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
1852                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1853                sys.exit(1)
1854            if 'p4ExitCode' in info:
1855                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1856                sys.exit(1)
1857
1858
1859            change = int(info["change"])
1860            if change > newestRevision:
1861                newestRevision = change
1862
1863            if info["action"] in self.delete_actions:
1864                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1865                #fileCnt = fileCnt + 1
1866                continue
1867
1868            for prop in ["depotFile", "rev", "action", "type" ]:
1869                details["%s%s" % (prop, fileCnt)] = info[prop]
1870
1871            fileCnt = fileCnt + 1
1872
1873        details["change"] = newestRevision
1874
1875        # Use time from top-most change so that all git-p4 clones of
1876        # the same p4 repo have the same commit SHA1s.
1877        res = p4CmdList("describe -s %d" % newestRevision)
1878        newestTime = None
1879        for r in res:
1880            if r.has_key('time'):
1881                newestTime = int(r['time'])
1882        if newestTime is None:
1883            die("\"describe -s\" on newest change %d did not give a time")
1884        details["time"] = newestTime
1885
1886        self.updateOptionDict(details)
1887        try:
1888            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1889        except IOError:
1890            print "IO error with git fast-import. Is your git version recent enough?"
1891            print self.gitError.read()
1892
1893
1894    def getClientSpec(self):
1895        specList = p4CmdList( "client -o" )
1896        temp = {}
1897        for entry in specList:
1898            for k,v in entry.iteritems():
1899                if k.startswith("View"):
1900
1901                    # p4 has these %%1 to %%9 arguments in specs to
1902                    # reorder paths; which we can't handle (yet :)
1903                    if re.search('%%\d', v) != None:
1904                        print "Sorry, can't handle %%n arguments in client specs"
1905                        sys.exit(1)
1906                    if re.search('\*', v) != None:
1907                        print "Sorry, can't handle * mappings in client specs"
1908                        sys.exit(1)
1909
1910                    if v.startswith('"'):
1911                        start = 1
1912                    else:
1913                        start = 0
1914                    index = v.find("...")
1915
1916                    # save the "client view"; i.e the RHS of the view
1917                    # line that tells the client where to put the
1918                    # files for this view.
1919
1920                    # check for individual file mappings - those which have no '...'
1921                    if  index < 0 :
1922                        v,cv = v.strip().split()
1923                        v = v[start:]
1924                        self.haveSingleFileClientViews = True
1925                    else:
1926                        cv = v[index+3:].strip() # +3 to remove previous '...'
1927                        cv=cv[:-3]
1928                        v = v[start:index]
1929
1930                    # now save the view; +index means included, -index
1931                    # means it should be filtered out.
1932                    if v.startswith("-"):
1933                        v = v[1:]
1934                        include = -len(v)
1935                    else:
1936                        include = len(v)
1937
1938                    # store the View #number for sorting
1939                    # and the View string itself (this last for documentation)
1940                    temp[v] = (include, cv, int(k[4:]),k)
1941
1942        self.clientSpecDirs = temp.items()
1943        # Perforce ViewNN with higher #numbers override those with lower
1944        # reverse sort on the View #number
1945        self.clientSpecDirs.sort( lambda x, y:  y[1][2] -  x[1][2]  )
1946        if self.verbose:
1947            for val in self.clientSpecDirs:
1948                    print "clientSpecDirs: %s %s" % (val[0],val[1])
1949
1950    def run(self, args):
1951        self.depotPaths = []
1952        self.changeRange = ""
1953        self.initialParent = ""
1954        self.previousDepotPaths = []
1955
1956        # map from branch depot path to parent branch
1957        self.knownBranches = {}
1958        self.initialParents = {}
1959        self.hasOrigin = originP4BranchesExist()
1960        if not self.syncWithOrigin:
1961            self.hasOrigin = False
1962
1963        if self.importIntoRemotes:
1964            self.refPrefix = "refs/remotes/p4/"
1965        else:
1966            self.refPrefix = "refs/heads/p4/"
1967
1968        if self.syncWithOrigin and self.hasOrigin:
1969            if not self.silent:
1970                print "Syncing with origin first by calling git fetch origin"
1971            system("git fetch origin")
1972
1973        if len(self.branch) == 0:
1974            self.branch = self.refPrefix + "master"
1975            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1976                system("git update-ref %s refs/heads/p4" % self.branch)
1977                system("git branch -D p4");
1978            # create it /after/ importing, when master exists
1979            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1980                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1981
1982        if not self.useClientSpec:
1983            if gitConfig("git-p4.useclientspec", "--bool") == "true":
1984                self.useClientSpec = True
1985        if self.useClientSpec:
1986            self.getClientSpec()
1987
1988        # TODO: should always look at previous commits,
1989        # merge with previous imports, if possible.
1990        if args == []:
1991            if self.hasOrigin:
1992                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1993            self.listExistingP4GitBranches()
1994
1995            if len(self.p4BranchesInGit) > 1:
1996                if not self.silent:
1997                    print "Importing from/into multiple branches"
1998                self.detectBranches = True
1999
2000            if self.verbose:
2001                print "branches: %s" % self.p4BranchesInGit
2002
2003            p4Change = 0
2004            for branch in self.p4BranchesInGit:
2005                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2006
2007                settings = extractSettingsGitLog(logMsg)
2008
2009                self.readOptions(settings)
2010                if (settings.has_key('depot-paths')
2011                    and settings.has_key ('change')):
2012                    change = int(settings['change']) + 1
2013                    p4Change = max(p4Change, change)
2014
2015                    depotPaths = sorted(settings['depot-paths'])
2016                    if self.previousDepotPaths == []:
2017                        self.previousDepotPaths = depotPaths
2018                    else:
2019                        paths = []
2020                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2021                            prev_list = prev.split("/")
2022                            cur_list = cur.split("/")
2023                            for i in range(0, min(len(cur_list), len(prev_list))):
2024                                if cur_list[i] <> prev_list[i]:
2025                                    i = i - 1
2026                                    break
2027
2028                            paths.append ("/".join(cur_list[:i + 1]))
2029
2030                        self.previousDepotPaths = paths
2031
2032            if p4Change > 0:
2033                self.depotPaths = sorted(self.previousDepotPaths)
2034                self.changeRange = "@%s,#head" % p4Change
2035                if not self.detectBranches:
2036                    self.initialParent = parseRevision(self.branch)
2037                if not self.silent and not self.detectBranches:
2038                    print "Performing incremental import into %s git branch" % self.branch
2039
2040        if not self.branch.startswith("refs/"):
2041            self.branch = "refs/heads/" + self.branch
2042
2043        if len(args) == 0 and self.depotPaths:
2044            if not self.silent:
2045                print "Depot paths: %s" % ' '.join(self.depotPaths)
2046        else:
2047            if self.depotPaths and self.depotPaths != args:
2048                print ("previous import used depot path %s and now %s was specified. "
2049                       "This doesn't work!" % (' '.join (self.depotPaths),
2050                                               ' '.join (args)))
2051                sys.exit(1)
2052
2053            self.depotPaths = sorted(args)
2054
2055        revision = ""
2056        self.users = {}
2057
2058        # Make sure no revision specifiers are used when --changesfile
2059        # is specified.
2060        bad_changesfile = False
2061        if len(self.changesFile) > 0:
2062            for p in self.depotPaths:
2063                if p.find("@") >= 0 or p.find("#") >= 0:
2064                    bad_changesfile = True
2065                    break
2066        if bad_changesfile:
2067            die("Option --changesfile is incompatible with revision specifiers")
2068
2069        newPaths = []
2070        for p in self.depotPaths:
2071            if p.find("@") != -1:
2072                atIdx = p.index("@")
2073                self.changeRange = p[atIdx:]
2074                if self.changeRange == "@all":
2075                    self.changeRange = ""
2076                elif ',' not in self.changeRange:
2077                    revision = self.changeRange
2078                    self.changeRange = ""
2079                p = p[:atIdx]
2080            elif p.find("#") != -1:
2081                hashIdx = p.index("#")
2082                revision = p[hashIdx:]
2083                p = p[:hashIdx]
2084            elif self.previousDepotPaths == []:
2085                # pay attention to changesfile, if given, else import
2086                # the entire p4 tree at the head revision
2087                if len(self.changesFile) == 0:
2088                    revision = "#head"
2089
2090            p = re.sub ("\.\.\.$", "", p)
2091            if not p.endswith("/"):
2092                p += "/"
2093
2094            newPaths.append(p)
2095
2096        self.depotPaths = newPaths
2097
2098
2099        self.loadUserMapFromCache()
2100        self.labels = {}
2101        if self.detectLabels:
2102            self.getLabels();
2103
2104        if self.detectBranches:
2105            ## FIXME - what's a P4 projectName ?
2106            self.projectName = self.guessProjectName()
2107
2108            if self.hasOrigin:
2109                self.getBranchMappingFromGitBranches()
2110            else:
2111                self.getBranchMapping()
2112            if self.verbose:
2113                print "p4-git branches: %s" % self.p4BranchesInGit
2114                print "initial parents: %s" % self.initialParents
2115            for b in self.p4BranchesInGit:
2116                if b != "master":
2117
2118                    ## FIXME
2119                    b = b[len(self.projectName):]
2120                self.createdBranches.add(b)
2121
2122        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2123
2124        importProcess = subprocess.Popen(["git", "fast-import"],
2125                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2126                                         stderr=subprocess.PIPE);
2127        self.gitOutput = importProcess.stdout
2128        self.gitStream = importProcess.stdin
2129        self.gitError = importProcess.stderr
2130
2131        if revision:
2132            self.importHeadRevision(revision)
2133        else:
2134            changes = []
2135
2136            if len(self.changesFile) > 0:
2137                output = open(self.changesFile).readlines()
2138                changeSet = set()
2139                for line in output:
2140                    changeSet.add(int(line))
2141
2142                for change in changeSet:
2143                    changes.append(change)
2144
2145                changes.sort()
2146            else:
2147                # catch "git-p4 sync" with no new branches, in a repo that
2148                # does not have any existing git-p4 branches
2149                if len(args) == 0 and not self.p4BranchesInGit:
2150                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2151                if self.verbose:
2152                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2153                                                              self.changeRange)
2154                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2155
2156                if len(self.maxChanges) > 0:
2157                    changes = changes[:min(int(self.maxChanges), len(changes))]
2158
2159            if len(changes) == 0:
2160                if not self.silent:
2161                    print "No changes to import!"
2162                return True
2163
2164            if not self.silent and not self.detectBranches:
2165                print "Import destination: %s" % self.branch
2166
2167            self.updatedBranches = set()
2168
2169            self.importChanges(changes)
2170
2171            if not self.silent:
2172                print ""
2173                if len(self.updatedBranches) > 0:
2174                    sys.stdout.write("Updated branches: ")
2175                    for b in self.updatedBranches:
2176                        sys.stdout.write("%s " % b)
2177                    sys.stdout.write("\n")
2178
2179        self.gitStream.close()
2180        if importProcess.wait() != 0:
2181            die("fast-import failed: %s" % self.gitError.read())
2182        self.gitOutput.close()
2183        self.gitError.close()
2184
2185        return True
2186
2187class P4Rebase(Command):
2188    def __init__(self):
2189        Command.__init__(self)
2190        self.options = [ ]
2191        self.description = ("Fetches the latest revision from perforce and "
2192                            + "rebases the current work (branch) against it")
2193        self.verbose = False
2194
2195    def run(self, args):
2196        sync = P4Sync()
2197        sync.run([])
2198
2199        return self.rebase()
2200
2201    def rebase(self):
2202        if os.system("git update-index --refresh") != 0:
2203            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2204        if len(read_pipe("git diff-index HEAD --")) > 0:
2205            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2206
2207        [upstream, settings] = findUpstreamBranchPoint()
2208        if len(upstream) == 0:
2209            die("Cannot find upstream branchpoint for rebase")
2210
2211        # the branchpoint may be p4/foo~3, so strip off the parent
2212        upstream = re.sub("~[0-9]+$", "", upstream)
2213
2214        print "Rebasing the current branch onto %s" % upstream
2215        oldHead = read_pipe("git rev-parse HEAD").strip()
2216        system("git rebase %s" % upstream)
2217        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2218        return True
2219
2220class P4Clone(P4Sync):
2221    def __init__(self):
2222        P4Sync.__init__(self)
2223        self.description = "Creates a new git repository and imports from Perforce into it"
2224        self.usage = "usage: %prog [options] //depot/path[@revRange]"
2225        self.options += [
2226            optparse.make_option("--destination", dest="cloneDestination",
2227                                 action='store', default=None,
2228                                 help="where to leave result of the clone"),
2229            optparse.make_option("-/", dest="cloneExclude",
2230                                 action="append", type="string",
2231                                 help="exclude depot path"),
2232            optparse.make_option("--bare", dest="cloneBare",
2233                                 action="store_true", default=False),
2234        ]
2235        self.cloneDestination = None
2236        self.needsGit = False
2237        self.cloneBare = False
2238
2239    # This is required for the "append" cloneExclude action
2240    def ensure_value(self, attr, value):
2241        if not hasattr(self, attr) or getattr(self, attr) is None:
2242            setattr(self, attr, value)
2243        return getattr(self, attr)
2244
2245    def defaultDestination(self, args):
2246        ## TODO: use common prefix of args?
2247        depotPath = args[0]
2248        depotDir = re.sub("(@[^@]*)$", "", depotPath)
2249        depotDir = re.sub("(#[^#]*)$", "", depotDir)
2250        depotDir = re.sub(r"\.\.\.$", "", depotDir)
2251        depotDir = re.sub(r"/$", "", depotDir)
2252        return os.path.split(depotDir)[1]
2253
2254    def run(self, args):
2255        if len(args) < 1:
2256            return False
2257
2258        if self.keepRepoPath and not self.cloneDestination:
2259            sys.stderr.write("Must specify destination for --keep-path\n")
2260            sys.exit(1)
2261
2262        depotPaths = args
2263
2264        if not self.cloneDestination and len(depotPaths) > 1:
2265            self.cloneDestination = depotPaths[-1]
2266            depotPaths = depotPaths[:-1]
2267
2268        self.cloneExclude = ["/"+p for p in self.cloneExclude]
2269        for p in depotPaths:
2270            if not p.startswith("//"):
2271                return False
2272
2273        if not self.cloneDestination:
2274            self.cloneDestination = self.defaultDestination(args)
2275
2276        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2277
2278        if not os.path.exists(self.cloneDestination):
2279            os.makedirs(self.cloneDestination)
2280        chdir(self.cloneDestination)
2281
2282        init_cmd = [ "git", "init" ]
2283        if self.cloneBare:
2284            init_cmd.append("--bare")
2285        subprocess.check_call(init_cmd)
2286
2287        if not P4Sync.run(self, depotPaths):
2288            return False
2289        if self.branch != "master":
2290            if self.importIntoRemotes:
2291                masterbranch = "refs/remotes/p4/master"
2292            else:
2293                masterbranch = "refs/heads/p4/master"
2294            if gitBranchExists(masterbranch):
2295                system("git branch master %s" % masterbranch)
2296                if not self.cloneBare:
2297                    system("git checkout -f")
2298            else:
2299                print "Could not detect main branch. No checkout/master branch created."
2300
2301        return True
2302
2303class P4Branches(Command):
2304    def __init__(self):
2305        Command.__init__(self)
2306        self.options = [ ]
2307        self.description = ("Shows the git branches that hold imports and their "
2308                            + "corresponding perforce depot paths")
2309        self.verbose = False
2310
2311    def run(self, args):
2312        if originP4BranchesExist():
2313            createOrUpdateBranchesFromOrigin()
2314
2315        cmdline = "git rev-parse --symbolic "
2316        cmdline += " --remotes"
2317
2318        for line in read_pipe_lines(cmdline):
2319            line = line.strip()
2320
2321            if not line.startswith('p4/') or line == "p4/HEAD":
2322                continue
2323            branch = line
2324
2325            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2326            settings = extractSettingsGitLog(log)
2327
2328            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2329        return True
2330
2331class HelpFormatter(optparse.IndentedHelpFormatter):
2332    def __init__(self):
2333        optparse.IndentedHelpFormatter.__init__(self)
2334
2335    def format_description(self, description):
2336        if description:
2337            return description + "\n"
2338        else:
2339            return ""
2340
2341def printUsage(commands):
2342    print "usage: %s <command> [options]" % sys.argv[0]
2343    print ""
2344    print "valid commands: %s" % ", ".join(commands)
2345    print ""
2346    print "Try %s <command> --help for command specific help." % sys.argv[0]
2347    print ""
2348
2349commands = {
2350    "debug" : P4Debug,
2351    "submit" : P4Submit,
2352    "commit" : P4Submit,
2353    "sync" : P4Sync,
2354    "rebase" : P4Rebase,
2355    "clone" : P4Clone,
2356    "rollback" : P4RollBack,
2357    "branches" : P4Branches
2358}
2359
2360
2361def main():
2362    if len(sys.argv[1:]) == 0:
2363        printUsage(commands.keys())
2364        sys.exit(2)
2365
2366    cmd = ""
2367    cmdName = sys.argv[1]
2368    try:
2369        klass = commands[cmdName]
2370        cmd = klass()
2371    except KeyError:
2372        print "unknown command %s" % cmdName
2373        print ""
2374        printUsage(commands.keys())
2375        sys.exit(2)
2376
2377    options = cmd.options
2378    cmd.gitdir = os.environ.get("GIT_DIR", None)
2379
2380    args = sys.argv[2:]
2381
2382    if len(options) > 0:
2383        if cmd.needsGit:
2384            options.append(optparse.make_option("--git-dir", dest="gitdir"))
2385
2386        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2387                                       options,
2388                                       description = cmd.description,
2389                                       formatter = HelpFormatter())
2390
2391        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2392    global verbose
2393    verbose = cmd.verbose
2394    if cmd.needsGit:
2395        if cmd.gitdir == None:
2396            cmd.gitdir = os.path.abspath(".git")
2397            if not isValidGitDir(cmd.gitdir):
2398                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2399                if os.path.exists(cmd.gitdir):
2400                    cdup = read_pipe("git rev-parse --show-cdup").strip()
2401                    if len(cdup) > 0:
2402                        chdir(cdup);
2403
2404        if not isValidGitDir(cmd.gitdir):
2405            if isValidGitDir(cmd.gitdir + "/.git"):
2406                cmd.gitdir += "/.git"
2407            else:
2408                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2409
2410        os.environ["GIT_DIR"] = cmd.gitdir
2411
2412    if not cmd.run(args):
2413        parser.print_help()
2414        sys.exit(2)
2415
2416
2417if __name__ == '__main__':
2418    main()