Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'nd/struct-pathspec' (d5c87a8)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    def __init__(self):
 838        Command.__init__(self)
 839        self.options = [
 840                optparse.make_option("--branch", dest="branch"),
 841                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                optparse.make_option("--changesfile", dest="changesFile"),
 843                optparse.make_option("--silent", dest="silent", action="store_true"),
 844                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                     help="Import into refs/heads/ , not refs/remotes"),
 848                optparse.make_option("--max-changes", dest="maxChanges"),
 849                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                     help="Only sync files that are included in the Perforce Client Spec")
 853        ]
 854        self.description = """Imports from Perforce into a git repository.\n
 855    example:
 856    //depot/my/project/ -- to import the current head
 857    //depot/my/project/@all -- to import everything
 858    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862        self.usage += " //depot/path[@revRange]"
 863        self.silent = False
 864        self.createdBranches = set()
 865        self.committedChanges = set()
 866        self.branch = ""
 867        self.detectBranches = False
 868        self.detectLabels = False
 869        self.changesFile = ""
 870        self.syncWithOrigin = True
 871        self.verbose = False
 872        self.importIntoRemotes = True
 873        self.maxChanges = ""
 874        self.isWindows = (platform.system() == "Windows")
 875        self.keepRepoPath = False
 876        self.depotPaths = None
 877        self.p4BranchesInGit = []
 878        self.cloneExclude = []
 879        self.useClientSpec = False
 880        self.clientSpecDirs = []
 881
 882        if gitConfig("git-p4.syncFromOrigin") == "false":
 883            self.syncWithOrigin = False
 884
 885    def extractFilesFromCommit(self, commit):
 886        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                             for path in self.cloneExclude]
 888        files = []
 889        fnum = 0
 890        while commit.has_key("depotFile%s" % fnum):
 891            path =  commit["depotFile%s" % fnum]
 892
 893            if [p for p in self.cloneExclude
 894                if path.startswith (p)]:
 895                found = False
 896            else:
 897                found = [p for p in self.depotPaths
 898                         if path.startswith (p)]
 899            if not found:
 900                fnum = fnum + 1
 901                continue
 902
 903            file = {}
 904            file["path"] = path
 905            file["rev"] = commit["rev%s" % fnum]
 906            file["action"] = commit["action%s" % fnum]
 907            file["type"] = commit["type%s" % fnum]
 908            files.append(file)
 909            fnum = fnum + 1
 910        return files
 911
 912    def stripRepoPath(self, path, prefixes):
 913        if self.useClientSpec:
 914
 915            # if using the client spec, we use the output directory
 916            # specified in the client.  For example, a view
 917            #   //depot/foo/branch/... //client/branch/foo/...
 918            # will end up putting all foo/branch files into
 919            #  branch/foo/
 920            for val in self.clientSpecDirs:
 921                if path.startswith(val[0]):
 922                    # replace the depot path with the client path
 923                    path = path.replace(val[0], val[1][1])
 924                    # now strip out the client (//client/...)
 925                    path = re.sub("^(//[^/]+/)", '', path)
 926                    # the rest is all path
 927                    return path
 928
 929        if self.keepRepoPath:
 930            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 931
 932        for p in prefixes:
 933            if path.startswith(p):
 934                path = path[len(p):]
 935
 936        return path
 937
 938    def splitFilesIntoBranches(self, commit):
 939        branches = {}
 940        fnum = 0
 941        while commit.has_key("depotFile%s" % fnum):
 942            path =  commit["depotFile%s" % fnum]
 943            found = [p for p in self.depotPaths
 944                     if path.startswith (p)]
 945            if not found:
 946                fnum = fnum + 1
 947                continue
 948
 949            file = {}
 950            file["path"] = path
 951            file["rev"] = commit["rev%s" % fnum]
 952            file["action"] = commit["action%s" % fnum]
 953            file["type"] = commit["type%s" % fnum]
 954            fnum = fnum + 1
 955
 956            relPath = self.stripRepoPath(path, self.depotPaths)
 957
 958            for branch in self.knownBranches.keys():
 959
 960                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 961                if relPath.startswith(branch + "/"):
 962                    if branch not in branches:
 963                        branches[branch] = []
 964                    branches[branch].append(file)
 965                    break
 966
 967        return branches
 968
 969    # output one file from the P4 stream
 970    # - helper for streamP4Files
 971
 972    def streamOneP4File(self, file, contents):
 973        if file["type"] == "apple":
 974            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 975                file['depotFile']
 976            return
 977
 978        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 979        if verbose:
 980            sys.stderr.write("%s\n" % relPath)
 981
 982        mode = "644"
 983        if isP4Exec(file["type"]):
 984            mode = "755"
 985        elif file["type"] == "symlink":
 986            mode = "120000"
 987            # p4 print on a symlink contains "target\n", so strip it off
 988            data = ''.join(contents)
 989            contents = [data[:-1]]
 990
 991        if self.isWindows and file["type"].endswith("text"):
 992            mangled = []
 993            for data in contents:
 994                data = data.replace("\r\n", "\n")
 995                mangled.append(data)
 996            contents = mangled
 997
 998        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 999            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1000        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1001            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1002
1003        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1004
1005        # total length...
1006        length = 0
1007        for d in contents:
1008            length = length + len(d)
1009
1010        self.gitStream.write("data %d\n" % length)
1011        for d in contents:
1012            self.gitStream.write(d)
1013        self.gitStream.write("\n")
1014
1015    def streamOneP4Deletion(self, file):
1016        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1017        if verbose:
1018            sys.stderr.write("delete %s\n" % relPath)
1019        self.gitStream.write("D %s\n" % relPath)
1020
1021    # handle another chunk of streaming data
1022    def streamP4FilesCb(self, marshalled):
1023
1024        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1025            # start of a new file - output the old one first
1026            self.streamOneP4File(self.stream_file, self.stream_contents)
1027            self.stream_file = {}
1028            self.stream_contents = []
1029            self.stream_have_file_info = False
1030
1031        # pick up the new file information... for the
1032        # 'data' field we need to append to our array
1033        for k in marshalled.keys():
1034            if k == 'data':
1035                self.stream_contents.append(marshalled['data'])
1036            else:
1037                self.stream_file[k] = marshalled[k]
1038
1039        self.stream_have_file_info = True
1040
1041    # Stream directly from "p4 files" into "git fast-import"
1042    def streamP4Files(self, files):
1043        filesForCommit = []
1044        filesToRead = []
1045        filesToDelete = []
1046
1047        for f in files:
1048            includeFile = True
1049            for val in self.clientSpecDirs:
1050                if f['path'].startswith(val[0]):
1051                    if val[1][0] <= 0:
1052                        includeFile = False
1053                    break
1054
1055            if includeFile:
1056                filesForCommit.append(f)
1057                if f['action'] not in ('delete', 'move/delete', 'purge'):
1058                    filesToRead.append(f)
1059                else:
1060                    filesToDelete.append(f)
1061
1062        # deleted files...
1063        for f in filesToDelete:
1064            self.streamOneP4Deletion(f)
1065
1066        if len(filesToRead) > 0:
1067            self.stream_file = {}
1068            self.stream_contents = []
1069            self.stream_have_file_info = False
1070
1071            # curry self argument
1072            def streamP4FilesCbSelf(entry):
1073                self.streamP4FilesCb(entry)
1074
1075            p4CmdList("-x - print",
1076                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1077                                                  for f in filesToRead]),
1078                cb=streamP4FilesCbSelf)
1079
1080            # do the last chunk
1081            if self.stream_file.has_key('depotFile'):
1082                self.streamOneP4File(self.stream_file, self.stream_contents)
1083
1084    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1085        epoch = details["time"]
1086        author = details["user"]
1087        self.branchPrefixes = branchPrefixes
1088
1089        if self.verbose:
1090            print "commit into %s" % branch
1091
1092        # start with reading files; if that fails, we should not
1093        # create a commit.
1094        new_files = []
1095        for f in files:
1096            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1097                new_files.append (f)
1098            else:
1099                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1100
1101        self.gitStream.write("commit %s\n" % branch)
1102#        gitStream.write("mark :%s\n" % details["change"])
1103        self.committedChanges.add(int(details["change"]))
1104        committer = ""
1105        if author not in self.users:
1106            self.getUserMapFromPerforceServer()
1107        if author in self.users:
1108            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1109        else:
1110            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1111
1112        self.gitStream.write("committer %s\n" % committer)
1113
1114        self.gitStream.write("data <<EOT\n")
1115        self.gitStream.write(details["desc"])
1116        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1117                             % (','.join (branchPrefixes), details["change"]))
1118        if len(details['options']) > 0:
1119            self.gitStream.write(": options = %s" % details['options'])
1120        self.gitStream.write("]\nEOT\n\n")
1121
1122        if len(parent) > 0:
1123            if self.verbose:
1124                print "parent %s" % parent
1125            self.gitStream.write("from %s\n" % parent)
1126
1127        self.streamP4Files(new_files)
1128        self.gitStream.write("\n")
1129
1130        change = int(details["change"])
1131
1132        if self.labels.has_key(change):
1133            label = self.labels[change]
1134            labelDetails = label[0]
1135            labelRevisions = label[1]
1136            if self.verbose:
1137                print "Change %s is labelled %s" % (change, labelDetails)
1138
1139            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1140                                                    for p in branchPrefixes]))
1141
1142            if len(files) == len(labelRevisions):
1143
1144                cleanedFiles = {}
1145                for info in files:
1146                    if info["action"] in ("delete", "purge"):
1147                        continue
1148                    cleanedFiles[info["depotFile"]] = info["rev"]
1149
1150                if cleanedFiles == labelRevisions:
1151                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1152                    self.gitStream.write("from %s\n" % branch)
1153
1154                    owner = labelDetails["Owner"]
1155                    tagger = ""
1156                    if author in self.users:
1157                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1158                    else:
1159                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1160                    self.gitStream.write("tagger %s\n" % tagger)
1161                    self.gitStream.write("data <<EOT\n")
1162                    self.gitStream.write(labelDetails["Description"])
1163                    self.gitStream.write("EOT\n\n")
1164
1165                else:
1166                    if not self.silent:
1167                        print ("Tag %s does not match with change %s: files do not match."
1168                               % (labelDetails["label"], change))
1169
1170            else:
1171                if not self.silent:
1172                    print ("Tag %s does not match with change %s: file count is different."
1173                           % (labelDetails["label"], change))
1174
1175    def getUserCacheFilename(self):
1176        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1177        return home + "/.gitp4-usercache.txt"
1178
1179    def getUserMapFromPerforceServer(self):
1180        if self.userMapFromPerforceServer:
1181            return
1182        self.users = {}
1183
1184        for output in p4CmdList("users"):
1185            if not output.has_key("User"):
1186                continue
1187            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1188
1189
1190        s = ''
1191        for (key, val) in self.users.items():
1192            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1193
1194        open(self.getUserCacheFilename(), "wb").write(s)
1195        self.userMapFromPerforceServer = True
1196
1197    def loadUserMapFromCache(self):
1198        self.users = {}
1199        self.userMapFromPerforceServer = False
1200        try:
1201            cache = open(self.getUserCacheFilename(), "rb")
1202            lines = cache.readlines()
1203            cache.close()
1204            for line in lines:
1205                entry = line.strip().split("\t")
1206                self.users[entry[0]] = entry[1]
1207        except IOError:
1208            self.getUserMapFromPerforceServer()
1209
1210    def getLabels(self):
1211        self.labels = {}
1212
1213        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1214        if len(l) > 0 and not self.silent:
1215            print "Finding files belonging to labels in %s" % `self.depotPaths`
1216
1217        for output in l:
1218            label = output["label"]
1219            revisions = {}
1220            newestChange = 0
1221            if self.verbose:
1222                print "Querying files for label %s" % label
1223            for file in p4CmdList("files "
1224                                  +  ' '.join (["%s...@%s" % (p, label)
1225                                                for p in self.depotPaths])):
1226                revisions[file["depotFile"]] = file["rev"]
1227                change = int(file["change"])
1228                if change > newestChange:
1229                    newestChange = change
1230
1231            self.labels[newestChange] = [output, revisions]
1232
1233        if self.verbose:
1234            print "Label changes: %s" % self.labels.keys()
1235
1236    def guessProjectName(self):
1237        for p in self.depotPaths:
1238            if p.endswith("/"):
1239                p = p[:-1]
1240            p = p[p.strip().rfind("/") + 1:]
1241            if not p.endswith("/"):
1242               p += "/"
1243            return p
1244
1245    def getBranchMapping(self):
1246        lostAndFoundBranches = set()
1247
1248        for info in p4CmdList("branches"):
1249            details = p4Cmd("branch -o %s" % info["branch"])
1250            viewIdx = 0
1251            while details.has_key("View%s" % viewIdx):
1252                paths = details["View%s" % viewIdx].split(" ")
1253                viewIdx = viewIdx + 1
1254                # require standard //depot/foo/... //depot/bar/... mapping
1255                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1256                    continue
1257                source = paths[0]
1258                destination = paths[1]
1259                ## HACK
1260                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1261                    source = source[len(self.depotPaths[0]):-4]
1262                    destination = destination[len(self.depotPaths[0]):-4]
1263
1264                    if destination in self.knownBranches:
1265                        if not self.silent:
1266                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1267                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1268                        continue
1269
1270                    self.knownBranches[destination] = source
1271
1272                    lostAndFoundBranches.discard(destination)
1273
1274                    if source not in self.knownBranches:
1275                        lostAndFoundBranches.add(source)
1276
1277
1278        for branch in lostAndFoundBranches:
1279            self.knownBranches[branch] = branch
1280
1281    def getBranchMappingFromGitBranches(self):
1282        branches = p4BranchesInGit(self.importIntoRemotes)
1283        for branch in branches.keys():
1284            if branch == "master":
1285                branch = "main"
1286            else:
1287                branch = branch[len(self.projectName):]
1288            self.knownBranches[branch] = branch
1289
1290    def listExistingP4GitBranches(self):
1291        # branches holds mapping from name to commit
1292        branches = p4BranchesInGit(self.importIntoRemotes)
1293        self.p4BranchesInGit = branches.keys()
1294        for branch in branches.keys():
1295            self.initialParents[self.refPrefix + branch] = branches[branch]
1296
1297    def updateOptionDict(self, d):
1298        option_keys = {}
1299        if self.keepRepoPath:
1300            option_keys['keepRepoPath'] = 1
1301
1302        d["options"] = ' '.join(sorted(option_keys.keys()))
1303
1304    def readOptions(self, d):
1305        self.keepRepoPath = (d.has_key('options')
1306                             and ('keepRepoPath' in d['options']))
1307
1308    def gitRefForBranch(self, branch):
1309        if branch == "main":
1310            return self.refPrefix + "master"
1311
1312        if len(branch) <= 0:
1313            return branch
1314
1315        return self.refPrefix + self.projectName + branch
1316
1317    def gitCommitByP4Change(self, ref, change):
1318        if self.verbose:
1319            print "looking in ref " + ref + " for change %s using bisect..." % change
1320
1321        earliestCommit = ""
1322        latestCommit = parseRevision(ref)
1323
1324        while True:
1325            if self.verbose:
1326                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1327            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1328            if len(next) == 0:
1329                if self.verbose:
1330                    print "argh"
1331                return ""
1332            log = extractLogMessageFromGitCommit(next)
1333            settings = extractSettingsGitLog(log)
1334            currentChange = int(settings['change'])
1335            if self.verbose:
1336                print "current change %s" % currentChange
1337
1338            if currentChange == change:
1339                if self.verbose:
1340                    print "found %s" % next
1341                return next
1342
1343            if currentChange < change:
1344                earliestCommit = "^%s" % next
1345            else:
1346                latestCommit = "%s" % next
1347
1348        return ""
1349
1350    def importNewBranch(self, branch, maxChange):
1351        # make fast-import flush all changes to disk and update the refs using the checkpoint
1352        # command so that we can try to find the branch parent in the git history
1353        self.gitStream.write("checkpoint\n\n");
1354        self.gitStream.flush();
1355        branchPrefix = self.depotPaths[0] + branch + "/"
1356        range = "@1,%s" % maxChange
1357        #print "prefix" + branchPrefix
1358        changes = p4ChangesForPaths([branchPrefix], range)
1359        if len(changes) <= 0:
1360            return False
1361        firstChange = changes[0]
1362        #print "first change in branch: %s" % firstChange
1363        sourceBranch = self.knownBranches[branch]
1364        sourceDepotPath = self.depotPaths[0] + sourceBranch
1365        sourceRef = self.gitRefForBranch(sourceBranch)
1366        #print "source " + sourceBranch
1367
1368        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1369        #print "branch parent: %s" % branchParentChange
1370        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1371        if len(gitParent) > 0:
1372            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1373            #print "parent git commit: %s" % gitParent
1374
1375        self.importChanges(changes)
1376        return True
1377
1378    def importChanges(self, changes):
1379        cnt = 1
1380        for change in changes:
1381            description = p4Cmd("describe %s" % change)
1382            self.updateOptionDict(description)
1383
1384            if not self.silent:
1385                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1386                sys.stdout.flush()
1387            cnt = cnt + 1
1388
1389            try:
1390                if self.detectBranches:
1391                    branches = self.splitFilesIntoBranches(description)
1392                    for branch in branches.keys():
1393                        ## HACK  --hwn
1394                        branchPrefix = self.depotPaths[0] + branch + "/"
1395
1396                        parent = ""
1397
1398                        filesForCommit = branches[branch]
1399
1400                        if self.verbose:
1401                            print "branch is %s" % branch
1402
1403                        self.updatedBranches.add(branch)
1404
1405                        if branch not in self.createdBranches:
1406                            self.createdBranches.add(branch)
1407                            parent = self.knownBranches[branch]
1408                            if parent == branch:
1409                                parent = ""
1410                            else:
1411                                fullBranch = self.projectName + branch
1412                                if fullBranch not in self.p4BranchesInGit:
1413                                    if not self.silent:
1414                                        print("\n    Importing new branch %s" % fullBranch);
1415                                    if self.importNewBranch(branch, change - 1):
1416                                        parent = ""
1417                                        self.p4BranchesInGit.append(fullBranch)
1418                                    if not self.silent:
1419                                        print("\n    Resuming with change %s" % change);
1420
1421                                if self.verbose:
1422                                    print "parent determined through known branches: %s" % parent
1423
1424                        branch = self.gitRefForBranch(branch)
1425                        parent = self.gitRefForBranch(parent)
1426
1427                        if self.verbose:
1428                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1429
1430                        if len(parent) == 0 and branch in self.initialParents:
1431                            parent = self.initialParents[branch]
1432                            del self.initialParents[branch]
1433
1434                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1435                else:
1436                    files = self.extractFilesFromCommit(description)
1437                    self.commit(description, files, self.branch, self.depotPaths,
1438                                self.initialParent)
1439                    self.initialParent = ""
1440            except IOError:
1441                print self.gitError.read()
1442                sys.exit(1)
1443
1444    def importHeadRevision(self, revision):
1445        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1446
1447        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1448        details["desc"] = ("Initial import of %s from the state at revision %s"
1449                           % (' '.join(self.depotPaths), revision))
1450        details["change"] = revision
1451        newestRevision = 0
1452
1453        fileCnt = 0
1454        for info in p4CmdList("files "
1455                              +  ' '.join(["%s...%s"
1456                                           % (p, revision)
1457                                           for p in self.depotPaths])):
1458
1459            if info['code'] == 'error':
1460                sys.stderr.write("p4 returned an error: %s\n"
1461                                 % info['data'])
1462                sys.exit(1)
1463
1464
1465            change = int(info["change"])
1466            if change > newestRevision:
1467                newestRevision = change
1468
1469            if info["action"] in ("delete", "purge"):
1470                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1471                #fileCnt = fileCnt + 1
1472                continue
1473
1474            for prop in ["depotFile", "rev", "action", "type" ]:
1475                details["%s%s" % (prop, fileCnt)] = info[prop]
1476
1477            fileCnt = fileCnt + 1
1478
1479        details["change"] = newestRevision
1480        self.updateOptionDict(details)
1481        try:
1482            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1483        except IOError:
1484            print "IO error with git fast-import. Is your git version recent enough?"
1485            print self.gitError.read()
1486
1487
1488    def getClientSpec(self):
1489        specList = p4CmdList( "client -o" )
1490        temp = {}
1491        for entry in specList:
1492            for k,v in entry.iteritems():
1493                if k.startswith("View"):
1494
1495                    # p4 has these %%1 to %%9 arguments in specs to
1496                    # reorder paths; which we can't handle (yet :)
1497                    if re.match('%%\d', v) != None:
1498                        print "Sorry, can't handle %%n arguments in client specs"
1499                        sys.exit(1)
1500
1501                    if v.startswith('"'):
1502                        start = 1
1503                    else:
1504                        start = 0
1505                    index = v.find("...")
1506
1507                    # save the "client view"; i.e the RHS of the view
1508                    # line that tells the client where to put the
1509                    # files for this view.
1510                    cv = v[index+3:].strip() # +3 to remove previous '...'
1511
1512                    # if the client view doesn't end with a
1513                    # ... wildcard, then we're going to mess up the
1514                    # output directory, so fail gracefully.
1515                    if not cv.endswith('...'):
1516                        print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1517                        sys.exit(1)
1518                    cv=cv[:-3]
1519
1520                    # now save the view; +index means included, -index
1521                    # means it should be filtered out.
1522                    v = v[start:index]
1523                    if v.startswith("-"):
1524                        v = v[1:]
1525                        include = -len(v)
1526                    else:
1527                        include = len(v)
1528
1529                    temp[v] = (include, cv)
1530
1531        self.clientSpecDirs = temp.items()
1532        self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1533
1534    def run(self, args):
1535        self.depotPaths = []
1536        self.changeRange = ""
1537        self.initialParent = ""
1538        self.previousDepotPaths = []
1539
1540        # map from branch depot path to parent branch
1541        self.knownBranches = {}
1542        self.initialParents = {}
1543        self.hasOrigin = originP4BranchesExist()
1544        if not self.syncWithOrigin:
1545            self.hasOrigin = False
1546
1547        if self.importIntoRemotes:
1548            self.refPrefix = "refs/remotes/p4/"
1549        else:
1550            self.refPrefix = "refs/heads/p4/"
1551
1552        if self.syncWithOrigin and self.hasOrigin:
1553            if not self.silent:
1554                print "Syncing with origin first by calling git fetch origin"
1555            system("git fetch origin")
1556
1557        if len(self.branch) == 0:
1558            self.branch = self.refPrefix + "master"
1559            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1560                system("git update-ref %s refs/heads/p4" % self.branch)
1561                system("git branch -D p4");
1562            # create it /after/ importing, when master exists
1563            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1564                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1565
1566        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1567            self.getClientSpec()
1568
1569        # TODO: should always look at previous commits,
1570        # merge with previous imports, if possible.
1571        if args == []:
1572            if self.hasOrigin:
1573                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1574            self.listExistingP4GitBranches()
1575
1576            if len(self.p4BranchesInGit) > 1:
1577                if not self.silent:
1578                    print "Importing from/into multiple branches"
1579                self.detectBranches = True
1580
1581            if self.verbose:
1582                print "branches: %s" % self.p4BranchesInGit
1583
1584            p4Change = 0
1585            for branch in self.p4BranchesInGit:
1586                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1587
1588                settings = extractSettingsGitLog(logMsg)
1589
1590                self.readOptions(settings)
1591                if (settings.has_key('depot-paths')
1592                    and settings.has_key ('change')):
1593                    change = int(settings['change']) + 1
1594                    p4Change = max(p4Change, change)
1595
1596                    depotPaths = sorted(settings['depot-paths'])
1597                    if self.previousDepotPaths == []:
1598                        self.previousDepotPaths = depotPaths
1599                    else:
1600                        paths = []
1601                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1602                            for i in range(0, min(len(cur), len(prev))):
1603                                if cur[i] <> prev[i]:
1604                                    i = i - 1
1605                                    break
1606
1607                            paths.append (cur[:i + 1])
1608
1609                        self.previousDepotPaths = paths
1610
1611            if p4Change > 0:
1612                self.depotPaths = sorted(self.previousDepotPaths)
1613                self.changeRange = "@%s,#head" % p4Change
1614                if not self.detectBranches:
1615                    self.initialParent = parseRevision(self.branch)
1616                if not self.silent and not self.detectBranches:
1617                    print "Performing incremental import into %s git branch" % self.branch
1618
1619        if not self.branch.startswith("refs/"):
1620            self.branch = "refs/heads/" + self.branch
1621
1622        if len(args) == 0 and self.depotPaths:
1623            if not self.silent:
1624                print "Depot paths: %s" % ' '.join(self.depotPaths)
1625        else:
1626            if self.depotPaths and self.depotPaths != args:
1627                print ("previous import used depot path %s and now %s was specified. "
1628                       "This doesn't work!" % (' '.join (self.depotPaths),
1629                                               ' '.join (args)))
1630                sys.exit(1)
1631
1632            self.depotPaths = sorted(args)
1633
1634        revision = ""
1635        self.users = {}
1636
1637        newPaths = []
1638        for p in self.depotPaths:
1639            if p.find("@") != -1:
1640                atIdx = p.index("@")
1641                self.changeRange = p[atIdx:]
1642                if self.changeRange == "@all":
1643                    self.changeRange = ""
1644                elif ',' not in self.changeRange:
1645                    revision = self.changeRange
1646                    self.changeRange = ""
1647                p = p[:atIdx]
1648            elif p.find("#") != -1:
1649                hashIdx = p.index("#")
1650                revision = p[hashIdx:]
1651                p = p[:hashIdx]
1652            elif self.previousDepotPaths == []:
1653                revision = "#head"
1654
1655            p = re.sub ("\.\.\.$", "", p)
1656            if not p.endswith("/"):
1657                p += "/"
1658
1659            newPaths.append(p)
1660
1661        self.depotPaths = newPaths
1662
1663
1664        self.loadUserMapFromCache()
1665        self.labels = {}
1666        if self.detectLabels:
1667            self.getLabels();
1668
1669        if self.detectBranches:
1670            ## FIXME - what's a P4 projectName ?
1671            self.projectName = self.guessProjectName()
1672
1673            if self.hasOrigin:
1674                self.getBranchMappingFromGitBranches()
1675            else:
1676                self.getBranchMapping()
1677            if self.verbose:
1678                print "p4-git branches: %s" % self.p4BranchesInGit
1679                print "initial parents: %s" % self.initialParents
1680            for b in self.p4BranchesInGit:
1681                if b != "master":
1682
1683                    ## FIXME
1684                    b = b[len(self.projectName):]
1685                self.createdBranches.add(b)
1686
1687        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1688
1689        importProcess = subprocess.Popen(["git", "fast-import"],
1690                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1691                                         stderr=subprocess.PIPE);
1692        self.gitOutput = importProcess.stdout
1693        self.gitStream = importProcess.stdin
1694        self.gitError = importProcess.stderr
1695
1696        if revision:
1697            self.importHeadRevision(revision)
1698        else:
1699            changes = []
1700
1701            if len(self.changesFile) > 0:
1702                output = open(self.changesFile).readlines()
1703                changeSet = set()
1704                for line in output:
1705                    changeSet.add(int(line))
1706
1707                for change in changeSet:
1708                    changes.append(change)
1709
1710                changes.sort()
1711            else:
1712                if self.verbose:
1713                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1714                                                              self.changeRange)
1715                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1716
1717                if len(self.maxChanges) > 0:
1718                    changes = changes[:min(int(self.maxChanges), len(changes))]
1719
1720            if len(changes) == 0:
1721                if not self.silent:
1722                    print "No changes to import!"
1723                return True
1724
1725            if not self.silent and not self.detectBranches:
1726                print "Import destination: %s" % self.branch
1727
1728            self.updatedBranches = set()
1729
1730            self.importChanges(changes)
1731
1732            if not self.silent:
1733                print ""
1734                if len(self.updatedBranches) > 0:
1735                    sys.stdout.write("Updated branches: ")
1736                    for b in self.updatedBranches:
1737                        sys.stdout.write("%s " % b)
1738                    sys.stdout.write("\n")
1739
1740        self.gitStream.close()
1741        if importProcess.wait() != 0:
1742            die("fast-import failed: %s" % self.gitError.read())
1743        self.gitOutput.close()
1744        self.gitError.close()
1745
1746        return True
1747
1748class P4Rebase(Command):
1749    def __init__(self):
1750        Command.__init__(self)
1751        self.options = [ ]
1752        self.description = ("Fetches the latest revision from perforce and "
1753                            + "rebases the current work (branch) against it")
1754        self.verbose = False
1755
1756    def run(self, args):
1757        sync = P4Sync()
1758        sync.run([])
1759
1760        return self.rebase()
1761
1762    def rebase(self):
1763        if os.system("git update-index --refresh") != 0:
1764            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1765        if len(read_pipe("git diff-index HEAD --")) > 0:
1766            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1767
1768        [upstream, settings] = findUpstreamBranchPoint()
1769        if len(upstream) == 0:
1770            die("Cannot find upstream branchpoint for rebase")
1771
1772        # the branchpoint may be p4/foo~3, so strip off the parent
1773        upstream = re.sub("~[0-9]+$", "", upstream)
1774
1775        print "Rebasing the current branch onto %s" % upstream
1776        oldHead = read_pipe("git rev-parse HEAD").strip()
1777        system("git rebase %s" % upstream)
1778        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1779        return True
1780
1781class P4Clone(P4Sync):
1782    def __init__(self):
1783        P4Sync.__init__(self)
1784        self.description = "Creates a new git repository and imports from Perforce into it"
1785        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1786        self.options += [
1787            optparse.make_option("--destination", dest="cloneDestination",
1788                                 action='store', default=None,
1789                                 help="where to leave result of the clone"),
1790            optparse.make_option("-/", dest="cloneExclude",
1791                                 action="append", type="string",
1792                                 help="exclude depot path")
1793        ]
1794        self.cloneDestination = None
1795        self.needsGit = False
1796
1797    # This is required for the "append" cloneExclude action
1798    def ensure_value(self, attr, value):
1799        if not hasattr(self, attr) or getattr(self, attr) is None:
1800            setattr(self, attr, value)
1801        return getattr(self, attr)
1802
1803    def defaultDestination(self, args):
1804        ## TODO: use common prefix of args?
1805        depotPath = args[0]
1806        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1807        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1808        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1809        depotDir = re.sub(r"/$", "", depotDir)
1810        return os.path.split(depotDir)[1]
1811
1812    def run(self, args):
1813        if len(args) < 1:
1814            return False
1815
1816        if self.keepRepoPath and not self.cloneDestination:
1817            sys.stderr.write("Must specify destination for --keep-path\n")
1818            sys.exit(1)
1819
1820        depotPaths = args
1821
1822        if not self.cloneDestination and len(depotPaths) > 1:
1823            self.cloneDestination = depotPaths[-1]
1824            depotPaths = depotPaths[:-1]
1825
1826        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1827        for p in depotPaths:
1828            if not p.startswith("//"):
1829                return False
1830
1831        if not self.cloneDestination:
1832            self.cloneDestination = self.defaultDestination(args)
1833
1834        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1835        if not os.path.exists(self.cloneDestination):
1836            os.makedirs(self.cloneDestination)
1837        chdir(self.cloneDestination)
1838        system("git init")
1839        self.gitdir = os.getcwd() + "/.git"
1840        if not P4Sync.run(self, depotPaths):
1841            return False
1842        if self.branch != "master":
1843            if self.importIntoRemotes:
1844                masterbranch = "refs/remotes/p4/master"
1845            else:
1846                masterbranch = "refs/heads/p4/master"
1847            if gitBranchExists(masterbranch):
1848                system("git branch master %s" % masterbranch)
1849                system("git checkout -f")
1850            else:
1851                print "Could not detect main branch. No checkout/master branch created."
1852
1853        return True
1854
1855class P4Branches(Command):
1856    def __init__(self):
1857        Command.__init__(self)
1858        self.options = [ ]
1859        self.description = ("Shows the git branches that hold imports and their "
1860                            + "corresponding perforce depot paths")
1861        self.verbose = False
1862
1863    def run(self, args):
1864        if originP4BranchesExist():
1865            createOrUpdateBranchesFromOrigin()
1866
1867        cmdline = "git rev-parse --symbolic "
1868        cmdline += " --remotes"
1869
1870        for line in read_pipe_lines(cmdline):
1871            line = line.strip()
1872
1873            if not line.startswith('p4/') or line == "p4/HEAD":
1874                continue
1875            branch = line
1876
1877            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1878            settings = extractSettingsGitLog(log)
1879
1880            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1881        return True
1882
1883class HelpFormatter(optparse.IndentedHelpFormatter):
1884    def __init__(self):
1885        optparse.IndentedHelpFormatter.__init__(self)
1886
1887    def format_description(self, description):
1888        if description:
1889            return description + "\n"
1890        else:
1891            return ""
1892
1893def printUsage(commands):
1894    print "usage: %s <command> [options]" % sys.argv[0]
1895    print ""
1896    print "valid commands: %s" % ", ".join(commands)
1897    print ""
1898    print "Try %s <command> --help for command specific help." % sys.argv[0]
1899    print ""
1900
1901commands = {
1902    "debug" : P4Debug,
1903    "submit" : P4Submit,
1904    "commit" : P4Submit,
1905    "sync" : P4Sync,
1906    "rebase" : P4Rebase,
1907    "clone" : P4Clone,
1908    "rollback" : P4RollBack,
1909    "branches" : P4Branches
1910}
1911
1912
1913def main():
1914    if len(sys.argv[1:]) == 0:
1915        printUsage(commands.keys())
1916        sys.exit(2)
1917
1918    cmd = ""
1919    cmdName = sys.argv[1]
1920    try:
1921        klass = commands[cmdName]
1922        cmd = klass()
1923    except KeyError:
1924        print "unknown command %s" % cmdName
1925        print ""
1926        printUsage(commands.keys())
1927        sys.exit(2)
1928
1929    options = cmd.options
1930    cmd.gitdir = os.environ.get("GIT_DIR", None)
1931
1932    args = sys.argv[2:]
1933
1934    if len(options) > 0:
1935        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1936
1937        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1938                                       options,
1939                                       description = cmd.description,
1940                                       formatter = HelpFormatter())
1941
1942        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1943    global verbose
1944    verbose = cmd.verbose
1945    if cmd.needsGit:
1946        if cmd.gitdir == None:
1947            cmd.gitdir = os.path.abspath(".git")
1948            if not isValidGitDir(cmd.gitdir):
1949                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1950                if os.path.exists(cmd.gitdir):
1951                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1952                    if len(cdup) > 0:
1953                        chdir(cdup);
1954
1955        if not isValidGitDir(cmd.gitdir):
1956            if isValidGitDir(cmd.gitdir + "/.git"):
1957                cmd.gitdir += "/.git"
1958            else:
1959                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1960
1961        os.environ["GIT_DIR"] = cmd.gitdir
1962
1963    if not cmd.run(args):
1964        parser.print_help()
1965
1966
1967if __name__ == '__main__':
1968    main()