Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'pw/p4' (8d3a362)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    delete_actions = ( "delete", "move/delete", "purge" )
 838
 839    def __init__(self):
 840        Command.__init__(self)
 841        self.options = [
 842                optparse.make_option("--branch", dest="branch"),
 843                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                optparse.make_option("--changesfile", dest="changesFile"),
 845                optparse.make_option("--silent", dest="silent", action="store_true"),
 846                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                     help="Import into refs/heads/ , not refs/remotes"),
 850                optparse.make_option("--max-changes", dest="maxChanges"),
 851                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                     help="Only sync files that are included in the Perforce Client Spec")
 855        ]
 856        self.description = """Imports from Perforce into a git repository.\n
 857    example:
 858    //depot/my/project/ -- to import the current head
 859    //depot/my/project/@all -- to import everything
 860    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864        self.usage += " //depot/path[@revRange]"
 865        self.silent = False
 866        self.createdBranches = set()
 867        self.committedChanges = set()
 868        self.branch = ""
 869        self.detectBranches = False
 870        self.detectLabels = False
 871        self.changesFile = ""
 872        self.syncWithOrigin = True
 873        self.verbose = False
 874        self.importIntoRemotes = True
 875        self.maxChanges = ""
 876        self.isWindows = (platform.system() == "Windows")
 877        self.keepRepoPath = False
 878        self.depotPaths = None
 879        self.p4BranchesInGit = []
 880        self.cloneExclude = []
 881        self.useClientSpec = False
 882        self.clientSpecDirs = []
 883
 884        if gitConfig("git-p4.syncFromOrigin") == "false":
 885            self.syncWithOrigin = False
 886
 887    #
 888    # P4 wildcards are not allowed in filenames.  P4 complains
 889    # if you simply add them, but you can force it with "-f", in
 890    # which case it translates them into %xx encoding internally.
 891    # Search for and fix just these four characters.  Do % last so
 892    # that fixing it does not inadvertently create new %-escapes.
 893    #
 894    def wildcard_decode(self, path):
 895        # Cannot have * in a filename in windows; untested as to
 896        # what p4 would do in such a case.
 897        if not self.isWindows:
 898            path = path.replace("%2A", "*")
 899        path = path.replace("%23", "#") \
 900                   .replace("%40", "@") \
 901                   .replace("%25", "%")
 902        return path
 903
 904    def extractFilesFromCommit(self, commit):
 905        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 906                             for path in self.cloneExclude]
 907        files = []
 908        fnum = 0
 909        while commit.has_key("depotFile%s" % fnum):
 910            path =  commit["depotFile%s" % fnum]
 911
 912            if [p for p in self.cloneExclude
 913                if path.startswith (p)]:
 914                found = False
 915            else:
 916                found = [p for p in self.depotPaths
 917                         if path.startswith (p)]
 918            if not found:
 919                fnum = fnum + 1
 920                continue
 921
 922            file = {}
 923            file["path"] = path
 924            file["rev"] = commit["rev%s" % fnum]
 925            file["action"] = commit["action%s" % fnum]
 926            file["type"] = commit["type%s" % fnum]
 927            files.append(file)
 928            fnum = fnum + 1
 929        return files
 930
 931    def stripRepoPath(self, path, prefixes):
 932        if self.useClientSpec:
 933
 934            # if using the client spec, we use the output directory
 935            # specified in the client.  For example, a view
 936            #   //depot/foo/branch/... //client/branch/foo/...
 937            # will end up putting all foo/branch files into
 938            #  branch/foo/
 939            for val in self.clientSpecDirs:
 940                if path.startswith(val[0]):
 941                    # replace the depot path with the client path
 942                    path = path.replace(val[0], val[1][1])
 943                    # now strip out the client (//client/...)
 944                    path = re.sub("^(//[^/]+/)", '', path)
 945                    # the rest is all path
 946                    return path
 947
 948        if self.keepRepoPath:
 949            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 950
 951        for p in prefixes:
 952            if path.startswith(p):
 953                path = path[len(p):]
 954
 955        return path
 956
 957    def splitFilesIntoBranches(self, commit):
 958        branches = {}
 959        fnum = 0
 960        while commit.has_key("depotFile%s" % fnum):
 961            path =  commit["depotFile%s" % fnum]
 962            found = [p for p in self.depotPaths
 963                     if path.startswith (p)]
 964            if not found:
 965                fnum = fnum + 1
 966                continue
 967
 968            file = {}
 969            file["path"] = path
 970            file["rev"] = commit["rev%s" % fnum]
 971            file["action"] = commit["action%s" % fnum]
 972            file["type"] = commit["type%s" % fnum]
 973            fnum = fnum + 1
 974
 975            relPath = self.stripRepoPath(path, self.depotPaths)
 976
 977            for branch in self.knownBranches.keys():
 978
 979                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 980                if relPath.startswith(branch + "/"):
 981                    if branch not in branches:
 982                        branches[branch] = []
 983                    branches[branch].append(file)
 984                    break
 985
 986        return branches
 987
 988    # output one file from the P4 stream
 989    # - helper for streamP4Files
 990
 991    def streamOneP4File(self, file, contents):
 992        if file["type"] == "apple":
 993            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 994                file['depotFile']
 995            return
 996
 997        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 998        relPath = self.wildcard_decode(relPath)
 999        if verbose:
1000            sys.stderr.write("%s\n" % relPath)
1001
1002        mode = "644"
1003        if isP4Exec(file["type"]):
1004            mode = "755"
1005        elif file["type"] == "symlink":
1006            mode = "120000"
1007            # p4 print on a symlink contains "target\n", so strip it off
1008            data = ''.join(contents)
1009            contents = [data[:-1]]
1010
1011        if self.isWindows and file["type"].endswith("text"):
1012            mangled = []
1013            for data in contents:
1014                data = data.replace("\r\n", "\n")
1015                mangled.append(data)
1016            contents = mangled
1017
1018        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1019            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1020        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1021            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1022
1023        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1024
1025        # total length...
1026        length = 0
1027        for d in contents:
1028            length = length + len(d)
1029
1030        self.gitStream.write("data %d\n" % length)
1031        for d in contents:
1032            self.gitStream.write(d)
1033        self.gitStream.write("\n")
1034
1035    def streamOneP4Deletion(self, file):
1036        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1037        if verbose:
1038            sys.stderr.write("delete %s\n" % relPath)
1039        self.gitStream.write("D %s\n" % relPath)
1040
1041    # handle another chunk of streaming data
1042    def streamP4FilesCb(self, marshalled):
1043
1044        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1045            # start of a new file - output the old one first
1046            self.streamOneP4File(self.stream_file, self.stream_contents)
1047            self.stream_file = {}
1048            self.stream_contents = []
1049            self.stream_have_file_info = False
1050
1051        # pick up the new file information... for the
1052        # 'data' field we need to append to our array
1053        for k in marshalled.keys():
1054            if k == 'data':
1055                self.stream_contents.append(marshalled['data'])
1056            else:
1057                self.stream_file[k] = marshalled[k]
1058
1059        self.stream_have_file_info = True
1060
1061    # Stream directly from "p4 files" into "git fast-import"
1062    def streamP4Files(self, files):
1063        filesForCommit = []
1064        filesToRead = []
1065        filesToDelete = []
1066
1067        for f in files:
1068            includeFile = True
1069            for val in self.clientSpecDirs:
1070                if f['path'].startswith(val[0]):
1071                    if val[1][0] <= 0:
1072                        includeFile = False
1073                    break
1074
1075            if includeFile:
1076                filesForCommit.append(f)
1077                if f['action'] in self.delete_actions:
1078                    filesToDelete.append(f)
1079                else:
1080                    filesToRead.append(f)
1081
1082        # deleted files...
1083        for f in filesToDelete:
1084            self.streamOneP4Deletion(f)
1085
1086        if len(filesToRead) > 0:
1087            self.stream_file = {}
1088            self.stream_contents = []
1089            self.stream_have_file_info = False
1090
1091            # curry self argument
1092            def streamP4FilesCbSelf(entry):
1093                self.streamP4FilesCb(entry)
1094
1095            p4CmdList("-x - print",
1096                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1097                                                  for f in filesToRead]),
1098                cb=streamP4FilesCbSelf)
1099
1100            # do the last chunk
1101            if self.stream_file.has_key('depotFile'):
1102                self.streamOneP4File(self.stream_file, self.stream_contents)
1103
1104    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1105        epoch = details["time"]
1106        author = details["user"]
1107        self.branchPrefixes = branchPrefixes
1108
1109        if self.verbose:
1110            print "commit into %s" % branch
1111
1112        # start with reading files; if that fails, we should not
1113        # create a commit.
1114        new_files = []
1115        for f in files:
1116            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1117                new_files.append (f)
1118            else:
1119                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1120
1121        self.gitStream.write("commit %s\n" % branch)
1122#        gitStream.write("mark :%s\n" % details["change"])
1123        self.committedChanges.add(int(details["change"]))
1124        committer = ""
1125        if author not in self.users:
1126            self.getUserMapFromPerforceServer()
1127        if author in self.users:
1128            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1129        else:
1130            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1131
1132        self.gitStream.write("committer %s\n" % committer)
1133
1134        self.gitStream.write("data <<EOT\n")
1135        self.gitStream.write(details["desc"])
1136        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1137                             % (','.join (branchPrefixes), details["change"]))
1138        if len(details['options']) > 0:
1139            self.gitStream.write(": options = %s" % details['options'])
1140        self.gitStream.write("]\nEOT\n\n")
1141
1142        if len(parent) > 0:
1143            if self.verbose:
1144                print "parent %s" % parent
1145            self.gitStream.write("from %s\n" % parent)
1146
1147        self.streamP4Files(new_files)
1148        self.gitStream.write("\n")
1149
1150        change = int(details["change"])
1151
1152        if self.labels.has_key(change):
1153            label = self.labels[change]
1154            labelDetails = label[0]
1155            labelRevisions = label[1]
1156            if self.verbose:
1157                print "Change %s is labelled %s" % (change, labelDetails)
1158
1159            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1160                                                    for p in branchPrefixes]))
1161
1162            if len(files) == len(labelRevisions):
1163
1164                cleanedFiles = {}
1165                for info in files:
1166                    if info["action"] in self.delete_actions:
1167                        continue
1168                    cleanedFiles[info["depotFile"]] = info["rev"]
1169
1170                if cleanedFiles == labelRevisions:
1171                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1172                    self.gitStream.write("from %s\n" % branch)
1173
1174                    owner = labelDetails["Owner"]
1175                    tagger = ""
1176                    if author in self.users:
1177                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1178                    else:
1179                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1180                    self.gitStream.write("tagger %s\n" % tagger)
1181                    self.gitStream.write("data <<EOT\n")
1182                    self.gitStream.write(labelDetails["Description"])
1183                    self.gitStream.write("EOT\n\n")
1184
1185                else:
1186                    if not self.silent:
1187                        print ("Tag %s does not match with change %s: files do not match."
1188                               % (labelDetails["label"], change))
1189
1190            else:
1191                if not self.silent:
1192                    print ("Tag %s does not match with change %s: file count is different."
1193                           % (labelDetails["label"], change))
1194
1195    def getUserCacheFilename(self):
1196        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1197        return home + "/.gitp4-usercache.txt"
1198
1199    def getUserMapFromPerforceServer(self):
1200        if self.userMapFromPerforceServer:
1201            return
1202        self.users = {}
1203
1204        for output in p4CmdList("users"):
1205            if not output.has_key("User"):
1206                continue
1207            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1208
1209
1210        s = ''
1211        for (key, val) in self.users.items():
1212            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1213
1214        open(self.getUserCacheFilename(), "wb").write(s)
1215        self.userMapFromPerforceServer = True
1216
1217    def loadUserMapFromCache(self):
1218        self.users = {}
1219        self.userMapFromPerforceServer = False
1220        try:
1221            cache = open(self.getUserCacheFilename(), "rb")
1222            lines = cache.readlines()
1223            cache.close()
1224            for line in lines:
1225                entry = line.strip().split("\t")
1226                self.users[entry[0]] = entry[1]
1227        except IOError:
1228            self.getUserMapFromPerforceServer()
1229
1230    def getLabels(self):
1231        self.labels = {}
1232
1233        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1234        if len(l) > 0 and not self.silent:
1235            print "Finding files belonging to labels in %s" % `self.depotPaths`
1236
1237        for output in l:
1238            label = output["label"]
1239            revisions = {}
1240            newestChange = 0
1241            if self.verbose:
1242                print "Querying files for label %s" % label
1243            for file in p4CmdList("files "
1244                                  +  ' '.join (["%s...@%s" % (p, label)
1245                                                for p in self.depotPaths])):
1246                revisions[file["depotFile"]] = file["rev"]
1247                change = int(file["change"])
1248                if change > newestChange:
1249                    newestChange = change
1250
1251            self.labels[newestChange] = [output, revisions]
1252
1253        if self.verbose:
1254            print "Label changes: %s" % self.labels.keys()
1255
1256    def guessProjectName(self):
1257        for p in self.depotPaths:
1258            if p.endswith("/"):
1259                p = p[:-1]
1260            p = p[p.strip().rfind("/") + 1:]
1261            if not p.endswith("/"):
1262               p += "/"
1263            return p
1264
1265    def getBranchMapping(self):
1266        lostAndFoundBranches = set()
1267
1268        for info in p4CmdList("branches"):
1269            details = p4Cmd("branch -o %s" % info["branch"])
1270            viewIdx = 0
1271            while details.has_key("View%s" % viewIdx):
1272                paths = details["View%s" % viewIdx].split(" ")
1273                viewIdx = viewIdx + 1
1274                # require standard //depot/foo/... //depot/bar/... mapping
1275                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1276                    continue
1277                source = paths[0]
1278                destination = paths[1]
1279                ## HACK
1280                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1281                    source = source[len(self.depotPaths[0]):-4]
1282                    destination = destination[len(self.depotPaths[0]):-4]
1283
1284                    if destination in self.knownBranches:
1285                        if not self.silent:
1286                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1287                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1288                        continue
1289
1290                    self.knownBranches[destination] = source
1291
1292                    lostAndFoundBranches.discard(destination)
1293
1294                    if source not in self.knownBranches:
1295                        lostAndFoundBranches.add(source)
1296
1297
1298        for branch in lostAndFoundBranches:
1299            self.knownBranches[branch] = branch
1300
1301    def getBranchMappingFromGitBranches(self):
1302        branches = p4BranchesInGit(self.importIntoRemotes)
1303        for branch in branches.keys():
1304            if branch == "master":
1305                branch = "main"
1306            else:
1307                branch = branch[len(self.projectName):]
1308            self.knownBranches[branch] = branch
1309
1310    def listExistingP4GitBranches(self):
1311        # branches holds mapping from name to commit
1312        branches = p4BranchesInGit(self.importIntoRemotes)
1313        self.p4BranchesInGit = branches.keys()
1314        for branch in branches.keys():
1315            self.initialParents[self.refPrefix + branch] = branches[branch]
1316
1317    def updateOptionDict(self, d):
1318        option_keys = {}
1319        if self.keepRepoPath:
1320            option_keys['keepRepoPath'] = 1
1321
1322        d["options"] = ' '.join(sorted(option_keys.keys()))
1323
1324    def readOptions(self, d):
1325        self.keepRepoPath = (d.has_key('options')
1326                             and ('keepRepoPath' in d['options']))
1327
1328    def gitRefForBranch(self, branch):
1329        if branch == "main":
1330            return self.refPrefix + "master"
1331
1332        if len(branch) <= 0:
1333            return branch
1334
1335        return self.refPrefix + self.projectName + branch
1336
1337    def gitCommitByP4Change(self, ref, change):
1338        if self.verbose:
1339            print "looking in ref " + ref + " for change %s using bisect..." % change
1340
1341        earliestCommit = ""
1342        latestCommit = parseRevision(ref)
1343
1344        while True:
1345            if self.verbose:
1346                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1347            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1348            if len(next) == 0:
1349                if self.verbose:
1350                    print "argh"
1351                return ""
1352            log = extractLogMessageFromGitCommit(next)
1353            settings = extractSettingsGitLog(log)
1354            currentChange = int(settings['change'])
1355            if self.verbose:
1356                print "current change %s" % currentChange
1357
1358            if currentChange == change:
1359                if self.verbose:
1360                    print "found %s" % next
1361                return next
1362
1363            if currentChange < change:
1364                earliestCommit = "^%s" % next
1365            else:
1366                latestCommit = "%s" % next
1367
1368        return ""
1369
1370    def importNewBranch(self, branch, maxChange):
1371        # make fast-import flush all changes to disk and update the refs using the checkpoint
1372        # command so that we can try to find the branch parent in the git history
1373        self.gitStream.write("checkpoint\n\n");
1374        self.gitStream.flush();
1375        branchPrefix = self.depotPaths[0] + branch + "/"
1376        range = "@1,%s" % maxChange
1377        #print "prefix" + branchPrefix
1378        changes = p4ChangesForPaths([branchPrefix], range)
1379        if len(changes) <= 0:
1380            return False
1381        firstChange = changes[0]
1382        #print "first change in branch: %s" % firstChange
1383        sourceBranch = self.knownBranches[branch]
1384        sourceDepotPath = self.depotPaths[0] + sourceBranch
1385        sourceRef = self.gitRefForBranch(sourceBranch)
1386        #print "source " + sourceBranch
1387
1388        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1389        #print "branch parent: %s" % branchParentChange
1390        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1391        if len(gitParent) > 0:
1392            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1393            #print "parent git commit: %s" % gitParent
1394
1395        self.importChanges(changes)
1396        return True
1397
1398    def importChanges(self, changes):
1399        cnt = 1
1400        for change in changes:
1401            description = p4Cmd("describe %s" % change)
1402            self.updateOptionDict(description)
1403
1404            if not self.silent:
1405                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1406                sys.stdout.flush()
1407            cnt = cnt + 1
1408
1409            try:
1410                if self.detectBranches:
1411                    branches = self.splitFilesIntoBranches(description)
1412                    for branch in branches.keys():
1413                        ## HACK  --hwn
1414                        branchPrefix = self.depotPaths[0] + branch + "/"
1415
1416                        parent = ""
1417
1418                        filesForCommit = branches[branch]
1419
1420                        if self.verbose:
1421                            print "branch is %s" % branch
1422
1423                        self.updatedBranches.add(branch)
1424
1425                        if branch not in self.createdBranches:
1426                            self.createdBranches.add(branch)
1427                            parent = self.knownBranches[branch]
1428                            if parent == branch:
1429                                parent = ""
1430                            else:
1431                                fullBranch = self.projectName + branch
1432                                if fullBranch not in self.p4BranchesInGit:
1433                                    if not self.silent:
1434                                        print("\n    Importing new branch %s" % fullBranch);
1435                                    if self.importNewBranch(branch, change - 1):
1436                                        parent = ""
1437                                        self.p4BranchesInGit.append(fullBranch)
1438                                    if not self.silent:
1439                                        print("\n    Resuming with change %s" % change);
1440
1441                                if self.verbose:
1442                                    print "parent determined through known branches: %s" % parent
1443
1444                        branch = self.gitRefForBranch(branch)
1445                        parent = self.gitRefForBranch(parent)
1446
1447                        if self.verbose:
1448                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1449
1450                        if len(parent) == 0 and branch in self.initialParents:
1451                            parent = self.initialParents[branch]
1452                            del self.initialParents[branch]
1453
1454                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1455                else:
1456                    files = self.extractFilesFromCommit(description)
1457                    self.commit(description, files, self.branch, self.depotPaths,
1458                                self.initialParent)
1459                    self.initialParent = ""
1460            except IOError:
1461                print self.gitError.read()
1462                sys.exit(1)
1463
1464    def importHeadRevision(self, revision):
1465        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1466
1467        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1468        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1469                           % (' '.join(self.depotPaths), revision))
1470        details["change"] = revision
1471        newestRevision = 0
1472
1473        fileCnt = 0
1474        for info in p4CmdList("files "
1475                              +  ' '.join(["%s...%s"
1476                                           % (p, revision)
1477                                           for p in self.depotPaths])):
1478
1479            if 'code' in info and info['code'] == 'error':
1480                sys.stderr.write("p4 returned an error: %s\n"
1481                                 % info['data'])
1482                if info['data'].find("must refer to client") >= 0:
1483                    sys.stderr.write("This particular p4 error is misleading.\n")
1484                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
1485                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1486                sys.exit(1)
1487            if 'p4ExitCode' in info:
1488                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1489                sys.exit(1)
1490
1491
1492            change = int(info["change"])
1493            if change > newestRevision:
1494                newestRevision = change
1495
1496            if info["action"] in self.delete_actions:
1497                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1498                #fileCnt = fileCnt + 1
1499                continue
1500
1501            for prop in ["depotFile", "rev", "action", "type" ]:
1502                details["%s%s" % (prop, fileCnt)] = info[prop]
1503
1504            fileCnt = fileCnt + 1
1505
1506        details["change"] = newestRevision
1507        self.updateOptionDict(details)
1508        try:
1509            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1510        except IOError:
1511            print "IO error with git fast-import. Is your git version recent enough?"
1512            print self.gitError.read()
1513
1514
1515    def getClientSpec(self):
1516        specList = p4CmdList( "client -o" )
1517        temp = {}
1518        for entry in specList:
1519            for k,v in entry.iteritems():
1520                if k.startswith("View"):
1521
1522                    # p4 has these %%1 to %%9 arguments in specs to
1523                    # reorder paths; which we can't handle (yet :)
1524                    if re.match('%%\d', v) != None:
1525                        print "Sorry, can't handle %%n arguments in client specs"
1526                        sys.exit(1)
1527
1528                    if v.startswith('"'):
1529                        start = 1
1530                    else:
1531                        start = 0
1532                    index = v.find("...")
1533
1534                    # save the "client view"; i.e the RHS of the view
1535                    # line that tells the client where to put the
1536                    # files for this view.
1537                    cv = v[index+3:].strip() # +3 to remove previous '...'
1538
1539                    # if the client view doesn't end with a
1540                    # ... wildcard, then we're going to mess up the
1541                    # output directory, so fail gracefully.
1542                    if not cv.endswith('...'):
1543                        print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1544                        sys.exit(1)
1545                    cv=cv[:-3]
1546
1547                    # now save the view; +index means included, -index
1548                    # means it should be filtered out.
1549                    v = v[start:index]
1550                    if v.startswith("-"):
1551                        v = v[1:]
1552                        include = -len(v)
1553                    else:
1554                        include = len(v)
1555
1556                    temp[v] = (include, cv)
1557
1558        self.clientSpecDirs = temp.items()
1559        self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1560
1561    def run(self, args):
1562        self.depotPaths = []
1563        self.changeRange = ""
1564        self.initialParent = ""
1565        self.previousDepotPaths = []
1566
1567        # map from branch depot path to parent branch
1568        self.knownBranches = {}
1569        self.initialParents = {}
1570        self.hasOrigin = originP4BranchesExist()
1571        if not self.syncWithOrigin:
1572            self.hasOrigin = False
1573
1574        if self.importIntoRemotes:
1575            self.refPrefix = "refs/remotes/p4/"
1576        else:
1577            self.refPrefix = "refs/heads/p4/"
1578
1579        if self.syncWithOrigin and self.hasOrigin:
1580            if not self.silent:
1581                print "Syncing with origin first by calling git fetch origin"
1582            system("git fetch origin")
1583
1584        if len(self.branch) == 0:
1585            self.branch = self.refPrefix + "master"
1586            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1587                system("git update-ref %s refs/heads/p4" % self.branch)
1588                system("git branch -D p4");
1589            # create it /after/ importing, when master exists
1590            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1591                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1592
1593        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1594            self.getClientSpec()
1595
1596        # TODO: should always look at previous commits,
1597        # merge with previous imports, if possible.
1598        if args == []:
1599            if self.hasOrigin:
1600                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1601            self.listExistingP4GitBranches()
1602
1603            if len(self.p4BranchesInGit) > 1:
1604                if not self.silent:
1605                    print "Importing from/into multiple branches"
1606                self.detectBranches = True
1607
1608            if self.verbose:
1609                print "branches: %s" % self.p4BranchesInGit
1610
1611            p4Change = 0
1612            for branch in self.p4BranchesInGit:
1613                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1614
1615                settings = extractSettingsGitLog(logMsg)
1616
1617                self.readOptions(settings)
1618                if (settings.has_key('depot-paths')
1619                    and settings.has_key ('change')):
1620                    change = int(settings['change']) + 1
1621                    p4Change = max(p4Change, change)
1622
1623                    depotPaths = sorted(settings['depot-paths'])
1624                    if self.previousDepotPaths == []:
1625                        self.previousDepotPaths = depotPaths
1626                    else:
1627                        paths = []
1628                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1629                            for i in range(0, min(len(cur), len(prev))):
1630                                if cur[i] <> prev[i]:
1631                                    i = i - 1
1632                                    break
1633
1634                            paths.append (cur[:i + 1])
1635
1636                        self.previousDepotPaths = paths
1637
1638            if p4Change > 0:
1639                self.depotPaths = sorted(self.previousDepotPaths)
1640                self.changeRange = "@%s,#head" % p4Change
1641                if not self.detectBranches:
1642                    self.initialParent = parseRevision(self.branch)
1643                if not self.silent and not self.detectBranches:
1644                    print "Performing incremental import into %s git branch" % self.branch
1645
1646        if not self.branch.startswith("refs/"):
1647            self.branch = "refs/heads/" + self.branch
1648
1649        if len(args) == 0 and self.depotPaths:
1650            if not self.silent:
1651                print "Depot paths: %s" % ' '.join(self.depotPaths)
1652        else:
1653            if self.depotPaths and self.depotPaths != args:
1654                print ("previous import used depot path %s and now %s was specified. "
1655                       "This doesn't work!" % (' '.join (self.depotPaths),
1656                                               ' '.join (args)))
1657                sys.exit(1)
1658
1659            self.depotPaths = sorted(args)
1660
1661        revision = ""
1662        self.users = {}
1663
1664        newPaths = []
1665        for p in self.depotPaths:
1666            if p.find("@") != -1:
1667                atIdx = p.index("@")
1668                self.changeRange = p[atIdx:]
1669                if self.changeRange == "@all":
1670                    self.changeRange = ""
1671                elif ',' not in self.changeRange:
1672                    revision = self.changeRange
1673                    self.changeRange = ""
1674                p = p[:atIdx]
1675            elif p.find("#") != -1:
1676                hashIdx = p.index("#")
1677                revision = p[hashIdx:]
1678                p = p[:hashIdx]
1679            elif self.previousDepotPaths == []:
1680                revision = "#head"
1681
1682            p = re.sub ("\.\.\.$", "", p)
1683            if not p.endswith("/"):
1684                p += "/"
1685
1686            newPaths.append(p)
1687
1688        self.depotPaths = newPaths
1689
1690
1691        self.loadUserMapFromCache()
1692        self.labels = {}
1693        if self.detectLabels:
1694            self.getLabels();
1695
1696        if self.detectBranches:
1697            ## FIXME - what's a P4 projectName ?
1698            self.projectName = self.guessProjectName()
1699
1700            if self.hasOrigin:
1701                self.getBranchMappingFromGitBranches()
1702            else:
1703                self.getBranchMapping()
1704            if self.verbose:
1705                print "p4-git branches: %s" % self.p4BranchesInGit
1706                print "initial parents: %s" % self.initialParents
1707            for b in self.p4BranchesInGit:
1708                if b != "master":
1709
1710                    ## FIXME
1711                    b = b[len(self.projectName):]
1712                self.createdBranches.add(b)
1713
1714        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1715
1716        importProcess = subprocess.Popen(["git", "fast-import"],
1717                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1718                                         stderr=subprocess.PIPE);
1719        self.gitOutput = importProcess.stdout
1720        self.gitStream = importProcess.stdin
1721        self.gitError = importProcess.stderr
1722
1723        if revision:
1724            self.importHeadRevision(revision)
1725        else:
1726            changes = []
1727
1728            if len(self.changesFile) > 0:
1729                output = open(self.changesFile).readlines()
1730                changeSet = set()
1731                for line in output:
1732                    changeSet.add(int(line))
1733
1734                for change in changeSet:
1735                    changes.append(change)
1736
1737                changes.sort()
1738            else:
1739                if not self.p4BranchesInGit:
1740                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1741                if self.verbose:
1742                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1743                                                              self.changeRange)
1744                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1745
1746                if len(self.maxChanges) > 0:
1747                    changes = changes[:min(int(self.maxChanges), len(changes))]
1748
1749            if len(changes) == 0:
1750                if not self.silent:
1751                    print "No changes to import!"
1752                return True
1753
1754            if not self.silent and not self.detectBranches:
1755                print "Import destination: %s" % self.branch
1756
1757            self.updatedBranches = set()
1758
1759            self.importChanges(changes)
1760
1761            if not self.silent:
1762                print ""
1763                if len(self.updatedBranches) > 0:
1764                    sys.stdout.write("Updated branches: ")
1765                    for b in self.updatedBranches:
1766                        sys.stdout.write("%s " % b)
1767                    sys.stdout.write("\n")
1768
1769        self.gitStream.close()
1770        if importProcess.wait() != 0:
1771            die("fast-import failed: %s" % self.gitError.read())
1772        self.gitOutput.close()
1773        self.gitError.close()
1774
1775        return True
1776
1777class P4Rebase(Command):
1778    def __init__(self):
1779        Command.__init__(self)
1780        self.options = [ ]
1781        self.description = ("Fetches the latest revision from perforce and "
1782                            + "rebases the current work (branch) against it")
1783        self.verbose = False
1784
1785    def run(self, args):
1786        sync = P4Sync()
1787        sync.run([])
1788
1789        return self.rebase()
1790
1791    def rebase(self):
1792        if os.system("git update-index --refresh") != 0:
1793            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1794        if len(read_pipe("git diff-index HEAD --")) > 0:
1795            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1796
1797        [upstream, settings] = findUpstreamBranchPoint()
1798        if len(upstream) == 0:
1799            die("Cannot find upstream branchpoint for rebase")
1800
1801        # the branchpoint may be p4/foo~3, so strip off the parent
1802        upstream = re.sub("~[0-9]+$", "", upstream)
1803
1804        print "Rebasing the current branch onto %s" % upstream
1805        oldHead = read_pipe("git rev-parse HEAD").strip()
1806        system("git rebase %s" % upstream)
1807        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1808        return True
1809
1810class P4Clone(P4Sync):
1811    def __init__(self):
1812        P4Sync.__init__(self)
1813        self.description = "Creates a new git repository and imports from Perforce into it"
1814        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1815        self.options += [
1816            optparse.make_option("--destination", dest="cloneDestination",
1817                                 action='store', default=None,
1818                                 help="where to leave result of the clone"),
1819            optparse.make_option("-/", dest="cloneExclude",
1820                                 action="append", type="string",
1821                                 help="exclude depot path"),
1822            optparse.make_option("--bare", dest="cloneBare",
1823                                 action="store_true", default=False),
1824        ]
1825        self.cloneDestination = None
1826        self.needsGit = False
1827        self.cloneBare = False
1828
1829    # This is required for the "append" cloneExclude action
1830    def ensure_value(self, attr, value):
1831        if not hasattr(self, attr) or getattr(self, attr) is None:
1832            setattr(self, attr, value)
1833        return getattr(self, attr)
1834
1835    def defaultDestination(self, args):
1836        ## TODO: use common prefix of args?
1837        depotPath = args[0]
1838        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1839        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1840        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1841        depotDir = re.sub(r"/$", "", depotDir)
1842        return os.path.split(depotDir)[1]
1843
1844    def run(self, args):
1845        if len(args) < 1:
1846            return False
1847
1848        if self.keepRepoPath and not self.cloneDestination:
1849            sys.stderr.write("Must specify destination for --keep-path\n")
1850            sys.exit(1)
1851
1852        depotPaths = args
1853
1854        if not self.cloneDestination and len(depotPaths) > 1:
1855            self.cloneDestination = depotPaths[-1]
1856            depotPaths = depotPaths[:-1]
1857
1858        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1859        for p in depotPaths:
1860            if not p.startswith("//"):
1861                return False
1862
1863        if not self.cloneDestination:
1864            self.cloneDestination = self.defaultDestination(args)
1865
1866        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1867
1868        if not os.path.exists(self.cloneDestination):
1869            os.makedirs(self.cloneDestination)
1870        chdir(self.cloneDestination)
1871
1872        init_cmd = [ "git", "init" ]
1873        if self.cloneBare:
1874            init_cmd.append("--bare")
1875        subprocess.check_call(init_cmd)
1876
1877        if not P4Sync.run(self, depotPaths):
1878            return False
1879        if self.branch != "master":
1880            if self.importIntoRemotes:
1881                masterbranch = "refs/remotes/p4/master"
1882            else:
1883                masterbranch = "refs/heads/p4/master"
1884            if gitBranchExists(masterbranch):
1885                system("git branch master %s" % masterbranch)
1886                if not self.cloneBare:
1887                    system("git checkout -f")
1888            else:
1889                print "Could not detect main branch. No checkout/master branch created."
1890
1891        return True
1892
1893class P4Branches(Command):
1894    def __init__(self):
1895        Command.__init__(self)
1896        self.options = [ ]
1897        self.description = ("Shows the git branches that hold imports and their "
1898                            + "corresponding perforce depot paths")
1899        self.verbose = False
1900
1901    def run(self, args):
1902        if originP4BranchesExist():
1903            createOrUpdateBranchesFromOrigin()
1904
1905        cmdline = "git rev-parse --symbolic "
1906        cmdline += " --remotes"
1907
1908        for line in read_pipe_lines(cmdline):
1909            line = line.strip()
1910
1911            if not line.startswith('p4/') or line == "p4/HEAD":
1912                continue
1913            branch = line
1914
1915            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1916            settings = extractSettingsGitLog(log)
1917
1918            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1919        return True
1920
1921class HelpFormatter(optparse.IndentedHelpFormatter):
1922    def __init__(self):
1923        optparse.IndentedHelpFormatter.__init__(self)
1924
1925    def format_description(self, description):
1926        if description:
1927            return description + "\n"
1928        else:
1929            return ""
1930
1931def printUsage(commands):
1932    print "usage: %s <command> [options]" % sys.argv[0]
1933    print ""
1934    print "valid commands: %s" % ", ".join(commands)
1935    print ""
1936    print "Try %s <command> --help for command specific help." % sys.argv[0]
1937    print ""
1938
1939commands = {
1940    "debug" : P4Debug,
1941    "submit" : P4Submit,
1942    "commit" : P4Submit,
1943    "sync" : P4Sync,
1944    "rebase" : P4Rebase,
1945    "clone" : P4Clone,
1946    "rollback" : P4RollBack,
1947    "branches" : P4Branches
1948}
1949
1950
1951def main():
1952    if len(sys.argv[1:]) == 0:
1953        printUsage(commands.keys())
1954        sys.exit(2)
1955
1956    cmd = ""
1957    cmdName = sys.argv[1]
1958    try:
1959        klass = commands[cmdName]
1960        cmd = klass()
1961    except KeyError:
1962        print "unknown command %s" % cmdName
1963        print ""
1964        printUsage(commands.keys())
1965        sys.exit(2)
1966
1967    options = cmd.options
1968    cmd.gitdir = os.environ.get("GIT_DIR", None)
1969
1970    args = sys.argv[2:]
1971
1972    if len(options) > 0:
1973        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1974
1975        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1976                                       options,
1977                                       description = cmd.description,
1978                                       formatter = HelpFormatter())
1979
1980        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1981    global verbose
1982    verbose = cmd.verbose
1983    if cmd.needsGit:
1984        if cmd.gitdir == None:
1985            cmd.gitdir = os.path.abspath(".git")
1986            if not isValidGitDir(cmd.gitdir):
1987                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1988                if os.path.exists(cmd.gitdir):
1989                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1990                    if len(cdup) > 0:
1991                        chdir(cdup);
1992
1993        if not isValidGitDir(cmd.gitdir):
1994            if isValidGitDir(cmd.gitdir + "/.git"):
1995                cmd.gitdir += "/.git"
1996            else:
1997                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1998
1999        os.environ["GIT_DIR"] = cmd.gitdir
2000
2001    if not cmd.run(args):
2002        parser.print_help()
2003
2004
2005if __name__ == '__main__':
2006    main()