Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: accommodate new move/delete type in p4 (56c0934)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    delete_actions = ( "delete", "move/delete", "purge" )
 838
 839    def __init__(self):
 840        Command.__init__(self)
 841        self.options = [
 842                optparse.make_option("--branch", dest="branch"),
 843                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                optparse.make_option("--changesfile", dest="changesFile"),
 845                optparse.make_option("--silent", dest="silent", action="store_true"),
 846                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                     help="Import into refs/heads/ , not refs/remotes"),
 850                optparse.make_option("--max-changes", dest="maxChanges"),
 851                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                     help="Only sync files that are included in the Perforce Client Spec")
 855        ]
 856        self.description = """Imports from Perforce into a git repository.\n
 857    example:
 858    //depot/my/project/ -- to import the current head
 859    //depot/my/project/@all -- to import everything
 860    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864        self.usage += " //depot/path[@revRange]"
 865        self.silent = False
 866        self.createdBranches = set()
 867        self.committedChanges = set()
 868        self.branch = ""
 869        self.detectBranches = False
 870        self.detectLabels = False
 871        self.changesFile = ""
 872        self.syncWithOrigin = True
 873        self.verbose = False
 874        self.importIntoRemotes = True
 875        self.maxChanges = ""
 876        self.isWindows = (platform.system() == "Windows")
 877        self.keepRepoPath = False
 878        self.depotPaths = None
 879        self.p4BranchesInGit = []
 880        self.cloneExclude = []
 881        self.useClientSpec = False
 882        self.clientSpecDirs = []
 883
 884        if gitConfig("git-p4.syncFromOrigin") == "false":
 885            self.syncWithOrigin = False
 886
 887    def extractFilesFromCommit(self, commit):
 888        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 889                             for path in self.cloneExclude]
 890        files = []
 891        fnum = 0
 892        while commit.has_key("depotFile%s" % fnum):
 893            path =  commit["depotFile%s" % fnum]
 894
 895            if [p for p in self.cloneExclude
 896                if path.startswith (p)]:
 897                found = False
 898            else:
 899                found = [p for p in self.depotPaths
 900                         if path.startswith (p)]
 901            if not found:
 902                fnum = fnum + 1
 903                continue
 904
 905            file = {}
 906            file["path"] = path
 907            file["rev"] = commit["rev%s" % fnum]
 908            file["action"] = commit["action%s" % fnum]
 909            file["type"] = commit["type%s" % fnum]
 910            files.append(file)
 911            fnum = fnum + 1
 912        return files
 913
 914    def stripRepoPath(self, path, prefixes):
 915        if self.keepRepoPath:
 916            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 917
 918        for p in prefixes:
 919            if path.startswith(p):
 920                path = path[len(p):]
 921
 922        return path
 923
 924    def splitFilesIntoBranches(self, commit):
 925        branches = {}
 926        fnum = 0
 927        while commit.has_key("depotFile%s" % fnum):
 928            path =  commit["depotFile%s" % fnum]
 929            found = [p for p in self.depotPaths
 930                     if path.startswith (p)]
 931            if not found:
 932                fnum = fnum + 1
 933                continue
 934
 935            file = {}
 936            file["path"] = path
 937            file["rev"] = commit["rev%s" % fnum]
 938            file["action"] = commit["action%s" % fnum]
 939            file["type"] = commit["type%s" % fnum]
 940            fnum = fnum + 1
 941
 942            relPath = self.stripRepoPath(path, self.depotPaths)
 943
 944            for branch in self.knownBranches.keys():
 945
 946                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 947                if relPath.startswith(branch + "/"):
 948                    if branch not in branches:
 949                        branches[branch] = []
 950                    branches[branch].append(file)
 951                    break
 952
 953        return branches
 954
 955    # output one file from the P4 stream
 956    # - helper for streamP4Files
 957
 958    def streamOneP4File(self, file, contents):
 959        if file["type"] == "apple":
 960            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 961                file['depotFile']
 962            return
 963
 964        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 965        if verbose:
 966            sys.stderr.write("%s\n" % relPath)
 967
 968        mode = "644"
 969        if isP4Exec(file["type"]):
 970            mode = "755"
 971        elif file["type"] == "symlink":
 972            mode = "120000"
 973            # p4 print on a symlink contains "target\n", so strip it off
 974            data = ''.join(contents)
 975            contents = [data[:-1]]
 976
 977        if self.isWindows and file["type"].endswith("text"):
 978            mangled = []
 979            for data in contents:
 980                data = data.replace("\r\n", "\n")
 981                mangled.append(data)
 982            contents = mangled
 983
 984        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 985            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 986        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 987            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 988
 989        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990
 991        # total length...
 992        length = 0
 993        for d in contents:
 994            length = length + len(d)
 995
 996        self.gitStream.write("data %d\n" % length)
 997        for d in contents:
 998            self.gitStream.write(d)
 999        self.gitStream.write("\n")
1000
1001    def streamOneP4Deletion(self, file):
1002        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1003        if verbose:
1004            sys.stderr.write("delete %s\n" % relPath)
1005        self.gitStream.write("D %s\n" % relPath)
1006
1007    # handle another chunk of streaming data
1008    def streamP4FilesCb(self, marshalled):
1009
1010        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1011            # start of a new file - output the old one first
1012            self.streamOneP4File(self.stream_file, self.stream_contents)
1013            self.stream_file = {}
1014            self.stream_contents = []
1015            self.stream_have_file_info = False
1016
1017        # pick up the new file information... for the
1018        # 'data' field we need to append to our array
1019        for k in marshalled.keys():
1020            if k == 'data':
1021                self.stream_contents.append(marshalled['data'])
1022            else:
1023                self.stream_file[k] = marshalled[k]
1024
1025        self.stream_have_file_info = True
1026
1027    # Stream directly from "p4 files" into "git fast-import"
1028    def streamP4Files(self, files):
1029        filesForCommit = []
1030        filesToRead = []
1031        filesToDelete = []
1032
1033        for f in files:
1034            includeFile = True
1035            for val in self.clientSpecDirs:
1036                if f['path'].startswith(val[0]):
1037                    if val[1] <= 0:
1038                        includeFile = False
1039                    break
1040
1041            if includeFile:
1042                filesForCommit.append(f)
1043                if f['action'] in self.delete_actions:
1044                    filesToDelete.append(f)
1045                else:
1046                    filesToRead.append(f)
1047
1048        # deleted files...
1049        for f in filesToDelete:
1050            self.streamOneP4Deletion(f)
1051
1052        if len(filesToRead) > 0:
1053            self.stream_file = {}
1054            self.stream_contents = []
1055            self.stream_have_file_info = False
1056
1057            # curry self argument
1058            def streamP4FilesCbSelf(entry):
1059                self.streamP4FilesCb(entry)
1060
1061            p4CmdList("-x - print",
1062                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1063                                                  for f in filesToRead]),
1064                cb=streamP4FilesCbSelf)
1065
1066            # do the last chunk
1067            if self.stream_file.has_key('depotFile'):
1068                self.streamOneP4File(self.stream_file, self.stream_contents)
1069
1070    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1071        epoch = details["time"]
1072        author = details["user"]
1073        self.branchPrefixes = branchPrefixes
1074
1075        if self.verbose:
1076            print "commit into %s" % branch
1077
1078        # start with reading files; if that fails, we should not
1079        # create a commit.
1080        new_files = []
1081        for f in files:
1082            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1083                new_files.append (f)
1084            else:
1085                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1086
1087        self.gitStream.write("commit %s\n" % branch)
1088#        gitStream.write("mark :%s\n" % details["change"])
1089        self.committedChanges.add(int(details["change"]))
1090        committer = ""
1091        if author not in self.users:
1092            self.getUserMapFromPerforceServer()
1093        if author in self.users:
1094            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1095        else:
1096            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1097
1098        self.gitStream.write("committer %s\n" % committer)
1099
1100        self.gitStream.write("data <<EOT\n")
1101        self.gitStream.write(details["desc"])
1102        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1103                             % (','.join (branchPrefixes), details["change"]))
1104        if len(details['options']) > 0:
1105            self.gitStream.write(": options = %s" % details['options'])
1106        self.gitStream.write("]\nEOT\n\n")
1107
1108        if len(parent) > 0:
1109            if self.verbose:
1110                print "parent %s" % parent
1111            self.gitStream.write("from %s\n" % parent)
1112
1113        self.streamP4Files(new_files)
1114        self.gitStream.write("\n")
1115
1116        change = int(details["change"])
1117
1118        if self.labels.has_key(change):
1119            label = self.labels[change]
1120            labelDetails = label[0]
1121            labelRevisions = label[1]
1122            if self.verbose:
1123                print "Change %s is labelled %s" % (change, labelDetails)
1124
1125            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1126                                                    for p in branchPrefixes]))
1127
1128            if len(files) == len(labelRevisions):
1129
1130                cleanedFiles = {}
1131                for info in files:
1132                    if info["action"] in self.delete_actions:
1133                        continue
1134                    cleanedFiles[info["depotFile"]] = info["rev"]
1135
1136                if cleanedFiles == labelRevisions:
1137                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1138                    self.gitStream.write("from %s\n" % branch)
1139
1140                    owner = labelDetails["Owner"]
1141                    tagger = ""
1142                    if author in self.users:
1143                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1144                    else:
1145                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1146                    self.gitStream.write("tagger %s\n" % tagger)
1147                    self.gitStream.write("data <<EOT\n")
1148                    self.gitStream.write(labelDetails["Description"])
1149                    self.gitStream.write("EOT\n\n")
1150
1151                else:
1152                    if not self.silent:
1153                        print ("Tag %s does not match with change %s: files do not match."
1154                               % (labelDetails["label"], change))
1155
1156            else:
1157                if not self.silent:
1158                    print ("Tag %s does not match with change %s: file count is different."
1159                           % (labelDetails["label"], change))
1160
1161    def getUserCacheFilename(self):
1162        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1163        return home + "/.gitp4-usercache.txt"
1164
1165    def getUserMapFromPerforceServer(self):
1166        if self.userMapFromPerforceServer:
1167            return
1168        self.users = {}
1169
1170        for output in p4CmdList("users"):
1171            if not output.has_key("User"):
1172                continue
1173            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174
1175
1176        s = ''
1177        for (key, val) in self.users.items():
1178            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1179
1180        open(self.getUserCacheFilename(), "wb").write(s)
1181        self.userMapFromPerforceServer = True
1182
1183    def loadUserMapFromCache(self):
1184        self.users = {}
1185        self.userMapFromPerforceServer = False
1186        try:
1187            cache = open(self.getUserCacheFilename(), "rb")
1188            lines = cache.readlines()
1189            cache.close()
1190            for line in lines:
1191                entry = line.strip().split("\t")
1192                self.users[entry[0]] = entry[1]
1193        except IOError:
1194            self.getUserMapFromPerforceServer()
1195
1196    def getLabels(self):
1197        self.labels = {}
1198
1199        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1200        if len(l) > 0 and not self.silent:
1201            print "Finding files belonging to labels in %s" % `self.depotPaths`
1202
1203        for output in l:
1204            label = output["label"]
1205            revisions = {}
1206            newestChange = 0
1207            if self.verbose:
1208                print "Querying files for label %s" % label
1209            for file in p4CmdList("files "
1210                                  +  ' '.join (["%s...@%s" % (p, label)
1211                                                for p in self.depotPaths])):
1212                revisions[file["depotFile"]] = file["rev"]
1213                change = int(file["change"])
1214                if change > newestChange:
1215                    newestChange = change
1216
1217            self.labels[newestChange] = [output, revisions]
1218
1219        if self.verbose:
1220            print "Label changes: %s" % self.labels.keys()
1221
1222    def guessProjectName(self):
1223        for p in self.depotPaths:
1224            if p.endswith("/"):
1225                p = p[:-1]
1226            p = p[p.strip().rfind("/") + 1:]
1227            if not p.endswith("/"):
1228               p += "/"
1229            return p
1230
1231    def getBranchMapping(self):
1232        lostAndFoundBranches = set()
1233
1234        for info in p4CmdList("branches"):
1235            details = p4Cmd("branch -o %s" % info["branch"])
1236            viewIdx = 0
1237            while details.has_key("View%s" % viewIdx):
1238                paths = details["View%s" % viewIdx].split(" ")
1239                viewIdx = viewIdx + 1
1240                # require standard //depot/foo/... //depot/bar/... mapping
1241                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1242                    continue
1243                source = paths[0]
1244                destination = paths[1]
1245                ## HACK
1246                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1247                    source = source[len(self.depotPaths[0]):-4]
1248                    destination = destination[len(self.depotPaths[0]):-4]
1249
1250                    if destination in self.knownBranches:
1251                        if not self.silent:
1252                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1253                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1254                        continue
1255
1256                    self.knownBranches[destination] = source
1257
1258                    lostAndFoundBranches.discard(destination)
1259
1260                    if source not in self.knownBranches:
1261                        lostAndFoundBranches.add(source)
1262
1263
1264        for branch in lostAndFoundBranches:
1265            self.knownBranches[branch] = branch
1266
1267    def getBranchMappingFromGitBranches(self):
1268        branches = p4BranchesInGit(self.importIntoRemotes)
1269        for branch in branches.keys():
1270            if branch == "master":
1271                branch = "main"
1272            else:
1273                branch = branch[len(self.projectName):]
1274            self.knownBranches[branch] = branch
1275
1276    def listExistingP4GitBranches(self):
1277        # branches holds mapping from name to commit
1278        branches = p4BranchesInGit(self.importIntoRemotes)
1279        self.p4BranchesInGit = branches.keys()
1280        for branch in branches.keys():
1281            self.initialParents[self.refPrefix + branch] = branches[branch]
1282
1283    def updateOptionDict(self, d):
1284        option_keys = {}
1285        if self.keepRepoPath:
1286            option_keys['keepRepoPath'] = 1
1287
1288        d["options"] = ' '.join(sorted(option_keys.keys()))
1289
1290    def readOptions(self, d):
1291        self.keepRepoPath = (d.has_key('options')
1292                             and ('keepRepoPath' in d['options']))
1293
1294    def gitRefForBranch(self, branch):
1295        if branch == "main":
1296            return self.refPrefix + "master"
1297
1298        if len(branch) <= 0:
1299            return branch
1300
1301        return self.refPrefix + self.projectName + branch
1302
1303    def gitCommitByP4Change(self, ref, change):
1304        if self.verbose:
1305            print "looking in ref " + ref + " for change %s using bisect..." % change
1306
1307        earliestCommit = ""
1308        latestCommit = parseRevision(ref)
1309
1310        while True:
1311            if self.verbose:
1312                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1313            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1314            if len(next) == 0:
1315                if self.verbose:
1316                    print "argh"
1317                return ""
1318            log = extractLogMessageFromGitCommit(next)
1319            settings = extractSettingsGitLog(log)
1320            currentChange = int(settings['change'])
1321            if self.verbose:
1322                print "current change %s" % currentChange
1323
1324            if currentChange == change:
1325                if self.verbose:
1326                    print "found %s" % next
1327                return next
1328
1329            if currentChange < change:
1330                earliestCommit = "^%s" % next
1331            else:
1332                latestCommit = "%s" % next
1333
1334        return ""
1335
1336    def importNewBranch(self, branch, maxChange):
1337        # make fast-import flush all changes to disk and update the refs using the checkpoint
1338        # command so that we can try to find the branch parent in the git history
1339        self.gitStream.write("checkpoint\n\n");
1340        self.gitStream.flush();
1341        branchPrefix = self.depotPaths[0] + branch + "/"
1342        range = "@1,%s" % maxChange
1343        #print "prefix" + branchPrefix
1344        changes = p4ChangesForPaths([branchPrefix], range)
1345        if len(changes) <= 0:
1346            return False
1347        firstChange = changes[0]
1348        #print "first change in branch: %s" % firstChange
1349        sourceBranch = self.knownBranches[branch]
1350        sourceDepotPath = self.depotPaths[0] + sourceBranch
1351        sourceRef = self.gitRefForBranch(sourceBranch)
1352        #print "source " + sourceBranch
1353
1354        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1355        #print "branch parent: %s" % branchParentChange
1356        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1357        if len(gitParent) > 0:
1358            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1359            #print "parent git commit: %s" % gitParent
1360
1361        self.importChanges(changes)
1362        return True
1363
1364    def importChanges(self, changes):
1365        cnt = 1
1366        for change in changes:
1367            description = p4Cmd("describe %s" % change)
1368            self.updateOptionDict(description)
1369
1370            if not self.silent:
1371                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1372                sys.stdout.flush()
1373            cnt = cnt + 1
1374
1375            try:
1376                if self.detectBranches:
1377                    branches = self.splitFilesIntoBranches(description)
1378                    for branch in branches.keys():
1379                        ## HACK  --hwn
1380                        branchPrefix = self.depotPaths[0] + branch + "/"
1381
1382                        parent = ""
1383
1384                        filesForCommit = branches[branch]
1385
1386                        if self.verbose:
1387                            print "branch is %s" % branch
1388
1389                        self.updatedBranches.add(branch)
1390
1391                        if branch not in self.createdBranches:
1392                            self.createdBranches.add(branch)
1393                            parent = self.knownBranches[branch]
1394                            if parent == branch:
1395                                parent = ""
1396                            else:
1397                                fullBranch = self.projectName + branch
1398                                if fullBranch not in self.p4BranchesInGit:
1399                                    if not self.silent:
1400                                        print("\n    Importing new branch %s" % fullBranch);
1401                                    if self.importNewBranch(branch, change - 1):
1402                                        parent = ""
1403                                        self.p4BranchesInGit.append(fullBranch)
1404                                    if not self.silent:
1405                                        print("\n    Resuming with change %s" % change);
1406
1407                                if self.verbose:
1408                                    print "parent determined through known branches: %s" % parent
1409
1410                        branch = self.gitRefForBranch(branch)
1411                        parent = self.gitRefForBranch(parent)
1412
1413                        if self.verbose:
1414                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1415
1416                        if len(parent) == 0 and branch in self.initialParents:
1417                            parent = self.initialParents[branch]
1418                            del self.initialParents[branch]
1419
1420                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1421                else:
1422                    files = self.extractFilesFromCommit(description)
1423                    self.commit(description, files, self.branch, self.depotPaths,
1424                                self.initialParent)
1425                    self.initialParent = ""
1426            except IOError:
1427                print self.gitError.read()
1428                sys.exit(1)
1429
1430    def importHeadRevision(self, revision):
1431        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1432
1433        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1434        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1435                           % (' '.join(self.depotPaths), revision))
1436        details["change"] = revision
1437        newestRevision = 0
1438
1439        fileCnt = 0
1440        for info in p4CmdList("files "
1441                              +  ' '.join(["%s...%s"
1442                                           % (p, revision)
1443                                           for p in self.depotPaths])):
1444
1445            if 'code' in info and info['code'] == 'error':
1446                sys.stderr.write("p4 returned an error: %s\n"
1447                                 % info['data'])
1448                sys.exit(1)
1449            if 'p4ExitCode' in info:
1450                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1451                sys.exit(1)
1452
1453
1454            change = int(info["change"])
1455            if change > newestRevision:
1456                newestRevision = change
1457
1458            if info["action"] in self.delete_actions:
1459                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1460                #fileCnt = fileCnt + 1
1461                continue
1462
1463            for prop in ["depotFile", "rev", "action", "type" ]:
1464                details["%s%s" % (prop, fileCnt)] = info[prop]
1465
1466            fileCnt = fileCnt + 1
1467
1468        details["change"] = newestRevision
1469        self.updateOptionDict(details)
1470        try:
1471            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1472        except IOError:
1473            print "IO error with git fast-import. Is your git version recent enough?"
1474            print self.gitError.read()
1475
1476
1477    def getClientSpec(self):
1478        specList = p4CmdList( "client -o" )
1479        temp = {}
1480        for entry in specList:
1481            for k,v in entry.iteritems():
1482                if k.startswith("View"):
1483                    if v.startswith('"'):
1484                        start = 1
1485                    else:
1486                        start = 0
1487                    index = v.find("...")
1488                    v = v[start:index]
1489                    if v.startswith("-"):
1490                        v = v[1:]
1491                        temp[v] = -len(v)
1492                    else:
1493                        temp[v] = len(v)
1494        self.clientSpecDirs = temp.items()
1495        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1496
1497    def run(self, args):
1498        self.depotPaths = []
1499        self.changeRange = ""
1500        self.initialParent = ""
1501        self.previousDepotPaths = []
1502
1503        # map from branch depot path to parent branch
1504        self.knownBranches = {}
1505        self.initialParents = {}
1506        self.hasOrigin = originP4BranchesExist()
1507        if not self.syncWithOrigin:
1508            self.hasOrigin = False
1509
1510        if self.importIntoRemotes:
1511            self.refPrefix = "refs/remotes/p4/"
1512        else:
1513            self.refPrefix = "refs/heads/p4/"
1514
1515        if self.syncWithOrigin and self.hasOrigin:
1516            if not self.silent:
1517                print "Syncing with origin first by calling git fetch origin"
1518            system("git fetch origin")
1519
1520        if len(self.branch) == 0:
1521            self.branch = self.refPrefix + "master"
1522            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1523                system("git update-ref %s refs/heads/p4" % self.branch)
1524                system("git branch -D p4");
1525            # create it /after/ importing, when master exists
1526            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1527                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1528
1529        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1530            self.getClientSpec()
1531
1532        # TODO: should always look at previous commits,
1533        # merge with previous imports, if possible.
1534        if args == []:
1535            if self.hasOrigin:
1536                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1537            self.listExistingP4GitBranches()
1538
1539            if len(self.p4BranchesInGit) > 1:
1540                if not self.silent:
1541                    print "Importing from/into multiple branches"
1542                self.detectBranches = True
1543
1544            if self.verbose:
1545                print "branches: %s" % self.p4BranchesInGit
1546
1547            p4Change = 0
1548            for branch in self.p4BranchesInGit:
1549                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1550
1551                settings = extractSettingsGitLog(logMsg)
1552
1553                self.readOptions(settings)
1554                if (settings.has_key('depot-paths')
1555                    and settings.has_key ('change')):
1556                    change = int(settings['change']) + 1
1557                    p4Change = max(p4Change, change)
1558
1559                    depotPaths = sorted(settings['depot-paths'])
1560                    if self.previousDepotPaths == []:
1561                        self.previousDepotPaths = depotPaths
1562                    else:
1563                        paths = []
1564                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1565                            for i in range(0, min(len(cur), len(prev))):
1566                                if cur[i] <> prev[i]:
1567                                    i = i - 1
1568                                    break
1569
1570                            paths.append (cur[:i + 1])
1571
1572                        self.previousDepotPaths = paths
1573
1574            if p4Change > 0:
1575                self.depotPaths = sorted(self.previousDepotPaths)
1576                self.changeRange = "@%s,#head" % p4Change
1577                if not self.detectBranches:
1578                    self.initialParent = parseRevision(self.branch)
1579                if not self.silent and not self.detectBranches:
1580                    print "Performing incremental import into %s git branch" % self.branch
1581
1582        if not self.branch.startswith("refs/"):
1583            self.branch = "refs/heads/" + self.branch
1584
1585        if len(args) == 0 and self.depotPaths:
1586            if not self.silent:
1587                print "Depot paths: %s" % ' '.join(self.depotPaths)
1588        else:
1589            if self.depotPaths and self.depotPaths != args:
1590                print ("previous import used depot path %s and now %s was specified. "
1591                       "This doesn't work!" % (' '.join (self.depotPaths),
1592                                               ' '.join (args)))
1593                sys.exit(1)
1594
1595            self.depotPaths = sorted(args)
1596
1597        revision = ""
1598        self.users = {}
1599
1600        newPaths = []
1601        for p in self.depotPaths:
1602            if p.find("@") != -1:
1603                atIdx = p.index("@")
1604                self.changeRange = p[atIdx:]
1605                if self.changeRange == "@all":
1606                    self.changeRange = ""
1607                elif ',' not in self.changeRange:
1608                    revision = self.changeRange
1609                    self.changeRange = ""
1610                p = p[:atIdx]
1611            elif p.find("#") != -1:
1612                hashIdx = p.index("#")
1613                revision = p[hashIdx:]
1614                p = p[:hashIdx]
1615            elif self.previousDepotPaths == []:
1616                revision = "#head"
1617
1618            p = re.sub ("\.\.\.$", "", p)
1619            if not p.endswith("/"):
1620                p += "/"
1621
1622            newPaths.append(p)
1623
1624        self.depotPaths = newPaths
1625
1626
1627        self.loadUserMapFromCache()
1628        self.labels = {}
1629        if self.detectLabels:
1630            self.getLabels();
1631
1632        if self.detectBranches:
1633            ## FIXME - what's a P4 projectName ?
1634            self.projectName = self.guessProjectName()
1635
1636            if self.hasOrigin:
1637                self.getBranchMappingFromGitBranches()
1638            else:
1639                self.getBranchMapping()
1640            if self.verbose:
1641                print "p4-git branches: %s" % self.p4BranchesInGit
1642                print "initial parents: %s" % self.initialParents
1643            for b in self.p4BranchesInGit:
1644                if b != "master":
1645
1646                    ## FIXME
1647                    b = b[len(self.projectName):]
1648                self.createdBranches.add(b)
1649
1650        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1651
1652        importProcess = subprocess.Popen(["git", "fast-import"],
1653                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1654                                         stderr=subprocess.PIPE);
1655        self.gitOutput = importProcess.stdout
1656        self.gitStream = importProcess.stdin
1657        self.gitError = importProcess.stderr
1658
1659        if revision:
1660            self.importHeadRevision(revision)
1661        else:
1662            changes = []
1663
1664            if len(self.changesFile) > 0:
1665                output = open(self.changesFile).readlines()
1666                changeSet = set()
1667                for line in output:
1668                    changeSet.add(int(line))
1669
1670                for change in changeSet:
1671                    changes.append(change)
1672
1673                changes.sort()
1674            else:
1675                if self.verbose:
1676                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1677                                                              self.changeRange)
1678                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1679
1680                if len(self.maxChanges) > 0:
1681                    changes = changes[:min(int(self.maxChanges), len(changes))]
1682
1683            if len(changes) == 0:
1684                if not self.silent:
1685                    print "No changes to import!"
1686                return True
1687
1688            if not self.silent and not self.detectBranches:
1689                print "Import destination: %s" % self.branch
1690
1691            self.updatedBranches = set()
1692
1693            self.importChanges(changes)
1694
1695            if not self.silent:
1696                print ""
1697                if len(self.updatedBranches) > 0:
1698                    sys.stdout.write("Updated branches: ")
1699                    for b in self.updatedBranches:
1700                        sys.stdout.write("%s " % b)
1701                    sys.stdout.write("\n")
1702
1703        self.gitStream.close()
1704        if importProcess.wait() != 0:
1705            die("fast-import failed: %s" % self.gitError.read())
1706        self.gitOutput.close()
1707        self.gitError.close()
1708
1709        return True
1710
1711class P4Rebase(Command):
1712    def __init__(self):
1713        Command.__init__(self)
1714        self.options = [ ]
1715        self.description = ("Fetches the latest revision from perforce and "
1716                            + "rebases the current work (branch) against it")
1717        self.verbose = False
1718
1719    def run(self, args):
1720        sync = P4Sync()
1721        sync.run([])
1722
1723        return self.rebase()
1724
1725    def rebase(self):
1726        if os.system("git update-index --refresh") != 0:
1727            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1728        if len(read_pipe("git diff-index HEAD --")) > 0:
1729            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1730
1731        [upstream, settings] = findUpstreamBranchPoint()
1732        if len(upstream) == 0:
1733            die("Cannot find upstream branchpoint for rebase")
1734
1735        # the branchpoint may be p4/foo~3, so strip off the parent
1736        upstream = re.sub("~[0-9]+$", "", upstream)
1737
1738        print "Rebasing the current branch onto %s" % upstream
1739        oldHead = read_pipe("git rev-parse HEAD").strip()
1740        system("git rebase %s" % upstream)
1741        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1742        return True
1743
1744class P4Clone(P4Sync):
1745    def __init__(self):
1746        P4Sync.__init__(self)
1747        self.description = "Creates a new git repository and imports from Perforce into it"
1748        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1749        self.options += [
1750            optparse.make_option("--destination", dest="cloneDestination",
1751                                 action='store', default=None,
1752                                 help="where to leave result of the clone"),
1753            optparse.make_option("-/", dest="cloneExclude",
1754                                 action="append", type="string",
1755                                 help="exclude depot path")
1756        ]
1757        self.cloneDestination = None
1758        self.needsGit = False
1759
1760    # This is required for the "append" cloneExclude action
1761    def ensure_value(self, attr, value):
1762        if not hasattr(self, attr) or getattr(self, attr) is None:
1763            setattr(self, attr, value)
1764        return getattr(self, attr)
1765
1766    def defaultDestination(self, args):
1767        ## TODO: use common prefix of args?
1768        depotPath = args[0]
1769        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1770        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1771        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1772        depotDir = re.sub(r"/$", "", depotDir)
1773        return os.path.split(depotDir)[1]
1774
1775    def run(self, args):
1776        if len(args) < 1:
1777            return False
1778
1779        if self.keepRepoPath and not self.cloneDestination:
1780            sys.stderr.write("Must specify destination for --keep-path\n")
1781            sys.exit(1)
1782
1783        depotPaths = args
1784
1785        if not self.cloneDestination and len(depotPaths) > 1:
1786            self.cloneDestination = depotPaths[-1]
1787            depotPaths = depotPaths[:-1]
1788
1789        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1790        for p in depotPaths:
1791            if not p.startswith("//"):
1792                return False
1793
1794        if not self.cloneDestination:
1795            self.cloneDestination = self.defaultDestination(args)
1796
1797        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1798        if not os.path.exists(self.cloneDestination):
1799            os.makedirs(self.cloneDestination)
1800        chdir(self.cloneDestination)
1801        system("git init")
1802        self.gitdir = os.getcwd() + "/.git"
1803        if not P4Sync.run(self, depotPaths):
1804            return False
1805        if self.branch != "master":
1806            if self.importIntoRemotes:
1807                masterbranch = "refs/remotes/p4/master"
1808            else:
1809                masterbranch = "refs/heads/p4/master"
1810            if gitBranchExists(masterbranch):
1811                system("git branch master %s" % masterbranch)
1812                system("git checkout -f")
1813            else:
1814                print "Could not detect main branch. No checkout/master branch created."
1815
1816        return True
1817
1818class P4Branches(Command):
1819    def __init__(self):
1820        Command.__init__(self)
1821        self.options = [ ]
1822        self.description = ("Shows the git branches that hold imports and their "
1823                            + "corresponding perforce depot paths")
1824        self.verbose = False
1825
1826    def run(self, args):
1827        if originP4BranchesExist():
1828            createOrUpdateBranchesFromOrigin()
1829
1830        cmdline = "git rev-parse --symbolic "
1831        cmdline += " --remotes"
1832
1833        for line in read_pipe_lines(cmdline):
1834            line = line.strip()
1835
1836            if not line.startswith('p4/') or line == "p4/HEAD":
1837                continue
1838            branch = line
1839
1840            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1841            settings = extractSettingsGitLog(log)
1842
1843            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1844        return True
1845
1846class HelpFormatter(optparse.IndentedHelpFormatter):
1847    def __init__(self):
1848        optparse.IndentedHelpFormatter.__init__(self)
1849
1850    def format_description(self, description):
1851        if description:
1852            return description + "\n"
1853        else:
1854            return ""
1855
1856def printUsage(commands):
1857    print "usage: %s <command> [options]" % sys.argv[0]
1858    print ""
1859    print "valid commands: %s" % ", ".join(commands)
1860    print ""
1861    print "Try %s <command> --help for command specific help." % sys.argv[0]
1862    print ""
1863
1864commands = {
1865    "debug" : P4Debug,
1866    "submit" : P4Submit,
1867    "commit" : P4Submit,
1868    "sync" : P4Sync,
1869    "rebase" : P4Rebase,
1870    "clone" : P4Clone,
1871    "rollback" : P4RollBack,
1872    "branches" : P4Branches
1873}
1874
1875
1876def main():
1877    if len(sys.argv[1:]) == 0:
1878        printUsage(commands.keys())
1879        sys.exit(2)
1880
1881    cmd = ""
1882    cmdName = sys.argv[1]
1883    try:
1884        klass = commands[cmdName]
1885        cmd = klass()
1886    except KeyError:
1887        print "unknown command %s" % cmdName
1888        print ""
1889        printUsage(commands.keys())
1890        sys.exit(2)
1891
1892    options = cmd.options
1893    cmd.gitdir = os.environ.get("GIT_DIR", None)
1894
1895    args = sys.argv[2:]
1896
1897    if len(options) > 0:
1898        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1899
1900        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1901                                       options,
1902                                       description = cmd.description,
1903                                       formatter = HelpFormatter())
1904
1905        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1906    global verbose
1907    verbose = cmd.verbose
1908    if cmd.needsGit:
1909        if cmd.gitdir == None:
1910            cmd.gitdir = os.path.abspath(".git")
1911            if not isValidGitDir(cmd.gitdir):
1912                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1913                if os.path.exists(cmd.gitdir):
1914                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1915                    if len(cdup) > 0:
1916                        chdir(cdup);
1917
1918        if not isValidGitDir(cmd.gitdir):
1919            if isValidGitDir(cmd.gitdir + "/.git"):
1920                cmd.gitdir += "/.git"
1921            else:
1922                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1923
1924        os.environ["GIT_DIR"] = cmd.gitdir
1925
1926    if not cmd.run(args):
1927        parser.print_help()
1928
1929
1930if __name__ == '__main__':
1931    main()