Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit count-objects: Add total pack size to verbose output (f223824)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def chdir(dir):
  20    if os.name == 'nt':
  21        os.environ['PWD']=dir
  22    os.chdir(dir)
  23
  24def die(msg):
  25    if verbose:
  26        raise Exception(msg)
  27    else:
  28        sys.stderr.write(msg + "\n")
  29        sys.exit(1)
  30
  31def write_pipe(c, str):
  32    if verbose:
  33        sys.stderr.write('Writing pipe: %s\n' % c)
  34
  35    pipe = os.popen(c, 'w')
  36    val = pipe.write(str)
  37    if pipe.close():
  38        die('Command failed: %s' % c)
  39
  40    return val
  41
  42def read_pipe(c, ignore_error=False):
  43    if verbose:
  44        sys.stderr.write('Reading pipe: %s\n' % c)
  45
  46    pipe = os.popen(c, 'rb')
  47    val = pipe.read()
  48    if pipe.close() and not ignore_error:
  49        die('Command failed: %s' % c)
  50
  51    return val
  52
  53
  54def read_pipe_lines(c):
  55    if verbose:
  56        sys.stderr.write('Reading pipe: %s\n' % c)
  57    ## todo: check return status
  58    pipe = os.popen(c, 'rb')
  59    val = pipe.readlines()
  60    if pipe.close():
  61        die('Command failed: %s' % c)
  62
  63    return val
  64
  65def system(cmd):
  66    if verbose:
  67        sys.stderr.write("executing %s\n" % cmd)
  68    if os.system(cmd) != 0:
  69        die("command failed: %s" % cmd)
  70
  71def isP4Exec(kind):
  72    """Determine if a Perforce 'kind' should have execute permission
  73
  74    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  75    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  76    a plus sign, it is also executable"""
  77    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  78
  79def setP4ExecBit(file, mode):
  80    # Reopens an already open file and changes the execute bit to match
  81    # the execute bit setting in the passed in mode.
  82
  83    p4Type = "+x"
  84
  85    if not isModeExec(mode):
  86        p4Type = getP4OpenedType(file)
  87        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  88        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  89        if p4Type[-1] == "+":
  90            p4Type = p4Type[0:-1]
  91
  92    system("p4 reopen -t %s %s" % (p4Type, file))
  93
  94def getP4OpenedType(file):
  95    # Returns the perforce file type for the given file.
  96
  97    result = read_pipe("p4 opened %s" % file)
  98    match = re.match(".*\((.+)\)\r?$", result)
  99    if match:
 100        return match.group(1)
 101    else:
 102        die("Could not determine file type for %s (result: '%s')" % (file, result))
 103
 104def diffTreePattern():
 105    # This is a simple generator for the diff tree regex pattern. This could be
 106    # a class variable if this and parseDiffTreeEntry were a part of a class.
 107    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 108    while True:
 109        yield pattern
 110
 111def parseDiffTreeEntry(entry):
 112    """Parses a single diff tree entry into its component elements.
 113
 114    See git-diff-tree(1) manpage for details about the format of the diff
 115    output. This method returns a dictionary with the following elements:
 116
 117    src_mode - The mode of the source file
 118    dst_mode - The mode of the destination file
 119    src_sha1 - The sha1 for the source file
 120    dst_sha1 - The sha1 fr the destination file
 121    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 122    status_score - The score for the status (applicable for 'C' and 'R'
 123                   statuses). This is None if there is no score.
 124    src - The path for the source file.
 125    dst - The path for the destination file. This is only present for
 126          copy or renames. If it is not present, this is None.
 127
 128    If the pattern is not matched, None is returned."""
 129
 130    match = diffTreePattern().next().match(entry)
 131    if match:
 132        return {
 133            'src_mode': match.group(1),
 134            'dst_mode': match.group(2),
 135            'src_sha1': match.group(3),
 136            'dst_sha1': match.group(4),
 137            'status': match.group(5),
 138            'status_score': match.group(6),
 139            'src': match.group(7),
 140            'dst': match.group(10)
 141        }
 142    return None
 143
 144def isModeExec(mode):
 145    # Returns True if the given git mode represents an executable file,
 146    # otherwise False.
 147    return mode[-3:] == "755"
 148
 149def isModeExecChanged(src_mode, dst_mode):
 150    return isModeExec(src_mode) != isModeExec(dst_mode)
 151
 152def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 153    cmd = "p4 -G %s" % cmd
 154    if verbose:
 155        sys.stderr.write("Opening pipe: %s\n" % cmd)
 156
 157    # Use a temporary file to avoid deadlocks without
 158    # subprocess.communicate(), which would put another copy
 159    # of stdout into memory.
 160    stdin_file = None
 161    if stdin is not None:
 162        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 163        stdin_file.write(stdin)
 164        stdin_file.flush()
 165        stdin_file.seek(0)
 166
 167    p4 = subprocess.Popen(cmd, shell=True,
 168                          stdin=stdin_file,
 169                          stdout=subprocess.PIPE)
 170
 171    result = []
 172    try:
 173        while True:
 174            entry = marshal.load(p4.stdout)
 175            result.append(entry)
 176    except EOFError:
 177        pass
 178    exitCode = p4.wait()
 179    if exitCode != 0:
 180        entry = {}
 181        entry["p4ExitCode"] = exitCode
 182        result.append(entry)
 183
 184    return result
 185
 186def p4Cmd(cmd):
 187    list = p4CmdList(cmd)
 188    result = {}
 189    for entry in list:
 190        result.update(entry)
 191    return result;
 192
 193def p4Where(depotPath):
 194    if not depotPath.endswith("/"):
 195        depotPath += "/"
 196    output = p4Cmd("where %s..." % depotPath)
 197    if output["code"] == "error":
 198        return ""
 199    clientPath = ""
 200    if "path" in output:
 201        clientPath = output.get("path")
 202    elif "data" in output:
 203        data = output.get("data")
 204        lastSpace = data.rfind(" ")
 205        clientPath = data[lastSpace + 1:]
 206
 207    if clientPath.endswith("..."):
 208        clientPath = clientPath[:-3]
 209    return clientPath
 210
 211def currentGitBranch():
 212    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 213
 214def isValidGitDir(path):
 215    if (os.path.exists(path + "/HEAD")
 216        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 217        return True;
 218    return False
 219
 220def parseRevision(ref):
 221    return read_pipe("git rev-parse %s" % ref).strip()
 222
 223def extractLogMessageFromGitCommit(commit):
 224    logMessage = ""
 225
 226    ## fixme: title is first line of commit, not 1st paragraph.
 227    foundTitle = False
 228    for log in read_pipe_lines("git cat-file commit %s" % commit):
 229       if not foundTitle:
 230           if len(log) == 1:
 231               foundTitle = True
 232           continue
 233
 234       logMessage += log
 235    return logMessage
 236
 237def extractSettingsGitLog(log):
 238    values = {}
 239    for line in log.split("\n"):
 240        line = line.strip()
 241        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 242        if not m:
 243            continue
 244
 245        assignments = m.group(1).split (':')
 246        for a in assignments:
 247            vals = a.split ('=')
 248            key = vals[0].strip()
 249            val = ('='.join (vals[1:])).strip()
 250            if val.endswith ('\"') and val.startswith('"'):
 251                val = val[1:-1]
 252
 253            values[key] = val
 254
 255    paths = values.get("depot-paths")
 256    if not paths:
 257        paths = values.get("depot-path")
 258    if paths:
 259        values['depot-paths'] = paths.split(',')
 260    return values
 261
 262def gitBranchExists(branch):
 263    proc = subprocess.Popen(["git", "rev-parse", branch],
 264                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 265    return proc.wait() == 0;
 266
 267def gitConfig(key):
 268    return read_pipe("git config %s" % key, ignore_error=True).strip()
 269
 270def p4BranchesInGit(branchesAreInRemotes = True):
 271    branches = {}
 272
 273    cmdline = "git rev-parse --symbolic "
 274    if branchesAreInRemotes:
 275        cmdline += " --remotes"
 276    else:
 277        cmdline += " --branches"
 278
 279    for line in read_pipe_lines(cmdline):
 280        line = line.strip()
 281
 282        ## only import to p4/
 283        if not line.startswith('p4/') or line == "p4/HEAD":
 284            continue
 285        branch = line
 286
 287        # strip off p4
 288        branch = re.sub ("^p4/", "", line)
 289
 290        branches[branch] = parseRevision(line)
 291    return branches
 292
 293def findUpstreamBranchPoint(head = "HEAD"):
 294    branches = p4BranchesInGit()
 295    # map from depot-path to branch name
 296    branchByDepotPath = {}
 297    for branch in branches.keys():
 298        tip = branches[branch]
 299        log = extractLogMessageFromGitCommit(tip)
 300        settings = extractSettingsGitLog(log)
 301        if settings.has_key("depot-paths"):
 302            paths = ",".join(settings["depot-paths"])
 303            branchByDepotPath[paths] = "remotes/p4/" + branch
 304
 305    settings = None
 306    parent = 0
 307    while parent < 65535:
 308        commit = head + "~%s" % parent
 309        log = extractLogMessageFromGitCommit(commit)
 310        settings = extractSettingsGitLog(log)
 311        if settings.has_key("depot-paths"):
 312            paths = ",".join(settings["depot-paths"])
 313            if branchByDepotPath.has_key(paths):
 314                return [branchByDepotPath[paths], settings]
 315
 316        parent = parent + 1
 317
 318    return ["", settings]
 319
 320def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 321    if not silent:
 322        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 323               % localRefPrefix)
 324
 325    originPrefix = "origin/p4/"
 326
 327    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 328        line = line.strip()
 329        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 330            continue
 331
 332        headName = line[len(originPrefix):]
 333        remoteHead = localRefPrefix + headName
 334        originHead = line
 335
 336        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 337        if (not original.has_key('depot-paths')
 338            or not original.has_key('change')):
 339            continue
 340
 341        update = False
 342        if not gitBranchExists(remoteHead):
 343            if verbose:
 344                print "creating %s" % remoteHead
 345            update = True
 346        else:
 347            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 348            if settings.has_key('change') > 0:
 349                if settings['depot-paths'] == original['depot-paths']:
 350                    originP4Change = int(original['change'])
 351                    p4Change = int(settings['change'])
 352                    if originP4Change > p4Change:
 353                        print ("%s (%s) is newer than %s (%s). "
 354                               "Updating p4 branch from origin."
 355                               % (originHead, originP4Change,
 356                                  remoteHead, p4Change))
 357                        update = True
 358                else:
 359                    print ("Ignoring: %s was imported from %s while "
 360                           "%s was imported from %s"
 361                           % (originHead, ','.join(original['depot-paths']),
 362                              remoteHead, ','.join(settings['depot-paths'])))
 363
 364        if update:
 365            system("git update-ref %s %s" % (remoteHead, originHead))
 366
 367def originP4BranchesExist():
 368        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 369
 370def p4ChangesForPaths(depotPaths, changeRange):
 371    assert depotPaths
 372    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 373                                                        for p in depotPaths]))
 374
 375    changes = []
 376    for line in output:
 377        changeNum = line.split(" ")[1]
 378        changes.append(int(changeNum))
 379
 380    changes.sort()
 381    return changes
 382
 383class Command:
 384    def __init__(self):
 385        self.usage = "usage: %prog [options]"
 386        self.needsGit = True
 387
 388class P4Debug(Command):
 389    def __init__(self):
 390        Command.__init__(self)
 391        self.options = [
 392            optparse.make_option("--verbose", dest="verbose", action="store_true",
 393                                 default=False),
 394            ]
 395        self.description = "A tool to debug the output of p4 -G."
 396        self.needsGit = False
 397        self.verbose = False
 398
 399    def run(self, args):
 400        j = 0
 401        for output in p4CmdList(" ".join(args)):
 402            print 'Element: %d' % j
 403            j += 1
 404            print output
 405        return True
 406
 407class P4RollBack(Command):
 408    def __init__(self):
 409        Command.__init__(self)
 410        self.options = [
 411            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 412            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 413        ]
 414        self.description = "A tool to debug the multi-branch import. Don't use :)"
 415        self.verbose = False
 416        self.rollbackLocalBranches = False
 417
 418    def run(self, args):
 419        if len(args) != 1:
 420            return False
 421        maxChange = int(args[0])
 422
 423        if "p4ExitCode" in p4Cmd("changes -m 1"):
 424            die("Problems executing p4");
 425
 426        if self.rollbackLocalBranches:
 427            refPrefix = "refs/heads/"
 428            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 429        else:
 430            refPrefix = "refs/remotes/"
 431            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 432
 433        for line in lines:
 434            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 435                line = line.strip()
 436                ref = refPrefix + line
 437                log = extractLogMessageFromGitCommit(ref)
 438                settings = extractSettingsGitLog(log)
 439
 440                depotPaths = settings['depot-paths']
 441                change = settings['change']
 442
 443                changed = False
 444
 445                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 446                                                           for p in depotPaths]))) == 0:
 447                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 448                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 449                    continue
 450
 451                while change and int(change) > maxChange:
 452                    changed = True
 453                    if self.verbose:
 454                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 455                    system("git update-ref %s \"%s^\"" % (ref, ref))
 456                    log = extractLogMessageFromGitCommit(ref)
 457                    settings =  extractSettingsGitLog(log)
 458
 459
 460                    depotPaths = settings['depot-paths']
 461                    change = settings['change']
 462
 463                if changed:
 464                    print "%s rewound to %s" % (ref, change)
 465
 466        return True
 467
 468class P4Submit(Command):
 469    def __init__(self):
 470        Command.__init__(self)
 471        self.options = [
 472                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 473                optparse.make_option("--origin", dest="origin"),
 474                optparse.make_option("-M", dest="detectRename", action="store_true"),
 475        ]
 476        self.description = "Submit changes from git to the perforce depot."
 477        self.usage += " [name of git branch to submit into perforce depot]"
 478        self.interactive = True
 479        self.origin = ""
 480        self.detectRename = False
 481        self.verbose = False
 482        self.isWindows = (platform.system() == "Windows")
 483
 484    def check(self):
 485        if len(p4CmdList("opened ...")) > 0:
 486            die("You have files opened with perforce! Close them before starting the sync.")
 487
 488    # replaces everything between 'Description:' and the next P4 submit template field with the
 489    # commit message
 490    def prepareLogMessage(self, template, message):
 491        result = ""
 492
 493        inDescriptionSection = False
 494
 495        for line in template.split("\n"):
 496            if line.startswith("#"):
 497                result += line + "\n"
 498                continue
 499
 500            if inDescriptionSection:
 501                if line.startswith("Files:"):
 502                    inDescriptionSection = False
 503                else:
 504                    continue
 505            else:
 506                if line.startswith("Description:"):
 507                    inDescriptionSection = True
 508                    line += "\n"
 509                    for messageLine in message.split("\n"):
 510                        line += "\t" + messageLine + "\n"
 511
 512            result += line + "\n"
 513
 514        return result
 515
 516    def prepareSubmitTemplate(self):
 517        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 518        template = ""
 519        inFilesSection = False
 520        for line in read_pipe_lines("p4 change -o"):
 521            if line.endswith("\r\n"):
 522                line = line[:-2] + "\n"
 523            if inFilesSection:
 524                if line.startswith("\t"):
 525                    # path starts and ends with a tab
 526                    path = line[1:]
 527                    lastTab = path.rfind("\t")
 528                    if lastTab != -1:
 529                        path = path[:lastTab]
 530                        if not path.startswith(self.depotPath):
 531                            continue
 532                else:
 533                    inFilesSection = False
 534            else:
 535                if line.startswith("Files:"):
 536                    inFilesSection = True
 537
 538            template += line
 539
 540        return template
 541
 542    def applyCommit(self, id):
 543        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 544        diffOpts = ("", "-M")[self.detectRename]
 545        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 546        filesToAdd = set()
 547        filesToDelete = set()
 548        editedFiles = set()
 549        filesToChangeExecBit = {}
 550        for line in diff:
 551            diff = parseDiffTreeEntry(line)
 552            modifier = diff['status']
 553            path = diff['src']
 554            if modifier == "M":
 555                system("p4 edit \"%s\"" % path)
 556                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 557                    filesToChangeExecBit[path] = diff['dst_mode']
 558                editedFiles.add(path)
 559            elif modifier == "A":
 560                filesToAdd.add(path)
 561                filesToChangeExecBit[path] = diff['dst_mode']
 562                if path in filesToDelete:
 563                    filesToDelete.remove(path)
 564            elif modifier == "D":
 565                filesToDelete.add(path)
 566                if path in filesToAdd:
 567                    filesToAdd.remove(path)
 568            elif modifier == "R":
 569                src, dest = diff['src'], diff['dst']
 570                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 571                system("p4 edit \"%s\"" % (dest))
 572                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 573                    filesToChangeExecBit[dest] = diff['dst_mode']
 574                os.unlink(dest)
 575                editedFiles.add(dest)
 576                filesToDelete.add(src)
 577            else:
 578                die("unknown modifier %s for %s" % (modifier, path))
 579
 580        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 581        patchcmd = diffcmd + " | git apply "
 582        tryPatchCmd = patchcmd + "--check -"
 583        applyPatchCmd = patchcmd + "--check --apply -"
 584
 585        if os.system(tryPatchCmd) != 0:
 586            print "Unfortunately applying the change failed!"
 587            print "What do you want to do?"
 588            response = "x"
 589            while response != "s" and response != "a" and response != "w":
 590                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 591                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 592            if response == "s":
 593                print "Skipping! Good luck with the next patches..."
 594                for f in editedFiles:
 595                    system("p4 revert \"%s\"" % f);
 596                for f in filesToAdd:
 597                    system("rm %s" %f)
 598                return
 599            elif response == "a":
 600                os.system(applyPatchCmd)
 601                if len(filesToAdd) > 0:
 602                    print "You may also want to call p4 add on the following files:"
 603                    print " ".join(filesToAdd)
 604                if len(filesToDelete):
 605                    print "The following files should be scheduled for deletion with p4 delete:"
 606                    print " ".join(filesToDelete)
 607                die("Please resolve and submit the conflict manually and "
 608                    + "continue afterwards with git-p4 submit --continue")
 609            elif response == "w":
 610                system(diffcmd + " > patch.txt")
 611                print "Patch saved to patch.txt in %s !" % self.clientPath
 612                die("Please resolve and submit the conflict manually and "
 613                    "continue afterwards with git-p4 submit --continue")
 614
 615        system(applyPatchCmd)
 616
 617        for f in filesToAdd:
 618            system("p4 add \"%s\"" % f)
 619        for f in filesToDelete:
 620            system("p4 revert \"%s\"" % f)
 621            system("p4 delete \"%s\"" % f)
 622
 623        # Set/clear executable bits
 624        for f in filesToChangeExecBit.keys():
 625            mode = filesToChangeExecBit[f]
 626            setP4ExecBit(f, mode)
 627
 628        logMessage = extractLogMessageFromGitCommit(id)
 629        logMessage = logMessage.strip()
 630
 631        template = self.prepareSubmitTemplate()
 632
 633        if self.interactive:
 634            submitTemplate = self.prepareLogMessage(template, logMessage)
 635            if os.environ.has_key("P4DIFF"):
 636                del(os.environ["P4DIFF"])
 637            diff = read_pipe("p4 diff -du ...")
 638
 639            newdiff = ""
 640            for newFile in filesToAdd:
 641                newdiff += "==== new file ====\n"
 642                newdiff += "--- /dev/null\n"
 643                newdiff += "+++ %s\n" % newFile
 644                f = open(newFile, "r")
 645                for line in f.readlines():
 646                    newdiff += "+" + line
 647                f.close()
 648
 649            separatorLine = "######## everything below this line is just the diff #######\n"
 650
 651            [handle, fileName] = tempfile.mkstemp()
 652            tmpFile = os.fdopen(handle, "w+")
 653            if self.isWindows:
 654                submitTemplate = submitTemplate.replace("\n", "\r\n")
 655                separatorLine = separatorLine.replace("\n", "\r\n")
 656                newdiff = newdiff.replace("\n", "\r\n")
 657            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 658            tmpFile.close()
 659            defaultEditor = "vi"
 660            if platform.system() == "Windows":
 661                defaultEditor = "notepad"
 662            if os.environ.has_key("P4EDITOR"):
 663                editor = os.environ.get("P4EDITOR")
 664            else:
 665                editor = os.environ.get("EDITOR", defaultEditor);
 666            system(editor + " " + fileName)
 667            tmpFile = open(fileName, "rb")
 668            message = tmpFile.read()
 669            tmpFile.close()
 670            os.remove(fileName)
 671            submitTemplate = message[:message.index(separatorLine)]
 672            if self.isWindows:
 673                submitTemplate = submitTemplate.replace("\r\n", "\n")
 674
 675            write_pipe("p4 submit -i", submitTemplate)
 676        else:
 677            fileName = "submit.txt"
 678            file = open(fileName, "w+")
 679            file.write(self.prepareLogMessage(template, logMessage))
 680            file.close()
 681            print ("Perforce submit template written as %s. "
 682                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 683                   % (fileName, fileName))
 684
 685    def run(self, args):
 686        if len(args) == 0:
 687            self.master = currentGitBranch()
 688            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 689                die("Detecting current git branch failed!")
 690        elif len(args) == 1:
 691            self.master = args[0]
 692        else:
 693            return False
 694
 695        allowSubmit = gitConfig("git-p4.allowSubmit")
 696        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 697            die("%s is not in git-p4.allowSubmit" % self.master)
 698
 699        [upstream, settings] = findUpstreamBranchPoint()
 700        self.depotPath = settings['depot-paths'][0]
 701        if len(self.origin) == 0:
 702            self.origin = upstream
 703
 704        if self.verbose:
 705            print "Origin branch is " + self.origin
 706
 707        if len(self.depotPath) == 0:
 708            print "Internal error: cannot locate perforce depot path from existing branches"
 709            sys.exit(128)
 710
 711        self.clientPath = p4Where(self.depotPath)
 712
 713        if len(self.clientPath) == 0:
 714            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 715            sys.exit(128)
 716
 717        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 718        self.oldWorkingDirectory = os.getcwd()
 719
 720        chdir(self.clientPath)
 721        print "Syncronizing p4 checkout..."
 722        system("p4 sync ...")
 723
 724        self.check()
 725
 726        commits = []
 727        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 728            commits.append(line.strip())
 729        commits.reverse()
 730
 731        while len(commits) > 0:
 732            commit = commits[0]
 733            commits = commits[1:]
 734            self.applyCommit(commit)
 735            if not self.interactive:
 736                break
 737
 738        if len(commits) == 0:
 739            print "All changes applied!"
 740            chdir(self.oldWorkingDirectory)
 741
 742            sync = P4Sync()
 743            sync.run([])
 744
 745            rebase = P4Rebase()
 746            rebase.rebase()
 747
 748        return True
 749
 750class P4Sync(Command):
 751    def __init__(self):
 752        Command.__init__(self)
 753        self.options = [
 754                optparse.make_option("--branch", dest="branch"),
 755                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 756                optparse.make_option("--changesfile", dest="changesFile"),
 757                optparse.make_option("--silent", dest="silent", action="store_true"),
 758                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 759                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 760                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 761                                     help="Import into refs/heads/ , not refs/remotes"),
 762                optparse.make_option("--max-changes", dest="maxChanges"),
 763                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 764                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 765                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 766                                     help="Only sync files that are included in the Perforce Client Spec")
 767        ]
 768        self.description = """Imports from Perforce into a git repository.\n
 769    example:
 770    //depot/my/project/ -- to import the current head
 771    //depot/my/project/@all -- to import everything
 772    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 773
 774    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 775
 776        self.usage += " //depot/path[@revRange]"
 777        self.silent = False
 778        self.createdBranches = Set()
 779        self.committedChanges = Set()
 780        self.branch = ""
 781        self.detectBranches = False
 782        self.detectLabels = False
 783        self.changesFile = ""
 784        self.syncWithOrigin = True
 785        self.verbose = False
 786        self.importIntoRemotes = True
 787        self.maxChanges = ""
 788        self.isWindows = (platform.system() == "Windows")
 789        self.keepRepoPath = False
 790        self.depotPaths = None
 791        self.p4BranchesInGit = []
 792        self.cloneExclude = []
 793        self.useClientSpec = False
 794        self.clientSpecDirs = []
 795
 796        if gitConfig("git-p4.syncFromOrigin") == "false":
 797            self.syncWithOrigin = False
 798
 799    def extractFilesFromCommit(self, commit):
 800        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 801                             for path in self.cloneExclude]
 802        files = []
 803        fnum = 0
 804        while commit.has_key("depotFile%s" % fnum):
 805            path =  commit["depotFile%s" % fnum]
 806
 807            if [p for p in self.cloneExclude
 808                if path.startswith (p)]:
 809                found = False
 810            else:
 811                found = [p for p in self.depotPaths
 812                         if path.startswith (p)]
 813            if not found:
 814                fnum = fnum + 1
 815                continue
 816
 817            file = {}
 818            file["path"] = path
 819            file["rev"] = commit["rev%s" % fnum]
 820            file["action"] = commit["action%s" % fnum]
 821            file["type"] = commit["type%s" % fnum]
 822            files.append(file)
 823            fnum = fnum + 1
 824        return files
 825
 826    def stripRepoPath(self, path, prefixes):
 827        if self.keepRepoPath:
 828            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 829
 830        for p in prefixes:
 831            if path.startswith(p):
 832                path = path[len(p):]
 833
 834        return path
 835
 836    def splitFilesIntoBranches(self, commit):
 837        branches = {}
 838        fnum = 0
 839        while commit.has_key("depotFile%s" % fnum):
 840            path =  commit["depotFile%s" % fnum]
 841            found = [p for p in self.depotPaths
 842                     if path.startswith (p)]
 843            if not found:
 844                fnum = fnum + 1
 845                continue
 846
 847            file = {}
 848            file["path"] = path
 849            file["rev"] = commit["rev%s" % fnum]
 850            file["action"] = commit["action%s" % fnum]
 851            file["type"] = commit["type%s" % fnum]
 852            fnum = fnum + 1
 853
 854            relPath = self.stripRepoPath(path, self.depotPaths)
 855
 856            for branch in self.knownBranches.keys():
 857
 858                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 859                if relPath.startswith(branch + "/"):
 860                    if branch not in branches:
 861                        branches[branch] = []
 862                    branches[branch].append(file)
 863                    break
 864
 865        return branches
 866
 867    ## Should move this out, doesn't use SELF.
 868    def readP4Files(self, files):
 869        filesForCommit = []
 870        filesToRead = []
 871
 872        for f in files:
 873            includeFile = True
 874            for val in self.clientSpecDirs:
 875                if f['path'].startswith(val[0]):
 876                    if val[1] <= 0:
 877                        includeFile = False
 878                    break
 879
 880            if includeFile:
 881                filesForCommit.append(f)
 882                if f['action'] != 'delete':
 883                    filesToRead.append(f)
 884
 885        filedata = []
 886        if len(filesToRead) > 0:
 887            filedata = p4CmdList('-x - print',
 888                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 889                                                  for f in filesToRead]),
 890                                 stdin_mode='w+')
 891
 892            if "p4ExitCode" in filedata[0]:
 893                die("Problems executing p4. Error: [%d]."
 894                    % (filedata[0]['p4ExitCode']));
 895
 896        j = 0;
 897        contents = {}
 898        while j < len(filedata):
 899            stat = filedata[j]
 900            j += 1
 901            text = [];
 902            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 903                text.append(filedata[j]['data'])
 904                j += 1
 905            text = ''.join(text)
 906
 907            if not stat.has_key('depotFile'):
 908                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 909                continue
 910
 911            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 912                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 913            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 914                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 915
 916            contents[stat['depotFile']] = text
 917
 918        for f in filesForCommit:
 919            path = f['path']
 920            if contents.has_key(path):
 921                f['data'] = contents[path]
 922
 923        return filesForCommit
 924
 925    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 926        epoch = details["time"]
 927        author = details["user"]
 928
 929        if self.verbose:
 930            print "commit into %s" % branch
 931
 932        # start with reading files; if that fails, we should not
 933        # create a commit.
 934        new_files = []
 935        for f in files:
 936            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 937                new_files.append (f)
 938            else:
 939                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 940        files = self.readP4Files(new_files)
 941
 942        self.gitStream.write("commit %s\n" % branch)
 943#        gitStream.write("mark :%s\n" % details["change"])
 944        self.committedChanges.add(int(details["change"]))
 945        committer = ""
 946        if author not in self.users:
 947            self.getUserMapFromPerforceServer()
 948        if author in self.users:
 949            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 950        else:
 951            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 952
 953        self.gitStream.write("committer %s\n" % committer)
 954
 955        self.gitStream.write("data <<EOT\n")
 956        self.gitStream.write(details["desc"])
 957        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 958                             % (','.join (branchPrefixes), details["change"]))
 959        if len(details['options']) > 0:
 960            self.gitStream.write(": options = %s" % details['options'])
 961        self.gitStream.write("]\nEOT\n\n")
 962
 963        if len(parent) > 0:
 964            if self.verbose:
 965                print "parent %s" % parent
 966            self.gitStream.write("from %s\n" % parent)
 967
 968        for file in files:
 969            if file["type"] == "apple":
 970                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 971                continue
 972
 973            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 974            if file["action"] == "delete":
 975                self.gitStream.write("D %s\n" % relPath)
 976            else:
 977                data = file['data']
 978
 979                mode = "644"
 980                if isP4Exec(file["type"]):
 981                    mode = "755"
 982                elif file["type"] == "symlink":
 983                    mode = "120000"
 984                    # p4 print on a symlink contains "target\n", so strip it off
 985                    data = data[:-1]
 986
 987                if self.isWindows and file["type"].endswith("text"):
 988                    data = data.replace("\r\n", "\n")
 989
 990                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 991                self.gitStream.write("data %s\n" % len(data))
 992                self.gitStream.write(data)
 993                self.gitStream.write("\n")
 994
 995        self.gitStream.write("\n")
 996
 997        change = int(details["change"])
 998
 999        if self.labels.has_key(change):
1000            label = self.labels[change]
1001            labelDetails = label[0]
1002            labelRevisions = label[1]
1003            if self.verbose:
1004                print "Change %s is labelled %s" % (change, labelDetails)
1005
1006            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1007                                                    for p in branchPrefixes]))
1008
1009            if len(files) == len(labelRevisions):
1010
1011                cleanedFiles = {}
1012                for info in files:
1013                    if info["action"] == "delete":
1014                        continue
1015                    cleanedFiles[info["depotFile"]] = info["rev"]
1016
1017                if cleanedFiles == labelRevisions:
1018                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1019                    self.gitStream.write("from %s\n" % branch)
1020
1021                    owner = labelDetails["Owner"]
1022                    tagger = ""
1023                    if author in self.users:
1024                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1025                    else:
1026                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1027                    self.gitStream.write("tagger %s\n" % tagger)
1028                    self.gitStream.write("data <<EOT\n")
1029                    self.gitStream.write(labelDetails["Description"])
1030                    self.gitStream.write("EOT\n\n")
1031
1032                else:
1033                    if not self.silent:
1034                        print ("Tag %s does not match with change %s: files do not match."
1035                               % (labelDetails["label"], change))
1036
1037            else:
1038                if not self.silent:
1039                    print ("Tag %s does not match with change %s: file count is different."
1040                           % (labelDetails["label"], change))
1041
1042    def getUserCacheFilename(self):
1043        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1044        return home + "/.gitp4-usercache.txt"
1045
1046    def getUserMapFromPerforceServer(self):
1047        if self.userMapFromPerforceServer:
1048            return
1049        self.users = {}
1050
1051        for output in p4CmdList("users"):
1052            if not output.has_key("User"):
1053                continue
1054            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1055
1056
1057        s = ''
1058        for (key, val) in self.users.items():
1059            s += "%s\t%s\n" % (key, val)
1060
1061        open(self.getUserCacheFilename(), "wb").write(s)
1062        self.userMapFromPerforceServer = True
1063
1064    def loadUserMapFromCache(self):
1065        self.users = {}
1066        self.userMapFromPerforceServer = False
1067        try:
1068            cache = open(self.getUserCacheFilename(), "rb")
1069            lines = cache.readlines()
1070            cache.close()
1071            for line in lines:
1072                entry = line.strip().split("\t")
1073                self.users[entry[0]] = entry[1]
1074        except IOError:
1075            self.getUserMapFromPerforceServer()
1076
1077    def getLabels(self):
1078        self.labels = {}
1079
1080        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1081        if len(l) > 0 and not self.silent:
1082            print "Finding files belonging to labels in %s" % `self.depotPaths`
1083
1084        for output in l:
1085            label = output["label"]
1086            revisions = {}
1087            newestChange = 0
1088            if self.verbose:
1089                print "Querying files for label %s" % label
1090            for file in p4CmdList("files "
1091                                  +  ' '.join (["%s...@%s" % (p, label)
1092                                                for p in self.depotPaths])):
1093                revisions[file["depotFile"]] = file["rev"]
1094                change = int(file["change"])
1095                if change > newestChange:
1096                    newestChange = change
1097
1098            self.labels[newestChange] = [output, revisions]
1099
1100        if self.verbose:
1101            print "Label changes: %s" % self.labels.keys()
1102
1103    def guessProjectName(self):
1104        for p in self.depotPaths:
1105            if p.endswith("/"):
1106                p = p[:-1]
1107            p = p[p.strip().rfind("/") + 1:]
1108            if not p.endswith("/"):
1109               p += "/"
1110            return p
1111
1112    def getBranchMapping(self):
1113        lostAndFoundBranches = set()
1114
1115        for info in p4CmdList("branches"):
1116            details = p4Cmd("branch -o %s" % info["branch"])
1117            viewIdx = 0
1118            while details.has_key("View%s" % viewIdx):
1119                paths = details["View%s" % viewIdx].split(" ")
1120                viewIdx = viewIdx + 1
1121                # require standard //depot/foo/... //depot/bar/... mapping
1122                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1123                    continue
1124                source = paths[0]
1125                destination = paths[1]
1126                ## HACK
1127                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1128                    source = source[len(self.depotPaths[0]):-4]
1129                    destination = destination[len(self.depotPaths[0]):-4]
1130
1131                    if destination in self.knownBranches:
1132                        if not self.silent:
1133                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1134                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1135                        continue
1136
1137                    self.knownBranches[destination] = source
1138
1139                    lostAndFoundBranches.discard(destination)
1140
1141                    if source not in self.knownBranches:
1142                        lostAndFoundBranches.add(source)
1143
1144
1145        for branch in lostAndFoundBranches:
1146            self.knownBranches[branch] = branch
1147
1148    def getBranchMappingFromGitBranches(self):
1149        branches = p4BranchesInGit(self.importIntoRemotes)
1150        for branch in branches.keys():
1151            if branch == "master":
1152                branch = "main"
1153            else:
1154                branch = branch[len(self.projectName):]
1155            self.knownBranches[branch] = branch
1156
1157    def listExistingP4GitBranches(self):
1158        # branches holds mapping from name to commit
1159        branches = p4BranchesInGit(self.importIntoRemotes)
1160        self.p4BranchesInGit = branches.keys()
1161        for branch in branches.keys():
1162            self.initialParents[self.refPrefix + branch] = branches[branch]
1163
1164    def updateOptionDict(self, d):
1165        option_keys = {}
1166        if self.keepRepoPath:
1167            option_keys['keepRepoPath'] = 1
1168
1169        d["options"] = ' '.join(sorted(option_keys.keys()))
1170
1171    def readOptions(self, d):
1172        self.keepRepoPath = (d.has_key('options')
1173                             and ('keepRepoPath' in d['options']))
1174
1175    def gitRefForBranch(self, branch):
1176        if branch == "main":
1177            return self.refPrefix + "master"
1178
1179        if len(branch) <= 0:
1180            return branch
1181
1182        return self.refPrefix + self.projectName + branch
1183
1184    def gitCommitByP4Change(self, ref, change):
1185        if self.verbose:
1186            print "looking in ref " + ref + " for change %s using bisect..." % change
1187
1188        earliestCommit = ""
1189        latestCommit = parseRevision(ref)
1190
1191        while True:
1192            if self.verbose:
1193                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1194            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1195            if len(next) == 0:
1196                if self.verbose:
1197                    print "argh"
1198                return ""
1199            log = extractLogMessageFromGitCommit(next)
1200            settings = extractSettingsGitLog(log)
1201            currentChange = int(settings['change'])
1202            if self.verbose:
1203                print "current change %s" % currentChange
1204
1205            if currentChange == change:
1206                if self.verbose:
1207                    print "found %s" % next
1208                return next
1209
1210            if currentChange < change:
1211                earliestCommit = "^%s" % next
1212            else:
1213                latestCommit = "%s" % next
1214
1215        return ""
1216
1217    def importNewBranch(self, branch, maxChange):
1218        # make fast-import flush all changes to disk and update the refs using the checkpoint
1219        # command so that we can try to find the branch parent in the git history
1220        self.gitStream.write("checkpoint\n\n");
1221        self.gitStream.flush();
1222        branchPrefix = self.depotPaths[0] + branch + "/"
1223        range = "@1,%s" % maxChange
1224        #print "prefix" + branchPrefix
1225        changes = p4ChangesForPaths([branchPrefix], range)
1226        if len(changes) <= 0:
1227            return False
1228        firstChange = changes[0]
1229        #print "first change in branch: %s" % firstChange
1230        sourceBranch = self.knownBranches[branch]
1231        sourceDepotPath = self.depotPaths[0] + sourceBranch
1232        sourceRef = self.gitRefForBranch(sourceBranch)
1233        #print "source " + sourceBranch
1234
1235        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1236        #print "branch parent: %s" % branchParentChange
1237        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1238        if len(gitParent) > 0:
1239            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1240            #print "parent git commit: %s" % gitParent
1241
1242        self.importChanges(changes)
1243        return True
1244
1245    def importChanges(self, changes):
1246        cnt = 1
1247        for change in changes:
1248            description = p4Cmd("describe %s" % change)
1249            self.updateOptionDict(description)
1250
1251            if not self.silent:
1252                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1253                sys.stdout.flush()
1254            cnt = cnt + 1
1255
1256            try:
1257                if self.detectBranches:
1258                    branches = self.splitFilesIntoBranches(description)
1259                    for branch in branches.keys():
1260                        ## HACK  --hwn
1261                        branchPrefix = self.depotPaths[0] + branch + "/"
1262
1263                        parent = ""
1264
1265                        filesForCommit = branches[branch]
1266
1267                        if self.verbose:
1268                            print "branch is %s" % branch
1269
1270                        self.updatedBranches.add(branch)
1271
1272                        if branch not in self.createdBranches:
1273                            self.createdBranches.add(branch)
1274                            parent = self.knownBranches[branch]
1275                            if parent == branch:
1276                                parent = ""
1277                            else:
1278                                fullBranch = self.projectName + branch
1279                                if fullBranch not in self.p4BranchesInGit:
1280                                    if not self.silent:
1281                                        print("\n    Importing new branch %s" % fullBranch);
1282                                    if self.importNewBranch(branch, change - 1):
1283                                        parent = ""
1284                                        self.p4BranchesInGit.append(fullBranch)
1285                                    if not self.silent:
1286                                        print("\n    Resuming with change %s" % change);
1287
1288                                if self.verbose:
1289                                    print "parent determined through known branches: %s" % parent
1290
1291                        branch = self.gitRefForBranch(branch)
1292                        parent = self.gitRefForBranch(parent)
1293
1294                        if self.verbose:
1295                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1296
1297                        if len(parent) == 0 and branch in self.initialParents:
1298                            parent = self.initialParents[branch]
1299                            del self.initialParents[branch]
1300
1301                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1302                else:
1303                    files = self.extractFilesFromCommit(description)
1304                    self.commit(description, files, self.branch, self.depotPaths,
1305                                self.initialParent)
1306                    self.initialParent = ""
1307            except IOError:
1308                print self.gitError.read()
1309                sys.exit(1)
1310
1311    def importHeadRevision(self, revision):
1312        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1313
1314        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1315        details["desc"] = ("Initial import of %s from the state at revision %s"
1316                           % (' '.join(self.depotPaths), revision))
1317        details["change"] = revision
1318        newestRevision = 0
1319
1320        fileCnt = 0
1321        for info in p4CmdList("files "
1322                              +  ' '.join(["%s...%s"
1323                                           % (p, revision)
1324                                           for p in self.depotPaths])):
1325
1326            if info['code'] == 'error':
1327                sys.stderr.write("p4 returned an error: %s\n"
1328                                 % info['data'])
1329                sys.exit(1)
1330
1331
1332            change = int(info["change"])
1333            if change > newestRevision:
1334                newestRevision = change
1335
1336            if info["action"] == "delete":
1337                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1338                #fileCnt = fileCnt + 1
1339                continue
1340
1341            for prop in ["depotFile", "rev", "action", "type" ]:
1342                details["%s%s" % (prop, fileCnt)] = info[prop]
1343
1344            fileCnt = fileCnt + 1
1345
1346        details["change"] = newestRevision
1347        self.updateOptionDict(details)
1348        try:
1349            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1350        except IOError:
1351            print "IO error with git fast-import. Is your git version recent enough?"
1352            print self.gitError.read()
1353
1354
1355    def getClientSpec(self):
1356        specList = p4CmdList( "client -o" )
1357        temp = {}
1358        for entry in specList:
1359            for k,v in entry.iteritems():
1360                if k.startswith("View"):
1361                    if v.startswith('"'):
1362                        start = 1
1363                    else:
1364                        start = 0
1365                    index = v.find("...")
1366                    v = v[start:index]
1367                    if v.startswith("-"):
1368                        v = v[1:]
1369                        temp[v] = -len(v)
1370                    else:
1371                        temp[v] = len(v)
1372        self.clientSpecDirs = temp.items()
1373        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1374
1375    def run(self, args):
1376        self.depotPaths = []
1377        self.changeRange = ""
1378        self.initialParent = ""
1379        self.previousDepotPaths = []
1380
1381        # map from branch depot path to parent branch
1382        self.knownBranches = {}
1383        self.initialParents = {}
1384        self.hasOrigin = originP4BranchesExist()
1385        if not self.syncWithOrigin:
1386            self.hasOrigin = False
1387
1388        if self.importIntoRemotes:
1389            self.refPrefix = "refs/remotes/p4/"
1390        else:
1391            self.refPrefix = "refs/heads/p4/"
1392
1393        if self.syncWithOrigin and self.hasOrigin:
1394            if not self.silent:
1395                print "Syncing with origin first by calling git fetch origin"
1396            system("git fetch origin")
1397
1398        if len(self.branch) == 0:
1399            self.branch = self.refPrefix + "master"
1400            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1401                system("git update-ref %s refs/heads/p4" % self.branch)
1402                system("git branch -D p4");
1403            # create it /after/ importing, when master exists
1404            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1405                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1406
1407        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1408            self.getClientSpec()
1409
1410        # TODO: should always look at previous commits,
1411        # merge with previous imports, if possible.
1412        if args == []:
1413            if self.hasOrigin:
1414                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1415            self.listExistingP4GitBranches()
1416
1417            if len(self.p4BranchesInGit) > 1:
1418                if not self.silent:
1419                    print "Importing from/into multiple branches"
1420                self.detectBranches = True
1421
1422            if self.verbose:
1423                print "branches: %s" % self.p4BranchesInGit
1424
1425            p4Change = 0
1426            for branch in self.p4BranchesInGit:
1427                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1428
1429                settings = extractSettingsGitLog(logMsg)
1430
1431                self.readOptions(settings)
1432                if (settings.has_key('depot-paths')
1433                    and settings.has_key ('change')):
1434                    change = int(settings['change']) + 1
1435                    p4Change = max(p4Change, change)
1436
1437                    depotPaths = sorted(settings['depot-paths'])
1438                    if self.previousDepotPaths == []:
1439                        self.previousDepotPaths = depotPaths
1440                    else:
1441                        paths = []
1442                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1443                            for i in range(0, min(len(cur), len(prev))):
1444                                if cur[i] <> prev[i]:
1445                                    i = i - 1
1446                                    break
1447
1448                            paths.append (cur[:i + 1])
1449
1450                        self.previousDepotPaths = paths
1451
1452            if p4Change > 0:
1453                self.depotPaths = sorted(self.previousDepotPaths)
1454                self.changeRange = "@%s,#head" % p4Change
1455                if not self.detectBranches:
1456                    self.initialParent = parseRevision(self.branch)
1457                if not self.silent and not self.detectBranches:
1458                    print "Performing incremental import into %s git branch" % self.branch
1459
1460        if not self.branch.startswith("refs/"):
1461            self.branch = "refs/heads/" + self.branch
1462
1463        if len(args) == 0 and self.depotPaths:
1464            if not self.silent:
1465                print "Depot paths: %s" % ' '.join(self.depotPaths)
1466        else:
1467            if self.depotPaths and self.depotPaths != args:
1468                print ("previous import used depot path %s and now %s was specified. "
1469                       "This doesn't work!" % (' '.join (self.depotPaths),
1470                                               ' '.join (args)))
1471                sys.exit(1)
1472
1473            self.depotPaths = sorted(args)
1474
1475        revision = ""
1476        self.users = {}
1477
1478        newPaths = []
1479        for p in self.depotPaths:
1480            if p.find("@") != -1:
1481                atIdx = p.index("@")
1482                self.changeRange = p[atIdx:]
1483                if self.changeRange == "@all":
1484                    self.changeRange = ""
1485                elif ',' not in self.changeRange:
1486                    revision = self.changeRange
1487                    self.changeRange = ""
1488                p = p[:atIdx]
1489            elif p.find("#") != -1:
1490                hashIdx = p.index("#")
1491                revision = p[hashIdx:]
1492                p = p[:hashIdx]
1493            elif self.previousDepotPaths == []:
1494                revision = "#head"
1495
1496            p = re.sub ("\.\.\.$", "", p)
1497            if not p.endswith("/"):
1498                p += "/"
1499
1500            newPaths.append(p)
1501
1502        self.depotPaths = newPaths
1503
1504
1505        self.loadUserMapFromCache()
1506        self.labels = {}
1507        if self.detectLabels:
1508            self.getLabels();
1509
1510        if self.detectBranches:
1511            ## FIXME - what's a P4 projectName ?
1512            self.projectName = self.guessProjectName()
1513
1514            if self.hasOrigin:
1515                self.getBranchMappingFromGitBranches()
1516            else:
1517                self.getBranchMapping()
1518            if self.verbose:
1519                print "p4-git branches: %s" % self.p4BranchesInGit
1520                print "initial parents: %s" % self.initialParents
1521            for b in self.p4BranchesInGit:
1522                if b != "master":
1523
1524                    ## FIXME
1525                    b = b[len(self.projectName):]
1526                self.createdBranches.add(b)
1527
1528        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1529
1530        importProcess = subprocess.Popen(["git", "fast-import"],
1531                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1532                                         stderr=subprocess.PIPE);
1533        self.gitOutput = importProcess.stdout
1534        self.gitStream = importProcess.stdin
1535        self.gitError = importProcess.stderr
1536
1537        if revision:
1538            self.importHeadRevision(revision)
1539        else:
1540            changes = []
1541
1542            if len(self.changesFile) > 0:
1543                output = open(self.changesFile).readlines()
1544                changeSet = Set()
1545                for line in output:
1546                    changeSet.add(int(line))
1547
1548                for change in changeSet:
1549                    changes.append(change)
1550
1551                changes.sort()
1552            else:
1553                if self.verbose:
1554                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1555                                                              self.changeRange)
1556                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1557
1558                if len(self.maxChanges) > 0:
1559                    changes = changes[:min(int(self.maxChanges), len(changes))]
1560
1561            if len(changes) == 0:
1562                if not self.silent:
1563                    print "No changes to import!"
1564                return True
1565
1566            if not self.silent and not self.detectBranches:
1567                print "Import destination: %s" % self.branch
1568
1569            self.updatedBranches = set()
1570
1571            self.importChanges(changes)
1572
1573            if not self.silent:
1574                print ""
1575                if len(self.updatedBranches) > 0:
1576                    sys.stdout.write("Updated branches: ")
1577                    for b in self.updatedBranches:
1578                        sys.stdout.write("%s " % b)
1579                    sys.stdout.write("\n")
1580
1581        self.gitStream.close()
1582        if importProcess.wait() != 0:
1583            die("fast-import failed: %s" % self.gitError.read())
1584        self.gitOutput.close()
1585        self.gitError.close()
1586
1587        return True
1588
1589class P4Rebase(Command):
1590    def __init__(self):
1591        Command.__init__(self)
1592        self.options = [ ]
1593        self.description = ("Fetches the latest revision from perforce and "
1594                            + "rebases the current work (branch) against it")
1595        self.verbose = False
1596
1597    def run(self, args):
1598        sync = P4Sync()
1599        sync.run([])
1600
1601        return self.rebase()
1602
1603    def rebase(self):
1604        if os.system("git update-index --refresh") != 0:
1605            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1606        if len(read_pipe("git diff-index HEAD --")) > 0:
1607            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1608
1609        [upstream, settings] = findUpstreamBranchPoint()
1610        if len(upstream) == 0:
1611            die("Cannot find upstream branchpoint for rebase")
1612
1613        # the branchpoint may be p4/foo~3, so strip off the parent
1614        upstream = re.sub("~[0-9]+$", "", upstream)
1615
1616        print "Rebasing the current branch onto %s" % upstream
1617        oldHead = read_pipe("git rev-parse HEAD").strip()
1618        system("git rebase %s" % upstream)
1619        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1620        return True
1621
1622class P4Clone(P4Sync):
1623    def __init__(self):
1624        P4Sync.__init__(self)
1625        self.description = "Creates a new git repository and imports from Perforce into it"
1626        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1627        self.options += [
1628            optparse.make_option("--destination", dest="cloneDestination",
1629                                 action='store', default=None,
1630                                 help="where to leave result of the clone"),
1631            optparse.make_option("-/", dest="cloneExclude",
1632                                 action="append", type="string",
1633                                 help="exclude depot path")
1634        ]
1635        self.cloneDestination = None
1636        self.needsGit = False
1637
1638    # This is required for the "append" cloneExclude action
1639    def ensure_value(self, attr, value):
1640        if not hasattr(self, attr) or getattr(self, attr) is None:
1641            setattr(self, attr, value)
1642        return getattr(self, attr)
1643
1644    def defaultDestination(self, args):
1645        ## TODO: use common prefix of args?
1646        depotPath = args[0]
1647        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1648        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1649        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1650        depotDir = re.sub(r"/$", "", depotDir)
1651        return os.path.split(depotDir)[1]
1652
1653    def run(self, args):
1654        if len(args) < 1:
1655            return False
1656
1657        if self.keepRepoPath and not self.cloneDestination:
1658            sys.stderr.write("Must specify destination for --keep-path\n")
1659            sys.exit(1)
1660
1661        depotPaths = args
1662
1663        if not self.cloneDestination and len(depotPaths) > 1:
1664            self.cloneDestination = depotPaths[-1]
1665            depotPaths = depotPaths[:-1]
1666
1667        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1668        for p in depotPaths:
1669            if not p.startswith("//"):
1670                return False
1671
1672        if not self.cloneDestination:
1673            self.cloneDestination = self.defaultDestination(args)
1674
1675        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1676        if not os.path.exists(self.cloneDestination):
1677            os.makedirs(self.cloneDestination)
1678        chdir(self.cloneDestination)
1679        system("git init")
1680        self.gitdir = os.getcwd() + "/.git"
1681        if not P4Sync.run(self, depotPaths):
1682            return False
1683        if self.branch != "master":
1684            if gitBranchExists("refs/remotes/p4/master"):
1685                system("git branch master refs/remotes/p4/master")
1686                system("git checkout -f")
1687            else:
1688                print "Could not detect main branch. No checkout/master branch created."
1689
1690        return True
1691
1692class P4Branches(Command):
1693    def __init__(self):
1694        Command.__init__(self)
1695        self.options = [ ]
1696        self.description = ("Shows the git branches that hold imports and their "
1697                            + "corresponding perforce depot paths")
1698        self.verbose = False
1699
1700    def run(self, args):
1701        if originP4BranchesExist():
1702            createOrUpdateBranchesFromOrigin()
1703
1704        cmdline = "git rev-parse --symbolic "
1705        cmdline += " --remotes"
1706
1707        for line in read_pipe_lines(cmdline):
1708            line = line.strip()
1709
1710            if not line.startswith('p4/') or line == "p4/HEAD":
1711                continue
1712            branch = line
1713
1714            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1715            settings = extractSettingsGitLog(log)
1716
1717            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1718        return True
1719
1720class HelpFormatter(optparse.IndentedHelpFormatter):
1721    def __init__(self):
1722        optparse.IndentedHelpFormatter.__init__(self)
1723
1724    def format_description(self, description):
1725        if description:
1726            return description + "\n"
1727        else:
1728            return ""
1729
1730def printUsage(commands):
1731    print "usage: %s <command> [options]" % sys.argv[0]
1732    print ""
1733    print "valid commands: %s" % ", ".join(commands)
1734    print ""
1735    print "Try %s <command> --help for command specific help." % sys.argv[0]
1736    print ""
1737
1738commands = {
1739    "debug" : P4Debug,
1740    "submit" : P4Submit,
1741    "commit" : P4Submit,
1742    "sync" : P4Sync,
1743    "rebase" : P4Rebase,
1744    "clone" : P4Clone,
1745    "rollback" : P4RollBack,
1746    "branches" : P4Branches
1747}
1748
1749
1750def main():
1751    if len(sys.argv[1:]) == 0:
1752        printUsage(commands.keys())
1753        sys.exit(2)
1754
1755    cmd = ""
1756    cmdName = sys.argv[1]
1757    try:
1758        klass = commands[cmdName]
1759        cmd = klass()
1760    except KeyError:
1761        print "unknown command %s" % cmdName
1762        print ""
1763        printUsage(commands.keys())
1764        sys.exit(2)
1765
1766    options = cmd.options
1767    cmd.gitdir = os.environ.get("GIT_DIR", None)
1768
1769    args = sys.argv[2:]
1770
1771    if len(options) > 0:
1772        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1773
1774        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1775                                       options,
1776                                       description = cmd.description,
1777                                       formatter = HelpFormatter())
1778
1779        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1780    global verbose
1781    verbose = cmd.verbose
1782    if cmd.needsGit:
1783        if cmd.gitdir == None:
1784            cmd.gitdir = os.path.abspath(".git")
1785            if not isValidGitDir(cmd.gitdir):
1786                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1787                if os.path.exists(cmd.gitdir):
1788                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1789                    if len(cdup) > 0:
1790                        chdir(cdup);
1791
1792        if not isValidGitDir(cmd.gitdir):
1793            if isValidGitDir(cmd.gitdir + "/.git"):
1794                cmd.gitdir += "/.git"
1795            else:
1796                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1797
1798        os.environ["GIT_DIR"] = cmd.gitdir
1799
1800    if not cmd.run(args):
1801        parser.print_help()
1802
1803
1804if __name__ == '__main__':
1805    main()