Andrew's git - gitweb.git/blob - contrib/fast-import/git-p4

   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if inFilesSection:
 517                if line.startswith("\t"):
 518                    # path starts and ends with a tab
 519                    path = line[1:]
 520                    lastTab = path.rfind("\t")
 521                    if lastTab != -1:
 522                        path = path[:lastTab]
 523                        if not path.startswith(self.depotPath):
 524                            continue
 525                else:
 526                    inFilesSection = False
 527            else:
 528                if line.startswith("Files:"):
 529                    inFilesSection = True
 530
 531            template += line
 532
 533        return template
 534
 535    def applyCommit(self, id):
 536        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537        diffOpts = ("", "-M")[self.detectRename]
 538        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539        filesToAdd = set()
 540        filesToDelete = set()
 541        editedFiles = set()
 542        filesToChangeExecBit = {}
 543        for line in diff:
 544            diff = parseDiffTreeEntry(line)
 545            modifier = diff['status']
 546            path = diff['src']
 547            if modifier == "M":
 548                system("p4 edit \"%s\"" % path)
 549                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                    filesToChangeExecBit[path] = diff['dst_mode']
 551                editedFiles.add(path)
 552            elif modifier == "A":
 553                filesToAdd.add(path)
 554                filesToChangeExecBit[path] = diff['dst_mode']
 555                if path in filesToDelete:
 556                    filesToDelete.remove(path)
 557            elif modifier == "D":
 558                filesToDelete.add(path)
 559                if path in filesToAdd:
 560                    filesToAdd.remove(path)
 561            elif modifier == "R":
 562                src, dest = diff['src'], diff['dst']
 563                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                system("p4 edit \"%s\"" % (dest))
 565                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                    filesToChangeExecBit[dest] = diff['dst_mode']
 567                os.unlink(dest)
 568                editedFiles.add(dest)
 569                filesToDelete.add(src)
 570            else:
 571                die("unknown modifier %s for %s" % (modifier, path))
 572
 573        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574        patchcmd = diffcmd + " | git apply "
 575        tryPatchCmd = patchcmd + "--check -"
 576        applyPatchCmd = patchcmd + "--check --apply -"
 577
 578        if os.system(tryPatchCmd) != 0:
 579            print "Unfortunately applying the change failed!"
 580            print "What do you want to do?"
 581            response = "x"
 582            while response != "s" and response != "a" and response != "w":
 583                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585            if response == "s":
 586                print "Skipping! Good luck with the next patches..."
 587                for f in editedFiles:
 588                    system("p4 revert \"%s\"" % f);
 589                for f in filesToAdd:
 590                    system("rm %s" %f)
 591                return
 592            elif response == "a":
 593                os.system(applyPatchCmd)
 594                if len(filesToAdd) > 0:
 595                    print "You may also want to call p4 add on the following files:"
 596                    print " ".join(filesToAdd)
 597                if len(filesToDelete):
 598                    print "The following files should be scheduled for deletion with p4 delete:"
 599                    print " ".join(filesToDelete)
 600                die("Please resolve and submit the conflict manually and "
 601                    + "continue afterwards with git-p4 submit --continue")
 602            elif response == "w":
 603                system(diffcmd + " > patch.txt")
 604                print "Patch saved to patch.txt in %s !" % self.clientPath
 605                die("Please resolve and submit the conflict manually and "
 606                    "continue afterwards with git-p4 submit --continue")
 607
 608        system(applyPatchCmd)
 609
 610        for f in filesToAdd:
 611            system("p4 add \"%s\"" % f)
 612        for f in filesToDelete:
 613            system("p4 revert \"%s\"" % f)
 614            system("p4 delete \"%s\"" % f)
 615
 616        # Set/clear executable bits
 617        for f in filesToChangeExecBit.keys():
 618            mode = filesToChangeExecBit[f]
 619            setP4ExecBit(f, mode)
 620
 621        logMessage = extractLogMessageFromGitCommit(id)
 622        if self.isWindows:
 623            logMessage = logMessage.replace("\n", "\r\n")
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            if os.environ.has_key("P4DIFF"):
 631                del(os.environ["P4DIFF"])
 632            diff = read_pipe("p4 diff -du ...")
 633
 634            for newFile in filesToAdd:
 635                diff += "==== new file ====\n"
 636                diff += "--- /dev/null\n"
 637                diff += "+++ %s\n" % newFile
 638                f = open(newFile, "r")
 639                for line in f.readlines():
 640                    diff += "+" + line
 641                f.close()
 642
 643            separatorLine = "######## everything below this line is just the diff #######"
 644            if platform.system() == "Windows":
 645                separatorLine += "\r"
 646            separatorLine += "\n"
 647
 648            [handle, fileName] = tempfile.mkstemp()
 649            tmpFile = os.fdopen(handle, "w+")
 650            tmpFile.write(submitTemplate + separatorLine + diff)
 651            tmpFile.close()
 652            defaultEditor = "vi"
 653            if platform.system() == "Windows":
 654                defaultEditor = "notepad"
 655            if os.environ.has_key("P4EDITOR"):
 656                editor = os.environ.get("P4EDITOR")
 657            else:
 658                editor = os.environ.get("EDITOR", defaultEditor);
 659            system(editor + " " + fileName)
 660            tmpFile = open(fileName, "rb")
 661            message = tmpFile.read()
 662            tmpFile.close()
 663            os.remove(fileName)
 664            submitTemplate = message[:message.index(separatorLine)]
 665            if self.isWindows:
 666                submitTemplate = submitTemplate.replace("\r\n", "\n")
 667
 668            write_pipe("p4 submit -i", submitTemplate)
 669        else:
 670            fileName = "submit.txt"
 671            file = open(fileName, "w+")
 672            file.write(self.prepareLogMessage(template, logMessage))
 673            file.close()
 674            print ("Perforce submit template written as %s. "
 675                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 676                   % (fileName, fileName))
 677
 678    def run(self, args):
 679        if len(args) == 0:
 680            self.master = currentGitBranch()
 681            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 682                die("Detecting current git branch failed!")
 683        elif len(args) == 1:
 684            self.master = args[0]
 685        else:
 686            return False
 687
 688        [upstream, settings] = findUpstreamBranchPoint()
 689        self.depotPath = settings['depot-paths'][0]
 690        if len(self.origin) == 0:
 691            self.origin = upstream
 692
 693        if self.verbose:
 694            print "Origin branch is " + self.origin
 695
 696        if len(self.depotPath) == 0:
 697            print "Internal error: cannot locate perforce depot path from existing branches"
 698            sys.exit(128)
 699
 700        self.clientPath = p4Where(self.depotPath)
 701
 702        if len(self.clientPath) == 0:
 703            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 704            sys.exit(128)
 705
 706        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 707        self.oldWorkingDirectory = os.getcwd()
 708
 709        os.chdir(self.clientPath)
 710        print "Syncronizing p4 checkout..."
 711        system("p4 sync ...")
 712
 713        self.check()
 714
 715        commits = []
 716        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 717            commits.append(line.strip())
 718        commits.reverse()
 719
 720        while len(commits) > 0:
 721            commit = commits[0]
 722            commits = commits[1:]
 723            self.applyCommit(commit)
 724            if not self.interactive:
 725                break
 726
 727        if len(commits) == 0:
 728            print "All changes applied!"
 729            os.chdir(self.oldWorkingDirectory)
 730
 731            sync = P4Sync()
 732            sync.run([])
 733
 734            rebase = P4Rebase()
 735            rebase.rebase()
 736
 737        return True
 738
 739class P4Sync(Command):
 740    def __init__(self):
 741        Command.__init__(self)
 742        self.options = [
 743                optparse.make_option("--branch", dest="branch"),
 744                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 745                optparse.make_option("--changesfile", dest="changesFile"),
 746                optparse.make_option("--silent", dest="silent", action="store_true"),
 747                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 748                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 749                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 750                                     help="Import into refs/heads/ , not refs/remotes"),
 751                optparse.make_option("--max-changes", dest="maxChanges"),
 752                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 753                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 754                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 755                                     help="Only sync files that are included in the Perforce Client Spec")
 756        ]
 757        self.description = """Imports from Perforce into a git repository.\n
 758    example:
 759    //depot/my/project/ -- to import the current head
 760    //depot/my/project/@all -- to import everything
 761    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 762
 763    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 764
 765        self.usage += " //depot/path[@revRange]"
 766        self.silent = False
 767        self.createdBranches = Set()
 768        self.committedChanges = Set()
 769        self.branch = ""
 770        self.detectBranches = False
 771        self.detectLabels = False
 772        self.changesFile = ""
 773        self.syncWithOrigin = True
 774        self.verbose = False
 775        self.importIntoRemotes = True
 776        self.maxChanges = ""
 777        self.isWindows = (platform.system() == "Windows")
 778        self.keepRepoPath = False
 779        self.depotPaths = None
 780        self.p4BranchesInGit = []
 781        self.cloneExclude = []
 782        self.useClientSpec = False
 783        self.clientSpecDirs = []
 784
 785        if gitConfig("git-p4.syncFromOrigin") == "false":
 786            self.syncWithOrigin = False
 787
 788    def extractFilesFromCommit(self, commit):
 789        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 790                             for path in self.cloneExclude]
 791        files = []
 792        fnum = 0
 793        while commit.has_key("depotFile%s" % fnum):
 794            path =  commit["depotFile%s" % fnum]
 795
 796            if [p for p in self.cloneExclude
 797                if path.startswith (p)]:
 798                found = False
 799            else:
 800                found = [p for p in self.depotPaths
 801                         if path.startswith (p)]
 802            if not found:
 803                fnum = fnum + 1
 804                continue
 805
 806            file = {}
 807            file["path"] = path
 808            file["rev"] = commit["rev%s" % fnum]
 809            file["action"] = commit["action%s" % fnum]
 810            file["type"] = commit["type%s" % fnum]
 811            files.append(file)
 812            fnum = fnum + 1
 813        return files
 814
 815    def stripRepoPath(self, path, prefixes):
 816        if self.keepRepoPath:
 817            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 818
 819        for p in prefixes:
 820            if path.startswith(p):
 821                path = path[len(p):]
 822
 823        return path
 824
 825    def splitFilesIntoBranches(self, commit):
 826        branches = {}
 827        fnum = 0
 828        while commit.has_key("depotFile%s" % fnum):
 829            path =  commit["depotFile%s" % fnum]
 830            found = [p for p in self.depotPaths
 831                     if path.startswith (p)]
 832            if not found:
 833                fnum = fnum + 1
 834                continue
 835
 836            file = {}
 837            file["path"] = path
 838            file["rev"] = commit["rev%s" % fnum]
 839            file["action"] = commit["action%s" % fnum]
 840            file["type"] = commit["type%s" % fnum]
 841            fnum = fnum + 1
 842
 843            relPath = self.stripRepoPath(path, self.depotPaths)
 844
 845            for branch in self.knownBranches.keys():
 846
 847                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 848                if relPath.startswith(branch + "/"):
 849                    if branch not in branches:
 850                        branches[branch] = []
 851                    branches[branch].append(file)
 852                    break
 853
 854        return branches
 855
 856    ## Should move this out, doesn't use SELF.
 857    def readP4Files(self, files):
 858        filesForCommit = []
 859        filesToRead = []
 860
 861        for f in files:
 862            includeFile = True
 863            for val in self.clientSpecDirs:
 864                if f['path'].startswith(val[0]):
 865                    if val[1] <= 0:
 866                        includeFile = False
 867                    break
 868
 869            if includeFile:
 870                filesForCommit.append(f)
 871                if f['action'] != 'delete':
 872                    filesToRead.append(f)
 873
 874        filedata = []
 875        if len(filesToRead) > 0:
 876            filedata = p4CmdList('-x - print',
 877                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 878                                                  for f in filesToRead]),
 879                                 stdin_mode='w+')
 880
 881            if "p4ExitCode" in filedata[0]:
 882                die("Problems executing p4. Error: [%d]."
 883                    % (filedata[0]['p4ExitCode']));
 884
 885        j = 0;
 886        contents = {}
 887        while j < len(filedata):
 888            stat = filedata[j]
 889            j += 1
 890            text = [];
 891            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 892                text.append(filedata[j]['data'])
 893                j += 1
 894            text = ''.join(text)
 895
 896            if not stat.has_key('depotFile'):
 897                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 898                continue
 899
 900            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 901                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 902            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 903                text = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 904
 905            contents[stat['depotFile']] = text
 906
 907        for f in filesForCommit:
 908            path = f['path']
 909            if contents.has_key(path):
 910                f['data'] = contents[path]
 911
 912        return filesForCommit
 913
 914    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 915        epoch = details["time"]
 916        author = details["user"]
 917
 918        if self.verbose:
 919            print "commit into %s" % branch
 920
 921        # start with reading files; if that fails, we should not
 922        # create a commit.
 923        new_files = []
 924        for f in files:
 925            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 926                new_files.append (f)
 927            else:
 928                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 929        files = self.readP4Files(new_files)
 930
 931        self.gitStream.write("commit %s\n" % branch)
 932#        gitStream.write("mark :%s\n" % details["change"])
 933        self.committedChanges.add(int(details["change"]))
 934        committer = ""
 935        if author not in self.users:
 936            self.getUserMapFromPerforceServer()
 937        if author in self.users:
 938            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 939        else:
 940            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 941
 942        self.gitStream.write("committer %s\n" % committer)
 943
 944        self.gitStream.write("data <<EOT\n")
 945        self.gitStream.write(details["desc"])
 946        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 947                             % (','.join (branchPrefixes), details["change"]))
 948        if len(details['options']) > 0:
 949            self.gitStream.write(": options = %s" % details['options'])
 950        self.gitStream.write("]\nEOT\n\n")
 951
 952        if len(parent) > 0:
 953            if self.verbose:
 954                print "parent %s" % parent
 955            self.gitStream.write("from %s\n" % parent)
 956
 957        for file in files:
 958            if file["type"] == "apple":
 959                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 960                continue
 961
 962            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 963            if file["action"] == "delete":
 964                self.gitStream.write("D %s\n" % relPath)
 965            else:
 966                data = file['data']
 967
 968                mode = "644"
 969                if isP4Exec(file["type"]):
 970                    mode = "755"
 971                elif file["type"] == "symlink":
 972                    mode = "120000"
 973                    # p4 print on a symlink contains "target\n", so strip it off
 974                    data = data[:-1]
 975
 976                if self.isWindows and file["type"].endswith("text"):
 977                    data = data.replace("\r\n", "\n")
 978
 979                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 980                self.gitStream.write("data %s\n" % len(data))
 981                self.gitStream.write(data)
 982                self.gitStream.write("\n")
 983
 984        self.gitStream.write("\n")
 985
 986        change = int(details["change"])
 987
 988        if self.labels.has_key(change):
 989            label = self.labels[change]
 990            labelDetails = label[0]
 991            labelRevisions = label[1]
 992            if self.verbose:
 993                print "Change %s is labelled %s" % (change, labelDetails)
 994
 995            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 996                                                    for p in branchPrefixes]))
 997
 998            if len(files) == len(labelRevisions):
 999
1000                cleanedFiles = {}
1001                for info in files:
1002                    if info["action"] == "delete":
1003                        continue
1004                    cleanedFiles[info["depotFile"]] = info["rev"]
1005
1006                if cleanedFiles == labelRevisions:
1007                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1008                    self.gitStream.write("from %s\n" % branch)
1009
1010                    owner = labelDetails["Owner"]
1011                    tagger = ""
1012                    if author in self.users:
1013                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1014                    else:
1015                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1016                    self.gitStream.write("tagger %s\n" % tagger)
1017                    self.gitStream.write("data <<EOT\n")
1018                    self.gitStream.write(labelDetails["Description"])
1019                    self.gitStream.write("EOT\n\n")
1020
1021                else:
1022                    if not self.silent:
1023                        print ("Tag %s does not match with change %s: files do not match."
1024                               % (labelDetails["label"], change))
1025
1026            else:
1027                if not self.silent:
1028                    print ("Tag %s does not match with change %s: file count is different."
1029                           % (labelDetails["label"], change))
1030
1031    def getUserCacheFilename(self):
1032        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1033        return home + "/.gitp4-usercache.txt"
1034
1035    def getUserMapFromPerforceServer(self):
1036        if self.userMapFromPerforceServer:
1037            return
1038        self.users = {}
1039
1040        for output in p4CmdList("users"):
1041            if not output.has_key("User"):
1042                continue
1043            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1044
1045
1046        s = ''
1047        for (key, val) in self.users.items():
1048            s += "%s\t%s\n" % (key, val)
1049
1050        open(self.getUserCacheFilename(), "wb").write(s)
1051        self.userMapFromPerforceServer = True
1052
1053    def loadUserMapFromCache(self):
1054        self.users = {}
1055        self.userMapFromPerforceServer = False
1056        try:
1057            cache = open(self.getUserCacheFilename(), "rb")
1058            lines = cache.readlines()
1059            cache.close()
1060            for line in lines:
1061                entry = line.strip().split("\t")
1062                self.users[entry[0]] = entry[1]
1063        except IOError:
1064            self.getUserMapFromPerforceServer()
1065
1066    def getLabels(self):
1067        self.labels = {}
1068
1069        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1070        if len(l) > 0 and not self.silent:
1071            print "Finding files belonging to labels in %s" % `self.depotPaths`
1072
1073        for output in l:
1074            label = output["label"]
1075            revisions = {}
1076            newestChange = 0
1077            if self.verbose:
1078                print "Querying files for label %s" % label
1079            for file in p4CmdList("files "
1080                                  +  ' '.join (["%s...@%s" % (p, label)
1081                                                for p in self.depotPaths])):
1082                revisions[file["depotFile"]] = file["rev"]
1083                change = int(file["change"])
1084                if change > newestChange:
1085                    newestChange = change
1086
1087            self.labels[newestChange] = [output, revisions]
1088
1089        if self.verbose:
1090            print "Label changes: %s" % self.labels.keys()
1091
1092    def guessProjectName(self):
1093        for p in self.depotPaths:
1094            if p.endswith("/"):
1095                p = p[:-1]
1096            p = p[p.strip().rfind("/") + 1:]
1097            if not p.endswith("/"):
1098               p += "/"
1099            return p
1100
1101    def getBranchMapping(self):
1102        lostAndFoundBranches = set()
1103
1104        for info in p4CmdList("branches"):
1105            details = p4Cmd("branch -o %s" % info["branch"])
1106            viewIdx = 0
1107            while details.has_key("View%s" % viewIdx):
1108                paths = details["View%s" % viewIdx].split(" ")
1109                viewIdx = viewIdx + 1
1110                # require standard //depot/foo/... //depot/bar/... mapping
1111                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1112                    continue
1113                source = paths[0]
1114                destination = paths[1]
1115                ## HACK
1116                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1117                    source = source[len(self.depotPaths[0]):-4]
1118                    destination = destination[len(self.depotPaths[0]):-4]
1119
1120                    if destination in self.knownBranches:
1121                        if not self.silent:
1122                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1123                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1124                        continue
1125
1126                    self.knownBranches[destination] = source
1127
1128                    lostAndFoundBranches.discard(destination)
1129
1130                    if source not in self.knownBranches:
1131                        lostAndFoundBranches.add(source)
1132
1133
1134        for branch in lostAndFoundBranches:
1135            self.knownBranches[branch] = branch
1136
1137    def getBranchMappingFromGitBranches(self):
1138        branches = p4BranchesInGit(self.importIntoRemotes)
1139        for branch in branches.keys():
1140            if branch == "master":
1141                branch = "main"
1142            else:
1143                branch = branch[len(self.projectName):]
1144            self.knownBranches[branch] = branch
1145
1146    def listExistingP4GitBranches(self):
1147        # branches holds mapping from name to commit
1148        branches = p4BranchesInGit(self.importIntoRemotes)
1149        self.p4BranchesInGit = branches.keys()
1150        for branch in branches.keys():
1151            self.initialParents[self.refPrefix + branch] = branches[branch]
1152
1153    def updateOptionDict(self, d):
1154        option_keys = {}
1155        if self.keepRepoPath:
1156            option_keys['keepRepoPath'] = 1
1157
1158        d["options"] = ' '.join(sorted(option_keys.keys()))
1159
1160    def readOptions(self, d):
1161        self.keepRepoPath = (d.has_key('options')
1162                             and ('keepRepoPath' in d['options']))
1163
1164    def gitRefForBranch(self, branch):
1165        if branch == "main":
1166            return self.refPrefix + "master"
1167
1168        if len(branch) <= 0:
1169            return branch
1170
1171        return self.refPrefix + self.projectName + branch
1172
1173    def gitCommitByP4Change(self, ref, change):
1174        if self.verbose:
1175            print "looking in ref " + ref + " for change %s using bisect..." % change
1176
1177        earliestCommit = ""
1178        latestCommit = parseRevision(ref)
1179
1180        while True:
1181            if self.verbose:
1182                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1183            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1184            if len(next) == 0:
1185                if self.verbose:
1186                    print "argh"
1187                return ""
1188            log = extractLogMessageFromGitCommit(next)
1189            settings = extractSettingsGitLog(log)
1190            currentChange = int(settings['change'])
1191            if self.verbose:
1192                print "current change %s" % currentChange
1193
1194            if currentChange == change:
1195                if self.verbose:
1196                    print "found %s" % next
1197                return next
1198
1199            if currentChange < change:
1200                earliestCommit = "^%s" % next
1201            else:
1202                latestCommit = "%s" % next
1203
1204        return ""
1205
1206    def importNewBranch(self, branch, maxChange):
1207        # make fast-import flush all changes to disk and update the refs using the checkpoint
1208        # command so that we can try to find the branch parent in the git history
1209        self.gitStream.write("checkpoint\n\n");
1210        self.gitStream.flush();
1211        branchPrefix = self.depotPaths[0] + branch + "/"
1212        range = "@1,%s" % maxChange
1213        #print "prefix" + branchPrefix
1214        changes = p4ChangesForPaths([branchPrefix], range)
1215        if len(changes) <= 0:
1216            return False
1217        firstChange = changes[0]
1218        #print "first change in branch: %s" % firstChange
1219        sourceBranch = self.knownBranches[branch]
1220        sourceDepotPath = self.depotPaths[0] + sourceBranch
1221        sourceRef = self.gitRefForBranch(sourceBranch)
1222        #print "source " + sourceBranch
1223
1224        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1225        #print "branch parent: %s" % branchParentChange
1226        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1227        if len(gitParent) > 0:
1228            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1229            #print "parent git commit: %s" % gitParent
1230
1231        self.importChanges(changes)
1232        return True
1233
1234    def importChanges(self, changes):
1235        cnt = 1
1236        for change in changes:
1237            description = p4Cmd("describe %s" % change)
1238            self.updateOptionDict(description)
1239
1240            if not self.silent:
1241                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1242                sys.stdout.flush()
1243            cnt = cnt + 1
1244
1245            try:
1246                if self.detectBranches:
1247                    branches = self.splitFilesIntoBranches(description)
1248                    for branch in branches.keys():
1249                        ## HACK  --hwn
1250                        branchPrefix = self.depotPaths[0] + branch + "/"
1251
1252                        parent = ""
1253
1254                        filesForCommit = branches[branch]
1255
1256                        if self.verbose:
1257                            print "branch is %s" % branch
1258
1259                        self.updatedBranches.add(branch)
1260
1261                        if branch not in self.createdBranches:
1262                            self.createdBranches.add(branch)
1263                            parent = self.knownBranches[branch]
1264                            if parent == branch:
1265                                parent = ""
1266                            else:
1267                                fullBranch = self.projectName + branch
1268                                if fullBranch not in self.p4BranchesInGit:
1269                                    if not self.silent:
1270                                        print("\n    Importing new branch %s" % fullBranch);
1271                                    if self.importNewBranch(branch, change - 1):
1272                                        parent = ""
1273                                        self.p4BranchesInGit.append(fullBranch)
1274                                    if not self.silent:
1275                                        print("\n    Resuming with change %s" % change);
1276
1277                                if self.verbose:
1278                                    print "parent determined through known branches: %s" % parent
1279
1280                        branch = self.gitRefForBranch(branch)
1281                        parent = self.gitRefForBranch(parent)
1282
1283                        if self.verbose:
1284                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1285
1286                        if len(parent) == 0 and branch in self.initialParents:
1287                            parent = self.initialParents[branch]
1288                            del self.initialParents[branch]
1289
1290                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1291                else:
1292                    files = self.extractFilesFromCommit(description)
1293                    self.commit(description, files, self.branch, self.depotPaths,
1294                                self.initialParent)
1295                    self.initialParent = ""
1296            except IOError:
1297                print self.gitError.read()
1298                sys.exit(1)
1299
1300    def importHeadRevision(self, revision):
1301        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1302
1303        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1304        details["desc"] = ("Initial import of %s from the state at revision %s"
1305                           % (' '.join(self.depotPaths), revision))
1306        details["change"] = revision
1307        newestRevision = 0
1308
1309        fileCnt = 0
1310        for info in p4CmdList("files "
1311                              +  ' '.join(["%s...%s"
1312                                           % (p, revision)
1313                                           for p in self.depotPaths])):
1314
1315            if info['code'] == 'error':
1316                sys.stderr.write("p4 returned an error: %s\n"
1317                                 % info['data'])
1318                sys.exit(1)
1319
1320
1321            change = int(info["change"])
1322            if change > newestRevision:
1323                newestRevision = change
1324
1325            if info["action"] == "delete":
1326                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1327                #fileCnt = fileCnt + 1
1328                continue
1329
1330            for prop in ["depotFile", "rev", "action", "type" ]:
1331                details["%s%s" % (prop, fileCnt)] = info[prop]
1332
1333            fileCnt = fileCnt + 1
1334
1335        details["change"] = newestRevision
1336        self.updateOptionDict(details)
1337        try:
1338            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1339        except IOError:
1340            print "IO error with git fast-import. Is your git version recent enough?"
1341            print self.gitError.read()
1342
1343
1344    def getClientSpec(self):
1345        specList = p4CmdList( "client -o" )
1346        temp = {}
1347        for entry in specList:
1348            for k,v in entry.iteritems():
1349                if k.startswith("View"):
1350                    if v.startswith('"'):
1351                        start = 1
1352                    else:
1353                        start = 0
1354                    index = v.find("...")
1355                    v = v[start:index]
1356                    if v.startswith("-"):
1357                        v = v[1:]
1358                        temp[v] = -len(v)
1359                    else:
1360                        temp[v] = len(v)
1361        self.clientSpecDirs = temp.items()
1362        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1363
1364    def run(self, args):
1365        self.depotPaths = []
1366        self.changeRange = ""
1367        self.initialParent = ""
1368        self.previousDepotPaths = []
1369
1370        # map from branch depot path to parent branch
1371        self.knownBranches = {}
1372        self.initialParents = {}
1373        self.hasOrigin = originP4BranchesExist()
1374        if not self.syncWithOrigin:
1375            self.hasOrigin = False
1376
1377        if self.importIntoRemotes:
1378            self.refPrefix = "refs/remotes/p4/"
1379        else:
1380            self.refPrefix = "refs/heads/p4/"
1381
1382        if self.syncWithOrigin and self.hasOrigin:
1383            if not self.silent:
1384                print "Syncing with origin first by calling git fetch origin"
1385            system("git fetch origin")
1386
1387        if len(self.branch) == 0:
1388            self.branch = self.refPrefix + "master"
1389            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1390                system("git update-ref %s refs/heads/p4" % self.branch)
1391                system("git branch -D p4");
1392            # create it /after/ importing, when master exists
1393            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1394                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1395
1396        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1397            self.getClientSpec()
1398
1399        # TODO: should always look at previous commits,
1400        # merge with previous imports, if possible.
1401        if args == []:
1402            if self.hasOrigin:
1403                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1404            self.listExistingP4GitBranches()
1405
1406            if len(self.p4BranchesInGit) > 1:
1407                if not self.silent:
1408                    print "Importing from/into multiple branches"
1409                self.detectBranches = True
1410
1411            if self.verbose:
1412                print "branches: %s" % self.p4BranchesInGit
1413
1414            p4Change = 0
1415            for branch in self.p4BranchesInGit:
1416                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1417
1418                settings = extractSettingsGitLog(logMsg)
1419
1420                self.readOptions(settings)
1421                if (settings.has_key('depot-paths')
1422                    and settings.has_key ('change')):
1423                    change = int(settings['change']) + 1
1424                    p4Change = max(p4Change, change)
1425
1426                    depotPaths = sorted(settings['depot-paths'])
1427                    if self.previousDepotPaths == []:
1428                        self.previousDepotPaths = depotPaths
1429                    else:
1430                        paths = []
1431                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1432                            for i in range(0, min(len(cur), len(prev))):
1433                                if cur[i] <> prev[i]:
1434                                    i = i - 1
1435                                    break
1436
1437                            paths.append (cur[:i + 1])
1438
1439                        self.previousDepotPaths = paths
1440
1441            if p4Change > 0:
1442                self.depotPaths = sorted(self.previousDepotPaths)
1443                self.changeRange = "@%s,#head" % p4Change
1444                if not self.detectBranches:
1445                    self.initialParent = parseRevision(self.branch)
1446                if not self.silent and not self.detectBranches:
1447                    print "Performing incremental import into %s git branch" % self.branch
1448
1449        if not self.branch.startswith("refs/"):
1450            self.branch = "refs/heads/" + self.branch
1451
1452        if len(args) == 0 and self.depotPaths:
1453            if not self.silent:
1454                print "Depot paths: %s" % ' '.join(self.depotPaths)
1455        else:
1456            if self.depotPaths and self.depotPaths != args:
1457                print ("previous import used depot path %s and now %s was specified. "
1458                       "This doesn't work!" % (' '.join (self.depotPaths),
1459                                               ' '.join (args)))
1460                sys.exit(1)
1461
1462            self.depotPaths = sorted(args)
1463
1464        revision = ""
1465        self.users = {}
1466
1467        newPaths = []
1468        for p in self.depotPaths:
1469            if p.find("@") != -1:
1470                atIdx = p.index("@")
1471                self.changeRange = p[atIdx:]
1472                if self.changeRange == "@all":
1473                    self.changeRange = ""
1474                elif ',' not in self.changeRange:
1475                    revision = self.changeRange
1476                    self.changeRange = ""
1477                p = p[:atIdx]
1478            elif p.find("#") != -1:
1479                hashIdx = p.index("#")
1480                revision = p[hashIdx:]
1481                p = p[:hashIdx]
1482            elif self.previousDepotPaths == []:
1483                revision = "#head"
1484
1485            p = re.sub ("\.\.\.$", "", p)
1486            if not p.endswith("/"):
1487                p += "/"
1488
1489            newPaths.append(p)
1490
1491        self.depotPaths = newPaths
1492
1493
1494        self.loadUserMapFromCache()
1495        self.labels = {}
1496        if self.detectLabels:
1497            self.getLabels();
1498
1499        if self.detectBranches:
1500            ## FIXME - what's a P4 projectName ?
1501            self.projectName = self.guessProjectName()
1502
1503            if self.hasOrigin:
1504                self.getBranchMappingFromGitBranches()
1505            else:
1506                self.getBranchMapping()
1507            if self.verbose:
1508                print "p4-git branches: %s" % self.p4BranchesInGit
1509                print "initial parents: %s" % self.initialParents
1510            for b in self.p4BranchesInGit:
1511                if b != "master":
1512
1513                    ## FIXME
1514                    b = b[len(self.projectName):]
1515                self.createdBranches.add(b)
1516
1517        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1518
1519        importProcess = subprocess.Popen(["git", "fast-import"],
1520                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1521                                         stderr=subprocess.PIPE);
1522        self.gitOutput = importProcess.stdout
1523        self.gitStream = importProcess.stdin
1524        self.gitError = importProcess.stderr
1525
1526        if revision:
1527            self.importHeadRevision(revision)
1528        else:
1529            changes = []
1530
1531            if len(self.changesFile) > 0:
1532                output = open(self.changesFile).readlines()
1533                changeSet = Set()
1534                for line in output:
1535                    changeSet.add(int(line))
1536
1537                for change in changeSet:
1538                    changes.append(change)
1539
1540                changes.sort()
1541            else:
1542                if self.verbose:
1543                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1544                                                              self.changeRange)
1545                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1546
1547                if len(self.maxChanges) > 0:
1548                    changes = changes[:min(int(self.maxChanges), len(changes))]
1549
1550            if len(changes) == 0:
1551                if not self.silent:
1552                    print "No changes to import!"
1553                return True
1554
1555            if not self.silent and not self.detectBranches:
1556                print "Import destination: %s" % self.branch
1557
1558            self.updatedBranches = set()
1559
1560            self.importChanges(changes)
1561
1562            if not self.silent:
1563                print ""
1564                if len(self.updatedBranches) > 0:
1565                    sys.stdout.write("Updated branches: ")
1566                    for b in self.updatedBranches:
1567                        sys.stdout.write("%s " % b)
1568                    sys.stdout.write("\n")
1569
1570        self.gitStream.close()
1571        if importProcess.wait() != 0:
1572            die("fast-import failed: %s" % self.gitError.read())
1573        self.gitOutput.close()
1574        self.gitError.close()
1575
1576        return True
1577
1578class P4Rebase(Command):
1579    def __init__(self):
1580        Command.__init__(self)
1581        self.options = [ ]
1582        self.description = ("Fetches the latest revision from perforce and "
1583                            + "rebases the current work (branch) against it")
1584        self.verbose = False
1585
1586    def run(self, args):
1587        sync = P4Sync()
1588        sync.run([])
1589
1590        return self.rebase()
1591
1592    def rebase(self):
1593        if os.system("git update-index --refresh") != 0:
1594            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1595        if len(read_pipe("git diff-index HEAD --")) > 0:
1596            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1597
1598        [upstream, settings] = findUpstreamBranchPoint()
1599        if len(upstream) == 0:
1600            die("Cannot find upstream branchpoint for rebase")
1601
1602        # the branchpoint may be p4/foo~3, so strip off the parent
1603        upstream = re.sub("~[0-9]+$", "", upstream)
1604
1605        print "Rebasing the current branch onto %s" % upstream
1606        oldHead = read_pipe("git rev-parse HEAD").strip()
1607        system("git rebase %s" % upstream)
1608        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1609        return True
1610
1611class P4Clone(P4Sync):
1612    def __init__(self):
1613        P4Sync.__init__(self)
1614        self.description = "Creates a new git repository and imports from Perforce into it"
1615        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1616        self.options += [
1617            optparse.make_option("--destination", dest="cloneDestination",
1618                                 action='store', default=None,
1619                                 help="where to leave result of the clone"),
1620            optparse.make_option("-/", dest="cloneExclude",
1621                                 action="append", type="string",
1622                                 help="exclude depot path")
1623        ]
1624        self.cloneDestination = None
1625        self.needsGit = False
1626
1627    # This is required for the "append" cloneExclude action
1628    def ensure_value(self, attr, value):
1629        if not hasattr(self, attr) or getattr(self, attr) is None:
1630            setattr(self, attr, value)
1631        return getattr(self, attr)
1632
1633    def defaultDestination(self, args):
1634        ## TODO: use common prefix of args?
1635        depotPath = args[0]
1636        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1637        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1638        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1639        depotDir = re.sub(r"/$", "", depotDir)
1640        return os.path.split(depotDir)[1]
1641
1642    def run(self, args):
1643        if len(args) < 1:
1644            return False
1645
1646        if self.keepRepoPath and not self.cloneDestination:
1647            sys.stderr.write("Must specify destination for --keep-path\n")
1648            sys.exit(1)
1649
1650        depotPaths = args
1651
1652        if not self.cloneDestination and len(depotPaths) > 1:
1653            self.cloneDestination = depotPaths[-1]
1654            depotPaths = depotPaths[:-1]
1655
1656        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1657        for p in depotPaths:
1658            if not p.startswith("//"):
1659                return False
1660
1661        if not self.cloneDestination:
1662            self.cloneDestination = self.defaultDestination(args)
1663
1664        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1665        if not os.path.exists(self.cloneDestination):
1666            os.makedirs(self.cloneDestination)
1667        os.chdir(self.cloneDestination)
1668        system("git init")
1669        self.gitdir = os.getcwd() + "/.git"
1670        if not P4Sync.run(self, depotPaths):
1671            return False
1672        if self.branch != "master":
1673            if gitBranchExists("refs/remotes/p4/master"):
1674                system("git branch master refs/remotes/p4/master")
1675                system("git checkout -f")
1676            else:
1677                print "Could not detect main branch. No checkout/master branch created."
1678
1679        return True
1680
1681class P4Branches(Command):
1682    def __init__(self):
1683        Command.__init__(self)
1684        self.options = [ ]
1685        self.description = ("Shows the git branches that hold imports and their "
1686                            + "corresponding perforce depot paths")
1687        self.verbose = False
1688
1689    def run(self, args):
1690        if originP4BranchesExist():
1691            createOrUpdateBranchesFromOrigin()
1692
1693        cmdline = "git rev-parse --symbolic "
1694        cmdline += " --remotes"
1695
1696        for line in read_pipe_lines(cmdline):
1697            line = line.strip()
1698
1699            if not line.startswith('p4/') or line == "p4/HEAD":
1700                continue
1701            branch = line
1702
1703            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1704            settings = extractSettingsGitLog(log)
1705
1706            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1707        return True
1708
1709class HelpFormatter(optparse.IndentedHelpFormatter):
1710    def __init__(self):
1711        optparse.IndentedHelpFormatter.__init__(self)
1712
1713    def format_description(self, description):
1714        if description:
1715            return description + "\n"
1716        else:
1717            return ""
1718
1719def printUsage(commands):
1720    print "usage: %s <command> [options]" % sys.argv[0]
1721    print ""
1722    print "valid commands: %s" % ", ".join(commands)
1723    print ""
1724    print "Try %s <command> --help for command specific help." % sys.argv[0]
1725    print ""
1726
1727commands = {
1728    "debug" : P4Debug,
1729    "submit" : P4Submit,
1730    "commit" : P4Submit,
1731    "sync" : P4Sync,
1732    "rebase" : P4Rebase,
1733    "clone" : P4Clone,
1734    "rollback" : P4RollBack,
1735    "branches" : P4Branches
1736}
1737
1738
1739def main():
1740    if len(sys.argv[1:]) == 0:
1741        printUsage(commands.keys())
1742        sys.exit(2)
1743
1744    cmd = ""
1745    cmdName = sys.argv[1]
1746    try:
1747        klass = commands[cmdName]
1748        cmd = klass()
1749    except KeyError:
1750        print "unknown command %s" % cmdName
1751        print ""
1752        printUsage(commands.keys())
1753        sys.exit(2)
1754
1755    options = cmd.options
1756    cmd.gitdir = os.environ.get("GIT_DIR", None)
1757
1758    args = sys.argv[2:]
1759
1760    if len(options) > 0:
1761        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1762
1763        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1764                                       options,
1765                                       description = cmd.description,
1766                                       formatter = HelpFormatter())
1767
1768        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1769    global verbose
1770    verbose = cmd.verbose
1771    if cmd.needsGit:
1772        if cmd.gitdir == None:
1773            cmd.gitdir = os.path.abspath(".git")
1774            if not isValidGitDir(cmd.gitdir):
1775                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1776                if os.path.exists(cmd.gitdir):
1777                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1778                    if len(cdup) > 0:
1779                        os.chdir(cdup);
1780
1781        if not isValidGitDir(cmd.gitdir):
1782            if isValidGitDir(cmd.gitdir + "/.git"):
1783                cmd.gitdir += "/.git"
1784            else:
1785                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1786
1787        os.environ["GIT_DIR"] = cmd.gitdir
1788
1789    if not cmd.run(args):
1790        parser.print_help()
1791
1792
1793if __name__ == '__main__':
1794    main()