Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: git-p4 submit cleanups. (4c750c0)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if inFilesSection:
 517                if line.startswith("\t"):
 518                    # path starts and ends with a tab
 519                    path = line[1:]
 520                    lastTab = path.rfind("\t")
 521                    if lastTab != -1:
 522                        path = path[:lastTab]
 523                        if not path.startswith(self.depotPath):
 524                            continue
 525                else:
 526                    inFilesSection = False
 527            else:
 528                if line.startswith("Files:"):
 529                    inFilesSection = True
 530
 531            template += line
 532
 533        return template
 534
 535    def applyCommit(self, id):
 536        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537        diffOpts = ("", "-M")[self.detectRename]
 538        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539        filesToAdd = set()
 540        filesToDelete = set()
 541        editedFiles = set()
 542        filesToChangeExecBit = {}
 543        for line in diff:
 544            diff = parseDiffTreeEntry(line)
 545            modifier = diff['status']
 546            path = diff['src']
 547            if modifier == "M":
 548                system("p4 edit \"%s\"" % path)
 549                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                    filesToChangeExecBit[path] = diff['dst_mode']
 551                editedFiles.add(path)
 552            elif modifier == "A":
 553                filesToAdd.add(path)
 554                filesToChangeExecBit[path] = diff['dst_mode']
 555                if path in filesToDelete:
 556                    filesToDelete.remove(path)
 557            elif modifier == "D":
 558                filesToDelete.add(path)
 559                if path in filesToAdd:
 560                    filesToAdd.remove(path)
 561            elif modifier == "R":
 562                src, dest = diff['src'], diff['dst']
 563                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                system("p4 edit \"%s\"" % (dest))
 565                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                    filesToChangeExecBit[dest] = diff['dst_mode']
 567                os.unlink(dest)
 568                editedFiles.add(dest)
 569                filesToDelete.add(src)
 570            else:
 571                die("unknown modifier %s for %s" % (modifier, path))
 572
 573        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574        patchcmd = diffcmd + " | git apply "
 575        tryPatchCmd = patchcmd + "--check -"
 576        applyPatchCmd = patchcmd + "--check --apply -"
 577
 578        if os.system(tryPatchCmd) != 0:
 579            print "Unfortunately applying the change failed!"
 580            print "What do you want to do?"
 581            response = "x"
 582            while response != "s" and response != "a" and response != "w":
 583                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585            if response == "s":
 586                print "Skipping! Good luck with the next patches..."
 587                for f in editedFiles:
 588                    system("p4 revert \"%s\"" % f);
 589                for f in filesToAdd:
 590                    system("rm %s" %f)
 591                return
 592            elif response == "a":
 593                os.system(applyPatchCmd)
 594                if len(filesToAdd) > 0:
 595                    print "You may also want to call p4 add on the following files:"
 596                    print " ".join(filesToAdd)
 597                if len(filesToDelete):
 598                    print "The following files should be scheduled for deletion with p4 delete:"
 599                    print " ".join(filesToDelete)
 600                die("Please resolve and submit the conflict manually and "
 601                    + "continue afterwards with git-p4 submit --continue")
 602            elif response == "w":
 603                system(diffcmd + " > patch.txt")
 604                print "Patch saved to patch.txt in %s !" % self.clientPath
 605                die("Please resolve and submit the conflict manually and "
 606                    "continue afterwards with git-p4 submit --continue")
 607
 608        system(applyPatchCmd)
 609
 610        for f in filesToAdd:
 611            system("p4 add \"%s\"" % f)
 612        for f in filesToDelete:
 613            system("p4 revert \"%s\"" % f)
 614            system("p4 delete \"%s\"" % f)
 615
 616        # Set/clear executable bits
 617        for f in filesToChangeExecBit.keys():
 618            mode = filesToChangeExecBit[f]
 619            setP4ExecBit(f, mode)
 620
 621        logMessage = extractLogMessageFromGitCommit(id)
 622        if self.isWindows:
 623            logMessage = logMessage.replace("\n", "\r\n")
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            diff = read_pipe("p4 diff -du ...")
 631
 632            for newFile in filesToAdd:
 633                diff += "==== new file ====\n"
 634                diff += "--- /dev/null\n"
 635                diff += "+++ %s\n" % newFile
 636                f = open(newFile, "r")
 637                for line in f.readlines():
 638                    diff += "+" + line
 639                f.close()
 640
 641            separatorLine = "######## everything below this line is just the diff #######"
 642            if platform.system() == "Windows":
 643                separatorLine += "\r"
 644            separatorLine += "\n"
 645
 646            [handle, fileName] = tempfile.mkstemp()
 647            tmpFile = os.fdopen(handle, "w+")
 648            tmpFile.write(submitTemplate + separatorLine + diff)
 649            tmpFile.close()
 650            defaultEditor = "vi"
 651            if platform.system() == "Windows":
 652                defaultEditor = "notepad"
 653            editor = os.environ.get("EDITOR", defaultEditor);
 654            system(editor + " " + fileName)
 655            tmpFile = open(fileName, "rb")
 656            message = tmpFile.read()
 657            tmpFile.close()
 658            os.remove(fileName)
 659            submitTemplate = message[:message.index(separatorLine)]
 660            if self.isWindows:
 661                submitTemplate = submitTemplate.replace("\r\n", "\n")
 662
 663            write_pipe("p4 submit -i", submitTemplate)
 664        else:
 665            fileName = "submit.txt"
 666            file = open(fileName, "w+")
 667            file.write(self.prepareLogMessage(template, logMessage))
 668            file.close()
 669            print ("Perforce submit template written as %s. "
 670                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 671                   % (fileName, fileName))
 672
 673    def run(self, args):
 674        if len(args) == 0:
 675            self.master = currentGitBranch()
 676            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 677                die("Detecting current git branch failed!")
 678        elif len(args) == 1:
 679            self.master = args[0]
 680        else:
 681            return False
 682
 683        [upstream, settings] = findUpstreamBranchPoint()
 684        self.depotPath = settings['depot-paths'][0]
 685        if len(self.origin) == 0:
 686            self.origin = upstream
 687
 688        if self.verbose:
 689            print "Origin branch is " + self.origin
 690
 691        if len(self.depotPath) == 0:
 692            print "Internal error: cannot locate perforce depot path from existing branches"
 693            sys.exit(128)
 694
 695        self.clientPath = p4Where(self.depotPath)
 696
 697        if len(self.clientPath) == 0:
 698            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 699            sys.exit(128)
 700
 701        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 702        self.oldWorkingDirectory = os.getcwd()
 703
 704        os.chdir(self.clientPath)
 705        print "Syncronizing p4 checkout..."
 706        system("p4 sync ...")
 707
 708        self.check()
 709
 710        commits = []
 711        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 712            commits.append(line.strip())
 713        commits.reverse()
 714
 715        while len(commits) > 0:
 716            commit = commits[0]
 717            commits = commits[1:]
 718            self.applyCommit(commit)
 719            if not self.interactive:
 720                break
 721
 722        if len(commits) == 0:
 723            print "All changes applied!"
 724            os.chdir(self.oldWorkingDirectory)
 725
 726            sync = P4Sync()
 727            sync.run([])
 728
 729            rebase = P4Rebase()
 730            rebase.rebase()
 731
 732        return True
 733
 734class P4Sync(Command):
 735    def __init__(self):
 736        Command.__init__(self)
 737        self.options = [
 738                optparse.make_option("--branch", dest="branch"),
 739                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 740                optparse.make_option("--changesfile", dest="changesFile"),
 741                optparse.make_option("--silent", dest="silent", action="store_true"),
 742                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 743                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 744                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 745                                     help="Import into refs/heads/ , not refs/remotes"),
 746                optparse.make_option("--max-changes", dest="maxChanges"),
 747                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 748                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 749        ]
 750        self.description = """Imports from Perforce into a git repository.\n
 751    example:
 752    //depot/my/project/ -- to import the current head
 753    //depot/my/project/@all -- to import everything
 754    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 755
 756    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 757
 758        self.usage += " //depot/path[@revRange]"
 759        self.silent = False
 760        self.createdBranches = Set()
 761        self.committedChanges = Set()
 762        self.branch = ""
 763        self.detectBranches = False
 764        self.detectLabels = False
 765        self.changesFile = ""
 766        self.syncWithOrigin = True
 767        self.verbose = False
 768        self.importIntoRemotes = True
 769        self.maxChanges = ""
 770        self.isWindows = (platform.system() == "Windows")
 771        self.keepRepoPath = False
 772        self.depotPaths = None
 773        self.p4BranchesInGit = []
 774        self.cloneExclude = []
 775
 776        if gitConfig("git-p4.syncFromOrigin") == "false":
 777            self.syncWithOrigin = False
 778
 779    def extractFilesFromCommit(self, commit):
 780        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 781                             for path in self.cloneExclude]
 782        files = []
 783        fnum = 0
 784        while commit.has_key("depotFile%s" % fnum):
 785            path =  commit["depotFile%s" % fnum]
 786
 787            if [p for p in self.cloneExclude
 788                if path.startswith (p)]:
 789                found = False
 790            else:
 791                found = [p for p in self.depotPaths
 792                         if path.startswith (p)]
 793            if not found:
 794                fnum = fnum + 1
 795                continue
 796
 797            file = {}
 798            file["path"] = path
 799            file["rev"] = commit["rev%s" % fnum]
 800            file["action"] = commit["action%s" % fnum]
 801            file["type"] = commit["type%s" % fnum]
 802            files.append(file)
 803            fnum = fnum + 1
 804        return files
 805
 806    def stripRepoPath(self, path, prefixes):
 807        if self.keepRepoPath:
 808            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 809
 810        for p in prefixes:
 811            if path.startswith(p):
 812                path = path[len(p):]
 813
 814        return path
 815
 816    def splitFilesIntoBranches(self, commit):
 817        branches = {}
 818        fnum = 0
 819        while commit.has_key("depotFile%s" % fnum):
 820            path =  commit["depotFile%s" % fnum]
 821            found = [p for p in self.depotPaths
 822                     if path.startswith (p)]
 823            if not found:
 824                fnum = fnum + 1
 825                continue
 826
 827            file = {}
 828            file["path"] = path
 829            file["rev"] = commit["rev%s" % fnum]
 830            file["action"] = commit["action%s" % fnum]
 831            file["type"] = commit["type%s" % fnum]
 832            fnum = fnum + 1
 833
 834            relPath = self.stripRepoPath(path, self.depotPaths)
 835
 836            for branch in self.knownBranches.keys():
 837
 838                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 839                if relPath.startswith(branch + "/"):
 840                    if branch not in branches:
 841                        branches[branch] = []
 842                    branches[branch].append(file)
 843                    break
 844
 845        return branches
 846
 847    ## Should move this out, doesn't use SELF.
 848    def readP4Files(self, files):
 849        files = [f for f in files
 850                 if f['action'] != 'delete']
 851
 852        if not files:
 853            return
 854
 855        filedata = p4CmdList('-x - print',
 856                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 857                                              for f in files]),
 858                             stdin_mode='w+')
 859        if "p4ExitCode" in filedata[0]:
 860            die("Problems executing p4. Error: [%d]."
 861                % (filedata[0]['p4ExitCode']));
 862
 863        j = 0;
 864        contents = {}
 865        while j < len(filedata):
 866            stat = filedata[j]
 867            j += 1
 868            text = ''
 869            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 870                tmp = filedata[j]['data']
 871                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 872                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 873                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 874                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 875                text += tmp
 876                j += 1
 877
 878
 879            if not stat.has_key('depotFile'):
 880                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 881                continue
 882
 883            contents[stat['depotFile']] = text
 884
 885        for f in files:
 886            assert not f.has_key('data')
 887            f['data'] = contents[f['path']]
 888
 889    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 890        epoch = details["time"]
 891        author = details["user"]
 892
 893        if self.verbose:
 894            print "commit into %s" % branch
 895
 896        # start with reading files; if that fails, we should not
 897        # create a commit.
 898        new_files = []
 899        for f in files:
 900            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 901                new_files.append (f)
 902            else:
 903                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 904        files = new_files
 905        self.readP4Files(files)
 906
 907
 908
 909
 910        self.gitStream.write("commit %s\n" % branch)
 911#        gitStream.write("mark :%s\n" % details["change"])
 912        self.committedChanges.add(int(details["change"]))
 913        committer = ""
 914        if author not in self.users:
 915            self.getUserMapFromPerforceServer()
 916        if author in self.users:
 917            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 918        else:
 919            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 920
 921        self.gitStream.write("committer %s\n" % committer)
 922
 923        self.gitStream.write("data <<EOT\n")
 924        self.gitStream.write(details["desc"])
 925        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 926                             % (','.join (branchPrefixes), details["change"]))
 927        if len(details['options']) > 0:
 928            self.gitStream.write(": options = %s" % details['options'])
 929        self.gitStream.write("]\nEOT\n\n")
 930
 931        if len(parent) > 0:
 932            if self.verbose:
 933                print "parent %s" % parent
 934            self.gitStream.write("from %s\n" % parent)
 935
 936        for file in files:
 937            if file["type"] == "apple":
 938                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 939                continue
 940
 941            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 942            if file["action"] == "delete":
 943                self.gitStream.write("D %s\n" % relPath)
 944            else:
 945                data = file['data']
 946
 947                mode = "644"
 948                if isP4Exec(file["type"]):
 949                    mode = "755"
 950                elif file["type"] == "symlink":
 951                    mode = "120000"
 952                    # p4 print on a symlink contains "target\n", so strip it off
 953                    data = data[:-1]
 954
 955                if self.isWindows and file["type"].endswith("text"):
 956                    data = data.replace("\r\n", "\n")
 957
 958                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 959                self.gitStream.write("data %s\n" % len(data))
 960                self.gitStream.write(data)
 961                self.gitStream.write("\n")
 962
 963        self.gitStream.write("\n")
 964
 965        change = int(details["change"])
 966
 967        if self.labels.has_key(change):
 968            label = self.labels[change]
 969            labelDetails = label[0]
 970            labelRevisions = label[1]
 971            if self.verbose:
 972                print "Change %s is labelled %s" % (change, labelDetails)
 973
 974            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 975                                                    for p in branchPrefixes]))
 976
 977            if len(files) == len(labelRevisions):
 978
 979                cleanedFiles = {}
 980                for info in files:
 981                    if info["action"] == "delete":
 982                        continue
 983                    cleanedFiles[info["depotFile"]] = info["rev"]
 984
 985                if cleanedFiles == labelRevisions:
 986                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 987                    self.gitStream.write("from %s\n" % branch)
 988
 989                    owner = labelDetails["Owner"]
 990                    tagger = ""
 991                    if author in self.users:
 992                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
 993                    else:
 994                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
 995                    self.gitStream.write("tagger %s\n" % tagger)
 996                    self.gitStream.write("data <<EOT\n")
 997                    self.gitStream.write(labelDetails["Description"])
 998                    self.gitStream.write("EOT\n\n")
 999
1000                else:
1001                    if not self.silent:
1002                        print ("Tag %s does not match with change %s: files do not match."
1003                               % (labelDetails["label"], change))
1004
1005            else:
1006                if not self.silent:
1007                    print ("Tag %s does not match with change %s: file count is different."
1008                           % (labelDetails["label"], change))
1009
1010    def getUserCacheFilename(self):
1011        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1012        return home + "/.gitp4-usercache.txt"
1013
1014    def getUserMapFromPerforceServer(self):
1015        if self.userMapFromPerforceServer:
1016            return
1017        self.users = {}
1018
1019        for output in p4CmdList("users"):
1020            if not output.has_key("User"):
1021                continue
1022            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1023
1024
1025        s = ''
1026        for (key, val) in self.users.items():
1027            s += "%s\t%s\n" % (key, val)
1028
1029        open(self.getUserCacheFilename(), "wb").write(s)
1030        self.userMapFromPerforceServer = True
1031
1032    def loadUserMapFromCache(self):
1033        self.users = {}
1034        self.userMapFromPerforceServer = False
1035        try:
1036            cache = open(self.getUserCacheFilename(), "rb")
1037            lines = cache.readlines()
1038            cache.close()
1039            for line in lines:
1040                entry = line.strip().split("\t")
1041                self.users[entry[0]] = entry[1]
1042        except IOError:
1043            self.getUserMapFromPerforceServer()
1044
1045    def getLabels(self):
1046        self.labels = {}
1047
1048        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1049        if len(l) > 0 and not self.silent:
1050            print "Finding files belonging to labels in %s" % `self.depotPaths`
1051
1052        for output in l:
1053            label = output["label"]
1054            revisions = {}
1055            newestChange = 0
1056            if self.verbose:
1057                print "Querying files for label %s" % label
1058            for file in p4CmdList("files "
1059                                  +  ' '.join (["%s...@%s" % (p, label)
1060                                                for p in self.depotPaths])):
1061                revisions[file["depotFile"]] = file["rev"]
1062                change = int(file["change"])
1063                if change > newestChange:
1064                    newestChange = change
1065
1066            self.labels[newestChange] = [output, revisions]
1067
1068        if self.verbose:
1069            print "Label changes: %s" % self.labels.keys()
1070
1071    def guessProjectName(self):
1072        for p in self.depotPaths:
1073            if p.endswith("/"):
1074                p = p[:-1]
1075            p = p[p.strip().rfind("/") + 1:]
1076            if not p.endswith("/"):
1077               p += "/"
1078            return p
1079
1080    def getBranchMapping(self):
1081        lostAndFoundBranches = set()
1082
1083        for info in p4CmdList("branches"):
1084            details = p4Cmd("branch -o %s" % info["branch"])
1085            viewIdx = 0
1086            while details.has_key("View%s" % viewIdx):
1087                paths = details["View%s" % viewIdx].split(" ")
1088                viewIdx = viewIdx + 1
1089                # require standard //depot/foo/... //depot/bar/... mapping
1090                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1091                    continue
1092                source = paths[0]
1093                destination = paths[1]
1094                ## HACK
1095                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1096                    source = source[len(self.depotPaths[0]):-4]
1097                    destination = destination[len(self.depotPaths[0]):-4]
1098
1099                    if destination in self.knownBranches:
1100                        if not self.silent:
1101                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1102                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1103                        continue
1104
1105                    self.knownBranches[destination] = source
1106
1107                    lostAndFoundBranches.discard(destination)
1108
1109                    if source not in self.knownBranches:
1110                        lostAndFoundBranches.add(source)
1111
1112
1113        for branch in lostAndFoundBranches:
1114            self.knownBranches[branch] = branch
1115
1116    def getBranchMappingFromGitBranches(self):
1117        branches = p4BranchesInGit(self.importIntoRemotes)
1118        for branch in branches.keys():
1119            if branch == "master":
1120                branch = "main"
1121            else:
1122                branch = branch[len(self.projectName):]
1123            self.knownBranches[branch] = branch
1124
1125    def listExistingP4GitBranches(self):
1126        # branches holds mapping from name to commit
1127        branches = p4BranchesInGit(self.importIntoRemotes)
1128        self.p4BranchesInGit = branches.keys()
1129        for branch in branches.keys():
1130            self.initialParents[self.refPrefix + branch] = branches[branch]
1131
1132    def updateOptionDict(self, d):
1133        option_keys = {}
1134        if self.keepRepoPath:
1135            option_keys['keepRepoPath'] = 1
1136
1137        d["options"] = ' '.join(sorted(option_keys.keys()))
1138
1139    def readOptions(self, d):
1140        self.keepRepoPath = (d.has_key('options')
1141                             and ('keepRepoPath' in d['options']))
1142
1143    def gitRefForBranch(self, branch):
1144        if branch == "main":
1145            return self.refPrefix + "master"
1146
1147        if len(branch) <= 0:
1148            return branch
1149
1150        return self.refPrefix + self.projectName + branch
1151
1152    def gitCommitByP4Change(self, ref, change):
1153        if self.verbose:
1154            print "looking in ref " + ref + " for change %s using bisect..." % change
1155
1156        earliestCommit = ""
1157        latestCommit = parseRevision(ref)
1158
1159        while True:
1160            if self.verbose:
1161                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1162            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1163            if len(next) == 0:
1164                if self.verbose:
1165                    print "argh"
1166                return ""
1167            log = extractLogMessageFromGitCommit(next)
1168            settings = extractSettingsGitLog(log)
1169            currentChange = int(settings['change'])
1170            if self.verbose:
1171                print "current change %s" % currentChange
1172
1173            if currentChange == change:
1174                if self.verbose:
1175                    print "found %s" % next
1176                return next
1177
1178            if currentChange < change:
1179                earliestCommit = "^%s" % next
1180            else:
1181                latestCommit = "%s" % next
1182
1183        return ""
1184
1185    def importNewBranch(self, branch, maxChange):
1186        # make fast-import flush all changes to disk and update the refs using the checkpoint
1187        # command so that we can try to find the branch parent in the git history
1188        self.gitStream.write("checkpoint\n\n");
1189        self.gitStream.flush();
1190        branchPrefix = self.depotPaths[0] + branch + "/"
1191        range = "@1,%s" % maxChange
1192        #print "prefix" + branchPrefix
1193        changes = p4ChangesForPaths([branchPrefix], range)
1194        if len(changes) <= 0:
1195            return False
1196        firstChange = changes[0]
1197        #print "first change in branch: %s" % firstChange
1198        sourceBranch = self.knownBranches[branch]
1199        sourceDepotPath = self.depotPaths[0] + sourceBranch
1200        sourceRef = self.gitRefForBranch(sourceBranch)
1201        #print "source " + sourceBranch
1202
1203        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1204        #print "branch parent: %s" % branchParentChange
1205        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1206        if len(gitParent) > 0:
1207            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1208            #print "parent git commit: %s" % gitParent
1209
1210        self.importChanges(changes)
1211        return True
1212
1213    def importChanges(self, changes):
1214        cnt = 1
1215        for change in changes:
1216            description = p4Cmd("describe %s" % change)
1217            self.updateOptionDict(description)
1218
1219            if not self.silent:
1220                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1221                sys.stdout.flush()
1222            cnt = cnt + 1
1223
1224            try:
1225                if self.detectBranches:
1226                    branches = self.splitFilesIntoBranches(description)
1227                    for branch in branches.keys():
1228                        ## HACK  --hwn
1229                        branchPrefix = self.depotPaths[0] + branch + "/"
1230
1231                        parent = ""
1232
1233                        filesForCommit = branches[branch]
1234
1235                        if self.verbose:
1236                            print "branch is %s" % branch
1237
1238                        self.updatedBranches.add(branch)
1239
1240                        if branch not in self.createdBranches:
1241                            self.createdBranches.add(branch)
1242                            parent = self.knownBranches[branch]
1243                            if parent == branch:
1244                                parent = ""
1245                            else:
1246                                fullBranch = self.projectName + branch
1247                                if fullBranch not in self.p4BranchesInGit:
1248                                    if not self.silent:
1249                                        print("\n    Importing new branch %s" % fullBranch);
1250                                    if self.importNewBranch(branch, change - 1):
1251                                        parent = ""
1252                                        self.p4BranchesInGit.append(fullBranch)
1253                                    if not self.silent:
1254                                        print("\n    Resuming with change %s" % change);
1255
1256                                if self.verbose:
1257                                    print "parent determined through known branches: %s" % parent
1258
1259                        branch = self.gitRefForBranch(branch)
1260                        parent = self.gitRefForBranch(parent)
1261
1262                        if self.verbose:
1263                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1264
1265                        if len(parent) == 0 and branch in self.initialParents:
1266                            parent = self.initialParents[branch]
1267                            del self.initialParents[branch]
1268
1269                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1270                else:
1271                    files = self.extractFilesFromCommit(description)
1272                    self.commit(description, files, self.branch, self.depotPaths,
1273                                self.initialParent)
1274                    self.initialParent = ""
1275            except IOError:
1276                print self.gitError.read()
1277                sys.exit(1)
1278
1279    def importHeadRevision(self, revision):
1280        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1281
1282        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1283        details["desc"] = ("Initial import of %s from the state at revision %s"
1284                           % (' '.join(self.depotPaths), revision))
1285        details["change"] = revision
1286        newestRevision = 0
1287
1288        fileCnt = 0
1289        for info in p4CmdList("files "
1290                              +  ' '.join(["%s...%s"
1291                                           % (p, revision)
1292                                           for p in self.depotPaths])):
1293
1294            if info['code'] == 'error':
1295                sys.stderr.write("p4 returned an error: %s\n"
1296                                 % info['data'])
1297                sys.exit(1)
1298
1299
1300            change = int(info["change"])
1301            if change > newestRevision:
1302                newestRevision = change
1303
1304            if info["action"] == "delete":
1305                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1306                #fileCnt = fileCnt + 1
1307                continue
1308
1309            for prop in ["depotFile", "rev", "action", "type" ]:
1310                details["%s%s" % (prop, fileCnt)] = info[prop]
1311
1312            fileCnt = fileCnt + 1
1313
1314        details["change"] = newestRevision
1315        self.updateOptionDict(details)
1316        try:
1317            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1318        except IOError:
1319            print "IO error with git fast-import. Is your git version recent enough?"
1320            print self.gitError.read()
1321
1322
1323    def run(self, args):
1324        self.depotPaths = []
1325        self.changeRange = ""
1326        self.initialParent = ""
1327        self.previousDepotPaths = []
1328
1329        # map from branch depot path to parent branch
1330        self.knownBranches = {}
1331        self.initialParents = {}
1332        self.hasOrigin = originP4BranchesExist()
1333        if not self.syncWithOrigin:
1334            self.hasOrigin = False
1335
1336        if self.importIntoRemotes:
1337            self.refPrefix = "refs/remotes/p4/"
1338        else:
1339            self.refPrefix = "refs/heads/p4/"
1340
1341        if self.syncWithOrigin and self.hasOrigin:
1342            if not self.silent:
1343                print "Syncing with origin first by calling git fetch origin"
1344            system("git fetch origin")
1345
1346        if len(self.branch) == 0:
1347            self.branch = self.refPrefix + "master"
1348            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1349                system("git update-ref %s refs/heads/p4" % self.branch)
1350                system("git branch -D p4");
1351            # create it /after/ importing, when master exists
1352            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1353                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1354
1355        # TODO: should always look at previous commits,
1356        # merge with previous imports, if possible.
1357        if args == []:
1358            if self.hasOrigin:
1359                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1360            self.listExistingP4GitBranches()
1361
1362            if len(self.p4BranchesInGit) > 1:
1363                if not self.silent:
1364                    print "Importing from/into multiple branches"
1365                self.detectBranches = True
1366
1367            if self.verbose:
1368                print "branches: %s" % self.p4BranchesInGit
1369
1370            p4Change = 0
1371            for branch in self.p4BranchesInGit:
1372                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1373
1374                settings = extractSettingsGitLog(logMsg)
1375
1376                self.readOptions(settings)
1377                if (settings.has_key('depot-paths')
1378                    and settings.has_key ('change')):
1379                    change = int(settings['change']) + 1
1380                    p4Change = max(p4Change, change)
1381
1382                    depotPaths = sorted(settings['depot-paths'])
1383                    if self.previousDepotPaths == []:
1384                        self.previousDepotPaths = depotPaths
1385                    else:
1386                        paths = []
1387                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1388                            for i in range(0, min(len(cur), len(prev))):
1389                                if cur[i] <> prev[i]:
1390                                    i = i - 1
1391                                    break
1392
1393                            paths.append (cur[:i + 1])
1394
1395                        self.previousDepotPaths = paths
1396
1397            if p4Change > 0:
1398                self.depotPaths = sorted(self.previousDepotPaths)
1399                self.changeRange = "@%s,#head" % p4Change
1400                if not self.detectBranches:
1401                    self.initialParent = parseRevision(self.branch)
1402                if not self.silent and not self.detectBranches:
1403                    print "Performing incremental import into %s git branch" % self.branch
1404
1405        if not self.branch.startswith("refs/"):
1406            self.branch = "refs/heads/" + self.branch
1407
1408        if len(args) == 0 and self.depotPaths:
1409            if not self.silent:
1410                print "Depot paths: %s" % ' '.join(self.depotPaths)
1411        else:
1412            if self.depotPaths and self.depotPaths != args:
1413                print ("previous import used depot path %s and now %s was specified. "
1414                       "This doesn't work!" % (' '.join (self.depotPaths),
1415                                               ' '.join (args)))
1416                sys.exit(1)
1417
1418            self.depotPaths = sorted(args)
1419
1420        revision = ""
1421        self.users = {}
1422
1423        newPaths = []
1424        for p in self.depotPaths:
1425            if p.find("@") != -1:
1426                atIdx = p.index("@")
1427                self.changeRange = p[atIdx:]
1428                if self.changeRange == "@all":
1429                    self.changeRange = ""
1430                elif ',' not in self.changeRange:
1431                    revision = self.changeRange
1432                    self.changeRange = ""
1433                p = p[:atIdx]
1434            elif p.find("#") != -1:
1435                hashIdx = p.index("#")
1436                revision = p[hashIdx:]
1437                p = p[:hashIdx]
1438            elif self.previousDepotPaths == []:
1439                revision = "#head"
1440
1441            p = re.sub ("\.\.\.$", "", p)
1442            if not p.endswith("/"):
1443                p += "/"
1444
1445            newPaths.append(p)
1446
1447        self.depotPaths = newPaths
1448
1449
1450        self.loadUserMapFromCache()
1451        self.labels = {}
1452        if self.detectLabels:
1453            self.getLabels();
1454
1455        if self.detectBranches:
1456            ## FIXME - what's a P4 projectName ?
1457            self.projectName = self.guessProjectName()
1458
1459            if self.hasOrigin:
1460                self.getBranchMappingFromGitBranches()
1461            else:
1462                self.getBranchMapping()
1463            if self.verbose:
1464                print "p4-git branches: %s" % self.p4BranchesInGit
1465                print "initial parents: %s" % self.initialParents
1466            for b in self.p4BranchesInGit:
1467                if b != "master":
1468
1469                    ## FIXME
1470                    b = b[len(self.projectName):]
1471                self.createdBranches.add(b)
1472
1473        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1474
1475        importProcess = subprocess.Popen(["git", "fast-import"],
1476                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1477                                         stderr=subprocess.PIPE);
1478        self.gitOutput = importProcess.stdout
1479        self.gitStream = importProcess.stdin
1480        self.gitError = importProcess.stderr
1481
1482        if revision:
1483            self.importHeadRevision(revision)
1484        else:
1485            changes = []
1486
1487            if len(self.changesFile) > 0:
1488                output = open(self.changesFile).readlines()
1489                changeSet = Set()
1490                for line in output:
1491                    changeSet.add(int(line))
1492
1493                for change in changeSet:
1494                    changes.append(change)
1495
1496                changes.sort()
1497            else:
1498                if self.verbose:
1499                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1500                                                              self.changeRange)
1501                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1502
1503                if len(self.maxChanges) > 0:
1504                    changes = changes[:min(int(self.maxChanges), len(changes))]
1505
1506            if len(changes) == 0:
1507                if not self.silent:
1508                    print "No changes to import!"
1509                return True
1510
1511            if not self.silent and not self.detectBranches:
1512                print "Import destination: %s" % self.branch
1513
1514            self.updatedBranches = set()
1515
1516            self.importChanges(changes)
1517
1518            if not self.silent:
1519                print ""
1520                if len(self.updatedBranches) > 0:
1521                    sys.stdout.write("Updated branches: ")
1522                    for b in self.updatedBranches:
1523                        sys.stdout.write("%s " % b)
1524                    sys.stdout.write("\n")
1525
1526        self.gitStream.close()
1527        if importProcess.wait() != 0:
1528            die("fast-import failed: %s" % self.gitError.read())
1529        self.gitOutput.close()
1530        self.gitError.close()
1531
1532        return True
1533
1534class P4Rebase(Command):
1535    def __init__(self):
1536        Command.__init__(self)
1537        self.options = [ ]
1538        self.description = ("Fetches the latest revision from perforce and "
1539                            + "rebases the current work (branch) against it")
1540        self.verbose = False
1541
1542    def run(self, args):
1543        sync = P4Sync()
1544        sync.run([])
1545
1546        return self.rebase()
1547
1548    def rebase(self):
1549        if os.system("git update-index --refresh") != 0:
1550            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1551        if len(read_pipe("git diff-index HEAD --")) > 0:
1552            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1553
1554        [upstream, settings] = findUpstreamBranchPoint()
1555        if len(upstream) == 0:
1556            die("Cannot find upstream branchpoint for rebase")
1557
1558        # the branchpoint may be p4/foo~3, so strip off the parent
1559        upstream = re.sub("~[0-9]+$", "", upstream)
1560
1561        print "Rebasing the current branch onto %s" % upstream
1562        oldHead = read_pipe("git rev-parse HEAD").strip()
1563        system("git rebase %s" % upstream)
1564        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1565        return True
1566
1567class P4Clone(P4Sync):
1568    def __init__(self):
1569        P4Sync.__init__(self)
1570        self.description = "Creates a new git repository and imports from Perforce into it"
1571        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1572        self.options += [
1573            optparse.make_option("--destination", dest="cloneDestination",
1574                                 action='store', default=None,
1575                                 help="where to leave result of the clone"),
1576            optparse.make_option("-/", dest="cloneExclude",
1577                                 action="append", type="string",
1578                                 help="exclude depot path")
1579        ]
1580        self.cloneDestination = None
1581        self.needsGit = False
1582
1583    # This is required for the "append" cloneExclude action
1584    def ensure_value(self, attr, value):
1585        if not hasattr(self, attr) or getattr(self, attr) is None:
1586            setattr(self, attr, value)
1587        return getattr(self, attr)
1588
1589    def defaultDestination(self, args):
1590        ## TODO: use common prefix of args?
1591        depotPath = args[0]
1592        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1593        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1594        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1595        depotDir = re.sub(r"/$", "", depotDir)
1596        return os.path.split(depotDir)[1]
1597
1598    def run(self, args):
1599        if len(args) < 1:
1600            return False
1601
1602        if self.keepRepoPath and not self.cloneDestination:
1603            sys.stderr.write("Must specify destination for --keep-path\n")
1604            sys.exit(1)
1605
1606        depotPaths = args
1607
1608        if not self.cloneDestination and len(depotPaths) > 1:
1609            self.cloneDestination = depotPaths[-1]
1610            depotPaths = depotPaths[:-1]
1611
1612        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1613        for p in depotPaths:
1614            if not p.startswith("//"):
1615                return False
1616
1617        if not self.cloneDestination:
1618            self.cloneDestination = self.defaultDestination(args)
1619
1620        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1621        if not os.path.exists(self.cloneDestination):
1622            os.makedirs(self.cloneDestination)
1623        os.chdir(self.cloneDestination)
1624        system("git init")
1625        self.gitdir = os.getcwd() + "/.git"
1626        if not P4Sync.run(self, depotPaths):
1627            return False
1628        if self.branch != "master":
1629            if gitBranchExists("refs/remotes/p4/master"):
1630                system("git branch master refs/remotes/p4/master")
1631                system("git checkout -f")
1632            else:
1633                print "Could not detect main branch. No checkout/master branch created."
1634
1635        return True
1636
1637class P4Branches(Command):
1638    def __init__(self):
1639        Command.__init__(self)
1640        self.options = [ ]
1641        self.description = ("Shows the git branches that hold imports and their "
1642                            + "corresponding perforce depot paths")
1643        self.verbose = False
1644
1645    def run(self, args):
1646        if originP4BranchesExist():
1647            createOrUpdateBranchesFromOrigin()
1648
1649        cmdline = "git rev-parse --symbolic "
1650        cmdline += " --remotes"
1651
1652        for line in read_pipe_lines(cmdline):
1653            line = line.strip()
1654
1655            if not line.startswith('p4/') or line == "p4/HEAD":
1656                continue
1657            branch = line
1658
1659            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1660            settings = extractSettingsGitLog(log)
1661
1662            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1663        return True
1664
1665class HelpFormatter(optparse.IndentedHelpFormatter):
1666    def __init__(self):
1667        optparse.IndentedHelpFormatter.__init__(self)
1668
1669    def format_description(self, description):
1670        if description:
1671            return description + "\n"
1672        else:
1673            return ""
1674
1675def printUsage(commands):
1676    print "usage: %s <command> [options]" % sys.argv[0]
1677    print ""
1678    print "valid commands: %s" % ", ".join(commands)
1679    print ""
1680    print "Try %s <command> --help for command specific help." % sys.argv[0]
1681    print ""
1682
1683commands = {
1684    "debug" : P4Debug,
1685    "submit" : P4Submit,
1686    "commit" : P4Submit,
1687    "sync" : P4Sync,
1688    "rebase" : P4Rebase,
1689    "clone" : P4Clone,
1690    "rollback" : P4RollBack,
1691    "branches" : P4Branches
1692}
1693
1694
1695def main():
1696    if len(sys.argv[1:]) == 0:
1697        printUsage(commands.keys())
1698        sys.exit(2)
1699
1700    cmd = ""
1701    cmdName = sys.argv[1]
1702    try:
1703        klass = commands[cmdName]
1704        cmd = klass()
1705    except KeyError:
1706        print "unknown command %s" % cmdName
1707        print ""
1708        printUsage(commands.keys())
1709        sys.exit(2)
1710
1711    options = cmd.options
1712    cmd.gitdir = os.environ.get("GIT_DIR", None)
1713
1714    args = sys.argv[2:]
1715
1716    if len(options) > 0:
1717        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1718
1719        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1720                                       options,
1721                                       description = cmd.description,
1722                                       formatter = HelpFormatter())
1723
1724        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1725    global verbose
1726    verbose = cmd.verbose
1727    if cmd.needsGit:
1728        if cmd.gitdir == None:
1729            cmd.gitdir = os.path.abspath(".git")
1730            if not isValidGitDir(cmd.gitdir):
1731                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1732                if os.path.exists(cmd.gitdir):
1733                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1734                    if len(cdup) > 0:
1735                        os.chdir(cdup);
1736
1737        if not isValidGitDir(cmd.gitdir):
1738            if isValidGitDir(cmd.gitdir + "/.git"):
1739                cmd.gitdir += "/.git"
1740            else:
1741                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1742
1743        os.environ["GIT_DIR"] = cmd.gitdir
1744
1745    if not cmd.run(args):
1746        parser.print_help()
1747
1748
1749if __name__ == '__main__':
1750    main()