Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' (6b48990)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if inFilesSection:
 517                if line.startswith("\t"):
 518                    # path starts and ends with a tab
 519                    path = line[1:]
 520                    lastTab = path.rfind("\t")
 521                    if lastTab != -1:
 522                        path = path[:lastTab]
 523                        if not path.startswith(self.depotPath):
 524                            continue
 525                else:
 526                    inFilesSection = False
 527            else:
 528                if line.startswith("Files:"):
 529                    inFilesSection = True
 530
 531            template += line
 532
 533        return template
 534
 535    def applyCommit(self, id):
 536        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537        diffOpts = ("", "-M")[self.detectRename]
 538        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539        filesToAdd = set()
 540        filesToDelete = set()
 541        editedFiles = set()
 542        filesToChangeExecBit = {}
 543        for line in diff:
 544            diff = parseDiffTreeEntry(line)
 545            modifier = diff['status']
 546            path = diff['src']
 547            if modifier == "M":
 548                system("p4 edit \"%s\"" % path)
 549                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                    filesToChangeExecBit[path] = diff['dst_mode']
 551                editedFiles.add(path)
 552            elif modifier == "A":
 553                filesToAdd.add(path)
 554                filesToChangeExecBit[path] = diff['dst_mode']
 555                if path in filesToDelete:
 556                    filesToDelete.remove(path)
 557            elif modifier == "D":
 558                filesToDelete.add(path)
 559                if path in filesToAdd:
 560                    filesToAdd.remove(path)
 561            elif modifier == "R":
 562                src, dest = diff['src'], diff['dst']
 563                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                system("p4 edit \"%s\"" % (dest))
 565                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                    filesToChangeExecBit[dest] = diff['dst_mode']
 567                os.unlink(dest)
 568                editedFiles.add(dest)
 569                filesToDelete.add(src)
 570            else:
 571                die("unknown modifier %s for %s" % (modifier, path))
 572
 573        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574        patchcmd = diffcmd + " | git apply "
 575        tryPatchCmd = patchcmd + "--check -"
 576        applyPatchCmd = patchcmd + "--check --apply -"
 577
 578        if os.system(tryPatchCmd) != 0:
 579            print "Unfortunately applying the change failed!"
 580            print "What do you want to do?"
 581            response = "x"
 582            while response != "s" and response != "a" and response != "w":
 583                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585            if response == "s":
 586                print "Skipping! Good luck with the next patches..."
 587                for f in editedFiles:
 588                    system("p4 revert \"%s\"" % f);
 589                for f in filesToAdd:
 590                    system("rm %s" %f)
 591                return
 592            elif response == "a":
 593                os.system(applyPatchCmd)
 594                if len(filesToAdd) > 0:
 595                    print "You may also want to call p4 add on the following files:"
 596                    print " ".join(filesToAdd)
 597                if len(filesToDelete):
 598                    print "The following files should be scheduled for deletion with p4 delete:"
 599                    print " ".join(filesToDelete)
 600                die("Please resolve and submit the conflict manually and "
 601                    + "continue afterwards with git-p4 submit --continue")
 602            elif response == "w":
 603                system(diffcmd + " > patch.txt")
 604                print "Patch saved to patch.txt in %s !" % self.clientPath
 605                die("Please resolve and submit the conflict manually and "
 606                    "continue afterwards with git-p4 submit --continue")
 607
 608        system(applyPatchCmd)
 609
 610        for f in filesToAdd:
 611            system("p4 add \"%s\"" % f)
 612        for f in filesToDelete:
 613            system("p4 revert \"%s\"" % f)
 614            system("p4 delete \"%s\"" % f)
 615
 616        # Set/clear executable bits
 617        for f in filesToChangeExecBit.keys():
 618            mode = filesToChangeExecBit[f]
 619            setP4ExecBit(f, mode)
 620
 621        logMessage = extractLogMessageFromGitCommit(id)
 622        if self.isWindows:
 623            logMessage = logMessage.replace("\n", "\r\n")
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            diff = read_pipe("p4 diff -du ...")
 631
 632            for newFile in filesToAdd:
 633                diff += "==== new file ====\n"
 634                diff += "--- /dev/null\n"
 635                diff += "+++ %s\n" % newFile
 636                f = open(newFile, "r")
 637                for line in f.readlines():
 638                    diff += "+" + line
 639                f.close()
 640
 641            separatorLine = "######## everything below this line is just the diff #######"
 642            if platform.system() == "Windows":
 643                separatorLine += "\r"
 644            separatorLine += "\n"
 645
 646            [handle, fileName] = tempfile.mkstemp()
 647            tmpFile = os.fdopen(handle, "w+")
 648            tmpFile.write(submitTemplate + separatorLine + diff)
 649            tmpFile.close()
 650            defaultEditor = "vi"
 651            if platform.system() == "Windows":
 652                defaultEditor = "notepad"
 653            editor = os.environ.get("EDITOR", defaultEditor);
 654            system(editor + " " + fileName)
 655            tmpFile = open(fileName, "rb")
 656            message = tmpFile.read()
 657            tmpFile.close()
 658            os.remove(fileName)
 659            submitTemplate = message[:message.index(separatorLine)]
 660            if self.isWindows:
 661                submitTemplate = submitTemplate.replace("\r\n", "\n")
 662
 663            write_pipe("p4 submit -i", submitTemplate)
 664        else:
 665            fileName = "submit.txt"
 666            file = open(fileName, "w+")
 667            file.write(self.prepareLogMessage(template, logMessage))
 668            file.close()
 669            print ("Perforce submit template written as %s. "
 670                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 671                   % (fileName, fileName))
 672
 673    def run(self, args):
 674        if len(args) == 0:
 675            self.master = currentGitBranch()
 676            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 677                die("Detecting current git branch failed!")
 678        elif len(args) == 1:
 679            self.master = args[0]
 680        else:
 681            return False
 682
 683        [upstream, settings] = findUpstreamBranchPoint()
 684        self.depotPath = settings['depot-paths'][0]
 685        if len(self.origin) == 0:
 686            self.origin = upstream
 687
 688        if self.verbose:
 689            print "Origin branch is " + self.origin
 690
 691        if len(self.depotPath) == 0:
 692            print "Internal error: cannot locate perforce depot path from existing branches"
 693            sys.exit(128)
 694
 695        self.clientPath = p4Where(self.depotPath)
 696
 697        if len(self.clientPath) == 0:
 698            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 699            sys.exit(128)
 700
 701        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 702        self.oldWorkingDirectory = os.getcwd()
 703
 704        os.chdir(self.clientPath)
 705        print "Syncronizing p4 checkout..."
 706        system("p4 sync ...")
 707
 708        self.check()
 709
 710        commits = []
 711        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 712            commits.append(line.strip())
 713        commits.reverse()
 714
 715        while len(commits) > 0:
 716            commit = commits[0]
 717            commits = commits[1:]
 718            self.applyCommit(commit)
 719            if not self.interactive:
 720                break
 721
 722        if len(commits) == 0:
 723            print "All changes applied!"
 724            os.chdir(self.oldWorkingDirectory)
 725
 726            sync = P4Sync()
 727            sync.run([])
 728
 729            rebase = P4Rebase()
 730            rebase.rebase()
 731
 732        return True
 733
 734class P4Sync(Command):
 735    def __init__(self):
 736        Command.__init__(self)
 737        self.options = [
 738                optparse.make_option("--branch", dest="branch"),
 739                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 740                optparse.make_option("--changesfile", dest="changesFile"),
 741                optparse.make_option("--silent", dest="silent", action="store_true"),
 742                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 743                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 744                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 745                                     help="Import into refs/heads/ , not refs/remotes"),
 746                optparse.make_option("--max-changes", dest="maxChanges"),
 747                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 748                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 749                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 750                                     help="Only sync files that are included in the Perforce Client Spec")
 751        ]
 752        self.description = """Imports from Perforce into a git repository.\n
 753    example:
 754    //depot/my/project/ -- to import the current head
 755    //depot/my/project/@all -- to import everything
 756    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 757
 758    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 759
 760        self.usage += " //depot/path[@revRange]"
 761        self.silent = False
 762        self.createdBranches = Set()
 763        self.committedChanges = Set()
 764        self.branch = ""
 765        self.detectBranches = False
 766        self.detectLabels = False
 767        self.changesFile = ""
 768        self.syncWithOrigin = True
 769        self.verbose = False
 770        self.importIntoRemotes = True
 771        self.maxChanges = ""
 772        self.isWindows = (platform.system() == "Windows")
 773        self.keepRepoPath = False
 774        self.depotPaths = None
 775        self.p4BranchesInGit = []
 776        self.cloneExclude = []
 777        self.useClientSpec = False
 778        self.clientSpecDirs = []
 779
 780        if gitConfig("git-p4.syncFromOrigin") == "false":
 781            self.syncWithOrigin = False
 782
 783    def extractFilesFromCommit(self, commit):
 784        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 785                             for path in self.cloneExclude]
 786        files = []
 787        fnum = 0
 788        while commit.has_key("depotFile%s" % fnum):
 789            path =  commit["depotFile%s" % fnum]
 790
 791            if [p for p in self.cloneExclude
 792                if path.startswith (p)]:
 793                found = False
 794            else:
 795                found = [p for p in self.depotPaths
 796                         if path.startswith (p)]
 797            if not found:
 798                fnum = fnum + 1
 799                continue
 800
 801            file = {}
 802            file["path"] = path
 803            file["rev"] = commit["rev%s" % fnum]
 804            file["action"] = commit["action%s" % fnum]
 805            file["type"] = commit["type%s" % fnum]
 806            files.append(file)
 807            fnum = fnum + 1
 808        return files
 809
 810    def stripRepoPath(self, path, prefixes):
 811        if self.keepRepoPath:
 812            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 813
 814        for p in prefixes:
 815            if path.startswith(p):
 816                path = path[len(p):]
 817
 818        return path
 819
 820    def splitFilesIntoBranches(self, commit):
 821        branches = {}
 822        fnum = 0
 823        while commit.has_key("depotFile%s" % fnum):
 824            path =  commit["depotFile%s" % fnum]
 825            found = [p for p in self.depotPaths
 826                     if path.startswith (p)]
 827            if not found:
 828                fnum = fnum + 1
 829                continue
 830
 831            file = {}
 832            file["path"] = path
 833            file["rev"] = commit["rev%s" % fnum]
 834            file["action"] = commit["action%s" % fnum]
 835            file["type"] = commit["type%s" % fnum]
 836            fnum = fnum + 1
 837
 838            relPath = self.stripRepoPath(path, self.depotPaths)
 839
 840            for branch in self.knownBranches.keys():
 841
 842                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 843                if relPath.startswith(branch + "/"):
 844                    if branch not in branches:
 845                        branches[branch] = []
 846                    branches[branch].append(file)
 847                    break
 848
 849        return branches
 850
 851    ## Should move this out, doesn't use SELF.
 852    def readP4Files(self, files):
 853        for f in files:
 854            for val in self.clientSpecDirs:
 855                if f['path'].startswith(val[0]):
 856                    if val[1] > 0:
 857                        f['include'] = True
 858                    else:
 859                        f['include'] = False
 860                    break
 861
 862        files = [f for f in files
 863                 if f['action'] != 'delete' and
 864                 (f.has_key('include') == False or f['include'] == True)]
 865
 866        if not files:
 867            return []
 868
 869        filedata = p4CmdList('-x - print',
 870                             stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 871                                              for f in files]),
 872                             stdin_mode='w+')
 873        if "p4ExitCode" in filedata[0]:
 874            die("Problems executing p4. Error: [%d]."
 875                % (filedata[0]['p4ExitCode']));
 876
 877        j = 0;
 878        contents = {}
 879        while j < len(filedata):
 880            stat = filedata[j]
 881            j += 1
 882            text = ''
 883            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 884                tmp = filedata[j]['data']
 885                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 886                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 887                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 888                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 889                text += tmp
 890                j += 1
 891
 892
 893            if not stat.has_key('depotFile'):
 894                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 895                continue
 896
 897            contents[stat['depotFile']] = text
 898
 899        for f in files:
 900            assert not f.has_key('data')
 901            f['data'] = contents[f['path']]
 902        return files
 903
 904    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 905        epoch = details["time"]
 906        author = details["user"]
 907
 908        if self.verbose:
 909            print "commit into %s" % branch
 910
 911        # start with reading files; if that fails, we should not
 912        # create a commit.
 913        new_files = []
 914        for f in files:
 915            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 916                new_files.append (f)
 917            else:
 918                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 919        files = self.readP4Files(new_files)
 920
 921        self.gitStream.write("commit %s\n" % branch)
 922#        gitStream.write("mark :%s\n" % details["change"])
 923        self.committedChanges.add(int(details["change"]))
 924        committer = ""
 925        if author not in self.users:
 926            self.getUserMapFromPerforceServer()
 927        if author in self.users:
 928            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 929        else:
 930            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 931
 932        self.gitStream.write("committer %s\n" % committer)
 933
 934        self.gitStream.write("data <<EOT\n")
 935        self.gitStream.write(details["desc"])
 936        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 937                             % (','.join (branchPrefixes), details["change"]))
 938        if len(details['options']) > 0:
 939            self.gitStream.write(": options = %s" % details['options'])
 940        self.gitStream.write("]\nEOT\n\n")
 941
 942        if len(parent) > 0:
 943            if self.verbose:
 944                print "parent %s" % parent
 945            self.gitStream.write("from %s\n" % parent)
 946
 947        for file in files:
 948            if file["type"] == "apple":
 949                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 950                continue
 951
 952            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 953            if file["action"] == "delete":
 954                self.gitStream.write("D %s\n" % relPath)
 955            else:
 956                data = file['data']
 957
 958                mode = "644"
 959                if isP4Exec(file["type"]):
 960                    mode = "755"
 961                elif file["type"] == "symlink":
 962                    mode = "120000"
 963                    # p4 print on a symlink contains "target\n", so strip it off
 964                    data = data[:-1]
 965
 966                if self.isWindows and file["type"].endswith("text"):
 967                    data = data.replace("\r\n", "\n")
 968
 969                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 970                self.gitStream.write("data %s\n" % len(data))
 971                self.gitStream.write(data)
 972                self.gitStream.write("\n")
 973
 974        self.gitStream.write("\n")
 975
 976        change = int(details["change"])
 977
 978        if self.labels.has_key(change):
 979            label = self.labels[change]
 980            labelDetails = label[0]
 981            labelRevisions = label[1]
 982            if self.verbose:
 983                print "Change %s is labelled %s" % (change, labelDetails)
 984
 985            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 986                                                    for p in branchPrefixes]))
 987
 988            if len(files) == len(labelRevisions):
 989
 990                cleanedFiles = {}
 991                for info in files:
 992                    if info["action"] == "delete":
 993                        continue
 994                    cleanedFiles[info["depotFile"]] = info["rev"]
 995
 996                if cleanedFiles == labelRevisions:
 997                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 998                    self.gitStream.write("from %s\n" % branch)
 999
1000                    owner = labelDetails["Owner"]
1001                    tagger = ""
1002                    if author in self.users:
1003                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1004                    else:
1005                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1006                    self.gitStream.write("tagger %s\n" % tagger)
1007                    self.gitStream.write("data <<EOT\n")
1008                    self.gitStream.write(labelDetails["Description"])
1009                    self.gitStream.write("EOT\n\n")
1010
1011                else:
1012                    if not self.silent:
1013                        print ("Tag %s does not match with change %s: files do not match."
1014                               % (labelDetails["label"], change))
1015
1016            else:
1017                if not self.silent:
1018                    print ("Tag %s does not match with change %s: file count is different."
1019                           % (labelDetails["label"], change))
1020
1021    def getUserCacheFilename(self):
1022        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1023        return home + "/.gitp4-usercache.txt"
1024
1025    def getUserMapFromPerforceServer(self):
1026        if self.userMapFromPerforceServer:
1027            return
1028        self.users = {}
1029
1030        for output in p4CmdList("users"):
1031            if not output.has_key("User"):
1032                continue
1033            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1034
1035
1036        s = ''
1037        for (key, val) in self.users.items():
1038            s += "%s\t%s\n" % (key, val)
1039
1040        open(self.getUserCacheFilename(), "wb").write(s)
1041        self.userMapFromPerforceServer = True
1042
1043    def loadUserMapFromCache(self):
1044        self.users = {}
1045        self.userMapFromPerforceServer = False
1046        try:
1047            cache = open(self.getUserCacheFilename(), "rb")
1048            lines = cache.readlines()
1049            cache.close()
1050            for line in lines:
1051                entry = line.strip().split("\t")
1052                self.users[entry[0]] = entry[1]
1053        except IOError:
1054            self.getUserMapFromPerforceServer()
1055
1056    def getLabels(self):
1057        self.labels = {}
1058
1059        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1060        if len(l) > 0 and not self.silent:
1061            print "Finding files belonging to labels in %s" % `self.depotPaths`
1062
1063        for output in l:
1064            label = output["label"]
1065            revisions = {}
1066            newestChange = 0
1067            if self.verbose:
1068                print "Querying files for label %s" % label
1069            for file in p4CmdList("files "
1070                                  +  ' '.join (["%s...@%s" % (p, label)
1071                                                for p in self.depotPaths])):
1072                revisions[file["depotFile"]] = file["rev"]
1073                change = int(file["change"])
1074                if change > newestChange:
1075                    newestChange = change
1076
1077            self.labels[newestChange] = [output, revisions]
1078
1079        if self.verbose:
1080            print "Label changes: %s" % self.labels.keys()
1081
1082    def guessProjectName(self):
1083        for p in self.depotPaths:
1084            if p.endswith("/"):
1085                p = p[:-1]
1086            p = p[p.strip().rfind("/") + 1:]
1087            if not p.endswith("/"):
1088               p += "/"
1089            return p
1090
1091    def getBranchMapping(self):
1092        lostAndFoundBranches = set()
1093
1094        for info in p4CmdList("branches"):
1095            details = p4Cmd("branch -o %s" % info["branch"])
1096            viewIdx = 0
1097            while details.has_key("View%s" % viewIdx):
1098                paths = details["View%s" % viewIdx].split(" ")
1099                viewIdx = viewIdx + 1
1100                # require standard //depot/foo/... //depot/bar/... mapping
1101                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1102                    continue
1103                source = paths[0]
1104                destination = paths[1]
1105                ## HACK
1106                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1107                    source = source[len(self.depotPaths[0]):-4]
1108                    destination = destination[len(self.depotPaths[0]):-4]
1109
1110                    if destination in self.knownBranches:
1111                        if not self.silent:
1112                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1113                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1114                        continue
1115
1116                    self.knownBranches[destination] = source
1117
1118                    lostAndFoundBranches.discard(destination)
1119
1120                    if source not in self.knownBranches:
1121                        lostAndFoundBranches.add(source)
1122
1123
1124        for branch in lostAndFoundBranches:
1125            self.knownBranches[branch] = branch
1126
1127    def getBranchMappingFromGitBranches(self):
1128        branches = p4BranchesInGit(self.importIntoRemotes)
1129        for branch in branches.keys():
1130            if branch == "master":
1131                branch = "main"
1132            else:
1133                branch = branch[len(self.projectName):]
1134            self.knownBranches[branch] = branch
1135
1136    def listExistingP4GitBranches(self):
1137        # branches holds mapping from name to commit
1138        branches = p4BranchesInGit(self.importIntoRemotes)
1139        self.p4BranchesInGit = branches.keys()
1140        for branch in branches.keys():
1141            self.initialParents[self.refPrefix + branch] = branches[branch]
1142
1143    def updateOptionDict(self, d):
1144        option_keys = {}
1145        if self.keepRepoPath:
1146            option_keys['keepRepoPath'] = 1
1147
1148        d["options"] = ' '.join(sorted(option_keys.keys()))
1149
1150    def readOptions(self, d):
1151        self.keepRepoPath = (d.has_key('options')
1152                             and ('keepRepoPath' in d['options']))
1153
1154    def gitRefForBranch(self, branch):
1155        if branch == "main":
1156            return self.refPrefix + "master"
1157
1158        if len(branch) <= 0:
1159            return branch
1160
1161        return self.refPrefix + self.projectName + branch
1162
1163    def gitCommitByP4Change(self, ref, change):
1164        if self.verbose:
1165            print "looking in ref " + ref + " for change %s using bisect..." % change
1166
1167        earliestCommit = ""
1168        latestCommit = parseRevision(ref)
1169
1170        while True:
1171            if self.verbose:
1172                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1173            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1174            if len(next) == 0:
1175                if self.verbose:
1176                    print "argh"
1177                return ""
1178            log = extractLogMessageFromGitCommit(next)
1179            settings = extractSettingsGitLog(log)
1180            currentChange = int(settings['change'])
1181            if self.verbose:
1182                print "current change %s" % currentChange
1183
1184            if currentChange == change:
1185                if self.verbose:
1186                    print "found %s" % next
1187                return next
1188
1189            if currentChange < change:
1190                earliestCommit = "^%s" % next
1191            else:
1192                latestCommit = "%s" % next
1193
1194        return ""
1195
1196    def importNewBranch(self, branch, maxChange):
1197        # make fast-import flush all changes to disk and update the refs using the checkpoint
1198        # command so that we can try to find the branch parent in the git history
1199        self.gitStream.write("checkpoint\n\n");
1200        self.gitStream.flush();
1201        branchPrefix = self.depotPaths[0] + branch + "/"
1202        range = "@1,%s" % maxChange
1203        #print "prefix" + branchPrefix
1204        changes = p4ChangesForPaths([branchPrefix], range)
1205        if len(changes) <= 0:
1206            return False
1207        firstChange = changes[0]
1208        #print "first change in branch: %s" % firstChange
1209        sourceBranch = self.knownBranches[branch]
1210        sourceDepotPath = self.depotPaths[0] + sourceBranch
1211        sourceRef = self.gitRefForBranch(sourceBranch)
1212        #print "source " + sourceBranch
1213
1214        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1215        #print "branch parent: %s" % branchParentChange
1216        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1217        if len(gitParent) > 0:
1218            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1219            #print "parent git commit: %s" % gitParent
1220
1221        self.importChanges(changes)
1222        return True
1223
1224    def importChanges(self, changes):
1225        cnt = 1
1226        for change in changes:
1227            description = p4Cmd("describe %s" % change)
1228            self.updateOptionDict(description)
1229
1230            if not self.silent:
1231                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1232                sys.stdout.flush()
1233            cnt = cnt + 1
1234
1235            try:
1236                if self.detectBranches:
1237                    branches = self.splitFilesIntoBranches(description)
1238                    for branch in branches.keys():
1239                        ## HACK  --hwn
1240                        branchPrefix = self.depotPaths[0] + branch + "/"
1241
1242                        parent = ""
1243
1244                        filesForCommit = branches[branch]
1245
1246                        if self.verbose:
1247                            print "branch is %s" % branch
1248
1249                        self.updatedBranches.add(branch)
1250
1251                        if branch not in self.createdBranches:
1252                            self.createdBranches.add(branch)
1253                            parent = self.knownBranches[branch]
1254                            if parent == branch:
1255                                parent = ""
1256                            else:
1257                                fullBranch = self.projectName + branch
1258                                if fullBranch not in self.p4BranchesInGit:
1259                                    if not self.silent:
1260                                        print("\n    Importing new branch %s" % fullBranch);
1261                                    if self.importNewBranch(branch, change - 1):
1262                                        parent = ""
1263                                        self.p4BranchesInGit.append(fullBranch)
1264                                    if not self.silent:
1265                                        print("\n    Resuming with change %s" % change);
1266
1267                                if self.verbose:
1268                                    print "parent determined through known branches: %s" % parent
1269
1270                        branch = self.gitRefForBranch(branch)
1271                        parent = self.gitRefForBranch(parent)
1272
1273                        if self.verbose:
1274                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1275
1276                        if len(parent) == 0 and branch in self.initialParents:
1277                            parent = self.initialParents[branch]
1278                            del self.initialParents[branch]
1279
1280                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1281                else:
1282                    files = self.extractFilesFromCommit(description)
1283                    self.commit(description, files, self.branch, self.depotPaths,
1284                                self.initialParent)
1285                    self.initialParent = ""
1286            except IOError:
1287                print self.gitError.read()
1288                sys.exit(1)
1289
1290    def importHeadRevision(self, revision):
1291        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1292
1293        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1294        details["desc"] = ("Initial import of %s from the state at revision %s"
1295                           % (' '.join(self.depotPaths), revision))
1296        details["change"] = revision
1297        newestRevision = 0
1298
1299        fileCnt = 0
1300        for info in p4CmdList("files "
1301                              +  ' '.join(["%s...%s"
1302                                           % (p, revision)
1303                                           for p in self.depotPaths])):
1304
1305            if info['code'] == 'error':
1306                sys.stderr.write("p4 returned an error: %s\n"
1307                                 % info['data'])
1308                sys.exit(1)
1309
1310
1311            change = int(info["change"])
1312            if change > newestRevision:
1313                newestRevision = change
1314
1315            if info["action"] == "delete":
1316                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1317                #fileCnt = fileCnt + 1
1318                continue
1319
1320            for prop in ["depotFile", "rev", "action", "type" ]:
1321                details["%s%s" % (prop, fileCnt)] = info[prop]
1322
1323            fileCnt = fileCnt + 1
1324
1325        details["change"] = newestRevision
1326        self.updateOptionDict(details)
1327        try:
1328            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1329        except IOError:
1330            print "IO error with git fast-import. Is your git version recent enough?"
1331            print self.gitError.read()
1332
1333
1334    def getClientSpec(self):
1335        specList = p4CmdList( "client -o" )
1336        temp = {}
1337        for entry in specList:
1338            for k,v in entry.iteritems():
1339                if k.startswith("View"):
1340                    if v.startswith('"'):
1341                        start = 1
1342                    else:
1343                        start = 0
1344                    index = v.find("...")
1345                    v = v[start:index]
1346                    if v.startswith("-"):
1347                        v = v[1:]
1348                        temp[v] = -len(v)
1349                    else:
1350                        temp[v] = len(v)
1351        self.clientSpecDirs = temp.items()
1352        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1353
1354    def run(self, args):
1355        self.depotPaths = []
1356        self.changeRange = ""
1357        self.initialParent = ""
1358        self.previousDepotPaths = []
1359
1360        # map from branch depot path to parent branch
1361        self.knownBranches = {}
1362        self.initialParents = {}
1363        self.hasOrigin = originP4BranchesExist()
1364        if not self.syncWithOrigin:
1365            self.hasOrigin = False
1366
1367        if self.importIntoRemotes:
1368            self.refPrefix = "refs/remotes/p4/"
1369        else:
1370            self.refPrefix = "refs/heads/p4/"
1371
1372        if self.syncWithOrigin and self.hasOrigin:
1373            if not self.silent:
1374                print "Syncing with origin first by calling git fetch origin"
1375            system("git fetch origin")
1376
1377        if len(self.branch) == 0:
1378            self.branch = self.refPrefix + "master"
1379            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1380                system("git update-ref %s refs/heads/p4" % self.branch)
1381                system("git branch -D p4");
1382            # create it /after/ importing, when master exists
1383            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1384                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1385
1386        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1387            self.getClientSpec()
1388
1389        # TODO: should always look at previous commits,
1390        # merge with previous imports, if possible.
1391        if args == []:
1392            if self.hasOrigin:
1393                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1394            self.listExistingP4GitBranches()
1395
1396            if len(self.p4BranchesInGit) > 1:
1397                if not self.silent:
1398                    print "Importing from/into multiple branches"
1399                self.detectBranches = True
1400
1401            if self.verbose:
1402                print "branches: %s" % self.p4BranchesInGit
1403
1404            p4Change = 0
1405            for branch in self.p4BranchesInGit:
1406                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1407
1408                settings = extractSettingsGitLog(logMsg)
1409
1410                self.readOptions(settings)
1411                if (settings.has_key('depot-paths')
1412                    and settings.has_key ('change')):
1413                    change = int(settings['change']) + 1
1414                    p4Change = max(p4Change, change)
1415
1416                    depotPaths = sorted(settings['depot-paths'])
1417                    if self.previousDepotPaths == []:
1418                        self.previousDepotPaths = depotPaths
1419                    else:
1420                        paths = []
1421                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1422                            for i in range(0, min(len(cur), len(prev))):
1423                                if cur[i] <> prev[i]:
1424                                    i = i - 1
1425                                    break
1426
1427                            paths.append (cur[:i + 1])
1428
1429                        self.previousDepotPaths = paths
1430
1431            if p4Change > 0:
1432                self.depotPaths = sorted(self.previousDepotPaths)
1433                self.changeRange = "@%s,#head" % p4Change
1434                if not self.detectBranches:
1435                    self.initialParent = parseRevision(self.branch)
1436                if not self.silent and not self.detectBranches:
1437                    print "Performing incremental import into %s git branch" % self.branch
1438
1439        if not self.branch.startswith("refs/"):
1440            self.branch = "refs/heads/" + self.branch
1441
1442        if len(args) == 0 and self.depotPaths:
1443            if not self.silent:
1444                print "Depot paths: %s" % ' '.join(self.depotPaths)
1445        else:
1446            if self.depotPaths and self.depotPaths != args:
1447                print ("previous import used depot path %s and now %s was specified. "
1448                       "This doesn't work!" % (' '.join (self.depotPaths),
1449                                               ' '.join (args)))
1450                sys.exit(1)
1451
1452            self.depotPaths = sorted(args)
1453
1454        revision = ""
1455        self.users = {}
1456
1457        newPaths = []
1458        for p in self.depotPaths:
1459            if p.find("@") != -1:
1460                atIdx = p.index("@")
1461                self.changeRange = p[atIdx:]
1462                if self.changeRange == "@all":
1463                    self.changeRange = ""
1464                elif ',' not in self.changeRange:
1465                    revision = self.changeRange
1466                    self.changeRange = ""
1467                p = p[:atIdx]
1468            elif p.find("#") != -1:
1469                hashIdx = p.index("#")
1470                revision = p[hashIdx:]
1471                p = p[:hashIdx]
1472            elif self.previousDepotPaths == []:
1473                revision = "#head"
1474
1475            p = re.sub ("\.\.\.$", "", p)
1476            if not p.endswith("/"):
1477                p += "/"
1478
1479            newPaths.append(p)
1480
1481        self.depotPaths = newPaths
1482
1483
1484        self.loadUserMapFromCache()
1485        self.labels = {}
1486        if self.detectLabels:
1487            self.getLabels();
1488
1489        if self.detectBranches:
1490            ## FIXME - what's a P4 projectName ?
1491            self.projectName = self.guessProjectName()
1492
1493            if self.hasOrigin:
1494                self.getBranchMappingFromGitBranches()
1495            else:
1496                self.getBranchMapping()
1497            if self.verbose:
1498                print "p4-git branches: %s" % self.p4BranchesInGit
1499                print "initial parents: %s" % self.initialParents
1500            for b in self.p4BranchesInGit:
1501                if b != "master":
1502
1503                    ## FIXME
1504                    b = b[len(self.projectName):]
1505                self.createdBranches.add(b)
1506
1507        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1508
1509        importProcess = subprocess.Popen(["git", "fast-import"],
1510                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1511                                         stderr=subprocess.PIPE);
1512        self.gitOutput = importProcess.stdout
1513        self.gitStream = importProcess.stdin
1514        self.gitError = importProcess.stderr
1515
1516        if revision:
1517            self.importHeadRevision(revision)
1518        else:
1519            changes = []
1520
1521            if len(self.changesFile) > 0:
1522                output = open(self.changesFile).readlines()
1523                changeSet = Set()
1524                for line in output:
1525                    changeSet.add(int(line))
1526
1527                for change in changeSet:
1528                    changes.append(change)
1529
1530                changes.sort()
1531            else:
1532                if self.verbose:
1533                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1534                                                              self.changeRange)
1535                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1536
1537                if len(self.maxChanges) > 0:
1538                    changes = changes[:min(int(self.maxChanges), len(changes))]
1539
1540            if len(changes) == 0:
1541                if not self.silent:
1542                    print "No changes to import!"
1543                return True
1544
1545            if not self.silent and not self.detectBranches:
1546                print "Import destination: %s" % self.branch
1547
1548            self.updatedBranches = set()
1549
1550            self.importChanges(changes)
1551
1552            if not self.silent:
1553                print ""
1554                if len(self.updatedBranches) > 0:
1555                    sys.stdout.write("Updated branches: ")
1556                    for b in self.updatedBranches:
1557                        sys.stdout.write("%s " % b)
1558                    sys.stdout.write("\n")
1559
1560        self.gitStream.close()
1561        if importProcess.wait() != 0:
1562            die("fast-import failed: %s" % self.gitError.read())
1563        self.gitOutput.close()
1564        self.gitError.close()
1565
1566        return True
1567
1568class P4Rebase(Command):
1569    def __init__(self):
1570        Command.__init__(self)
1571        self.options = [ ]
1572        self.description = ("Fetches the latest revision from perforce and "
1573                            + "rebases the current work (branch) against it")
1574        self.verbose = False
1575
1576    def run(self, args):
1577        sync = P4Sync()
1578        sync.run([])
1579
1580        return self.rebase()
1581
1582    def rebase(self):
1583        if os.system("git update-index --refresh") != 0:
1584            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1585        if len(read_pipe("git diff-index HEAD --")) > 0:
1586            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1587
1588        [upstream, settings] = findUpstreamBranchPoint()
1589        if len(upstream) == 0:
1590            die("Cannot find upstream branchpoint for rebase")
1591
1592        # the branchpoint may be p4/foo~3, so strip off the parent
1593        upstream = re.sub("~[0-9]+$", "", upstream)
1594
1595        print "Rebasing the current branch onto %s" % upstream
1596        oldHead = read_pipe("git rev-parse HEAD").strip()
1597        system("git rebase %s" % upstream)
1598        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1599        return True
1600
1601class P4Clone(P4Sync):
1602    def __init__(self):
1603        P4Sync.__init__(self)
1604        self.description = "Creates a new git repository and imports from Perforce into it"
1605        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1606        self.options += [
1607            optparse.make_option("--destination", dest="cloneDestination",
1608                                 action='store', default=None,
1609                                 help="where to leave result of the clone"),
1610            optparse.make_option("-/", dest="cloneExclude",
1611                                 action="append", type="string",
1612                                 help="exclude depot path")
1613        ]
1614        self.cloneDestination = None
1615        self.needsGit = False
1616
1617    # This is required for the "append" cloneExclude action
1618    def ensure_value(self, attr, value):
1619        if not hasattr(self, attr) or getattr(self, attr) is None:
1620            setattr(self, attr, value)
1621        return getattr(self, attr)
1622
1623    def defaultDestination(self, args):
1624        ## TODO: use common prefix of args?
1625        depotPath = args[0]
1626        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1627        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1628        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1629        depotDir = re.sub(r"/$", "", depotDir)
1630        return os.path.split(depotDir)[1]
1631
1632    def run(self, args):
1633        if len(args) < 1:
1634            return False
1635
1636        if self.keepRepoPath and not self.cloneDestination:
1637            sys.stderr.write("Must specify destination for --keep-path\n")
1638            sys.exit(1)
1639
1640        depotPaths = args
1641
1642        if not self.cloneDestination and len(depotPaths) > 1:
1643            self.cloneDestination = depotPaths[-1]
1644            depotPaths = depotPaths[:-1]
1645
1646        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1647        for p in depotPaths:
1648            if not p.startswith("//"):
1649                return False
1650
1651        if not self.cloneDestination:
1652            self.cloneDestination = self.defaultDestination(args)
1653
1654        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1655        if not os.path.exists(self.cloneDestination):
1656            os.makedirs(self.cloneDestination)
1657        os.chdir(self.cloneDestination)
1658        system("git init")
1659        self.gitdir = os.getcwd() + "/.git"
1660        if not P4Sync.run(self, depotPaths):
1661            return False
1662        if self.branch != "master":
1663            if gitBranchExists("refs/remotes/p4/master"):
1664                system("git branch master refs/remotes/p4/master")
1665                system("git checkout -f")
1666            else:
1667                print "Could not detect main branch. No checkout/master branch created."
1668
1669        return True
1670
1671class P4Branches(Command):
1672    def __init__(self):
1673        Command.__init__(self)
1674        self.options = [ ]
1675        self.description = ("Shows the git branches that hold imports and their "
1676                            + "corresponding perforce depot paths")
1677        self.verbose = False
1678
1679    def run(self, args):
1680        if originP4BranchesExist():
1681            createOrUpdateBranchesFromOrigin()
1682
1683        cmdline = "git rev-parse --symbolic "
1684        cmdline += " --remotes"
1685
1686        for line in read_pipe_lines(cmdline):
1687            line = line.strip()
1688
1689            if not line.startswith('p4/') or line == "p4/HEAD":
1690                continue
1691            branch = line
1692
1693            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1694            settings = extractSettingsGitLog(log)
1695
1696            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1697        return True
1698
1699class HelpFormatter(optparse.IndentedHelpFormatter):
1700    def __init__(self):
1701        optparse.IndentedHelpFormatter.__init__(self)
1702
1703    def format_description(self, description):
1704        if description:
1705            return description + "\n"
1706        else:
1707            return ""
1708
1709def printUsage(commands):
1710    print "usage: %s <command> [options]" % sys.argv[0]
1711    print ""
1712    print "valid commands: %s" % ", ".join(commands)
1713    print ""
1714    print "Try %s <command> --help for command specific help." % sys.argv[0]
1715    print ""
1716
1717commands = {
1718    "debug" : P4Debug,
1719    "submit" : P4Submit,
1720    "commit" : P4Submit,
1721    "sync" : P4Sync,
1722    "rebase" : P4Rebase,
1723    "clone" : P4Clone,
1724    "rollback" : P4RollBack,
1725    "branches" : P4Branches
1726}
1727
1728
1729def main():
1730    if len(sys.argv[1:]) == 0:
1731        printUsage(commands.keys())
1732        sys.exit(2)
1733
1734    cmd = ""
1735    cmdName = sys.argv[1]
1736    try:
1737        klass = commands[cmdName]
1738        cmd = klass()
1739    except KeyError:
1740        print "unknown command %s" % cmdName
1741        print ""
1742        printUsage(commands.keys())
1743        sys.exit(2)
1744
1745    options = cmd.options
1746    cmd.gitdir = os.environ.get("GIT_DIR", None)
1747
1748    args = sys.argv[2:]
1749
1750    if len(options) > 0:
1751        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1752
1753        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1754                                       options,
1755                                       description = cmd.description,
1756                                       formatter = HelpFormatter())
1757
1758        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1759    global verbose
1760    verbose = cmd.verbose
1761    if cmd.needsGit:
1762        if cmd.gitdir == None:
1763            cmd.gitdir = os.path.abspath(".git")
1764            if not isValidGitDir(cmd.gitdir):
1765                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1766                if os.path.exists(cmd.gitdir):
1767                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1768                    if len(cdup) > 0:
1769                        os.chdir(cdup);
1770
1771        if not isValidGitDir(cmd.gitdir):
1772            if isValidGitDir(cmd.gitdir + "/.git"):
1773                cmd.gitdir += "/.git"
1774            else:
1775                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1776
1777        os.environ["GIT_DIR"] = cmd.gitdir
1778
1779    if not cmd.run(args):
1780        parser.print_help()
1781
1782
1783if __name__ == '__main__':
1784    main()