Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'jc/am-options' (d8af75d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    depotPath = depotPath + "..."
 249    outputList = p4CmdList("where %s" % depotPath)
 250    output = None
 251    for entry in outputList:
 252        if entry["depotFile"] == depotPath:
 253            output = entry
 254            break
 255    if output == None:
 256        return ""
 257    if output["code"] == "error":
 258        return ""
 259    clientPath = ""
 260    if "path" in output:
 261        clientPath = output.get("path")
 262    elif "data" in output:
 263        data = output.get("data")
 264        lastSpace = data.rfind(" ")
 265        clientPath = data[lastSpace + 1:]
 266
 267    if clientPath.endswith("..."):
 268        clientPath = clientPath[:-3]
 269    return clientPath
 270
 271def currentGitBranch():
 272    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 273
 274def isValidGitDir(path):
 275    if (os.path.exists(path + "/HEAD")
 276        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 277        return True;
 278    return False
 279
 280def parseRevision(ref):
 281    return read_pipe("git rev-parse %s" % ref).strip()
 282
 283def extractLogMessageFromGitCommit(commit):
 284    logMessage = ""
 285
 286    ## fixme: title is first line of commit, not 1st paragraph.
 287    foundTitle = False
 288    for log in read_pipe_lines("git cat-file commit %s" % commit):
 289       if not foundTitle:
 290           if len(log) == 1:
 291               foundTitle = True
 292           continue
 293
 294       logMessage += log
 295    return logMessage
 296
 297def extractSettingsGitLog(log):
 298    values = {}
 299    for line in log.split("\n"):
 300        line = line.strip()
 301        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 302        if not m:
 303            continue
 304
 305        assignments = m.group(1).split (':')
 306        for a in assignments:
 307            vals = a.split ('=')
 308            key = vals[0].strip()
 309            val = ('='.join (vals[1:])).strip()
 310            if val.endswith ('\"') and val.startswith('"'):
 311                val = val[1:-1]
 312
 313            values[key] = val
 314
 315    paths = values.get("depot-paths")
 316    if not paths:
 317        paths = values.get("depot-path")
 318    if paths:
 319        values['depot-paths'] = paths.split(',')
 320    return values
 321
 322def gitBranchExists(branch):
 323    proc = subprocess.Popen(["git", "rev-parse", branch],
 324                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 325    return proc.wait() == 0;
 326
 327_gitConfig = {}
 328def gitConfig(key):
 329    if not _gitConfig.has_key(key):
 330        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 331    return _gitConfig[key]
 332
 333def p4BranchesInGit(branchesAreInRemotes = True):
 334    branches = {}
 335
 336    cmdline = "git rev-parse --symbolic "
 337    if branchesAreInRemotes:
 338        cmdline += " --remotes"
 339    else:
 340        cmdline += " --branches"
 341
 342    for line in read_pipe_lines(cmdline):
 343        line = line.strip()
 344
 345        ## only import to p4/
 346        if not line.startswith('p4/') or line == "p4/HEAD":
 347            continue
 348        branch = line
 349
 350        # strip off p4
 351        branch = re.sub ("^p4/", "", line)
 352
 353        branches[branch] = parseRevision(line)
 354    return branches
 355
 356def findUpstreamBranchPoint(head = "HEAD"):
 357    branches = p4BranchesInGit()
 358    # map from depot-path to branch name
 359    branchByDepotPath = {}
 360    for branch in branches.keys():
 361        tip = branches[branch]
 362        log = extractLogMessageFromGitCommit(tip)
 363        settings = extractSettingsGitLog(log)
 364        if settings.has_key("depot-paths"):
 365            paths = ",".join(settings["depot-paths"])
 366            branchByDepotPath[paths] = "remotes/p4/" + branch
 367
 368    settings = None
 369    parent = 0
 370    while parent < 65535:
 371        commit = head + "~%s" % parent
 372        log = extractLogMessageFromGitCommit(commit)
 373        settings = extractSettingsGitLog(log)
 374        if settings.has_key("depot-paths"):
 375            paths = ",".join(settings["depot-paths"])
 376            if branchByDepotPath.has_key(paths):
 377                return [branchByDepotPath[paths], settings]
 378
 379        parent = parent + 1
 380
 381    return ["", settings]
 382
 383def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 384    if not silent:
 385        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 386               % localRefPrefix)
 387
 388    originPrefix = "origin/p4/"
 389
 390    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 391        line = line.strip()
 392        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 393            continue
 394
 395        headName = line[len(originPrefix):]
 396        remoteHead = localRefPrefix + headName
 397        originHead = line
 398
 399        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 400        if (not original.has_key('depot-paths')
 401            or not original.has_key('change')):
 402            continue
 403
 404        update = False
 405        if not gitBranchExists(remoteHead):
 406            if verbose:
 407                print "creating %s" % remoteHead
 408            update = True
 409        else:
 410            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 411            if settings.has_key('change') > 0:
 412                if settings['depot-paths'] == original['depot-paths']:
 413                    originP4Change = int(original['change'])
 414                    p4Change = int(settings['change'])
 415                    if originP4Change > p4Change:
 416                        print ("%s (%s) is newer than %s (%s). "
 417                               "Updating p4 branch from origin."
 418                               % (originHead, originP4Change,
 419                                  remoteHead, p4Change))
 420                        update = True
 421                else:
 422                    print ("Ignoring: %s was imported from %s while "
 423                           "%s was imported from %s"
 424                           % (originHead, ','.join(original['depot-paths']),
 425                              remoteHead, ','.join(settings['depot-paths'])))
 426
 427        if update:
 428            system("git update-ref %s %s" % (remoteHead, originHead))
 429
 430def originP4BranchesExist():
 431        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 432
 433def p4ChangesForPaths(depotPaths, changeRange):
 434    assert depotPaths
 435    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 436                                                        for p in depotPaths]))
 437
 438    changes = []
 439    for line in output:
 440        changeNum = line.split(" ")[1]
 441        changes.append(int(changeNum))
 442
 443    changes.sort()
 444    return changes
 445
 446class Command:
 447    def __init__(self):
 448        self.usage = "usage: %prog [options]"
 449        self.needsGit = True
 450
 451class P4Debug(Command):
 452    def __init__(self):
 453        Command.__init__(self)
 454        self.options = [
 455            optparse.make_option("--verbose", dest="verbose", action="store_true",
 456                                 default=False),
 457            ]
 458        self.description = "A tool to debug the output of p4 -G."
 459        self.needsGit = False
 460        self.verbose = False
 461
 462    def run(self, args):
 463        j = 0
 464        for output in p4CmdList(" ".join(args)):
 465            print 'Element: %d' % j
 466            j += 1
 467            print output
 468        return True
 469
 470class P4RollBack(Command):
 471    def __init__(self):
 472        Command.__init__(self)
 473        self.options = [
 474            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 475            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 476        ]
 477        self.description = "A tool to debug the multi-branch import. Don't use :)"
 478        self.verbose = False
 479        self.rollbackLocalBranches = False
 480
 481    def run(self, args):
 482        if len(args) != 1:
 483            return False
 484        maxChange = int(args[0])
 485
 486        if "p4ExitCode" in p4Cmd("changes -m 1"):
 487            die("Problems executing p4");
 488
 489        if self.rollbackLocalBranches:
 490            refPrefix = "refs/heads/"
 491            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 492        else:
 493            refPrefix = "refs/remotes/"
 494            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 495
 496        for line in lines:
 497            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 498                line = line.strip()
 499                ref = refPrefix + line
 500                log = extractLogMessageFromGitCommit(ref)
 501                settings = extractSettingsGitLog(log)
 502
 503                depotPaths = settings['depot-paths']
 504                change = settings['change']
 505
 506                changed = False
 507
 508                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 509                                                           for p in depotPaths]))) == 0:
 510                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 511                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 512                    continue
 513
 514                while change and int(change) > maxChange:
 515                    changed = True
 516                    if self.verbose:
 517                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 518                    system("git update-ref %s \"%s^\"" % (ref, ref))
 519                    log = extractLogMessageFromGitCommit(ref)
 520                    settings =  extractSettingsGitLog(log)
 521
 522
 523                    depotPaths = settings['depot-paths']
 524                    change = settings['change']
 525
 526                if changed:
 527                    print "%s rewound to %s" % (ref, change)
 528
 529        return True
 530
 531class P4Submit(Command):
 532    def __init__(self):
 533        Command.__init__(self)
 534        self.options = [
 535                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 536                optparse.make_option("--origin", dest="origin"),
 537                optparse.make_option("-M", dest="detectRename", action="store_true"),
 538        ]
 539        self.description = "Submit changes from git to the perforce depot."
 540        self.usage += " [name of git branch to submit into perforce depot]"
 541        self.interactive = True
 542        self.origin = ""
 543        self.detectRename = False
 544        self.verbose = False
 545        self.isWindows = (platform.system() == "Windows")
 546
 547    def check(self):
 548        if len(p4CmdList("opened ...")) > 0:
 549            die("You have files opened with perforce! Close them before starting the sync.")
 550
 551    # replaces everything between 'Description:' and the next P4 submit template field with the
 552    # commit message
 553    def prepareLogMessage(self, template, message):
 554        result = ""
 555
 556        inDescriptionSection = False
 557
 558        for line in template.split("\n"):
 559            if line.startswith("#"):
 560                result += line + "\n"
 561                continue
 562
 563            if inDescriptionSection:
 564                if line.startswith("Files:"):
 565                    inDescriptionSection = False
 566                else:
 567                    continue
 568            else:
 569                if line.startswith("Description:"):
 570                    inDescriptionSection = True
 571                    line += "\n"
 572                    for messageLine in message.split("\n"):
 573                        line += "\t" + messageLine + "\n"
 574
 575            result += line + "\n"
 576
 577        return result
 578
 579    def prepareSubmitTemplate(self):
 580        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 581        template = ""
 582        inFilesSection = False
 583        for line in p4_read_pipe_lines("change -o"):
 584            if line.endswith("\r\n"):
 585                line = line[:-2] + "\n"
 586            if inFilesSection:
 587                if line.startswith("\t"):
 588                    # path starts and ends with a tab
 589                    path = line[1:]
 590                    lastTab = path.rfind("\t")
 591                    if lastTab != -1:
 592                        path = path[:lastTab]
 593                        if not path.startswith(self.depotPath):
 594                            continue
 595                else:
 596                    inFilesSection = False
 597            else:
 598                if line.startswith("Files:"):
 599                    inFilesSection = True
 600
 601            template += line
 602
 603        return template
 604
 605    def applyCommit(self, id):
 606        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 607        diffOpts = ("", "-M")[self.detectRename]
 608        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 609        filesToAdd = set()
 610        filesToDelete = set()
 611        editedFiles = set()
 612        filesToChangeExecBit = {}
 613        for line in diff:
 614            diff = parseDiffTreeEntry(line)
 615            modifier = diff['status']
 616            path = diff['src']
 617            if modifier == "M":
 618                p4_system("edit \"%s\"" % path)
 619                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 620                    filesToChangeExecBit[path] = diff['dst_mode']
 621                editedFiles.add(path)
 622            elif modifier == "A":
 623                filesToAdd.add(path)
 624                filesToChangeExecBit[path] = diff['dst_mode']
 625                if path in filesToDelete:
 626                    filesToDelete.remove(path)
 627            elif modifier == "D":
 628                filesToDelete.add(path)
 629                if path in filesToAdd:
 630                    filesToAdd.remove(path)
 631            elif modifier == "R":
 632                src, dest = diff['src'], diff['dst']
 633                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 634                p4_system("edit \"%s\"" % (dest))
 635                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 636                    filesToChangeExecBit[dest] = diff['dst_mode']
 637                os.unlink(dest)
 638                editedFiles.add(dest)
 639                filesToDelete.add(src)
 640            else:
 641                die("unknown modifier %s for %s" % (modifier, path))
 642
 643        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 644        patchcmd = diffcmd + " | git apply "
 645        tryPatchCmd = patchcmd + "--check -"
 646        applyPatchCmd = patchcmd + "--check --apply -"
 647
 648        if os.system(tryPatchCmd) != 0:
 649            print "Unfortunately applying the change failed!"
 650            print "What do you want to do?"
 651            response = "x"
 652            while response != "s" and response != "a" and response != "w":
 653                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 654                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 655            if response == "s":
 656                print "Skipping! Good luck with the next patches..."
 657                for f in editedFiles:
 658                    p4_system("revert \"%s\"" % f);
 659                for f in filesToAdd:
 660                    system("rm %s" %f)
 661                return
 662            elif response == "a":
 663                os.system(applyPatchCmd)
 664                if len(filesToAdd) > 0:
 665                    print "You may also want to call p4 add on the following files:"
 666                    print " ".join(filesToAdd)
 667                if len(filesToDelete):
 668                    print "The following files should be scheduled for deletion with p4 delete:"
 669                    print " ".join(filesToDelete)
 670                die("Please resolve and submit the conflict manually and "
 671                    + "continue afterwards with git-p4 submit --continue")
 672            elif response == "w":
 673                system(diffcmd + " > patch.txt")
 674                print "Patch saved to patch.txt in %s !" % self.clientPath
 675                die("Please resolve and submit the conflict manually and "
 676                    "continue afterwards with git-p4 submit --continue")
 677
 678        system(applyPatchCmd)
 679
 680        for f in filesToAdd:
 681            p4_system("add \"%s\"" % f)
 682        for f in filesToDelete:
 683            p4_system("revert \"%s\"" % f)
 684            p4_system("delete \"%s\"" % f)
 685
 686        # Set/clear executable bits
 687        for f in filesToChangeExecBit.keys():
 688            mode = filesToChangeExecBit[f]
 689            setP4ExecBit(f, mode)
 690
 691        logMessage = extractLogMessageFromGitCommit(id)
 692        logMessage = logMessage.strip()
 693
 694        template = self.prepareSubmitTemplate()
 695
 696        if self.interactive:
 697            submitTemplate = self.prepareLogMessage(template, logMessage)
 698            if os.environ.has_key("P4DIFF"):
 699                del(os.environ["P4DIFF"])
 700            diff = p4_read_pipe("diff -du ...")
 701
 702            newdiff = ""
 703            for newFile in filesToAdd:
 704                newdiff += "==== new file ====\n"
 705                newdiff += "--- /dev/null\n"
 706                newdiff += "+++ %s\n" % newFile
 707                f = open(newFile, "r")
 708                for line in f.readlines():
 709                    newdiff += "+" + line
 710                f.close()
 711
 712            separatorLine = "######## everything below this line is just the diff #######\n"
 713
 714            [handle, fileName] = tempfile.mkstemp()
 715            tmpFile = os.fdopen(handle, "w+")
 716            if self.isWindows:
 717                submitTemplate = submitTemplate.replace("\n", "\r\n")
 718                separatorLine = separatorLine.replace("\n", "\r\n")
 719                newdiff = newdiff.replace("\n", "\r\n")
 720            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 721            tmpFile.close()
 722            mtime = os.stat(fileName).st_mtime
 723            defaultEditor = "vi"
 724            if platform.system() == "Windows":
 725                defaultEditor = "notepad"
 726            if os.environ.has_key("P4EDITOR"):
 727                editor = os.environ.get("P4EDITOR")
 728            else:
 729                editor = os.environ.get("EDITOR", defaultEditor);
 730            system(editor + " " + fileName)
 731
 732            response = "y"
 733            if os.stat(fileName).st_mtime <= mtime:
 734                response = "x"
 735                while response != "y" and response != "n":
 736                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 737
 738            if response == "y":
 739                tmpFile = open(fileName, "rb")
 740                message = tmpFile.read()
 741                tmpFile.close()
 742                submitTemplate = message[:message.index(separatorLine)]
 743                if self.isWindows:
 744                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 745                p4_write_pipe("submit -i", submitTemplate)
 746            else:
 747                for f in editedFiles:
 748                    p4_system("revert \"%s\"" % f);
 749                for f in filesToAdd:
 750                    p4_system("revert \"%s\"" % f);
 751                    system("rm %s" %f)
 752
 753            os.remove(fileName)
 754        else:
 755            fileName = "submit.txt"
 756            file = open(fileName, "w+")
 757            file.write(self.prepareLogMessage(template, logMessage))
 758            file.close()
 759            print ("Perforce submit template written as %s. "
 760                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 761                   % (fileName, fileName))
 762
 763    def run(self, args):
 764        if len(args) == 0:
 765            self.master = currentGitBranch()
 766            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 767                die("Detecting current git branch failed!")
 768        elif len(args) == 1:
 769            self.master = args[0]
 770        else:
 771            return False
 772
 773        allowSubmit = gitConfig("git-p4.allowSubmit")
 774        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 775            die("%s is not in git-p4.allowSubmit" % self.master)
 776
 777        [upstream, settings] = findUpstreamBranchPoint()
 778        self.depotPath = settings['depot-paths'][0]
 779        if len(self.origin) == 0:
 780            self.origin = upstream
 781
 782        if self.verbose:
 783            print "Origin branch is " + self.origin
 784
 785        if len(self.depotPath) == 0:
 786            print "Internal error: cannot locate perforce depot path from existing branches"
 787            sys.exit(128)
 788
 789        self.clientPath = p4Where(self.depotPath)
 790
 791        if len(self.clientPath) == 0:
 792            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 793            sys.exit(128)
 794
 795        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 796        self.oldWorkingDirectory = os.getcwd()
 797
 798        chdir(self.clientPath)
 799        print "Syncronizing p4 checkout..."
 800        p4_system("sync ...")
 801
 802        self.check()
 803
 804        commits = []
 805        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 806            commits.append(line.strip())
 807        commits.reverse()
 808
 809        while len(commits) > 0:
 810            commit = commits[0]
 811            commits = commits[1:]
 812            self.applyCommit(commit)
 813            if not self.interactive:
 814                break
 815
 816        if len(commits) == 0:
 817            print "All changes applied!"
 818            chdir(self.oldWorkingDirectory)
 819
 820            sync = P4Sync()
 821            sync.run([])
 822
 823            rebase = P4Rebase()
 824            rebase.rebase()
 825
 826        return True
 827
 828class P4Sync(Command):
 829    def __init__(self):
 830        Command.__init__(self)
 831        self.options = [
 832                optparse.make_option("--branch", dest="branch"),
 833                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 834                optparse.make_option("--changesfile", dest="changesFile"),
 835                optparse.make_option("--silent", dest="silent", action="store_true"),
 836                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 837                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 838                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 839                                     help="Import into refs/heads/ , not refs/remotes"),
 840                optparse.make_option("--max-changes", dest="maxChanges"),
 841                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 842                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 843                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 844                                     help="Only sync files that are included in the Perforce Client Spec")
 845        ]
 846        self.description = """Imports from Perforce into a git repository.\n
 847    example:
 848    //depot/my/project/ -- to import the current head
 849    //depot/my/project/@all -- to import everything
 850    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 851
 852    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 853
 854        self.usage += " //depot/path[@revRange]"
 855        self.silent = False
 856        self.createdBranches = Set()
 857        self.committedChanges = Set()
 858        self.branch = ""
 859        self.detectBranches = False
 860        self.detectLabels = False
 861        self.changesFile = ""
 862        self.syncWithOrigin = True
 863        self.verbose = False
 864        self.importIntoRemotes = True
 865        self.maxChanges = ""
 866        self.isWindows = (platform.system() == "Windows")
 867        self.keepRepoPath = False
 868        self.depotPaths = None
 869        self.p4BranchesInGit = []
 870        self.cloneExclude = []
 871        self.useClientSpec = False
 872        self.clientSpecDirs = []
 873
 874        if gitConfig("git-p4.syncFromOrigin") == "false":
 875            self.syncWithOrigin = False
 876
 877    def extractFilesFromCommit(self, commit):
 878        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 879                             for path in self.cloneExclude]
 880        files = []
 881        fnum = 0
 882        while commit.has_key("depotFile%s" % fnum):
 883            path =  commit["depotFile%s" % fnum]
 884
 885            if [p for p in self.cloneExclude
 886                if path.startswith (p)]:
 887                found = False
 888            else:
 889                found = [p for p in self.depotPaths
 890                         if path.startswith (p)]
 891            if not found:
 892                fnum = fnum + 1
 893                continue
 894
 895            file = {}
 896            file["path"] = path
 897            file["rev"] = commit["rev%s" % fnum]
 898            file["action"] = commit["action%s" % fnum]
 899            file["type"] = commit["type%s" % fnum]
 900            files.append(file)
 901            fnum = fnum + 1
 902        return files
 903
 904    def stripRepoPath(self, path, prefixes):
 905        if self.keepRepoPath:
 906            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908        for p in prefixes:
 909            if path.startswith(p):
 910                path = path[len(p):]
 911
 912        return path
 913
 914    def splitFilesIntoBranches(self, commit):
 915        branches = {}
 916        fnum = 0
 917        while commit.has_key("depotFile%s" % fnum):
 918            path =  commit["depotFile%s" % fnum]
 919            found = [p for p in self.depotPaths
 920                     if path.startswith (p)]
 921            if not found:
 922                fnum = fnum + 1
 923                continue
 924
 925            file = {}
 926            file["path"] = path
 927            file["rev"] = commit["rev%s" % fnum]
 928            file["action"] = commit["action%s" % fnum]
 929            file["type"] = commit["type%s" % fnum]
 930            fnum = fnum + 1
 931
 932            relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934            for branch in self.knownBranches.keys():
 935
 936                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                if relPath.startswith(branch + "/"):
 938                    if branch not in branches:
 939                        branches[branch] = []
 940                    branches[branch].append(file)
 941                    break
 942
 943        return branches
 944
 945    ## Should move this out, doesn't use SELF.
 946    def readP4Files(self, files):
 947        filesForCommit = []
 948        filesToRead = []
 949
 950        for f in files:
 951            includeFile = True
 952            for val in self.clientSpecDirs:
 953                if f['path'].startswith(val[0]):
 954                    if val[1] <= 0:
 955                        includeFile = False
 956                    break
 957
 958            if includeFile:
 959                filesForCommit.append(f)
 960                if f['action'] not in ('delete', 'purge'):
 961                    filesToRead.append(f)
 962
 963        filedata = []
 964        if len(filesToRead) > 0:
 965            filedata = p4CmdList('-x - print',
 966                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 967                                                  for f in filesToRead]),
 968                                 stdin_mode='w+')
 969
 970            if "p4ExitCode" in filedata[0]:
 971                die("Problems executing p4. Error: [%d]."
 972                    % (filedata[0]['p4ExitCode']));
 973
 974        j = 0;
 975        contents = {}
 976        while j < len(filedata):
 977            stat = filedata[j]
 978            j += 1
 979            text = ''
 980            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 981                text += filedata[j]['data']
 982                del filedata[j]['data']
 983                j += 1
 984
 985            if not stat.has_key('depotFile'):
 986                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 987                continue
 988
 989            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 990                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 991            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 992                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
 993
 994            contents[stat['depotFile']] = text
 995
 996        for f in filesForCommit:
 997            path = f['path']
 998            if contents.has_key(path):
 999                f['data'] = contents[path]
1000
1001        return filesForCommit
1002
1003    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1004        epoch = details["time"]
1005        author = details["user"]
1006
1007        if self.verbose:
1008            print "commit into %s" % branch
1009
1010        # start with reading files; if that fails, we should not
1011        # create a commit.
1012        new_files = []
1013        for f in files:
1014            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1015                new_files.append (f)
1016            else:
1017                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1018        files = self.readP4Files(new_files)
1019
1020        self.gitStream.write("commit %s\n" % branch)
1021#        gitStream.write("mark :%s\n" % details["change"])
1022        self.committedChanges.add(int(details["change"]))
1023        committer = ""
1024        if author not in self.users:
1025            self.getUserMapFromPerforceServer()
1026        if author in self.users:
1027            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1028        else:
1029            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1030
1031        self.gitStream.write("committer %s\n" % committer)
1032
1033        self.gitStream.write("data <<EOT\n")
1034        self.gitStream.write(details["desc"])
1035        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1036                             % (','.join (branchPrefixes), details["change"]))
1037        if len(details['options']) > 0:
1038            self.gitStream.write(": options = %s" % details['options'])
1039        self.gitStream.write("]\nEOT\n\n")
1040
1041        if len(parent) > 0:
1042            if self.verbose:
1043                print "parent %s" % parent
1044            self.gitStream.write("from %s\n" % parent)
1045
1046        for file in files:
1047            if file["type"] == "apple":
1048                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1049                continue
1050
1051            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1052            if file["action"] in ("delete", "purge"):
1053                self.gitStream.write("D %s\n" % relPath)
1054            else:
1055                data = file['data']
1056
1057                mode = "644"
1058                if isP4Exec(file["type"]):
1059                    mode = "755"
1060                elif file["type"] == "symlink":
1061                    mode = "120000"
1062                    # p4 print on a symlink contains "target\n", so strip it off
1063                    data = data[:-1]
1064
1065                if self.isWindows and file["type"].endswith("text"):
1066                    data = data.replace("\r\n", "\n")
1067
1068                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1069                self.gitStream.write("data %s\n" % len(data))
1070                self.gitStream.write(data)
1071                self.gitStream.write("\n")
1072
1073        self.gitStream.write("\n")
1074
1075        change = int(details["change"])
1076
1077        if self.labels.has_key(change):
1078            label = self.labels[change]
1079            labelDetails = label[0]
1080            labelRevisions = label[1]
1081            if self.verbose:
1082                print "Change %s is labelled %s" % (change, labelDetails)
1083
1084            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1085                                                    for p in branchPrefixes]))
1086
1087            if len(files) == len(labelRevisions):
1088
1089                cleanedFiles = {}
1090                for info in files:
1091                    if info["action"] in ("delete", "purge"):
1092                        continue
1093                    cleanedFiles[info["depotFile"]] = info["rev"]
1094
1095                if cleanedFiles == labelRevisions:
1096                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1097                    self.gitStream.write("from %s\n" % branch)
1098
1099                    owner = labelDetails["Owner"]
1100                    tagger = ""
1101                    if author in self.users:
1102                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1103                    else:
1104                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1105                    self.gitStream.write("tagger %s\n" % tagger)
1106                    self.gitStream.write("data <<EOT\n")
1107                    self.gitStream.write(labelDetails["Description"])
1108                    self.gitStream.write("EOT\n\n")
1109
1110                else:
1111                    if not self.silent:
1112                        print ("Tag %s does not match with change %s: files do not match."
1113                               % (labelDetails["label"], change))
1114
1115            else:
1116                if not self.silent:
1117                    print ("Tag %s does not match with change %s: file count is different."
1118                           % (labelDetails["label"], change))
1119
1120    def getUserCacheFilename(self):
1121        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1122        return home + "/.gitp4-usercache.txt"
1123
1124    def getUserMapFromPerforceServer(self):
1125        if self.userMapFromPerforceServer:
1126            return
1127        self.users = {}
1128
1129        for output in p4CmdList("users"):
1130            if not output.has_key("User"):
1131                continue
1132            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1133
1134
1135        s = ''
1136        for (key, val) in self.users.items():
1137            s += "%s\t%s\n" % (key, val)
1138
1139        open(self.getUserCacheFilename(), "wb").write(s)
1140        self.userMapFromPerforceServer = True
1141
1142    def loadUserMapFromCache(self):
1143        self.users = {}
1144        self.userMapFromPerforceServer = False
1145        try:
1146            cache = open(self.getUserCacheFilename(), "rb")
1147            lines = cache.readlines()
1148            cache.close()
1149            for line in lines:
1150                entry = line.strip().split("\t")
1151                self.users[entry[0]] = entry[1]
1152        except IOError:
1153            self.getUserMapFromPerforceServer()
1154
1155    def getLabels(self):
1156        self.labels = {}
1157
1158        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1159        if len(l) > 0 and not self.silent:
1160            print "Finding files belonging to labels in %s" % `self.depotPaths`
1161
1162        for output in l:
1163            label = output["label"]
1164            revisions = {}
1165            newestChange = 0
1166            if self.verbose:
1167                print "Querying files for label %s" % label
1168            for file in p4CmdList("files "
1169                                  +  ' '.join (["%s...@%s" % (p, label)
1170                                                for p in self.depotPaths])):
1171                revisions[file["depotFile"]] = file["rev"]
1172                change = int(file["change"])
1173                if change > newestChange:
1174                    newestChange = change
1175
1176            self.labels[newestChange] = [output, revisions]
1177
1178        if self.verbose:
1179            print "Label changes: %s" % self.labels.keys()
1180
1181    def guessProjectName(self):
1182        for p in self.depotPaths:
1183            if p.endswith("/"):
1184                p = p[:-1]
1185            p = p[p.strip().rfind("/") + 1:]
1186            if not p.endswith("/"):
1187               p += "/"
1188            return p
1189
1190    def getBranchMapping(self):
1191        lostAndFoundBranches = set()
1192
1193        for info in p4CmdList("branches"):
1194            details = p4Cmd("branch -o %s" % info["branch"])
1195            viewIdx = 0
1196            while details.has_key("View%s" % viewIdx):
1197                paths = details["View%s" % viewIdx].split(" ")
1198                viewIdx = viewIdx + 1
1199                # require standard //depot/foo/... //depot/bar/... mapping
1200                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1201                    continue
1202                source = paths[0]
1203                destination = paths[1]
1204                ## HACK
1205                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1206                    source = source[len(self.depotPaths[0]):-4]
1207                    destination = destination[len(self.depotPaths[0]):-4]
1208
1209                    if destination in self.knownBranches:
1210                        if not self.silent:
1211                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1212                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1213                        continue
1214
1215                    self.knownBranches[destination] = source
1216
1217                    lostAndFoundBranches.discard(destination)
1218
1219                    if source not in self.knownBranches:
1220                        lostAndFoundBranches.add(source)
1221
1222
1223        for branch in lostAndFoundBranches:
1224            self.knownBranches[branch] = branch
1225
1226    def getBranchMappingFromGitBranches(self):
1227        branches = p4BranchesInGit(self.importIntoRemotes)
1228        for branch in branches.keys():
1229            if branch == "master":
1230                branch = "main"
1231            else:
1232                branch = branch[len(self.projectName):]
1233            self.knownBranches[branch] = branch
1234
1235    def listExistingP4GitBranches(self):
1236        # branches holds mapping from name to commit
1237        branches = p4BranchesInGit(self.importIntoRemotes)
1238        self.p4BranchesInGit = branches.keys()
1239        for branch in branches.keys():
1240            self.initialParents[self.refPrefix + branch] = branches[branch]
1241
1242    def updateOptionDict(self, d):
1243        option_keys = {}
1244        if self.keepRepoPath:
1245            option_keys['keepRepoPath'] = 1
1246
1247        d["options"] = ' '.join(sorted(option_keys.keys()))
1248
1249    def readOptions(self, d):
1250        self.keepRepoPath = (d.has_key('options')
1251                             and ('keepRepoPath' in d['options']))
1252
1253    def gitRefForBranch(self, branch):
1254        if branch == "main":
1255            return self.refPrefix + "master"
1256
1257        if len(branch) <= 0:
1258            return branch
1259
1260        return self.refPrefix + self.projectName + branch
1261
1262    def gitCommitByP4Change(self, ref, change):
1263        if self.verbose:
1264            print "looking in ref " + ref + " for change %s using bisect..." % change
1265
1266        earliestCommit = ""
1267        latestCommit = parseRevision(ref)
1268
1269        while True:
1270            if self.verbose:
1271                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1272            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1273            if len(next) == 0:
1274                if self.verbose:
1275                    print "argh"
1276                return ""
1277            log = extractLogMessageFromGitCommit(next)
1278            settings = extractSettingsGitLog(log)
1279            currentChange = int(settings['change'])
1280            if self.verbose:
1281                print "current change %s" % currentChange
1282
1283            if currentChange == change:
1284                if self.verbose:
1285                    print "found %s" % next
1286                return next
1287
1288            if currentChange < change:
1289                earliestCommit = "^%s" % next
1290            else:
1291                latestCommit = "%s" % next
1292
1293        return ""
1294
1295    def importNewBranch(self, branch, maxChange):
1296        # make fast-import flush all changes to disk and update the refs using the checkpoint
1297        # command so that we can try to find the branch parent in the git history
1298        self.gitStream.write("checkpoint\n\n");
1299        self.gitStream.flush();
1300        branchPrefix = self.depotPaths[0] + branch + "/"
1301        range = "@1,%s" % maxChange
1302        #print "prefix" + branchPrefix
1303        changes = p4ChangesForPaths([branchPrefix], range)
1304        if len(changes) <= 0:
1305            return False
1306        firstChange = changes[0]
1307        #print "first change in branch: %s" % firstChange
1308        sourceBranch = self.knownBranches[branch]
1309        sourceDepotPath = self.depotPaths[0] + sourceBranch
1310        sourceRef = self.gitRefForBranch(sourceBranch)
1311        #print "source " + sourceBranch
1312
1313        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1314        #print "branch parent: %s" % branchParentChange
1315        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1316        if len(gitParent) > 0:
1317            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1318            #print "parent git commit: %s" % gitParent
1319
1320        self.importChanges(changes)
1321        return True
1322
1323    def importChanges(self, changes):
1324        cnt = 1
1325        for change in changes:
1326            description = p4Cmd("describe %s" % change)
1327            self.updateOptionDict(description)
1328
1329            if not self.silent:
1330                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1331                sys.stdout.flush()
1332            cnt = cnt + 1
1333
1334            try:
1335                if self.detectBranches:
1336                    branches = self.splitFilesIntoBranches(description)
1337                    for branch in branches.keys():
1338                        ## HACK  --hwn
1339                        branchPrefix = self.depotPaths[0] + branch + "/"
1340
1341                        parent = ""
1342
1343                        filesForCommit = branches[branch]
1344
1345                        if self.verbose:
1346                            print "branch is %s" % branch
1347
1348                        self.updatedBranches.add(branch)
1349
1350                        if branch not in self.createdBranches:
1351                            self.createdBranches.add(branch)
1352                            parent = self.knownBranches[branch]
1353                            if parent == branch:
1354                                parent = ""
1355                            else:
1356                                fullBranch = self.projectName + branch
1357                                if fullBranch not in self.p4BranchesInGit:
1358                                    if not self.silent:
1359                                        print("\n    Importing new branch %s" % fullBranch);
1360                                    if self.importNewBranch(branch, change - 1):
1361                                        parent = ""
1362                                        self.p4BranchesInGit.append(fullBranch)
1363                                    if not self.silent:
1364                                        print("\n    Resuming with change %s" % change);
1365
1366                                if self.verbose:
1367                                    print "parent determined through known branches: %s" % parent
1368
1369                        branch = self.gitRefForBranch(branch)
1370                        parent = self.gitRefForBranch(parent)
1371
1372                        if self.verbose:
1373                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1374
1375                        if len(parent) == 0 and branch in self.initialParents:
1376                            parent = self.initialParents[branch]
1377                            del self.initialParents[branch]
1378
1379                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1380                else:
1381                    files = self.extractFilesFromCommit(description)
1382                    self.commit(description, files, self.branch, self.depotPaths,
1383                                self.initialParent)
1384                    self.initialParent = ""
1385            except IOError:
1386                print self.gitError.read()
1387                sys.exit(1)
1388
1389    def importHeadRevision(self, revision):
1390        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1391
1392        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1393        details["desc"] = ("Initial import of %s from the state at revision %s"
1394                           % (' '.join(self.depotPaths), revision))
1395        details["change"] = revision
1396        newestRevision = 0
1397
1398        fileCnt = 0
1399        for info in p4CmdList("files "
1400                              +  ' '.join(["%s...%s"
1401                                           % (p, revision)
1402                                           for p in self.depotPaths])):
1403
1404            if info['code'] == 'error':
1405                sys.stderr.write("p4 returned an error: %s\n"
1406                                 % info['data'])
1407                sys.exit(1)
1408
1409
1410            change = int(info["change"])
1411            if change > newestRevision:
1412                newestRevision = change
1413
1414            if info["action"] in ("delete", "purge"):
1415                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1416                #fileCnt = fileCnt + 1
1417                continue
1418
1419            for prop in ["depotFile", "rev", "action", "type" ]:
1420                details["%s%s" % (prop, fileCnt)] = info[prop]
1421
1422            fileCnt = fileCnt + 1
1423
1424        details["change"] = newestRevision
1425        self.updateOptionDict(details)
1426        try:
1427            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1428        except IOError:
1429            print "IO error with git fast-import. Is your git version recent enough?"
1430            print self.gitError.read()
1431
1432
1433    def getClientSpec(self):
1434        specList = p4CmdList( "client -o" )
1435        temp = {}
1436        for entry in specList:
1437            for k,v in entry.iteritems():
1438                if k.startswith("View"):
1439                    if v.startswith('"'):
1440                        start = 1
1441                    else:
1442                        start = 0
1443                    index = v.find("...")
1444                    v = v[start:index]
1445                    if v.startswith("-"):
1446                        v = v[1:]
1447                        temp[v] = -len(v)
1448                    else:
1449                        temp[v] = len(v)
1450        self.clientSpecDirs = temp.items()
1451        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1452
1453    def run(self, args):
1454        self.depotPaths = []
1455        self.changeRange = ""
1456        self.initialParent = ""
1457        self.previousDepotPaths = []
1458
1459        # map from branch depot path to parent branch
1460        self.knownBranches = {}
1461        self.initialParents = {}
1462        self.hasOrigin = originP4BranchesExist()
1463        if not self.syncWithOrigin:
1464            self.hasOrigin = False
1465
1466        if self.importIntoRemotes:
1467            self.refPrefix = "refs/remotes/p4/"
1468        else:
1469            self.refPrefix = "refs/heads/p4/"
1470
1471        if self.syncWithOrigin and self.hasOrigin:
1472            if not self.silent:
1473                print "Syncing with origin first by calling git fetch origin"
1474            system("git fetch origin")
1475
1476        if len(self.branch) == 0:
1477            self.branch = self.refPrefix + "master"
1478            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1479                system("git update-ref %s refs/heads/p4" % self.branch)
1480                system("git branch -D p4");
1481            # create it /after/ importing, when master exists
1482            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1483                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1484
1485        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1486            self.getClientSpec()
1487
1488        # TODO: should always look at previous commits,
1489        # merge with previous imports, if possible.
1490        if args == []:
1491            if self.hasOrigin:
1492                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1493            self.listExistingP4GitBranches()
1494
1495            if len(self.p4BranchesInGit) > 1:
1496                if not self.silent:
1497                    print "Importing from/into multiple branches"
1498                self.detectBranches = True
1499
1500            if self.verbose:
1501                print "branches: %s" % self.p4BranchesInGit
1502
1503            p4Change = 0
1504            for branch in self.p4BranchesInGit:
1505                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1506
1507                settings = extractSettingsGitLog(logMsg)
1508
1509                self.readOptions(settings)
1510                if (settings.has_key('depot-paths')
1511                    and settings.has_key ('change')):
1512                    change = int(settings['change']) + 1
1513                    p4Change = max(p4Change, change)
1514
1515                    depotPaths = sorted(settings['depot-paths'])
1516                    if self.previousDepotPaths == []:
1517                        self.previousDepotPaths = depotPaths
1518                    else:
1519                        paths = []
1520                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1521                            for i in range(0, min(len(cur), len(prev))):
1522                                if cur[i] <> prev[i]:
1523                                    i = i - 1
1524                                    break
1525
1526                            paths.append (cur[:i + 1])
1527
1528                        self.previousDepotPaths = paths
1529
1530            if p4Change > 0:
1531                self.depotPaths = sorted(self.previousDepotPaths)
1532                self.changeRange = "@%s,#head" % p4Change
1533                if not self.detectBranches:
1534                    self.initialParent = parseRevision(self.branch)
1535                if not self.silent and not self.detectBranches:
1536                    print "Performing incremental import into %s git branch" % self.branch
1537
1538        if not self.branch.startswith("refs/"):
1539            self.branch = "refs/heads/" + self.branch
1540
1541        if len(args) == 0 and self.depotPaths:
1542            if not self.silent:
1543                print "Depot paths: %s" % ' '.join(self.depotPaths)
1544        else:
1545            if self.depotPaths and self.depotPaths != args:
1546                print ("previous import used depot path %s and now %s was specified. "
1547                       "This doesn't work!" % (' '.join (self.depotPaths),
1548                                               ' '.join (args)))
1549                sys.exit(1)
1550
1551            self.depotPaths = sorted(args)
1552
1553        revision = ""
1554        self.users = {}
1555
1556        newPaths = []
1557        for p in self.depotPaths:
1558            if p.find("@") != -1:
1559                atIdx = p.index("@")
1560                self.changeRange = p[atIdx:]
1561                if self.changeRange == "@all":
1562                    self.changeRange = ""
1563                elif ',' not in self.changeRange:
1564                    revision = self.changeRange
1565                    self.changeRange = ""
1566                p = p[:atIdx]
1567            elif p.find("#") != -1:
1568                hashIdx = p.index("#")
1569                revision = p[hashIdx:]
1570                p = p[:hashIdx]
1571            elif self.previousDepotPaths == []:
1572                revision = "#head"
1573
1574            p = re.sub ("\.\.\.$", "", p)
1575            if not p.endswith("/"):
1576                p += "/"
1577
1578            newPaths.append(p)
1579
1580        self.depotPaths = newPaths
1581
1582
1583        self.loadUserMapFromCache()
1584        self.labels = {}
1585        if self.detectLabels:
1586            self.getLabels();
1587
1588        if self.detectBranches:
1589            ## FIXME - what's a P4 projectName ?
1590            self.projectName = self.guessProjectName()
1591
1592            if self.hasOrigin:
1593                self.getBranchMappingFromGitBranches()
1594            else:
1595                self.getBranchMapping()
1596            if self.verbose:
1597                print "p4-git branches: %s" % self.p4BranchesInGit
1598                print "initial parents: %s" % self.initialParents
1599            for b in self.p4BranchesInGit:
1600                if b != "master":
1601
1602                    ## FIXME
1603                    b = b[len(self.projectName):]
1604                self.createdBranches.add(b)
1605
1606        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1607
1608        importProcess = subprocess.Popen(["git", "fast-import"],
1609                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1610                                         stderr=subprocess.PIPE);
1611        self.gitOutput = importProcess.stdout
1612        self.gitStream = importProcess.stdin
1613        self.gitError = importProcess.stderr
1614
1615        if revision:
1616            self.importHeadRevision(revision)
1617        else:
1618            changes = []
1619
1620            if len(self.changesFile) > 0:
1621                output = open(self.changesFile).readlines()
1622                changeSet = Set()
1623                for line in output:
1624                    changeSet.add(int(line))
1625
1626                for change in changeSet:
1627                    changes.append(change)
1628
1629                changes.sort()
1630            else:
1631                if self.verbose:
1632                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1633                                                              self.changeRange)
1634                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1635
1636                if len(self.maxChanges) > 0:
1637                    changes = changes[:min(int(self.maxChanges), len(changes))]
1638
1639            if len(changes) == 0:
1640                if not self.silent:
1641                    print "No changes to import!"
1642                return True
1643
1644            if not self.silent and not self.detectBranches:
1645                print "Import destination: %s" % self.branch
1646
1647            self.updatedBranches = set()
1648
1649            self.importChanges(changes)
1650
1651            if not self.silent:
1652                print ""
1653                if len(self.updatedBranches) > 0:
1654                    sys.stdout.write("Updated branches: ")
1655                    for b in self.updatedBranches:
1656                        sys.stdout.write("%s " % b)
1657                    sys.stdout.write("\n")
1658
1659        self.gitStream.close()
1660        if importProcess.wait() != 0:
1661            die("fast-import failed: %s" % self.gitError.read())
1662        self.gitOutput.close()
1663        self.gitError.close()
1664
1665        return True
1666
1667class P4Rebase(Command):
1668    def __init__(self):
1669        Command.__init__(self)
1670        self.options = [ ]
1671        self.description = ("Fetches the latest revision from perforce and "
1672                            + "rebases the current work (branch) against it")
1673        self.verbose = False
1674
1675    def run(self, args):
1676        sync = P4Sync()
1677        sync.run([])
1678
1679        return self.rebase()
1680
1681    def rebase(self):
1682        if os.system("git update-index --refresh") != 0:
1683            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1684        if len(read_pipe("git diff-index HEAD --")) > 0:
1685            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1686
1687        [upstream, settings] = findUpstreamBranchPoint()
1688        if len(upstream) == 0:
1689            die("Cannot find upstream branchpoint for rebase")
1690
1691        # the branchpoint may be p4/foo~3, so strip off the parent
1692        upstream = re.sub("~[0-9]+$", "", upstream)
1693
1694        print "Rebasing the current branch onto %s" % upstream
1695        oldHead = read_pipe("git rev-parse HEAD").strip()
1696        system("git rebase %s" % upstream)
1697        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1698        return True
1699
1700class P4Clone(P4Sync):
1701    def __init__(self):
1702        P4Sync.__init__(self)
1703        self.description = "Creates a new git repository and imports from Perforce into it"
1704        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1705        self.options += [
1706            optparse.make_option("--destination", dest="cloneDestination",
1707                                 action='store', default=None,
1708                                 help="where to leave result of the clone"),
1709            optparse.make_option("-/", dest="cloneExclude",
1710                                 action="append", type="string",
1711                                 help="exclude depot path")
1712        ]
1713        self.cloneDestination = None
1714        self.needsGit = False
1715
1716    # This is required for the "append" cloneExclude action
1717    def ensure_value(self, attr, value):
1718        if not hasattr(self, attr) or getattr(self, attr) is None:
1719            setattr(self, attr, value)
1720        return getattr(self, attr)
1721
1722    def defaultDestination(self, args):
1723        ## TODO: use common prefix of args?
1724        depotPath = args[0]
1725        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1726        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1727        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1728        depotDir = re.sub(r"/$", "", depotDir)
1729        return os.path.split(depotDir)[1]
1730
1731    def run(self, args):
1732        if len(args) < 1:
1733            return False
1734
1735        if self.keepRepoPath and not self.cloneDestination:
1736            sys.stderr.write("Must specify destination for --keep-path\n")
1737            sys.exit(1)
1738
1739        depotPaths = args
1740
1741        if not self.cloneDestination and len(depotPaths) > 1:
1742            self.cloneDestination = depotPaths[-1]
1743            depotPaths = depotPaths[:-1]
1744
1745        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1746        for p in depotPaths:
1747            if not p.startswith("//"):
1748                return False
1749
1750        if not self.cloneDestination:
1751            self.cloneDestination = self.defaultDestination(args)
1752
1753        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1754        if not os.path.exists(self.cloneDestination):
1755            os.makedirs(self.cloneDestination)
1756        chdir(self.cloneDestination)
1757        system("git init")
1758        self.gitdir = os.getcwd() + "/.git"
1759        if not P4Sync.run(self, depotPaths):
1760            return False
1761        if self.branch != "master":
1762            if self.importIntoRemotes:
1763                masterbranch = "refs/remotes/p4/master"
1764            else:
1765                masterbranch = "refs/heads/p4/master"
1766            if gitBranchExists(masterbranch):
1767                system("git branch master %s" % masterbranch)
1768                system("git checkout -f")
1769            else:
1770                print "Could not detect main branch. No checkout/master branch created."
1771
1772        return True
1773
1774class P4Branches(Command):
1775    def __init__(self):
1776        Command.__init__(self)
1777        self.options = [ ]
1778        self.description = ("Shows the git branches that hold imports and their "
1779                            + "corresponding perforce depot paths")
1780        self.verbose = False
1781
1782    def run(self, args):
1783        if originP4BranchesExist():
1784            createOrUpdateBranchesFromOrigin()
1785
1786        cmdline = "git rev-parse --symbolic "
1787        cmdline += " --remotes"
1788
1789        for line in read_pipe_lines(cmdline):
1790            line = line.strip()
1791
1792            if not line.startswith('p4/') or line == "p4/HEAD":
1793                continue
1794            branch = line
1795
1796            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1797            settings = extractSettingsGitLog(log)
1798
1799            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1800        return True
1801
1802class HelpFormatter(optparse.IndentedHelpFormatter):
1803    def __init__(self):
1804        optparse.IndentedHelpFormatter.__init__(self)
1805
1806    def format_description(self, description):
1807        if description:
1808            return description + "\n"
1809        else:
1810            return ""
1811
1812def printUsage(commands):
1813    print "usage: %s <command> [options]" % sys.argv[0]
1814    print ""
1815    print "valid commands: %s" % ", ".join(commands)
1816    print ""
1817    print "Try %s <command> --help for command specific help." % sys.argv[0]
1818    print ""
1819
1820commands = {
1821    "debug" : P4Debug,
1822    "submit" : P4Submit,
1823    "commit" : P4Submit,
1824    "sync" : P4Sync,
1825    "rebase" : P4Rebase,
1826    "clone" : P4Clone,
1827    "rollback" : P4RollBack,
1828    "branches" : P4Branches
1829}
1830
1831
1832def main():
1833    if len(sys.argv[1:]) == 0:
1834        printUsage(commands.keys())
1835        sys.exit(2)
1836
1837    cmd = ""
1838    cmdName = sys.argv[1]
1839    try:
1840        klass = commands[cmdName]
1841        cmd = klass()
1842    except KeyError:
1843        print "unknown command %s" % cmdName
1844        print ""
1845        printUsage(commands.keys())
1846        sys.exit(2)
1847
1848    options = cmd.options
1849    cmd.gitdir = os.environ.get("GIT_DIR", None)
1850
1851    args = sys.argv[2:]
1852
1853    if len(options) > 0:
1854        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1855
1856        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1857                                       options,
1858                                       description = cmd.description,
1859                                       formatter = HelpFormatter())
1860
1861        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1862    global verbose
1863    verbose = cmd.verbose
1864    if cmd.needsGit:
1865        if cmd.gitdir == None:
1866            cmd.gitdir = os.path.abspath(".git")
1867            if not isValidGitDir(cmd.gitdir):
1868                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1869                if os.path.exists(cmd.gitdir):
1870                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1871                    if len(cdup) > 0:
1872                        chdir(cdup);
1873
1874        if not isValidGitDir(cmd.gitdir):
1875            if isValidGitDir(cmd.gitdir + "/.git"):
1876                cmd.gitdir += "/.git"
1877            else:
1878                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1879
1880        os.environ["GIT_DIR"] = cmd.gitdir
1881
1882    if not cmd.run(args):
1883        parser.print_help()
1884
1885
1886if __name__ == '__main__':
1887    main()