Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'ml/http' (c535d76)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19
  20def p4_build_cmd(cmd):
  21    """Build a suitable p4 command line.
  22
  23    This consolidates building and returning a p4 command line into one
  24    location. It means that hooking into the environment, or other configuration
  25    can be done more easily.
  26    """
  27    real_cmd = "%s " % "p4"
  28
  29    user = gitConfig("git-p4.user")
  30    if len(user) > 0:
  31        real_cmd += "-u %s " % user
  32
  33    password = gitConfig("git-p4.password")
  34    if len(password) > 0:
  35        real_cmd += "-P %s " % password
  36
  37    port = gitConfig("git-p4.port")
  38    if len(port) > 0:
  39        real_cmd += "-p %s " % port
  40
  41    host = gitConfig("git-p4.host")
  42    if len(host) > 0:
  43        real_cmd += "-h %s " % host
  44
  45    client = gitConfig("git-p4.client")
  46    if len(client) > 0:
  47        real_cmd += "-c %s " % client
  48
  49    real_cmd += "%s" % (cmd)
  50    if verbose:
  51        print real_cmd
  52    return real_cmd
  53
  54def chdir(dir):
  55    if os.name == 'nt':
  56        os.environ['PWD']=dir
  57    os.chdir(dir)
  58
  59def die(msg):
  60    if verbose:
  61        raise Exception(msg)
  62    else:
  63        sys.stderr.write(msg + "\n")
  64        sys.exit(1)
  65
  66def write_pipe(c, str):
  67    if verbose:
  68        sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70    pipe = os.popen(c, 'w')
  71    val = pipe.write(str)
  72    if pipe.close():
  73        die('Command failed: %s' % c)
  74
  75    return val
  76
  77def p4_write_pipe(c, str):
  78    real_cmd = p4_build_cmd(c)
  79    return write_pipe(real_cmd, str)
  80
  81def read_pipe(c, ignore_error=False):
  82    if verbose:
  83        sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85    pipe = os.popen(c, 'rb')
  86    val = pipe.read()
  87    if pipe.close() and not ignore_error:
  88        die('Command failed: %s' % c)
  89
  90    return val
  91
  92def p4_read_pipe(c, ignore_error=False):
  93    real_cmd = p4_build_cmd(c)
  94    return read_pipe(real_cmd, ignore_error)
  95
  96def read_pipe_lines(c):
  97    if verbose:
  98        sys.stderr.write('Reading pipe: %s\n' % c)
  99    ## todo: check return status
 100    pipe = os.popen(c, 'rb')
 101    val = pipe.readlines()
 102    if pipe.close():
 103        die('Command failed: %s' % c)
 104
 105    return val
 106
 107def p4_read_pipe_lines(c):
 108    """Specifically invoke p4 on the command supplied. """
 109    real_cmd = p4_build_cmd(c)
 110    return read_pipe_lines(real_cmd)
 111
 112def system(cmd):
 113    if verbose:
 114        sys.stderr.write("executing %s\n" % cmd)
 115    if os.system(cmd) != 0:
 116        die("command failed: %s" % cmd)
 117
 118def p4_system(cmd):
 119    """Specifically invoke p4 as the system command. """
 120    real_cmd = p4_build_cmd(cmd)
 121    return system(real_cmd)
 122
 123def isP4Exec(kind):
 124    """Determine if a Perforce 'kind' should have execute permission
 125
 126    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128    a plus sign, it is also executable"""
 129    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131def setP4ExecBit(file, mode):
 132    # Reopens an already open file and changes the execute bit to match
 133    # the execute bit setting in the passed in mode.
 134
 135    p4Type = "+x"
 136
 137    if not isModeExec(mode):
 138        p4Type = getP4OpenedType(file)
 139        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141        if p4Type[-1] == "+":
 142            p4Type = p4Type[0:-1]
 143
 144    p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146def getP4OpenedType(file):
 147    # Returns the perforce file type for the given file.
 148
 149    result = p4_read_pipe("opened %s" % file)
 150    match = re.match(".*\((.+)\)\r?$", result)
 151    if match:
 152        return match.group(1)
 153    else:
 154        die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156def diffTreePattern():
 157    # This is a simple generator for the diff tree regex pattern. This could be
 158    # a class variable if this and parseDiffTreeEntry were a part of a class.
 159    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160    while True:
 161        yield pattern
 162
 163def parseDiffTreeEntry(entry):
 164    """Parses a single diff tree entry into its component elements.
 165
 166    See git-diff-tree(1) manpage for details about the format of the diff
 167    output. This method returns a dictionary with the following elements:
 168
 169    src_mode - The mode of the source file
 170    dst_mode - The mode of the destination file
 171    src_sha1 - The sha1 for the source file
 172    dst_sha1 - The sha1 fr the destination file
 173    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174    status_score - The score for the status (applicable for 'C' and 'R'
 175                   statuses). This is None if there is no score.
 176    src - The path for the source file.
 177    dst - The path for the destination file. This is only present for
 178          copy or renames. If it is not present, this is None.
 179
 180    If the pattern is not matched, None is returned."""
 181
 182    match = diffTreePattern().next().match(entry)
 183    if match:
 184        return {
 185            'src_mode': match.group(1),
 186            'dst_mode': match.group(2),
 187            'src_sha1': match.group(3),
 188            'dst_sha1': match.group(4),
 189            'status': match.group(5),
 190            'status_score': match.group(6),
 191            'src': match.group(7),
 192            'dst': match.group(10)
 193        }
 194    return None
 195
 196def isModeExec(mode):
 197    # Returns True if the given git mode represents an executable file,
 198    # otherwise False.
 199    return mode[-3:] == "755"
 200
 201def isModeExecChanged(src_mode, dst_mode):
 202    return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205    cmd = p4_build_cmd("-G %s" % (cmd))
 206    if verbose:
 207        sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209    # Use a temporary file to avoid deadlocks without
 210    # subprocess.communicate(), which would put another copy
 211    # of stdout into memory.
 212    stdin_file = None
 213    if stdin is not None:
 214        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215        stdin_file.write(stdin)
 216        stdin_file.flush()
 217        stdin_file.seek(0)
 218
 219    p4 = subprocess.Popen(cmd, shell=True,
 220                          stdin=stdin_file,
 221                          stdout=subprocess.PIPE)
 222
 223    result = []
 224    try:
 225        while True:
 226            entry = marshal.load(p4.stdout)
 227            result.append(entry)
 228    except EOFError:
 229        pass
 230    exitCode = p4.wait()
 231    if exitCode != 0:
 232        entry = {}
 233        entry["p4ExitCode"] = exitCode
 234        result.append(entry)
 235
 236    return result
 237
 238def p4Cmd(cmd):
 239    list = p4CmdList(cmd)
 240    result = {}
 241    for entry in list:
 242        result.update(entry)
 243    return result;
 244
 245def p4Where(depotPath):
 246    if not depotPath.endswith("/"):
 247        depotPath += "/"
 248    depotPath = depotPath + "..."
 249    outputList = p4CmdList("where %s" % depotPath)
 250    output = None
 251    for entry in outputList:
 252        if "depotFile" in entry:
 253            if entry["depotFile"] == depotPath:
 254                output = entry
 255                break
 256        elif "data" in entry:
 257            data = entry.get("data")
 258            space = data.find(" ")
 259            if data[:space] == depotPath:
 260                output = entry
 261                break
 262    if output == None:
 263        return ""
 264    if output["code"] == "error":
 265        return ""
 266    clientPath = ""
 267    if "path" in output:
 268        clientPath = output.get("path")
 269    elif "data" in output:
 270        data = output.get("data")
 271        lastSpace = data.rfind(" ")
 272        clientPath = data[lastSpace + 1:]
 273
 274    if clientPath.endswith("..."):
 275        clientPath = clientPath[:-3]
 276    return clientPath
 277
 278def currentGitBranch():
 279    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 280
 281def isValidGitDir(path):
 282    if (os.path.exists(path + "/HEAD")
 283        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 284        return True;
 285    return False
 286
 287def parseRevision(ref):
 288    return read_pipe("git rev-parse %s" % ref).strip()
 289
 290def extractLogMessageFromGitCommit(commit):
 291    logMessage = ""
 292
 293    ## fixme: title is first line of commit, not 1st paragraph.
 294    foundTitle = False
 295    for log in read_pipe_lines("git cat-file commit %s" % commit):
 296       if not foundTitle:
 297           if len(log) == 1:
 298               foundTitle = True
 299           continue
 300
 301       logMessage += log
 302    return logMessage
 303
 304def extractSettingsGitLog(log):
 305    values = {}
 306    for line in log.split("\n"):
 307        line = line.strip()
 308        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 309        if not m:
 310            continue
 311
 312        assignments = m.group(1).split (':')
 313        for a in assignments:
 314            vals = a.split ('=')
 315            key = vals[0].strip()
 316            val = ('='.join (vals[1:])).strip()
 317            if val.endswith ('\"') and val.startswith('"'):
 318                val = val[1:-1]
 319
 320            values[key] = val
 321
 322    paths = values.get("depot-paths")
 323    if not paths:
 324        paths = values.get("depot-path")
 325    if paths:
 326        values['depot-paths'] = paths.split(',')
 327    return values
 328
 329def gitBranchExists(branch):
 330    proc = subprocess.Popen(["git", "rev-parse", branch],
 331                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 332    return proc.wait() == 0;
 333
 334_gitConfig = {}
 335def gitConfig(key):
 336    if not _gitConfig.has_key(key):
 337        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 338    return _gitConfig[key]
 339
 340def p4BranchesInGit(branchesAreInRemotes = True):
 341    branches = {}
 342
 343    cmdline = "git rev-parse --symbolic "
 344    if branchesAreInRemotes:
 345        cmdline += " --remotes"
 346    else:
 347        cmdline += " --branches"
 348
 349    for line in read_pipe_lines(cmdline):
 350        line = line.strip()
 351
 352        ## only import to p4/
 353        if not line.startswith('p4/') or line == "p4/HEAD":
 354            continue
 355        branch = line
 356
 357        # strip off p4
 358        branch = re.sub ("^p4/", "", line)
 359
 360        branches[branch] = parseRevision(line)
 361    return branches
 362
 363def findUpstreamBranchPoint(head = "HEAD"):
 364    branches = p4BranchesInGit()
 365    # map from depot-path to branch name
 366    branchByDepotPath = {}
 367    for branch in branches.keys():
 368        tip = branches[branch]
 369        log = extractLogMessageFromGitCommit(tip)
 370        settings = extractSettingsGitLog(log)
 371        if settings.has_key("depot-paths"):
 372            paths = ",".join(settings["depot-paths"])
 373            branchByDepotPath[paths] = "remotes/p4/" + branch
 374
 375    settings = None
 376    parent = 0
 377    while parent < 65535:
 378        commit = head + "~%s" % parent
 379        log = extractLogMessageFromGitCommit(commit)
 380        settings = extractSettingsGitLog(log)
 381        if settings.has_key("depot-paths"):
 382            paths = ",".join(settings["depot-paths"])
 383            if branchByDepotPath.has_key(paths):
 384                return [branchByDepotPath[paths], settings]
 385
 386        parent = parent + 1
 387
 388    return ["", settings]
 389
 390def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 391    if not silent:
 392        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 393               % localRefPrefix)
 394
 395    originPrefix = "origin/p4/"
 396
 397    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 398        line = line.strip()
 399        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 400            continue
 401
 402        headName = line[len(originPrefix):]
 403        remoteHead = localRefPrefix + headName
 404        originHead = line
 405
 406        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 407        if (not original.has_key('depot-paths')
 408            or not original.has_key('change')):
 409            continue
 410
 411        update = False
 412        if not gitBranchExists(remoteHead):
 413            if verbose:
 414                print "creating %s" % remoteHead
 415            update = True
 416        else:
 417            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 418            if settings.has_key('change') > 0:
 419                if settings['depot-paths'] == original['depot-paths']:
 420                    originP4Change = int(original['change'])
 421                    p4Change = int(settings['change'])
 422                    if originP4Change > p4Change:
 423                        print ("%s (%s) is newer than %s (%s). "
 424                               "Updating p4 branch from origin."
 425                               % (originHead, originP4Change,
 426                                  remoteHead, p4Change))
 427                        update = True
 428                else:
 429                    print ("Ignoring: %s was imported from %s while "
 430                           "%s was imported from %s"
 431                           % (originHead, ','.join(original['depot-paths']),
 432                              remoteHead, ','.join(settings['depot-paths'])))
 433
 434        if update:
 435            system("git update-ref %s %s" % (remoteHead, originHead))
 436
 437def originP4BranchesExist():
 438        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 439
 440def p4ChangesForPaths(depotPaths, changeRange):
 441    assert depotPaths
 442    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 443                                                        for p in depotPaths]))
 444
 445    changes = {}
 446    for line in output:
 447        changeNum = int(line.split(" ")[1])
 448        changes[changeNum] = True
 449
 450    changelist = changes.keys()
 451    changelist.sort()
 452    return changelist
 453
 454class Command:
 455    def __init__(self):
 456        self.usage = "usage: %prog [options]"
 457        self.needsGit = True
 458
 459class P4Debug(Command):
 460    def __init__(self):
 461        Command.__init__(self)
 462        self.options = [
 463            optparse.make_option("--verbose", dest="verbose", action="store_true",
 464                                 default=False),
 465            ]
 466        self.description = "A tool to debug the output of p4 -G."
 467        self.needsGit = False
 468        self.verbose = False
 469
 470    def run(self, args):
 471        j = 0
 472        for output in p4CmdList(" ".join(args)):
 473            print 'Element: %d' % j
 474            j += 1
 475            print output
 476        return True
 477
 478class P4RollBack(Command):
 479    def __init__(self):
 480        Command.__init__(self)
 481        self.options = [
 482            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 483            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 484        ]
 485        self.description = "A tool to debug the multi-branch import. Don't use :)"
 486        self.verbose = False
 487        self.rollbackLocalBranches = False
 488
 489    def run(self, args):
 490        if len(args) != 1:
 491            return False
 492        maxChange = int(args[0])
 493
 494        if "p4ExitCode" in p4Cmd("changes -m 1"):
 495            die("Problems executing p4");
 496
 497        if self.rollbackLocalBranches:
 498            refPrefix = "refs/heads/"
 499            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 500        else:
 501            refPrefix = "refs/remotes/"
 502            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 503
 504        for line in lines:
 505            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 506                line = line.strip()
 507                ref = refPrefix + line
 508                log = extractLogMessageFromGitCommit(ref)
 509                settings = extractSettingsGitLog(log)
 510
 511                depotPaths = settings['depot-paths']
 512                change = settings['change']
 513
 514                changed = False
 515
 516                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 517                                                           for p in depotPaths]))) == 0:
 518                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 519                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 520                    continue
 521
 522                while change and int(change) > maxChange:
 523                    changed = True
 524                    if self.verbose:
 525                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 526                    system("git update-ref %s \"%s^\"" % (ref, ref))
 527                    log = extractLogMessageFromGitCommit(ref)
 528                    settings =  extractSettingsGitLog(log)
 529
 530
 531                    depotPaths = settings['depot-paths']
 532                    change = settings['change']
 533
 534                if changed:
 535                    print "%s rewound to %s" % (ref, change)
 536
 537        return True
 538
 539class P4Submit(Command):
 540    def __init__(self):
 541        Command.__init__(self)
 542        self.options = [
 543                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 544                optparse.make_option("--origin", dest="origin"),
 545                optparse.make_option("-M", dest="detectRename", action="store_true"),
 546        ]
 547        self.description = "Submit changes from git to the perforce depot."
 548        self.usage += " [name of git branch to submit into perforce depot]"
 549        self.interactive = True
 550        self.origin = ""
 551        self.detectRename = False
 552        self.verbose = False
 553        self.isWindows = (platform.system() == "Windows")
 554
 555    def check(self):
 556        if len(p4CmdList("opened ...")) > 0:
 557            die("You have files opened with perforce! Close them before starting the sync.")
 558
 559    # replaces everything between 'Description:' and the next P4 submit template field with the
 560    # commit message
 561    def prepareLogMessage(self, template, message):
 562        result = ""
 563
 564        inDescriptionSection = False
 565
 566        for line in template.split("\n"):
 567            if line.startswith("#"):
 568                result += line + "\n"
 569                continue
 570
 571            if inDescriptionSection:
 572                if line.startswith("Files:"):
 573                    inDescriptionSection = False
 574                else:
 575                    continue
 576            else:
 577                if line.startswith("Description:"):
 578                    inDescriptionSection = True
 579                    line += "\n"
 580                    for messageLine in message.split("\n"):
 581                        line += "\t" + messageLine + "\n"
 582
 583            result += line + "\n"
 584
 585        return result
 586
 587    def prepareSubmitTemplate(self):
 588        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 589        template = ""
 590        inFilesSection = False
 591        for line in p4_read_pipe_lines("change -o"):
 592            if line.endswith("\r\n"):
 593                line = line[:-2] + "\n"
 594            if inFilesSection:
 595                if line.startswith("\t"):
 596                    # path starts and ends with a tab
 597                    path = line[1:]
 598                    lastTab = path.rfind("\t")
 599                    if lastTab != -1:
 600                        path = path[:lastTab]
 601                        if not path.startswith(self.depotPath):
 602                            continue
 603                else:
 604                    inFilesSection = False
 605            else:
 606                if line.startswith("Files:"):
 607                    inFilesSection = True
 608
 609            template += line
 610
 611        return template
 612
 613    def applyCommit(self, id):
 614        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 615        diffOpts = ("", "-M")[self.detectRename]
 616        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 617        filesToAdd = set()
 618        filesToDelete = set()
 619        editedFiles = set()
 620        filesToChangeExecBit = {}
 621        for line in diff:
 622            diff = parseDiffTreeEntry(line)
 623            modifier = diff['status']
 624            path = diff['src']
 625            if modifier == "M":
 626                p4_system("edit \"%s\"" % path)
 627                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 628                    filesToChangeExecBit[path] = diff['dst_mode']
 629                editedFiles.add(path)
 630            elif modifier == "A":
 631                filesToAdd.add(path)
 632                filesToChangeExecBit[path] = diff['dst_mode']
 633                if path in filesToDelete:
 634                    filesToDelete.remove(path)
 635            elif modifier == "D":
 636                filesToDelete.add(path)
 637                if path in filesToAdd:
 638                    filesToAdd.remove(path)
 639            elif modifier == "R":
 640                src, dest = diff['src'], diff['dst']
 641                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 642                p4_system("edit \"%s\"" % (dest))
 643                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 644                    filesToChangeExecBit[dest] = diff['dst_mode']
 645                os.unlink(dest)
 646                editedFiles.add(dest)
 647                filesToDelete.add(src)
 648            else:
 649                die("unknown modifier %s for %s" % (modifier, path))
 650
 651        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 652        patchcmd = diffcmd + " | git apply "
 653        tryPatchCmd = patchcmd + "--check -"
 654        applyPatchCmd = patchcmd + "--check --apply -"
 655
 656        if os.system(tryPatchCmd) != 0:
 657            print "Unfortunately applying the change failed!"
 658            print "What do you want to do?"
 659            response = "x"
 660            while response != "s" and response != "a" and response != "w":
 661                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 662                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 663            if response == "s":
 664                print "Skipping! Good luck with the next patches..."
 665                for f in editedFiles:
 666                    p4_system("revert \"%s\"" % f);
 667                for f in filesToAdd:
 668                    system("rm %s" %f)
 669                return
 670            elif response == "a":
 671                os.system(applyPatchCmd)
 672                if len(filesToAdd) > 0:
 673                    print "You may also want to call p4 add on the following files:"
 674                    print " ".join(filesToAdd)
 675                if len(filesToDelete):
 676                    print "The following files should be scheduled for deletion with p4 delete:"
 677                    print " ".join(filesToDelete)
 678                die("Please resolve and submit the conflict manually and "
 679                    + "continue afterwards with git-p4 submit --continue")
 680            elif response == "w":
 681                system(diffcmd + " > patch.txt")
 682                print "Patch saved to patch.txt in %s !" % self.clientPath
 683                die("Please resolve and submit the conflict manually and "
 684                    "continue afterwards with git-p4 submit --continue")
 685
 686        system(applyPatchCmd)
 687
 688        for f in filesToAdd:
 689            p4_system("add \"%s\"" % f)
 690        for f in filesToDelete:
 691            p4_system("revert \"%s\"" % f)
 692            p4_system("delete \"%s\"" % f)
 693
 694        # Set/clear executable bits
 695        for f in filesToChangeExecBit.keys():
 696            mode = filesToChangeExecBit[f]
 697            setP4ExecBit(f, mode)
 698
 699        logMessage = extractLogMessageFromGitCommit(id)
 700        logMessage = logMessage.strip()
 701
 702        template = self.prepareSubmitTemplate()
 703
 704        if self.interactive:
 705            submitTemplate = self.prepareLogMessage(template, logMessage)
 706            if os.environ.has_key("P4DIFF"):
 707                del(os.environ["P4DIFF"])
 708            diff = p4_read_pipe("diff -du ...")
 709
 710            newdiff = ""
 711            for newFile in filesToAdd:
 712                newdiff += "==== new file ====\n"
 713                newdiff += "--- /dev/null\n"
 714                newdiff += "+++ %s\n" % newFile
 715                f = open(newFile, "r")
 716                for line in f.readlines():
 717                    newdiff += "+" + line
 718                f.close()
 719
 720            separatorLine = "######## everything below this line is just the diff #######\n"
 721
 722            [handle, fileName] = tempfile.mkstemp()
 723            tmpFile = os.fdopen(handle, "w+")
 724            if self.isWindows:
 725                submitTemplate = submitTemplate.replace("\n", "\r\n")
 726                separatorLine = separatorLine.replace("\n", "\r\n")
 727                newdiff = newdiff.replace("\n", "\r\n")
 728            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 729            tmpFile.close()
 730            mtime = os.stat(fileName).st_mtime
 731            defaultEditor = "vi"
 732            if platform.system() == "Windows":
 733                defaultEditor = "notepad"
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = os.environ.get("EDITOR", defaultEditor);
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Syncronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    def __init__(self):
 838        Command.__init__(self)
 839        self.options = [
 840                optparse.make_option("--branch", dest="branch"),
 841                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                optparse.make_option("--changesfile", dest="changesFile"),
 843                optparse.make_option("--silent", dest="silent", action="store_true"),
 844                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                     help="Import into refs/heads/ , not refs/remotes"),
 848                optparse.make_option("--max-changes", dest="maxChanges"),
 849                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                     help="Only sync files that are included in the Perforce Client Spec")
 853        ]
 854        self.description = """Imports from Perforce into a git repository.\n
 855    example:
 856    //depot/my/project/ -- to import the current head
 857    //depot/my/project/@all -- to import everything
 858    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862        self.usage += " //depot/path[@revRange]"
 863        self.silent = False
 864        self.createdBranches = Set()
 865        self.committedChanges = Set()
 866        self.branch = ""
 867        self.detectBranches = False
 868        self.detectLabels = False
 869        self.changesFile = ""
 870        self.syncWithOrigin = True
 871        self.verbose = False
 872        self.importIntoRemotes = True
 873        self.maxChanges = ""
 874        self.isWindows = (platform.system() == "Windows")
 875        self.keepRepoPath = False
 876        self.depotPaths = None
 877        self.p4BranchesInGit = []
 878        self.cloneExclude = []
 879        self.useClientSpec = False
 880        self.clientSpecDirs = []
 881
 882        if gitConfig("git-p4.syncFromOrigin") == "false":
 883            self.syncWithOrigin = False
 884
 885    def extractFilesFromCommit(self, commit):
 886        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                             for path in self.cloneExclude]
 888        files = []
 889        fnum = 0
 890        while commit.has_key("depotFile%s" % fnum):
 891            path =  commit["depotFile%s" % fnum]
 892
 893            if [p for p in self.cloneExclude
 894                if path.startswith (p)]:
 895                found = False
 896            else:
 897                found = [p for p in self.depotPaths
 898                         if path.startswith (p)]
 899            if not found:
 900                fnum = fnum + 1
 901                continue
 902
 903            file = {}
 904            file["path"] = path
 905            file["rev"] = commit["rev%s" % fnum]
 906            file["action"] = commit["action%s" % fnum]
 907            file["type"] = commit["type%s" % fnum]
 908            files.append(file)
 909            fnum = fnum + 1
 910        return files
 911
 912    def stripRepoPath(self, path, prefixes):
 913        if self.keepRepoPath:
 914            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 915
 916        for p in prefixes:
 917            if path.startswith(p):
 918                path = path[len(p):]
 919
 920        return path
 921
 922    def splitFilesIntoBranches(self, commit):
 923        branches = {}
 924        fnum = 0
 925        while commit.has_key("depotFile%s" % fnum):
 926            path =  commit["depotFile%s" % fnum]
 927            found = [p for p in self.depotPaths
 928                     if path.startswith (p)]
 929            if not found:
 930                fnum = fnum + 1
 931                continue
 932
 933            file = {}
 934            file["path"] = path
 935            file["rev"] = commit["rev%s" % fnum]
 936            file["action"] = commit["action%s" % fnum]
 937            file["type"] = commit["type%s" % fnum]
 938            fnum = fnum + 1
 939
 940            relPath = self.stripRepoPath(path, self.depotPaths)
 941
 942            for branch in self.knownBranches.keys():
 943
 944                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 945                if relPath.startswith(branch + "/"):
 946                    if branch not in branches:
 947                        branches[branch] = []
 948                    branches[branch].append(file)
 949                    break
 950
 951        return branches
 952
 953    ## Should move this out, doesn't use SELF.
 954    def readP4Files(self, files):
 955        filesForCommit = []
 956        filesToRead = []
 957
 958        for f in files:
 959            includeFile = True
 960            for val in self.clientSpecDirs:
 961                if f['path'].startswith(val[0]):
 962                    if val[1] <= 0:
 963                        includeFile = False
 964                    break
 965
 966            if includeFile:
 967                filesForCommit.append(f)
 968                if f['action'] not in ('delete', 'purge'):
 969                    filesToRead.append(f)
 970
 971        filedata = []
 972        if len(filesToRead) > 0:
 973            filedata = p4CmdList('-x - print',
 974                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 975                                                  for f in filesToRead]),
 976                                 stdin_mode='w+')
 977
 978            if "p4ExitCode" in filedata[0]:
 979                die("Problems executing p4. Error: [%d]."
 980                    % (filedata[0]['p4ExitCode']));
 981
 982        j = 0;
 983        contents = {}
 984        while j < len(filedata):
 985            stat = filedata[j]
 986            j += 1
 987            text = ''
 988            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 989                text += filedata[j]['data']
 990                del filedata[j]['data']
 991                j += 1
 992
 993            if not stat.has_key('depotFile'):
 994                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 995                continue
 996
 997            if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 998                text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 999            elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1000                text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
1001
1002            contents[stat['depotFile']] = text
1003
1004        for f in filesForCommit:
1005            path = f['path']
1006            if contents.has_key(path):
1007                f['data'] = contents[path]
1008
1009        return filesForCommit
1010
1011    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1012        epoch = details["time"]
1013        author = details["user"]
1014
1015        if self.verbose:
1016            print "commit into %s" % branch
1017
1018        # start with reading files; if that fails, we should not
1019        # create a commit.
1020        new_files = []
1021        for f in files:
1022            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1023                new_files.append (f)
1024            else:
1025                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1026        files = self.readP4Files(new_files)
1027
1028        self.gitStream.write("commit %s\n" % branch)
1029#        gitStream.write("mark :%s\n" % details["change"])
1030        self.committedChanges.add(int(details["change"]))
1031        committer = ""
1032        if author not in self.users:
1033            self.getUserMapFromPerforceServer()
1034        if author in self.users:
1035            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1036        else:
1037            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1038
1039        self.gitStream.write("committer %s\n" % committer)
1040
1041        self.gitStream.write("data <<EOT\n")
1042        self.gitStream.write(details["desc"])
1043        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1044                             % (','.join (branchPrefixes), details["change"]))
1045        if len(details['options']) > 0:
1046            self.gitStream.write(": options = %s" % details['options'])
1047        self.gitStream.write("]\nEOT\n\n")
1048
1049        if len(parent) > 0:
1050            if self.verbose:
1051                print "parent %s" % parent
1052            self.gitStream.write("from %s\n" % parent)
1053
1054        for file in files:
1055            if file["type"] == "apple":
1056                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1057                continue
1058
1059            relPath = self.stripRepoPath(file['path'], branchPrefixes)
1060            if file["action"] in ("delete", "purge"):
1061                self.gitStream.write("D %s\n" % relPath)
1062            else:
1063                data = file['data']
1064
1065                mode = "644"
1066                if isP4Exec(file["type"]):
1067                    mode = "755"
1068                elif file["type"] == "symlink":
1069                    mode = "120000"
1070                    # p4 print on a symlink contains "target\n", so strip it off
1071                    data = data[:-1]
1072
1073                if self.isWindows and file["type"].endswith("text"):
1074                    data = data.replace("\r\n", "\n")
1075
1076                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1077                self.gitStream.write("data %s\n" % len(data))
1078                self.gitStream.write(data)
1079                self.gitStream.write("\n")
1080
1081        self.gitStream.write("\n")
1082
1083        change = int(details["change"])
1084
1085        if self.labels.has_key(change):
1086            label = self.labels[change]
1087            labelDetails = label[0]
1088            labelRevisions = label[1]
1089            if self.verbose:
1090                print "Change %s is labelled %s" % (change, labelDetails)
1091
1092            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1093                                                    for p in branchPrefixes]))
1094
1095            if len(files) == len(labelRevisions):
1096
1097                cleanedFiles = {}
1098                for info in files:
1099                    if info["action"] in ("delete", "purge"):
1100                        continue
1101                    cleanedFiles[info["depotFile"]] = info["rev"]
1102
1103                if cleanedFiles == labelRevisions:
1104                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1105                    self.gitStream.write("from %s\n" % branch)
1106
1107                    owner = labelDetails["Owner"]
1108                    tagger = ""
1109                    if author in self.users:
1110                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1111                    else:
1112                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1113                    self.gitStream.write("tagger %s\n" % tagger)
1114                    self.gitStream.write("data <<EOT\n")
1115                    self.gitStream.write(labelDetails["Description"])
1116                    self.gitStream.write("EOT\n\n")
1117
1118                else:
1119                    if not self.silent:
1120                        print ("Tag %s does not match with change %s: files do not match."
1121                               % (labelDetails["label"], change))
1122
1123            else:
1124                if not self.silent:
1125                    print ("Tag %s does not match with change %s: file count is different."
1126                           % (labelDetails["label"], change))
1127
1128    def getUserCacheFilename(self):
1129        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1130        return home + "/.gitp4-usercache.txt"
1131
1132    def getUserMapFromPerforceServer(self):
1133        if self.userMapFromPerforceServer:
1134            return
1135        self.users = {}
1136
1137        for output in p4CmdList("users"):
1138            if not output.has_key("User"):
1139                continue
1140            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1141
1142
1143        s = ''
1144        for (key, val) in self.users.items():
1145            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1146
1147        open(self.getUserCacheFilename(), "wb").write(s)
1148        self.userMapFromPerforceServer = True
1149
1150    def loadUserMapFromCache(self):
1151        self.users = {}
1152        self.userMapFromPerforceServer = False
1153        try:
1154            cache = open(self.getUserCacheFilename(), "rb")
1155            lines = cache.readlines()
1156            cache.close()
1157            for line in lines:
1158                entry = line.strip().split("\t")
1159                self.users[entry[0]] = entry[1]
1160        except IOError:
1161            self.getUserMapFromPerforceServer()
1162
1163    def getLabels(self):
1164        self.labels = {}
1165
1166        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1167        if len(l) > 0 and not self.silent:
1168            print "Finding files belonging to labels in %s" % `self.depotPaths`
1169
1170        for output in l:
1171            label = output["label"]
1172            revisions = {}
1173            newestChange = 0
1174            if self.verbose:
1175                print "Querying files for label %s" % label
1176            for file in p4CmdList("files "
1177                                  +  ' '.join (["%s...@%s" % (p, label)
1178                                                for p in self.depotPaths])):
1179                revisions[file["depotFile"]] = file["rev"]
1180                change = int(file["change"])
1181                if change > newestChange:
1182                    newestChange = change
1183
1184            self.labels[newestChange] = [output, revisions]
1185
1186        if self.verbose:
1187            print "Label changes: %s" % self.labels.keys()
1188
1189    def guessProjectName(self):
1190        for p in self.depotPaths:
1191            if p.endswith("/"):
1192                p = p[:-1]
1193            p = p[p.strip().rfind("/") + 1:]
1194            if not p.endswith("/"):
1195               p += "/"
1196            return p
1197
1198    def getBranchMapping(self):
1199        lostAndFoundBranches = set()
1200
1201        for info in p4CmdList("branches"):
1202            details = p4Cmd("branch -o %s" % info["branch"])
1203            viewIdx = 0
1204            while details.has_key("View%s" % viewIdx):
1205                paths = details["View%s" % viewIdx].split(" ")
1206                viewIdx = viewIdx + 1
1207                # require standard //depot/foo/... //depot/bar/... mapping
1208                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1209                    continue
1210                source = paths[0]
1211                destination = paths[1]
1212                ## HACK
1213                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1214                    source = source[len(self.depotPaths[0]):-4]
1215                    destination = destination[len(self.depotPaths[0]):-4]
1216
1217                    if destination in self.knownBranches:
1218                        if not self.silent:
1219                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1220                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1221                        continue
1222
1223                    self.knownBranches[destination] = source
1224
1225                    lostAndFoundBranches.discard(destination)
1226
1227                    if source not in self.knownBranches:
1228                        lostAndFoundBranches.add(source)
1229
1230
1231        for branch in lostAndFoundBranches:
1232            self.knownBranches[branch] = branch
1233
1234    def getBranchMappingFromGitBranches(self):
1235        branches = p4BranchesInGit(self.importIntoRemotes)
1236        for branch in branches.keys():
1237            if branch == "master":
1238                branch = "main"
1239            else:
1240                branch = branch[len(self.projectName):]
1241            self.knownBranches[branch] = branch
1242
1243    def listExistingP4GitBranches(self):
1244        # branches holds mapping from name to commit
1245        branches = p4BranchesInGit(self.importIntoRemotes)
1246        self.p4BranchesInGit = branches.keys()
1247        for branch in branches.keys():
1248            self.initialParents[self.refPrefix + branch] = branches[branch]
1249
1250    def updateOptionDict(self, d):
1251        option_keys = {}
1252        if self.keepRepoPath:
1253            option_keys['keepRepoPath'] = 1
1254
1255        d["options"] = ' '.join(sorted(option_keys.keys()))
1256
1257    def readOptions(self, d):
1258        self.keepRepoPath = (d.has_key('options')
1259                             and ('keepRepoPath' in d['options']))
1260
1261    def gitRefForBranch(self, branch):
1262        if branch == "main":
1263            return self.refPrefix + "master"
1264
1265        if len(branch) <= 0:
1266            return branch
1267
1268        return self.refPrefix + self.projectName + branch
1269
1270    def gitCommitByP4Change(self, ref, change):
1271        if self.verbose:
1272            print "looking in ref " + ref + " for change %s using bisect..." % change
1273
1274        earliestCommit = ""
1275        latestCommit = parseRevision(ref)
1276
1277        while True:
1278            if self.verbose:
1279                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1280            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1281            if len(next) == 0:
1282                if self.verbose:
1283                    print "argh"
1284                return ""
1285            log = extractLogMessageFromGitCommit(next)
1286            settings = extractSettingsGitLog(log)
1287            currentChange = int(settings['change'])
1288            if self.verbose:
1289                print "current change %s" % currentChange
1290
1291            if currentChange == change:
1292                if self.verbose:
1293                    print "found %s" % next
1294                return next
1295
1296            if currentChange < change:
1297                earliestCommit = "^%s" % next
1298            else:
1299                latestCommit = "%s" % next
1300
1301        return ""
1302
1303    def importNewBranch(self, branch, maxChange):
1304        # make fast-import flush all changes to disk and update the refs using the checkpoint
1305        # command so that we can try to find the branch parent in the git history
1306        self.gitStream.write("checkpoint\n\n");
1307        self.gitStream.flush();
1308        branchPrefix = self.depotPaths[0] + branch + "/"
1309        range = "@1,%s" % maxChange
1310        #print "prefix" + branchPrefix
1311        changes = p4ChangesForPaths([branchPrefix], range)
1312        if len(changes) <= 0:
1313            return False
1314        firstChange = changes[0]
1315        #print "first change in branch: %s" % firstChange
1316        sourceBranch = self.knownBranches[branch]
1317        sourceDepotPath = self.depotPaths[0] + sourceBranch
1318        sourceRef = self.gitRefForBranch(sourceBranch)
1319        #print "source " + sourceBranch
1320
1321        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1322        #print "branch parent: %s" % branchParentChange
1323        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1324        if len(gitParent) > 0:
1325            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1326            #print "parent git commit: %s" % gitParent
1327
1328        self.importChanges(changes)
1329        return True
1330
1331    def importChanges(self, changes):
1332        cnt = 1
1333        for change in changes:
1334            description = p4Cmd("describe %s" % change)
1335            self.updateOptionDict(description)
1336
1337            if not self.silent:
1338                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1339                sys.stdout.flush()
1340            cnt = cnt + 1
1341
1342            try:
1343                if self.detectBranches:
1344                    branches = self.splitFilesIntoBranches(description)
1345                    for branch in branches.keys():
1346                        ## HACK  --hwn
1347                        branchPrefix = self.depotPaths[0] + branch + "/"
1348
1349                        parent = ""
1350
1351                        filesForCommit = branches[branch]
1352
1353                        if self.verbose:
1354                            print "branch is %s" % branch
1355
1356                        self.updatedBranches.add(branch)
1357
1358                        if branch not in self.createdBranches:
1359                            self.createdBranches.add(branch)
1360                            parent = self.knownBranches[branch]
1361                            if parent == branch:
1362                                parent = ""
1363                            else:
1364                                fullBranch = self.projectName + branch
1365                                if fullBranch not in self.p4BranchesInGit:
1366                                    if not self.silent:
1367                                        print("\n    Importing new branch %s" % fullBranch);
1368                                    if self.importNewBranch(branch, change - 1):
1369                                        parent = ""
1370                                        self.p4BranchesInGit.append(fullBranch)
1371                                    if not self.silent:
1372                                        print("\n    Resuming with change %s" % change);
1373
1374                                if self.verbose:
1375                                    print "parent determined through known branches: %s" % parent
1376
1377                        branch = self.gitRefForBranch(branch)
1378                        parent = self.gitRefForBranch(parent)
1379
1380                        if self.verbose:
1381                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1382
1383                        if len(parent) == 0 and branch in self.initialParents:
1384                            parent = self.initialParents[branch]
1385                            del self.initialParents[branch]
1386
1387                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1388                else:
1389                    files = self.extractFilesFromCommit(description)
1390                    self.commit(description, files, self.branch, self.depotPaths,
1391                                self.initialParent)
1392                    self.initialParent = ""
1393            except IOError:
1394                print self.gitError.read()
1395                sys.exit(1)
1396
1397    def importHeadRevision(self, revision):
1398        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1399
1400        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1401        details["desc"] = ("Initial import of %s from the state at revision %s"
1402                           % (' '.join(self.depotPaths), revision))
1403        details["change"] = revision
1404        newestRevision = 0
1405
1406        fileCnt = 0
1407        for info in p4CmdList("files "
1408                              +  ' '.join(["%s...%s"
1409                                           % (p, revision)
1410                                           for p in self.depotPaths])):
1411
1412            if info['code'] == 'error':
1413                sys.stderr.write("p4 returned an error: %s\n"
1414                                 % info['data'])
1415                sys.exit(1)
1416
1417
1418            change = int(info["change"])
1419            if change > newestRevision:
1420                newestRevision = change
1421
1422            if info["action"] in ("delete", "purge"):
1423                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1424                #fileCnt = fileCnt + 1
1425                continue
1426
1427            for prop in ["depotFile", "rev", "action", "type" ]:
1428                details["%s%s" % (prop, fileCnt)] = info[prop]
1429
1430            fileCnt = fileCnt + 1
1431
1432        details["change"] = newestRevision
1433        self.updateOptionDict(details)
1434        try:
1435            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1436        except IOError:
1437            print "IO error with git fast-import. Is your git version recent enough?"
1438            print self.gitError.read()
1439
1440
1441    def getClientSpec(self):
1442        specList = p4CmdList( "client -o" )
1443        temp = {}
1444        for entry in specList:
1445            for k,v in entry.iteritems():
1446                if k.startswith("View"):
1447                    if v.startswith('"'):
1448                        start = 1
1449                    else:
1450                        start = 0
1451                    index = v.find("...")
1452                    v = v[start:index]
1453                    if v.startswith("-"):
1454                        v = v[1:]
1455                        temp[v] = -len(v)
1456                    else:
1457                        temp[v] = len(v)
1458        self.clientSpecDirs = temp.items()
1459        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1460
1461    def run(self, args):
1462        self.depotPaths = []
1463        self.changeRange = ""
1464        self.initialParent = ""
1465        self.previousDepotPaths = []
1466
1467        # map from branch depot path to parent branch
1468        self.knownBranches = {}
1469        self.initialParents = {}
1470        self.hasOrigin = originP4BranchesExist()
1471        if not self.syncWithOrigin:
1472            self.hasOrigin = False
1473
1474        if self.importIntoRemotes:
1475            self.refPrefix = "refs/remotes/p4/"
1476        else:
1477            self.refPrefix = "refs/heads/p4/"
1478
1479        if self.syncWithOrigin and self.hasOrigin:
1480            if not self.silent:
1481                print "Syncing with origin first by calling git fetch origin"
1482            system("git fetch origin")
1483
1484        if len(self.branch) == 0:
1485            self.branch = self.refPrefix + "master"
1486            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1487                system("git update-ref %s refs/heads/p4" % self.branch)
1488                system("git branch -D p4");
1489            # create it /after/ importing, when master exists
1490            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1491                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1492
1493        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1494            self.getClientSpec()
1495
1496        # TODO: should always look at previous commits,
1497        # merge with previous imports, if possible.
1498        if args == []:
1499            if self.hasOrigin:
1500                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1501            self.listExistingP4GitBranches()
1502
1503            if len(self.p4BranchesInGit) > 1:
1504                if not self.silent:
1505                    print "Importing from/into multiple branches"
1506                self.detectBranches = True
1507
1508            if self.verbose:
1509                print "branches: %s" % self.p4BranchesInGit
1510
1511            p4Change = 0
1512            for branch in self.p4BranchesInGit:
1513                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1514
1515                settings = extractSettingsGitLog(logMsg)
1516
1517                self.readOptions(settings)
1518                if (settings.has_key('depot-paths')
1519                    and settings.has_key ('change')):
1520                    change = int(settings['change']) + 1
1521                    p4Change = max(p4Change, change)
1522
1523                    depotPaths = sorted(settings['depot-paths'])
1524                    if self.previousDepotPaths == []:
1525                        self.previousDepotPaths = depotPaths
1526                    else:
1527                        paths = []
1528                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1529                            for i in range(0, min(len(cur), len(prev))):
1530                                if cur[i] <> prev[i]:
1531                                    i = i - 1
1532                                    break
1533
1534                            paths.append (cur[:i + 1])
1535
1536                        self.previousDepotPaths = paths
1537
1538            if p4Change > 0:
1539                self.depotPaths = sorted(self.previousDepotPaths)
1540                self.changeRange = "@%s,#head" % p4Change
1541                if not self.detectBranches:
1542                    self.initialParent = parseRevision(self.branch)
1543                if not self.silent and not self.detectBranches:
1544                    print "Performing incremental import into %s git branch" % self.branch
1545
1546        if not self.branch.startswith("refs/"):
1547            self.branch = "refs/heads/" + self.branch
1548
1549        if len(args) == 0 and self.depotPaths:
1550            if not self.silent:
1551                print "Depot paths: %s" % ' '.join(self.depotPaths)
1552        else:
1553            if self.depotPaths and self.depotPaths != args:
1554                print ("previous import used depot path %s and now %s was specified. "
1555                       "This doesn't work!" % (' '.join (self.depotPaths),
1556                                               ' '.join (args)))
1557                sys.exit(1)
1558
1559            self.depotPaths = sorted(args)
1560
1561        revision = ""
1562        self.users = {}
1563
1564        newPaths = []
1565        for p in self.depotPaths:
1566            if p.find("@") != -1:
1567                atIdx = p.index("@")
1568                self.changeRange = p[atIdx:]
1569                if self.changeRange == "@all":
1570                    self.changeRange = ""
1571                elif ',' not in self.changeRange:
1572                    revision = self.changeRange
1573                    self.changeRange = ""
1574                p = p[:atIdx]
1575            elif p.find("#") != -1:
1576                hashIdx = p.index("#")
1577                revision = p[hashIdx:]
1578                p = p[:hashIdx]
1579            elif self.previousDepotPaths == []:
1580                revision = "#head"
1581
1582            p = re.sub ("\.\.\.$", "", p)
1583            if not p.endswith("/"):
1584                p += "/"
1585
1586            newPaths.append(p)
1587
1588        self.depotPaths = newPaths
1589
1590
1591        self.loadUserMapFromCache()
1592        self.labels = {}
1593        if self.detectLabels:
1594            self.getLabels();
1595
1596        if self.detectBranches:
1597            ## FIXME - what's a P4 projectName ?
1598            self.projectName = self.guessProjectName()
1599
1600            if self.hasOrigin:
1601                self.getBranchMappingFromGitBranches()
1602            else:
1603                self.getBranchMapping()
1604            if self.verbose:
1605                print "p4-git branches: %s" % self.p4BranchesInGit
1606                print "initial parents: %s" % self.initialParents
1607            for b in self.p4BranchesInGit:
1608                if b != "master":
1609
1610                    ## FIXME
1611                    b = b[len(self.projectName):]
1612                self.createdBranches.add(b)
1613
1614        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1615
1616        importProcess = subprocess.Popen(["git", "fast-import"],
1617                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1618                                         stderr=subprocess.PIPE);
1619        self.gitOutput = importProcess.stdout
1620        self.gitStream = importProcess.stdin
1621        self.gitError = importProcess.stderr
1622
1623        if revision:
1624            self.importHeadRevision(revision)
1625        else:
1626            changes = []
1627
1628            if len(self.changesFile) > 0:
1629                output = open(self.changesFile).readlines()
1630                changeSet = Set()
1631                for line in output:
1632                    changeSet.add(int(line))
1633
1634                for change in changeSet:
1635                    changes.append(change)
1636
1637                changes.sort()
1638            else:
1639                if self.verbose:
1640                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1641                                                              self.changeRange)
1642                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1643
1644                if len(self.maxChanges) > 0:
1645                    changes = changes[:min(int(self.maxChanges), len(changes))]
1646
1647            if len(changes) == 0:
1648                if not self.silent:
1649                    print "No changes to import!"
1650                return True
1651
1652            if not self.silent and not self.detectBranches:
1653                print "Import destination: %s" % self.branch
1654
1655            self.updatedBranches = set()
1656
1657            self.importChanges(changes)
1658
1659            if not self.silent:
1660                print ""
1661                if len(self.updatedBranches) > 0:
1662                    sys.stdout.write("Updated branches: ")
1663                    for b in self.updatedBranches:
1664                        sys.stdout.write("%s " % b)
1665                    sys.stdout.write("\n")
1666
1667        self.gitStream.close()
1668        if importProcess.wait() != 0:
1669            die("fast-import failed: %s" % self.gitError.read())
1670        self.gitOutput.close()
1671        self.gitError.close()
1672
1673        return True
1674
1675class P4Rebase(Command):
1676    def __init__(self):
1677        Command.__init__(self)
1678        self.options = [ ]
1679        self.description = ("Fetches the latest revision from perforce and "
1680                            + "rebases the current work (branch) against it")
1681        self.verbose = False
1682
1683    def run(self, args):
1684        sync = P4Sync()
1685        sync.run([])
1686
1687        return self.rebase()
1688
1689    def rebase(self):
1690        if os.system("git update-index --refresh") != 0:
1691            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1692        if len(read_pipe("git diff-index HEAD --")) > 0:
1693            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1694
1695        [upstream, settings] = findUpstreamBranchPoint()
1696        if len(upstream) == 0:
1697            die("Cannot find upstream branchpoint for rebase")
1698
1699        # the branchpoint may be p4/foo~3, so strip off the parent
1700        upstream = re.sub("~[0-9]+$", "", upstream)
1701
1702        print "Rebasing the current branch onto %s" % upstream
1703        oldHead = read_pipe("git rev-parse HEAD").strip()
1704        system("git rebase %s" % upstream)
1705        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1706        return True
1707
1708class P4Clone(P4Sync):
1709    def __init__(self):
1710        P4Sync.__init__(self)
1711        self.description = "Creates a new git repository and imports from Perforce into it"
1712        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1713        self.options += [
1714            optparse.make_option("--destination", dest="cloneDestination",
1715                                 action='store', default=None,
1716                                 help="where to leave result of the clone"),
1717            optparse.make_option("-/", dest="cloneExclude",
1718                                 action="append", type="string",
1719                                 help="exclude depot path")
1720        ]
1721        self.cloneDestination = None
1722        self.needsGit = False
1723
1724    # This is required for the "append" cloneExclude action
1725    def ensure_value(self, attr, value):
1726        if not hasattr(self, attr) or getattr(self, attr) is None:
1727            setattr(self, attr, value)
1728        return getattr(self, attr)
1729
1730    def defaultDestination(self, args):
1731        ## TODO: use common prefix of args?
1732        depotPath = args[0]
1733        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1734        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1735        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1736        depotDir = re.sub(r"/$", "", depotDir)
1737        return os.path.split(depotDir)[1]
1738
1739    def run(self, args):
1740        if len(args) < 1:
1741            return False
1742
1743        if self.keepRepoPath and not self.cloneDestination:
1744            sys.stderr.write("Must specify destination for --keep-path\n")
1745            sys.exit(1)
1746
1747        depotPaths = args
1748
1749        if not self.cloneDestination and len(depotPaths) > 1:
1750            self.cloneDestination = depotPaths[-1]
1751            depotPaths = depotPaths[:-1]
1752
1753        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1754        for p in depotPaths:
1755            if not p.startswith("//"):
1756                return False
1757
1758        if not self.cloneDestination:
1759            self.cloneDestination = self.defaultDestination(args)
1760
1761        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1762        if not os.path.exists(self.cloneDestination):
1763            os.makedirs(self.cloneDestination)
1764        chdir(self.cloneDestination)
1765        system("git init")
1766        self.gitdir = os.getcwd() + "/.git"
1767        if not P4Sync.run(self, depotPaths):
1768            return False
1769        if self.branch != "master":
1770            if self.importIntoRemotes:
1771                masterbranch = "refs/remotes/p4/master"
1772            else:
1773                masterbranch = "refs/heads/p4/master"
1774            if gitBranchExists(masterbranch):
1775                system("git branch master %s" % masterbranch)
1776                system("git checkout -f")
1777            else:
1778                print "Could not detect main branch. No checkout/master branch created."
1779
1780        return True
1781
1782class P4Branches(Command):
1783    def __init__(self):
1784        Command.__init__(self)
1785        self.options = [ ]
1786        self.description = ("Shows the git branches that hold imports and their "
1787                            + "corresponding perforce depot paths")
1788        self.verbose = False
1789
1790    def run(self, args):
1791        if originP4BranchesExist():
1792            createOrUpdateBranchesFromOrigin()
1793
1794        cmdline = "git rev-parse --symbolic "
1795        cmdline += " --remotes"
1796
1797        for line in read_pipe_lines(cmdline):
1798            line = line.strip()
1799
1800            if not line.startswith('p4/') or line == "p4/HEAD":
1801                continue
1802            branch = line
1803
1804            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1805            settings = extractSettingsGitLog(log)
1806
1807            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1808        return True
1809
1810class HelpFormatter(optparse.IndentedHelpFormatter):
1811    def __init__(self):
1812        optparse.IndentedHelpFormatter.__init__(self)
1813
1814    def format_description(self, description):
1815        if description:
1816            return description + "\n"
1817        else:
1818            return ""
1819
1820def printUsage(commands):
1821    print "usage: %s <command> [options]" % sys.argv[0]
1822    print ""
1823    print "valid commands: %s" % ", ".join(commands)
1824    print ""
1825    print "Try %s <command> --help for command specific help." % sys.argv[0]
1826    print ""
1827
1828commands = {
1829    "debug" : P4Debug,
1830    "submit" : P4Submit,
1831    "commit" : P4Submit,
1832    "sync" : P4Sync,
1833    "rebase" : P4Rebase,
1834    "clone" : P4Clone,
1835    "rollback" : P4RollBack,
1836    "branches" : P4Branches
1837}
1838
1839
1840def main():
1841    if len(sys.argv[1:]) == 0:
1842        printUsage(commands.keys())
1843        sys.exit(2)
1844
1845    cmd = ""
1846    cmdName = sys.argv[1]
1847    try:
1848        klass = commands[cmdName]
1849        cmd = klass()
1850    except KeyError:
1851        print "unknown command %s" % cmdName
1852        print ""
1853        printUsage(commands.keys())
1854        sys.exit(2)
1855
1856    options = cmd.options
1857    cmd.gitdir = os.environ.get("GIT_DIR", None)
1858
1859    args = sys.argv[2:]
1860
1861    if len(options) > 0:
1862        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1863
1864        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1865                                       options,
1866                                       description = cmd.description,
1867                                       formatter = HelpFormatter())
1868
1869        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1870    global verbose
1871    verbose = cmd.verbose
1872    if cmd.needsGit:
1873        if cmd.gitdir == None:
1874            cmd.gitdir = os.path.abspath(".git")
1875            if not isValidGitDir(cmd.gitdir):
1876                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1877                if os.path.exists(cmd.gitdir):
1878                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1879                    if len(cdup) > 0:
1880                        chdir(cdup);
1881
1882        if not isValidGitDir(cmd.gitdir):
1883            if isValidGitDir(cmd.gitdir + "/.git"):
1884                cmd.gitdir += "/.git"
1885            else:
1886                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1887
1888        os.environ["GIT_DIR"] = cmd.gitdir
1889
1890    if not cmd.run(args):
1891        parser.print_help()
1892
1893
1894if __name__ == '__main__':
1895    main()