Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'maint' (87b3c01)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = p4_read_pipe("diff -du ...")
 710
 711            newdiff = ""
 712            for newFile in filesToAdd:
 713                newdiff += "==== new file ====\n"
 714                newdiff += "--- /dev/null\n"
 715                newdiff += "+++ %s\n" % newFile
 716                f = open(newFile, "r")
 717                for line in f.readlines():
 718                    newdiff += "+" + line
 719                f.close()
 720
 721            separatorLine = "######## everything below this line is just the diff #######\n"
 722
 723            [handle, fileName] = tempfile.mkstemp()
 724            tmpFile = os.fdopen(handle, "w+")
 725            if self.isWindows:
 726                submitTemplate = submitTemplate.replace("\n", "\r\n")
 727                separatorLine = separatorLine.replace("\n", "\r\n")
 728                newdiff = newdiff.replace("\n", "\r\n")
 729            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 730            tmpFile.close()
 731            mtime = os.stat(fileName).st_mtime
 732            if os.environ.has_key("P4EDITOR"):
 733                editor = os.environ.get("P4EDITOR")
 734            else:
 735                editor = read_pipe("git var GIT_EDITOR").strip()
 736            system(editor + " " + fileName)
 737
 738            response = "y"
 739            if os.stat(fileName).st_mtime <= mtime:
 740                response = "x"
 741                while response != "y" and response != "n":
 742                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 743
 744            if response == "y":
 745                tmpFile = open(fileName, "rb")
 746                message = tmpFile.read()
 747                tmpFile.close()
 748                submitTemplate = message[:message.index(separatorLine)]
 749                if self.isWindows:
 750                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 751                p4_write_pipe("submit -i", submitTemplate)
 752            else:
 753                for f in editedFiles:
 754                    p4_system("revert \"%s\"" % f);
 755                for f in filesToAdd:
 756                    p4_system("revert \"%s\"" % f);
 757                    system("rm %s" %f)
 758
 759            os.remove(fileName)
 760        else:
 761            fileName = "submit.txt"
 762            file = open(fileName, "w+")
 763            file.write(self.prepareLogMessage(template, logMessage))
 764            file.close()
 765            print ("Perforce submit template written as %s. "
 766                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 767                   % (fileName, fileName))
 768
 769    def run(self, args):
 770        if len(args) == 0:
 771            self.master = currentGitBranch()
 772            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 773                die("Detecting current git branch failed!")
 774        elif len(args) == 1:
 775            self.master = args[0]
 776        else:
 777            return False
 778
 779        allowSubmit = gitConfig("git-p4.allowSubmit")
 780        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 781            die("%s is not in git-p4.allowSubmit" % self.master)
 782
 783        [upstream, settings] = findUpstreamBranchPoint()
 784        self.depotPath = settings['depot-paths'][0]
 785        if len(self.origin) == 0:
 786            self.origin = upstream
 787
 788        if self.verbose:
 789            print "Origin branch is " + self.origin
 790
 791        if len(self.depotPath) == 0:
 792            print "Internal error: cannot locate perforce depot path from existing branches"
 793            sys.exit(128)
 794
 795        self.clientPath = p4Where(self.depotPath)
 796
 797        if len(self.clientPath) == 0:
 798            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 799            sys.exit(128)
 800
 801        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 802        self.oldWorkingDirectory = os.getcwd()
 803
 804        chdir(self.clientPath)
 805        print "Synchronizing p4 checkout..."
 806        p4_system("sync ...")
 807
 808        self.check()
 809
 810        commits = []
 811        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 812            commits.append(line.strip())
 813        commits.reverse()
 814
 815        while len(commits) > 0:
 816            commit = commits[0]
 817            commits = commits[1:]
 818            self.applyCommit(commit)
 819            if not self.interactive:
 820                break
 821
 822        if len(commits) == 0:
 823            print "All changes applied!"
 824            chdir(self.oldWorkingDirectory)
 825
 826            sync = P4Sync()
 827            sync.run([])
 828
 829            rebase = P4Rebase()
 830            rebase.rebase()
 831
 832        return True
 833
 834class P4Sync(Command):
 835    def __init__(self):
 836        Command.__init__(self)
 837        self.options = [
 838                optparse.make_option("--branch", dest="branch"),
 839                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 840                optparse.make_option("--changesfile", dest="changesFile"),
 841                optparse.make_option("--silent", dest="silent", action="store_true"),
 842                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 843                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 844                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 845                                     help="Import into refs/heads/ , not refs/remotes"),
 846                optparse.make_option("--max-changes", dest="maxChanges"),
 847                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 848                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 849                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 850                                     help="Only sync files that are included in the Perforce Client Spec")
 851        ]
 852        self.description = """Imports from Perforce into a git repository.\n
 853    example:
 854    //depot/my/project/ -- to import the current head
 855    //depot/my/project/@all -- to import everything
 856    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 857
 858    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 859
 860        self.usage += " //depot/path[@revRange]"
 861        self.silent = False
 862        self.createdBranches = set()
 863        self.committedChanges = set()
 864        self.branch = ""
 865        self.detectBranches = False
 866        self.detectLabels = False
 867        self.changesFile = ""
 868        self.syncWithOrigin = True
 869        self.verbose = False
 870        self.importIntoRemotes = True
 871        self.maxChanges = ""
 872        self.isWindows = (platform.system() == "Windows")
 873        self.keepRepoPath = False
 874        self.depotPaths = None
 875        self.p4BranchesInGit = []
 876        self.cloneExclude = []
 877        self.useClientSpec = False
 878        self.clientSpecDirs = []
 879
 880        if gitConfig("git-p4.syncFromOrigin") == "false":
 881            self.syncWithOrigin = False
 882
 883    def extractFilesFromCommit(self, commit):
 884        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 885                             for path in self.cloneExclude]
 886        files = []
 887        fnum = 0
 888        while commit.has_key("depotFile%s" % fnum):
 889            path =  commit["depotFile%s" % fnum]
 890
 891            if [p for p in self.cloneExclude
 892                if path.startswith (p)]:
 893                found = False
 894            else:
 895                found = [p for p in self.depotPaths
 896                         if path.startswith (p)]
 897            if not found:
 898                fnum = fnum + 1
 899                continue
 900
 901            file = {}
 902            file["path"] = path
 903            file["rev"] = commit["rev%s" % fnum]
 904            file["action"] = commit["action%s" % fnum]
 905            file["type"] = commit["type%s" % fnum]
 906            files.append(file)
 907            fnum = fnum + 1
 908        return files
 909
 910    def stripRepoPath(self, path, prefixes):
 911        if self.keepRepoPath:
 912            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 913
 914        for p in prefixes:
 915            if path.startswith(p):
 916                path = path[len(p):]
 917
 918        return path
 919
 920    def splitFilesIntoBranches(self, commit):
 921        branches = {}
 922        fnum = 0
 923        while commit.has_key("depotFile%s" % fnum):
 924            path =  commit["depotFile%s" % fnum]
 925            found = [p for p in self.depotPaths
 926                     if path.startswith (p)]
 927            if not found:
 928                fnum = fnum + 1
 929                continue
 930
 931            file = {}
 932            file["path"] = path
 933            file["rev"] = commit["rev%s" % fnum]
 934            file["action"] = commit["action%s" % fnum]
 935            file["type"] = commit["type%s" % fnum]
 936            fnum = fnum + 1
 937
 938            relPath = self.stripRepoPath(path, self.depotPaths)
 939
 940            for branch in self.knownBranches.keys():
 941
 942                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 943                if relPath.startswith(branch + "/"):
 944                    if branch not in branches:
 945                        branches[branch] = []
 946                    branches[branch].append(file)
 947                    break
 948
 949        return branches
 950
 951    # output one file from the P4 stream
 952    # - helper for streamP4Files
 953
 954    def streamOneP4File(self, file, contents):
 955        if file["type"] == "apple":
 956            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 957                file['depotFile']
 958            return
 959
 960        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 961        if verbose:
 962            sys.stderr.write("%s\n" % relPath)
 963
 964        mode = "644"
 965        if isP4Exec(file["type"]):
 966            mode = "755"
 967        elif file["type"] == "symlink":
 968            mode = "120000"
 969            # p4 print on a symlink contains "target\n", so strip it off
 970            data = ''.join(contents)
 971            contents = [data[:-1]]
 972
 973        if self.isWindows and file["type"].endswith("text"):
 974            mangled = []
 975            for data in contents:
 976                data = data.replace("\r\n", "\n")
 977                mangled.append(data)
 978            contents = mangled
 979
 980        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 981            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 982        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 983            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 984
 985        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 986
 987        # total length...
 988        length = 0
 989        for d in contents:
 990            length = length + len(d)
 991
 992        self.gitStream.write("data %d\n" % length)
 993        for d in contents:
 994            self.gitStream.write(d)
 995        self.gitStream.write("\n")
 996
 997    def streamOneP4Deletion(self, file):
 998        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
 999        if verbose:
1000            sys.stderr.write("delete %s\n" % relPath)
1001        self.gitStream.write("D %s\n" % relPath)
1002
1003    # handle another chunk of streaming data
1004    def streamP4FilesCb(self, marshalled):
1005
1006        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1007            # start of a new file - output the old one first
1008            self.streamOneP4File(self.stream_file, self.stream_contents)
1009            self.stream_file = {}
1010            self.stream_contents = []
1011            self.stream_have_file_info = False
1012
1013        # pick up the new file information... for the
1014        # 'data' field we need to append to our array
1015        for k in marshalled.keys():
1016            if k == 'data':
1017                self.stream_contents.append(marshalled['data'])
1018            else:
1019                self.stream_file[k] = marshalled[k]
1020
1021        self.stream_have_file_info = True
1022
1023    # Stream directly from "p4 files" into "git fast-import"
1024    def streamP4Files(self, files):
1025        filesForCommit = []
1026        filesToRead = []
1027        filesToDelete = []
1028
1029        for f in files:
1030            includeFile = True
1031            for val in self.clientSpecDirs:
1032                if f['path'].startswith(val[0]):
1033                    if val[1] <= 0:
1034                        includeFile = False
1035                    break
1036
1037            if includeFile:
1038                filesForCommit.append(f)
1039                if f['action'] not in ('delete', 'move/delete', 'purge'):
1040                    filesToRead.append(f)
1041                else:
1042                    filesToDelete.append(f)
1043
1044        # deleted files...
1045        for f in filesToDelete:
1046            self.streamOneP4Deletion(f)
1047
1048        if len(filesToRead) > 0:
1049            self.stream_file = {}
1050            self.stream_contents = []
1051            self.stream_have_file_info = False
1052
1053            # curry self argument
1054            def streamP4FilesCbSelf(entry):
1055                self.streamP4FilesCb(entry)
1056
1057            p4CmdList("-x - print",
1058                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1059                                                  for f in filesToRead]),
1060                cb=streamP4FilesCbSelf)
1061
1062            # do the last chunk
1063            if self.stream_file.has_key('depotFile'):
1064                self.streamOneP4File(self.stream_file, self.stream_contents)
1065
1066    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1067        epoch = details["time"]
1068        author = details["user"]
1069        self.branchPrefixes = branchPrefixes
1070
1071        if self.verbose:
1072            print "commit into %s" % branch
1073
1074        # start with reading files; if that fails, we should not
1075        # create a commit.
1076        new_files = []
1077        for f in files:
1078            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1079                new_files.append (f)
1080            else:
1081                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1082
1083        self.gitStream.write("commit %s\n" % branch)
1084#        gitStream.write("mark :%s\n" % details["change"])
1085        self.committedChanges.add(int(details["change"]))
1086        committer = ""
1087        if author not in self.users:
1088            self.getUserMapFromPerforceServer()
1089        if author in self.users:
1090            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1091        else:
1092            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1093
1094        self.gitStream.write("committer %s\n" % committer)
1095
1096        self.gitStream.write("data <<EOT\n")
1097        self.gitStream.write(details["desc"])
1098        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1099                             % (','.join (branchPrefixes), details["change"]))
1100        if len(details['options']) > 0:
1101            self.gitStream.write(": options = %s" % details['options'])
1102        self.gitStream.write("]\nEOT\n\n")
1103
1104        if len(parent) > 0:
1105            if self.verbose:
1106                print "parent %s" % parent
1107            self.gitStream.write("from %s\n" % parent)
1108
1109        self.streamP4Files(new_files)
1110        self.gitStream.write("\n")
1111
1112        change = int(details["change"])
1113
1114        if self.labels.has_key(change):
1115            label = self.labels[change]
1116            labelDetails = label[0]
1117            labelRevisions = label[1]
1118            if self.verbose:
1119                print "Change %s is labelled %s" % (change, labelDetails)
1120
1121            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1122                                                    for p in branchPrefixes]))
1123
1124            if len(files) == len(labelRevisions):
1125
1126                cleanedFiles = {}
1127                for info in files:
1128                    if info["action"] in ("delete", "purge"):
1129                        continue
1130                    cleanedFiles[info["depotFile"]] = info["rev"]
1131
1132                if cleanedFiles == labelRevisions:
1133                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1134                    self.gitStream.write("from %s\n" % branch)
1135
1136                    owner = labelDetails["Owner"]
1137                    tagger = ""
1138                    if author in self.users:
1139                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1140                    else:
1141                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1142                    self.gitStream.write("tagger %s\n" % tagger)
1143                    self.gitStream.write("data <<EOT\n")
1144                    self.gitStream.write(labelDetails["Description"])
1145                    self.gitStream.write("EOT\n\n")
1146
1147                else:
1148                    if not self.silent:
1149                        print ("Tag %s does not match with change %s: files do not match."
1150                               % (labelDetails["label"], change))
1151
1152            else:
1153                if not self.silent:
1154                    print ("Tag %s does not match with change %s: file count is different."
1155                           % (labelDetails["label"], change))
1156
1157    def getUserCacheFilename(self):
1158        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1159        return home + "/.gitp4-usercache.txt"
1160
1161    def getUserMapFromPerforceServer(self):
1162        if self.userMapFromPerforceServer:
1163            return
1164        self.users = {}
1165
1166        for output in p4CmdList("users"):
1167            if not output.has_key("User"):
1168                continue
1169            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1170
1171
1172        s = ''
1173        for (key, val) in self.users.items():
1174            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1175
1176        open(self.getUserCacheFilename(), "wb").write(s)
1177        self.userMapFromPerforceServer = True
1178
1179    def loadUserMapFromCache(self):
1180        self.users = {}
1181        self.userMapFromPerforceServer = False
1182        try:
1183            cache = open(self.getUserCacheFilename(), "rb")
1184            lines = cache.readlines()
1185            cache.close()
1186            for line in lines:
1187                entry = line.strip().split("\t")
1188                self.users[entry[0]] = entry[1]
1189        except IOError:
1190            self.getUserMapFromPerforceServer()
1191
1192    def getLabels(self):
1193        self.labels = {}
1194
1195        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1196        if len(l) > 0 and not self.silent:
1197            print "Finding files belonging to labels in %s" % `self.depotPaths`
1198
1199        for output in l:
1200            label = output["label"]
1201            revisions = {}
1202            newestChange = 0
1203            if self.verbose:
1204                print "Querying files for label %s" % label
1205            for file in p4CmdList("files "
1206                                  +  ' '.join (["%s...@%s" % (p, label)
1207                                                for p in self.depotPaths])):
1208                revisions[file["depotFile"]] = file["rev"]
1209                change = int(file["change"])
1210                if change > newestChange:
1211                    newestChange = change
1212
1213            self.labels[newestChange] = [output, revisions]
1214
1215        if self.verbose:
1216            print "Label changes: %s" % self.labels.keys()
1217
1218    def guessProjectName(self):
1219        for p in self.depotPaths:
1220            if p.endswith("/"):
1221                p = p[:-1]
1222            p = p[p.strip().rfind("/") + 1:]
1223            if not p.endswith("/"):
1224               p += "/"
1225            return p
1226
1227    def getBranchMapping(self):
1228        lostAndFoundBranches = set()
1229
1230        for info in p4CmdList("branches"):
1231            details = p4Cmd("branch -o %s" % info["branch"])
1232            viewIdx = 0
1233            while details.has_key("View%s" % viewIdx):
1234                paths = details["View%s" % viewIdx].split(" ")
1235                viewIdx = viewIdx + 1
1236                # require standard //depot/foo/... //depot/bar/... mapping
1237                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1238                    continue
1239                source = paths[0]
1240                destination = paths[1]
1241                ## HACK
1242                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1243                    source = source[len(self.depotPaths[0]):-4]
1244                    destination = destination[len(self.depotPaths[0]):-4]
1245
1246                    if destination in self.knownBranches:
1247                        if not self.silent:
1248                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1249                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1250                        continue
1251
1252                    self.knownBranches[destination] = source
1253
1254                    lostAndFoundBranches.discard(destination)
1255
1256                    if source not in self.knownBranches:
1257                        lostAndFoundBranches.add(source)
1258
1259
1260        for branch in lostAndFoundBranches:
1261            self.knownBranches[branch] = branch
1262
1263    def getBranchMappingFromGitBranches(self):
1264        branches = p4BranchesInGit(self.importIntoRemotes)
1265        for branch in branches.keys():
1266            if branch == "master":
1267                branch = "main"
1268            else:
1269                branch = branch[len(self.projectName):]
1270            self.knownBranches[branch] = branch
1271
1272    def listExistingP4GitBranches(self):
1273        # branches holds mapping from name to commit
1274        branches = p4BranchesInGit(self.importIntoRemotes)
1275        self.p4BranchesInGit = branches.keys()
1276        for branch in branches.keys():
1277            self.initialParents[self.refPrefix + branch] = branches[branch]
1278
1279    def updateOptionDict(self, d):
1280        option_keys = {}
1281        if self.keepRepoPath:
1282            option_keys['keepRepoPath'] = 1
1283
1284        d["options"] = ' '.join(sorted(option_keys.keys()))
1285
1286    def readOptions(self, d):
1287        self.keepRepoPath = (d.has_key('options')
1288                             and ('keepRepoPath' in d['options']))
1289
1290    def gitRefForBranch(self, branch):
1291        if branch == "main":
1292            return self.refPrefix + "master"
1293
1294        if len(branch) <= 0:
1295            return branch
1296
1297        return self.refPrefix + self.projectName + branch
1298
1299    def gitCommitByP4Change(self, ref, change):
1300        if self.verbose:
1301            print "looking in ref " + ref + " for change %s using bisect..." % change
1302
1303        earliestCommit = ""
1304        latestCommit = parseRevision(ref)
1305
1306        while True:
1307            if self.verbose:
1308                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1309            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1310            if len(next) == 0:
1311                if self.verbose:
1312                    print "argh"
1313                return ""
1314            log = extractLogMessageFromGitCommit(next)
1315            settings = extractSettingsGitLog(log)
1316            currentChange = int(settings['change'])
1317            if self.verbose:
1318                print "current change %s" % currentChange
1319
1320            if currentChange == change:
1321                if self.verbose:
1322                    print "found %s" % next
1323                return next
1324
1325            if currentChange < change:
1326                earliestCommit = "^%s" % next
1327            else:
1328                latestCommit = "%s" % next
1329
1330        return ""
1331
1332    def importNewBranch(self, branch, maxChange):
1333        # make fast-import flush all changes to disk and update the refs using the checkpoint
1334        # command so that we can try to find the branch parent in the git history
1335        self.gitStream.write("checkpoint\n\n");
1336        self.gitStream.flush();
1337        branchPrefix = self.depotPaths[0] + branch + "/"
1338        range = "@1,%s" % maxChange
1339        #print "prefix" + branchPrefix
1340        changes = p4ChangesForPaths([branchPrefix], range)
1341        if len(changes) <= 0:
1342            return False
1343        firstChange = changes[0]
1344        #print "first change in branch: %s" % firstChange
1345        sourceBranch = self.knownBranches[branch]
1346        sourceDepotPath = self.depotPaths[0] + sourceBranch
1347        sourceRef = self.gitRefForBranch(sourceBranch)
1348        #print "source " + sourceBranch
1349
1350        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1351        #print "branch parent: %s" % branchParentChange
1352        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1353        if len(gitParent) > 0:
1354            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1355            #print "parent git commit: %s" % gitParent
1356
1357        self.importChanges(changes)
1358        return True
1359
1360    def importChanges(self, changes):
1361        cnt = 1
1362        for change in changes:
1363            description = p4Cmd("describe %s" % change)
1364            self.updateOptionDict(description)
1365
1366            if not self.silent:
1367                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1368                sys.stdout.flush()
1369            cnt = cnt + 1
1370
1371            try:
1372                if self.detectBranches:
1373                    branches = self.splitFilesIntoBranches(description)
1374                    for branch in branches.keys():
1375                        ## HACK  --hwn
1376                        branchPrefix = self.depotPaths[0] + branch + "/"
1377
1378                        parent = ""
1379
1380                        filesForCommit = branches[branch]
1381
1382                        if self.verbose:
1383                            print "branch is %s" % branch
1384
1385                        self.updatedBranches.add(branch)
1386
1387                        if branch not in self.createdBranches:
1388                            self.createdBranches.add(branch)
1389                            parent = self.knownBranches[branch]
1390                            if parent == branch:
1391                                parent = ""
1392                            else:
1393                                fullBranch = self.projectName + branch
1394                                if fullBranch not in self.p4BranchesInGit:
1395                                    if not self.silent:
1396                                        print("\n    Importing new branch %s" % fullBranch);
1397                                    if self.importNewBranch(branch, change - 1):
1398                                        parent = ""
1399                                        self.p4BranchesInGit.append(fullBranch)
1400                                    if not self.silent:
1401                                        print("\n    Resuming with change %s" % change);
1402
1403                                if self.verbose:
1404                                    print "parent determined through known branches: %s" % parent
1405
1406                        branch = self.gitRefForBranch(branch)
1407                        parent = self.gitRefForBranch(parent)
1408
1409                        if self.verbose:
1410                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1411
1412                        if len(parent) == 0 and branch in self.initialParents:
1413                            parent = self.initialParents[branch]
1414                            del self.initialParents[branch]
1415
1416                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1417                else:
1418                    files = self.extractFilesFromCommit(description)
1419                    self.commit(description, files, self.branch, self.depotPaths,
1420                                self.initialParent)
1421                    self.initialParent = ""
1422            except IOError:
1423                print self.gitError.read()
1424                sys.exit(1)
1425
1426    def importHeadRevision(self, revision):
1427        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1428
1429        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1430        details["desc"] = ("Initial import of %s from the state at revision %s"
1431                           % (' '.join(self.depotPaths), revision))
1432        details["change"] = revision
1433        newestRevision = 0
1434
1435        fileCnt = 0
1436        for info in p4CmdList("files "
1437                              +  ' '.join(["%s...%s"
1438                                           % (p, revision)
1439                                           for p in self.depotPaths])):
1440
1441            if info['code'] == 'error':
1442                sys.stderr.write("p4 returned an error: %s\n"
1443                                 % info['data'])
1444                sys.exit(1)
1445
1446
1447            change = int(info["change"])
1448            if change > newestRevision:
1449                newestRevision = change
1450
1451            if info["action"] in ("delete", "purge"):
1452                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1453                #fileCnt = fileCnt + 1
1454                continue
1455
1456            for prop in ["depotFile", "rev", "action", "type" ]:
1457                details["%s%s" % (prop, fileCnt)] = info[prop]
1458
1459            fileCnt = fileCnt + 1
1460
1461        details["change"] = newestRevision
1462        self.updateOptionDict(details)
1463        try:
1464            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1465        except IOError:
1466            print "IO error with git fast-import. Is your git version recent enough?"
1467            print self.gitError.read()
1468
1469
1470    def getClientSpec(self):
1471        specList = p4CmdList( "client -o" )
1472        temp = {}
1473        for entry in specList:
1474            for k,v in entry.iteritems():
1475                if k.startswith("View"):
1476                    if v.startswith('"'):
1477                        start = 1
1478                    else:
1479                        start = 0
1480                    index = v.find("...")
1481                    v = v[start:index]
1482                    if v.startswith("-"):
1483                        v = v[1:]
1484                        temp[v] = -len(v)
1485                    else:
1486                        temp[v] = len(v)
1487        self.clientSpecDirs = temp.items()
1488        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1489
1490    def run(self, args):
1491        self.depotPaths = []
1492        self.changeRange = ""
1493        self.initialParent = ""
1494        self.previousDepotPaths = []
1495
1496        # map from branch depot path to parent branch
1497        self.knownBranches = {}
1498        self.initialParents = {}
1499        self.hasOrigin = originP4BranchesExist()
1500        if not self.syncWithOrigin:
1501            self.hasOrigin = False
1502
1503        if self.importIntoRemotes:
1504            self.refPrefix = "refs/remotes/p4/"
1505        else:
1506            self.refPrefix = "refs/heads/p4/"
1507
1508        if self.syncWithOrigin and self.hasOrigin:
1509            if not self.silent:
1510                print "Syncing with origin first by calling git fetch origin"
1511            system("git fetch origin")
1512
1513        if len(self.branch) == 0:
1514            self.branch = self.refPrefix + "master"
1515            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1516                system("git update-ref %s refs/heads/p4" % self.branch)
1517                system("git branch -D p4");
1518            # create it /after/ importing, when master exists
1519            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1520                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1521
1522        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1523            self.getClientSpec()
1524
1525        # TODO: should always look at previous commits,
1526        # merge with previous imports, if possible.
1527        if args == []:
1528            if self.hasOrigin:
1529                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1530            self.listExistingP4GitBranches()
1531
1532            if len(self.p4BranchesInGit) > 1:
1533                if not self.silent:
1534                    print "Importing from/into multiple branches"
1535                self.detectBranches = True
1536
1537            if self.verbose:
1538                print "branches: %s" % self.p4BranchesInGit
1539
1540            p4Change = 0
1541            for branch in self.p4BranchesInGit:
1542                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1543
1544                settings = extractSettingsGitLog(logMsg)
1545
1546                self.readOptions(settings)
1547                if (settings.has_key('depot-paths')
1548                    and settings.has_key ('change')):
1549                    change = int(settings['change']) + 1
1550                    p4Change = max(p4Change, change)
1551
1552                    depotPaths = sorted(settings['depot-paths'])
1553                    if self.previousDepotPaths == []:
1554                        self.previousDepotPaths = depotPaths
1555                    else:
1556                        paths = []
1557                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1558                            for i in range(0, min(len(cur), len(prev))):
1559                                if cur[i] <> prev[i]:
1560                                    i = i - 1
1561                                    break
1562
1563                            paths.append (cur[:i + 1])
1564
1565                        self.previousDepotPaths = paths
1566
1567            if p4Change > 0:
1568                self.depotPaths = sorted(self.previousDepotPaths)
1569                self.changeRange = "@%s,#head" % p4Change
1570                if not self.detectBranches:
1571                    self.initialParent = parseRevision(self.branch)
1572                if not self.silent and not self.detectBranches:
1573                    print "Performing incremental import into %s git branch" % self.branch
1574
1575        if not self.branch.startswith("refs/"):
1576            self.branch = "refs/heads/" + self.branch
1577
1578        if len(args) == 0 and self.depotPaths:
1579            if not self.silent:
1580                print "Depot paths: %s" % ' '.join(self.depotPaths)
1581        else:
1582            if self.depotPaths and self.depotPaths != args:
1583                print ("previous import used depot path %s and now %s was specified. "
1584                       "This doesn't work!" % (' '.join (self.depotPaths),
1585                                               ' '.join (args)))
1586                sys.exit(1)
1587
1588            self.depotPaths = sorted(args)
1589
1590        revision = ""
1591        self.users = {}
1592
1593        newPaths = []
1594        for p in self.depotPaths:
1595            if p.find("@") != -1:
1596                atIdx = p.index("@")
1597                self.changeRange = p[atIdx:]
1598                if self.changeRange == "@all":
1599                    self.changeRange = ""
1600                elif ',' not in self.changeRange:
1601                    revision = self.changeRange
1602                    self.changeRange = ""
1603                p = p[:atIdx]
1604            elif p.find("#") != -1:
1605                hashIdx = p.index("#")
1606                revision = p[hashIdx:]
1607                p = p[:hashIdx]
1608            elif self.previousDepotPaths == []:
1609                revision = "#head"
1610
1611            p = re.sub ("\.\.\.$", "", p)
1612            if not p.endswith("/"):
1613                p += "/"
1614
1615            newPaths.append(p)
1616
1617        self.depotPaths = newPaths
1618
1619
1620        self.loadUserMapFromCache()
1621        self.labels = {}
1622        if self.detectLabels:
1623            self.getLabels();
1624
1625        if self.detectBranches:
1626            ## FIXME - what's a P4 projectName ?
1627            self.projectName = self.guessProjectName()
1628
1629            if self.hasOrigin:
1630                self.getBranchMappingFromGitBranches()
1631            else:
1632                self.getBranchMapping()
1633            if self.verbose:
1634                print "p4-git branches: %s" % self.p4BranchesInGit
1635                print "initial parents: %s" % self.initialParents
1636            for b in self.p4BranchesInGit:
1637                if b != "master":
1638
1639                    ## FIXME
1640                    b = b[len(self.projectName):]
1641                self.createdBranches.add(b)
1642
1643        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1644
1645        importProcess = subprocess.Popen(["git", "fast-import"],
1646                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1647                                         stderr=subprocess.PIPE);
1648        self.gitOutput = importProcess.stdout
1649        self.gitStream = importProcess.stdin
1650        self.gitError = importProcess.stderr
1651
1652        if revision:
1653            self.importHeadRevision(revision)
1654        else:
1655            changes = []
1656
1657            if len(self.changesFile) > 0:
1658                output = open(self.changesFile).readlines()
1659                changeSet = set()
1660                for line in output:
1661                    changeSet.add(int(line))
1662
1663                for change in changeSet:
1664                    changes.append(change)
1665
1666                changes.sort()
1667            else:
1668                if self.verbose:
1669                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1670                                                              self.changeRange)
1671                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1672
1673                if len(self.maxChanges) > 0:
1674                    changes = changes[:min(int(self.maxChanges), len(changes))]
1675
1676            if len(changes) == 0:
1677                if not self.silent:
1678                    print "No changes to import!"
1679                return True
1680
1681            if not self.silent and not self.detectBranches:
1682                print "Import destination: %s" % self.branch
1683
1684            self.updatedBranches = set()
1685
1686            self.importChanges(changes)
1687
1688            if not self.silent:
1689                print ""
1690                if len(self.updatedBranches) > 0:
1691                    sys.stdout.write("Updated branches: ")
1692                    for b in self.updatedBranches:
1693                        sys.stdout.write("%s " % b)
1694                    sys.stdout.write("\n")
1695
1696        self.gitStream.close()
1697        if importProcess.wait() != 0:
1698            die("fast-import failed: %s" % self.gitError.read())
1699        self.gitOutput.close()
1700        self.gitError.close()
1701
1702        return True
1703
1704class P4Rebase(Command):
1705    def __init__(self):
1706        Command.__init__(self)
1707        self.options = [ ]
1708        self.description = ("Fetches the latest revision from perforce and "
1709                            + "rebases the current work (branch) against it")
1710        self.verbose = False
1711
1712    def run(self, args):
1713        sync = P4Sync()
1714        sync.run([])
1715
1716        return self.rebase()
1717
1718    def rebase(self):
1719        if os.system("git update-index --refresh") != 0:
1720            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1721        if len(read_pipe("git diff-index HEAD --")) > 0:
1722            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1723
1724        [upstream, settings] = findUpstreamBranchPoint()
1725        if len(upstream) == 0:
1726            die("Cannot find upstream branchpoint for rebase")
1727
1728        # the branchpoint may be p4/foo~3, so strip off the parent
1729        upstream = re.sub("~[0-9]+$", "", upstream)
1730
1731        print "Rebasing the current branch onto %s" % upstream
1732        oldHead = read_pipe("git rev-parse HEAD").strip()
1733        system("git rebase %s" % upstream)
1734        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1735        return True
1736
1737class P4Clone(P4Sync):
1738    def __init__(self):
1739        P4Sync.__init__(self)
1740        self.description = "Creates a new git repository and imports from Perforce into it"
1741        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1742        self.options += [
1743            optparse.make_option("--destination", dest="cloneDestination",
1744                                 action='store', default=None,
1745                                 help="where to leave result of the clone"),
1746            optparse.make_option("-/", dest="cloneExclude",
1747                                 action="append", type="string",
1748                                 help="exclude depot path")
1749        ]
1750        self.cloneDestination = None
1751        self.needsGit = False
1752
1753    # This is required for the "append" cloneExclude action
1754    def ensure_value(self, attr, value):
1755        if not hasattr(self, attr) or getattr(self, attr) is None:
1756            setattr(self, attr, value)
1757        return getattr(self, attr)
1758
1759    def defaultDestination(self, args):
1760        ## TODO: use common prefix of args?
1761        depotPath = args[0]
1762        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1763        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1764        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1765        depotDir = re.sub(r"/$", "", depotDir)
1766        return os.path.split(depotDir)[1]
1767
1768    def run(self, args):
1769        if len(args) < 1:
1770            return False
1771
1772        if self.keepRepoPath and not self.cloneDestination:
1773            sys.stderr.write("Must specify destination for --keep-path\n")
1774            sys.exit(1)
1775
1776        depotPaths = args
1777
1778        if not self.cloneDestination and len(depotPaths) > 1:
1779            self.cloneDestination = depotPaths[-1]
1780            depotPaths = depotPaths[:-1]
1781
1782        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1783        for p in depotPaths:
1784            if not p.startswith("//"):
1785                return False
1786
1787        if not self.cloneDestination:
1788            self.cloneDestination = self.defaultDestination(args)
1789
1790        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1791        if not os.path.exists(self.cloneDestination):
1792            os.makedirs(self.cloneDestination)
1793        chdir(self.cloneDestination)
1794        system("git init")
1795        self.gitdir = os.getcwd() + "/.git"
1796        if not P4Sync.run(self, depotPaths):
1797            return False
1798        if self.branch != "master":
1799            if self.importIntoRemotes:
1800                masterbranch = "refs/remotes/p4/master"
1801            else:
1802                masterbranch = "refs/heads/p4/master"
1803            if gitBranchExists(masterbranch):
1804                system("git branch master %s" % masterbranch)
1805                system("git checkout -f")
1806            else:
1807                print "Could not detect main branch. No checkout/master branch created."
1808
1809        return True
1810
1811class P4Branches(Command):
1812    def __init__(self):
1813        Command.__init__(self)
1814        self.options = [ ]
1815        self.description = ("Shows the git branches that hold imports and their "
1816                            + "corresponding perforce depot paths")
1817        self.verbose = False
1818
1819    def run(self, args):
1820        if originP4BranchesExist():
1821            createOrUpdateBranchesFromOrigin()
1822
1823        cmdline = "git rev-parse --symbolic "
1824        cmdline += " --remotes"
1825
1826        for line in read_pipe_lines(cmdline):
1827            line = line.strip()
1828
1829            if not line.startswith('p4/') or line == "p4/HEAD":
1830                continue
1831            branch = line
1832
1833            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1834            settings = extractSettingsGitLog(log)
1835
1836            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1837        return True
1838
1839class HelpFormatter(optparse.IndentedHelpFormatter):
1840    def __init__(self):
1841        optparse.IndentedHelpFormatter.__init__(self)
1842
1843    def format_description(self, description):
1844        if description:
1845            return description + "\n"
1846        else:
1847            return ""
1848
1849def printUsage(commands):
1850    print "usage: %s <command> [options]" % sys.argv[0]
1851    print ""
1852    print "valid commands: %s" % ", ".join(commands)
1853    print ""
1854    print "Try %s <command> --help for command specific help." % sys.argv[0]
1855    print ""
1856
1857commands = {
1858    "debug" : P4Debug,
1859    "submit" : P4Submit,
1860    "commit" : P4Submit,
1861    "sync" : P4Sync,
1862    "rebase" : P4Rebase,
1863    "clone" : P4Clone,
1864    "rollback" : P4RollBack,
1865    "branches" : P4Branches
1866}
1867
1868
1869def main():
1870    if len(sys.argv[1:]) == 0:
1871        printUsage(commands.keys())
1872        sys.exit(2)
1873
1874    cmd = ""
1875    cmdName = sys.argv[1]
1876    try:
1877        klass = commands[cmdName]
1878        cmd = klass()
1879    except KeyError:
1880        print "unknown command %s" % cmdName
1881        print ""
1882        printUsage(commands.keys())
1883        sys.exit(2)
1884
1885    options = cmd.options
1886    cmd.gitdir = os.environ.get("GIT_DIR", None)
1887
1888    args = sys.argv[2:]
1889
1890    if len(options) > 0:
1891        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1892
1893        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1894                                       options,
1895                                       description = cmd.description,
1896                                       formatter = HelpFormatter())
1897
1898        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1899    global verbose
1900    verbose = cmd.verbose
1901    if cmd.needsGit:
1902        if cmd.gitdir == None:
1903            cmd.gitdir = os.path.abspath(".git")
1904            if not isValidGitDir(cmd.gitdir):
1905                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1906                if os.path.exists(cmd.gitdir):
1907                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1908                    if len(cdup) > 0:
1909                        chdir(cdup);
1910
1911        if not isValidGitDir(cmd.gitdir):
1912            if isValidGitDir(cmd.gitdir + "/.git"):
1913                cmd.gitdir += "/.git"
1914            else:
1915                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1916
1917        os.environ["GIT_DIR"] = cmd.gitdir
1918
1919    if not cmd.run(args):
1920        parser.print_help()
1921
1922
1923if __name__ == '__main__':
1924    main()