Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'jl/maint-pull-tags-doc' (03276d9)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    def __init__(self):
 838        Command.__init__(self)
 839        self.options = [
 840                optparse.make_option("--branch", dest="branch"),
 841                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                optparse.make_option("--changesfile", dest="changesFile"),
 843                optparse.make_option("--silent", dest="silent", action="store_true"),
 844                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                     help="Import into refs/heads/ , not refs/remotes"),
 848                optparse.make_option("--max-changes", dest="maxChanges"),
 849                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                     help="Only sync files that are included in the Perforce Client Spec")
 853        ]
 854        self.description = """Imports from Perforce into a git repository.\n
 855    example:
 856    //depot/my/project/ -- to import the current head
 857    //depot/my/project/@all -- to import everything
 858    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862        self.usage += " //depot/path[@revRange]"
 863        self.silent = False
 864        self.createdBranches = set()
 865        self.committedChanges = set()
 866        self.branch = ""
 867        self.detectBranches = False
 868        self.detectLabels = False
 869        self.changesFile = ""
 870        self.syncWithOrigin = True
 871        self.verbose = False
 872        self.importIntoRemotes = True
 873        self.maxChanges = ""
 874        self.isWindows = (platform.system() == "Windows")
 875        self.keepRepoPath = False
 876        self.depotPaths = None
 877        self.p4BranchesInGit = []
 878        self.cloneExclude = []
 879        self.useClientSpec = False
 880        self.clientSpecDirs = []
 881
 882        if gitConfig("git-p4.syncFromOrigin") == "false":
 883            self.syncWithOrigin = False
 884
 885    def extractFilesFromCommit(self, commit):
 886        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                             for path in self.cloneExclude]
 888        files = []
 889        fnum = 0
 890        while commit.has_key("depotFile%s" % fnum):
 891            path =  commit["depotFile%s" % fnum]
 892
 893            if [p for p in self.cloneExclude
 894                if path.startswith (p)]:
 895                found = False
 896            else:
 897                found = [p for p in self.depotPaths
 898                         if path.startswith (p)]
 899            if not found:
 900                fnum = fnum + 1
 901                continue
 902
 903            file = {}
 904            file["path"] = path
 905            file["rev"] = commit["rev%s" % fnum]
 906            file["action"] = commit["action%s" % fnum]
 907            file["type"] = commit["type%s" % fnum]
 908            files.append(file)
 909            fnum = fnum + 1
 910        return files
 911
 912    def stripRepoPath(self, path, prefixes):
 913        if self.keepRepoPath:
 914            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 915
 916        for p in prefixes:
 917            if path.startswith(p):
 918                path = path[len(p):]
 919
 920        return path
 921
 922    def splitFilesIntoBranches(self, commit):
 923        branches = {}
 924        fnum = 0
 925        while commit.has_key("depotFile%s" % fnum):
 926            path =  commit["depotFile%s" % fnum]
 927            found = [p for p in self.depotPaths
 928                     if path.startswith (p)]
 929            if not found:
 930                fnum = fnum + 1
 931                continue
 932
 933            file = {}
 934            file["path"] = path
 935            file["rev"] = commit["rev%s" % fnum]
 936            file["action"] = commit["action%s" % fnum]
 937            file["type"] = commit["type%s" % fnum]
 938            fnum = fnum + 1
 939
 940            relPath = self.stripRepoPath(path, self.depotPaths)
 941
 942            for branch in self.knownBranches.keys():
 943
 944                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 945                if relPath.startswith(branch + "/"):
 946                    if branch not in branches:
 947                        branches[branch] = []
 948                    branches[branch].append(file)
 949                    break
 950
 951        return branches
 952
 953    # output one file from the P4 stream
 954    # - helper for streamP4Files
 955
 956    def streamOneP4File(self, file, contents):
 957        if file["type"] == "apple":
 958            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 959                file['depotFile']
 960            return
 961
 962        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 963        if verbose:
 964            sys.stderr.write("%s\n" % relPath)
 965
 966        mode = "644"
 967        if isP4Exec(file["type"]):
 968            mode = "755"
 969        elif file["type"] == "symlink":
 970            mode = "120000"
 971            # p4 print on a symlink contains "target\n", so strip it off
 972            data = ''.join(contents)
 973            contents = [data[:-1]]
 974
 975        if self.isWindows and file["type"].endswith("text"):
 976            mangled = []
 977            for data in contents:
 978                data = data.replace("\r\n", "\n")
 979                mangled.append(data)
 980            contents = mangled
 981
 982        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 983            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 984        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 985            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 986
 987        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 988
 989        # total length...
 990        length = 0
 991        for d in contents:
 992            length = length + len(d)
 993
 994        self.gitStream.write("data %d\n" % length)
 995        for d in contents:
 996            self.gitStream.write(d)
 997        self.gitStream.write("\n")
 998
 999    def streamOneP4Deletion(self, file):
1000        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1001        if verbose:
1002            sys.stderr.write("delete %s\n" % relPath)
1003        self.gitStream.write("D %s\n" % relPath)
1004
1005    # handle another chunk of streaming data
1006    def streamP4FilesCb(self, marshalled):
1007
1008        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1009            # start of a new file - output the old one first
1010            self.streamOneP4File(self.stream_file, self.stream_contents)
1011            self.stream_file = {}
1012            self.stream_contents = []
1013            self.stream_have_file_info = False
1014
1015        # pick up the new file information... for the
1016        # 'data' field we need to append to our array
1017        for k in marshalled.keys():
1018            if k == 'data':
1019                self.stream_contents.append(marshalled['data'])
1020            else:
1021                self.stream_file[k] = marshalled[k]
1022
1023        self.stream_have_file_info = True
1024
1025    # Stream directly from "p4 files" into "git fast-import"
1026    def streamP4Files(self, files):
1027        filesForCommit = []
1028        filesToRead = []
1029        filesToDelete = []
1030
1031        for f in files:
1032            includeFile = True
1033            for val in self.clientSpecDirs:
1034                if f['path'].startswith(val[0]):
1035                    if val[1] <= 0:
1036                        includeFile = False
1037                    break
1038
1039            if includeFile:
1040                filesForCommit.append(f)
1041                if f['action'] not in ('delete', 'move/delete', 'purge'):
1042                    filesToRead.append(f)
1043                else:
1044                    filesToDelete.append(f)
1045
1046        # deleted files...
1047        for f in filesToDelete:
1048            self.streamOneP4Deletion(f)
1049
1050        if len(filesToRead) > 0:
1051            self.stream_file = {}
1052            self.stream_contents = []
1053            self.stream_have_file_info = False
1054
1055            # curry self argument
1056            def streamP4FilesCbSelf(entry):
1057                self.streamP4FilesCb(entry)
1058
1059            p4CmdList("-x - print",
1060                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1061                                                  for f in filesToRead]),
1062                cb=streamP4FilesCbSelf)
1063
1064            # do the last chunk
1065            if self.stream_file.has_key('depotFile'):
1066                self.streamOneP4File(self.stream_file, self.stream_contents)
1067
1068    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1069        epoch = details["time"]
1070        author = details["user"]
1071        self.branchPrefixes = branchPrefixes
1072
1073        if self.verbose:
1074            print "commit into %s" % branch
1075
1076        # start with reading files; if that fails, we should not
1077        # create a commit.
1078        new_files = []
1079        for f in files:
1080            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1081                new_files.append (f)
1082            else:
1083                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1084
1085        self.gitStream.write("commit %s\n" % branch)
1086#        gitStream.write("mark :%s\n" % details["change"])
1087        self.committedChanges.add(int(details["change"]))
1088        committer = ""
1089        if author not in self.users:
1090            self.getUserMapFromPerforceServer()
1091        if author in self.users:
1092            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1093        else:
1094            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1095
1096        self.gitStream.write("committer %s\n" % committer)
1097
1098        self.gitStream.write("data <<EOT\n")
1099        self.gitStream.write(details["desc"])
1100        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1101                             % (','.join (branchPrefixes), details["change"]))
1102        if len(details['options']) > 0:
1103            self.gitStream.write(": options = %s" % details['options'])
1104        self.gitStream.write("]\nEOT\n\n")
1105
1106        if len(parent) > 0:
1107            if self.verbose:
1108                print "parent %s" % parent
1109            self.gitStream.write("from %s\n" % parent)
1110
1111        self.streamP4Files(new_files)
1112        self.gitStream.write("\n")
1113
1114        change = int(details["change"])
1115
1116        if self.labels.has_key(change):
1117            label = self.labels[change]
1118            labelDetails = label[0]
1119            labelRevisions = label[1]
1120            if self.verbose:
1121                print "Change %s is labelled %s" % (change, labelDetails)
1122
1123            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1124                                                    for p in branchPrefixes]))
1125
1126            if len(files) == len(labelRevisions):
1127
1128                cleanedFiles = {}
1129                for info in files:
1130                    if info["action"] in ("delete", "purge"):
1131                        continue
1132                    cleanedFiles[info["depotFile"]] = info["rev"]
1133
1134                if cleanedFiles == labelRevisions:
1135                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1136                    self.gitStream.write("from %s\n" % branch)
1137
1138                    owner = labelDetails["Owner"]
1139                    tagger = ""
1140                    if author in self.users:
1141                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1142                    else:
1143                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1144                    self.gitStream.write("tagger %s\n" % tagger)
1145                    self.gitStream.write("data <<EOT\n")
1146                    self.gitStream.write(labelDetails["Description"])
1147                    self.gitStream.write("EOT\n\n")
1148
1149                else:
1150                    if not self.silent:
1151                        print ("Tag %s does not match with change %s: files do not match."
1152                               % (labelDetails["label"], change))
1153
1154            else:
1155                if not self.silent:
1156                    print ("Tag %s does not match with change %s: file count is different."
1157                           % (labelDetails["label"], change))
1158
1159    def getUserCacheFilename(self):
1160        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1161        return home + "/.gitp4-usercache.txt"
1162
1163    def getUserMapFromPerforceServer(self):
1164        if self.userMapFromPerforceServer:
1165            return
1166        self.users = {}
1167
1168        for output in p4CmdList("users"):
1169            if not output.has_key("User"):
1170                continue
1171            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1172
1173
1174        s = ''
1175        for (key, val) in self.users.items():
1176            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1177
1178        open(self.getUserCacheFilename(), "wb").write(s)
1179        self.userMapFromPerforceServer = True
1180
1181    def loadUserMapFromCache(self):
1182        self.users = {}
1183        self.userMapFromPerforceServer = False
1184        try:
1185            cache = open(self.getUserCacheFilename(), "rb")
1186            lines = cache.readlines()
1187            cache.close()
1188            for line in lines:
1189                entry = line.strip().split("\t")
1190                self.users[entry[0]] = entry[1]
1191        except IOError:
1192            self.getUserMapFromPerforceServer()
1193
1194    def getLabels(self):
1195        self.labels = {}
1196
1197        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1198        if len(l) > 0 and not self.silent:
1199            print "Finding files belonging to labels in %s" % `self.depotPaths`
1200
1201        for output in l:
1202            label = output["label"]
1203            revisions = {}
1204            newestChange = 0
1205            if self.verbose:
1206                print "Querying files for label %s" % label
1207            for file in p4CmdList("files "
1208                                  +  ' '.join (["%s...@%s" % (p, label)
1209                                                for p in self.depotPaths])):
1210                revisions[file["depotFile"]] = file["rev"]
1211                change = int(file["change"])
1212                if change > newestChange:
1213                    newestChange = change
1214
1215            self.labels[newestChange] = [output, revisions]
1216
1217        if self.verbose:
1218            print "Label changes: %s" % self.labels.keys()
1219
1220    def guessProjectName(self):
1221        for p in self.depotPaths:
1222            if p.endswith("/"):
1223                p = p[:-1]
1224            p = p[p.strip().rfind("/") + 1:]
1225            if not p.endswith("/"):
1226               p += "/"
1227            return p
1228
1229    def getBranchMapping(self):
1230        lostAndFoundBranches = set()
1231
1232        for info in p4CmdList("branches"):
1233            details = p4Cmd("branch -o %s" % info["branch"])
1234            viewIdx = 0
1235            while details.has_key("View%s" % viewIdx):
1236                paths = details["View%s" % viewIdx].split(" ")
1237                viewIdx = viewIdx + 1
1238                # require standard //depot/foo/... //depot/bar/... mapping
1239                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1240                    continue
1241                source = paths[0]
1242                destination = paths[1]
1243                ## HACK
1244                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1245                    source = source[len(self.depotPaths[0]):-4]
1246                    destination = destination[len(self.depotPaths[0]):-4]
1247
1248                    if destination in self.knownBranches:
1249                        if not self.silent:
1250                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1251                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1252                        continue
1253
1254                    self.knownBranches[destination] = source
1255
1256                    lostAndFoundBranches.discard(destination)
1257
1258                    if source not in self.knownBranches:
1259                        lostAndFoundBranches.add(source)
1260
1261
1262        for branch in lostAndFoundBranches:
1263            self.knownBranches[branch] = branch
1264
1265    def getBranchMappingFromGitBranches(self):
1266        branches = p4BranchesInGit(self.importIntoRemotes)
1267        for branch in branches.keys():
1268            if branch == "master":
1269                branch = "main"
1270            else:
1271                branch = branch[len(self.projectName):]
1272            self.knownBranches[branch] = branch
1273
1274    def listExistingP4GitBranches(self):
1275        # branches holds mapping from name to commit
1276        branches = p4BranchesInGit(self.importIntoRemotes)
1277        self.p4BranchesInGit = branches.keys()
1278        for branch in branches.keys():
1279            self.initialParents[self.refPrefix + branch] = branches[branch]
1280
1281    def updateOptionDict(self, d):
1282        option_keys = {}
1283        if self.keepRepoPath:
1284            option_keys['keepRepoPath'] = 1
1285
1286        d["options"] = ' '.join(sorted(option_keys.keys()))
1287
1288    def readOptions(self, d):
1289        self.keepRepoPath = (d.has_key('options')
1290                             and ('keepRepoPath' in d['options']))
1291
1292    def gitRefForBranch(self, branch):
1293        if branch == "main":
1294            return self.refPrefix + "master"
1295
1296        if len(branch) <= 0:
1297            return branch
1298
1299        return self.refPrefix + self.projectName + branch
1300
1301    def gitCommitByP4Change(self, ref, change):
1302        if self.verbose:
1303            print "looking in ref " + ref + " for change %s using bisect..." % change
1304
1305        earliestCommit = ""
1306        latestCommit = parseRevision(ref)
1307
1308        while True:
1309            if self.verbose:
1310                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1311            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1312            if len(next) == 0:
1313                if self.verbose:
1314                    print "argh"
1315                return ""
1316            log = extractLogMessageFromGitCommit(next)
1317            settings = extractSettingsGitLog(log)
1318            currentChange = int(settings['change'])
1319            if self.verbose:
1320                print "current change %s" % currentChange
1321
1322            if currentChange == change:
1323                if self.verbose:
1324                    print "found %s" % next
1325                return next
1326
1327            if currentChange < change:
1328                earliestCommit = "^%s" % next
1329            else:
1330                latestCommit = "%s" % next
1331
1332        return ""
1333
1334    def importNewBranch(self, branch, maxChange):
1335        # make fast-import flush all changes to disk and update the refs using the checkpoint
1336        # command so that we can try to find the branch parent in the git history
1337        self.gitStream.write("checkpoint\n\n");
1338        self.gitStream.flush();
1339        branchPrefix = self.depotPaths[0] + branch + "/"
1340        range = "@1,%s" % maxChange
1341        #print "prefix" + branchPrefix
1342        changes = p4ChangesForPaths([branchPrefix], range)
1343        if len(changes) <= 0:
1344            return False
1345        firstChange = changes[0]
1346        #print "first change in branch: %s" % firstChange
1347        sourceBranch = self.knownBranches[branch]
1348        sourceDepotPath = self.depotPaths[0] + sourceBranch
1349        sourceRef = self.gitRefForBranch(sourceBranch)
1350        #print "source " + sourceBranch
1351
1352        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1353        #print "branch parent: %s" % branchParentChange
1354        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1355        if len(gitParent) > 0:
1356            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1357            #print "parent git commit: %s" % gitParent
1358
1359        self.importChanges(changes)
1360        return True
1361
1362    def importChanges(self, changes):
1363        cnt = 1
1364        for change in changes:
1365            description = p4Cmd("describe %s" % change)
1366            self.updateOptionDict(description)
1367
1368            if not self.silent:
1369                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1370                sys.stdout.flush()
1371            cnt = cnt + 1
1372
1373            try:
1374                if self.detectBranches:
1375                    branches = self.splitFilesIntoBranches(description)
1376                    for branch in branches.keys():
1377                        ## HACK  --hwn
1378                        branchPrefix = self.depotPaths[0] + branch + "/"
1379
1380                        parent = ""
1381
1382                        filesForCommit = branches[branch]
1383
1384                        if self.verbose:
1385                            print "branch is %s" % branch
1386
1387                        self.updatedBranches.add(branch)
1388
1389                        if branch not in self.createdBranches:
1390                            self.createdBranches.add(branch)
1391                            parent = self.knownBranches[branch]
1392                            if parent == branch:
1393                                parent = ""
1394                            else:
1395                                fullBranch = self.projectName + branch
1396                                if fullBranch not in self.p4BranchesInGit:
1397                                    if not self.silent:
1398                                        print("\n    Importing new branch %s" % fullBranch);
1399                                    if self.importNewBranch(branch, change - 1):
1400                                        parent = ""
1401                                        self.p4BranchesInGit.append(fullBranch)
1402                                    if not self.silent:
1403                                        print("\n    Resuming with change %s" % change);
1404
1405                                if self.verbose:
1406                                    print "parent determined through known branches: %s" % parent
1407
1408                        branch = self.gitRefForBranch(branch)
1409                        parent = self.gitRefForBranch(parent)
1410
1411                        if self.verbose:
1412                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1413
1414                        if len(parent) == 0 and branch in self.initialParents:
1415                            parent = self.initialParents[branch]
1416                            del self.initialParents[branch]
1417
1418                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1419                else:
1420                    files = self.extractFilesFromCommit(description)
1421                    self.commit(description, files, self.branch, self.depotPaths,
1422                                self.initialParent)
1423                    self.initialParent = ""
1424            except IOError:
1425                print self.gitError.read()
1426                sys.exit(1)
1427
1428    def importHeadRevision(self, revision):
1429        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1430
1431        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1432        details["desc"] = ("Initial import of %s from the state at revision %s"
1433                           % (' '.join(self.depotPaths), revision))
1434        details["change"] = revision
1435        newestRevision = 0
1436
1437        fileCnt = 0
1438        for info in p4CmdList("files "
1439                              +  ' '.join(["%s...%s"
1440                                           % (p, revision)
1441                                           for p in self.depotPaths])):
1442
1443            if info['code'] == 'error':
1444                sys.stderr.write("p4 returned an error: %s\n"
1445                                 % info['data'])
1446                sys.exit(1)
1447
1448
1449            change = int(info["change"])
1450            if change > newestRevision:
1451                newestRevision = change
1452
1453            if info["action"] in ("delete", "purge"):
1454                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1455                #fileCnt = fileCnt + 1
1456                continue
1457
1458            for prop in ["depotFile", "rev", "action", "type" ]:
1459                details["%s%s" % (prop, fileCnt)] = info[prop]
1460
1461            fileCnt = fileCnt + 1
1462
1463        details["change"] = newestRevision
1464        self.updateOptionDict(details)
1465        try:
1466            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1467        except IOError:
1468            print "IO error with git fast-import. Is your git version recent enough?"
1469            print self.gitError.read()
1470
1471
1472    def getClientSpec(self):
1473        specList = p4CmdList( "client -o" )
1474        temp = {}
1475        for entry in specList:
1476            for k,v in entry.iteritems():
1477                if k.startswith("View"):
1478                    if v.startswith('"'):
1479                        start = 1
1480                    else:
1481                        start = 0
1482                    index = v.find("...")
1483                    v = v[start:index]
1484                    if v.startswith("-"):
1485                        v = v[1:]
1486                        temp[v] = -len(v)
1487                    else:
1488                        temp[v] = len(v)
1489        self.clientSpecDirs = temp.items()
1490        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1491
1492    def run(self, args):
1493        self.depotPaths = []
1494        self.changeRange = ""
1495        self.initialParent = ""
1496        self.previousDepotPaths = []
1497
1498        # map from branch depot path to parent branch
1499        self.knownBranches = {}
1500        self.initialParents = {}
1501        self.hasOrigin = originP4BranchesExist()
1502        if not self.syncWithOrigin:
1503            self.hasOrigin = False
1504
1505        if self.importIntoRemotes:
1506            self.refPrefix = "refs/remotes/p4/"
1507        else:
1508            self.refPrefix = "refs/heads/p4/"
1509
1510        if self.syncWithOrigin and self.hasOrigin:
1511            if not self.silent:
1512                print "Syncing with origin first by calling git fetch origin"
1513            system("git fetch origin")
1514
1515        if len(self.branch) == 0:
1516            self.branch = self.refPrefix + "master"
1517            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1518                system("git update-ref %s refs/heads/p4" % self.branch)
1519                system("git branch -D p4");
1520            # create it /after/ importing, when master exists
1521            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1522                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1523
1524        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1525            self.getClientSpec()
1526
1527        # TODO: should always look at previous commits,
1528        # merge with previous imports, if possible.
1529        if args == []:
1530            if self.hasOrigin:
1531                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1532            self.listExistingP4GitBranches()
1533
1534            if len(self.p4BranchesInGit) > 1:
1535                if not self.silent:
1536                    print "Importing from/into multiple branches"
1537                self.detectBranches = True
1538
1539            if self.verbose:
1540                print "branches: %s" % self.p4BranchesInGit
1541
1542            p4Change = 0
1543            for branch in self.p4BranchesInGit:
1544                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1545
1546                settings = extractSettingsGitLog(logMsg)
1547
1548                self.readOptions(settings)
1549                if (settings.has_key('depot-paths')
1550                    and settings.has_key ('change')):
1551                    change = int(settings['change']) + 1
1552                    p4Change = max(p4Change, change)
1553
1554                    depotPaths = sorted(settings['depot-paths'])
1555                    if self.previousDepotPaths == []:
1556                        self.previousDepotPaths = depotPaths
1557                    else:
1558                        paths = []
1559                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1560                            for i in range(0, min(len(cur), len(prev))):
1561                                if cur[i] <> prev[i]:
1562                                    i = i - 1
1563                                    break
1564
1565                            paths.append (cur[:i + 1])
1566
1567                        self.previousDepotPaths = paths
1568
1569            if p4Change > 0:
1570                self.depotPaths = sorted(self.previousDepotPaths)
1571                self.changeRange = "@%s,#head" % p4Change
1572                if not self.detectBranches:
1573                    self.initialParent = parseRevision(self.branch)
1574                if not self.silent and not self.detectBranches:
1575                    print "Performing incremental import into %s git branch" % self.branch
1576
1577        if not self.branch.startswith("refs/"):
1578            self.branch = "refs/heads/" + self.branch
1579
1580        if len(args) == 0 and self.depotPaths:
1581            if not self.silent:
1582                print "Depot paths: %s" % ' '.join(self.depotPaths)
1583        else:
1584            if self.depotPaths and self.depotPaths != args:
1585                print ("previous import used depot path %s and now %s was specified. "
1586                       "This doesn't work!" % (' '.join (self.depotPaths),
1587                                               ' '.join (args)))
1588                sys.exit(1)
1589
1590            self.depotPaths = sorted(args)
1591
1592        revision = ""
1593        self.users = {}
1594
1595        newPaths = []
1596        for p in self.depotPaths:
1597            if p.find("@") != -1:
1598                atIdx = p.index("@")
1599                self.changeRange = p[atIdx:]
1600                if self.changeRange == "@all":
1601                    self.changeRange = ""
1602                elif ',' not in self.changeRange:
1603                    revision = self.changeRange
1604                    self.changeRange = ""
1605                p = p[:atIdx]
1606            elif p.find("#") != -1:
1607                hashIdx = p.index("#")
1608                revision = p[hashIdx:]
1609                p = p[:hashIdx]
1610            elif self.previousDepotPaths == []:
1611                revision = "#head"
1612
1613            p = re.sub ("\.\.\.$", "", p)
1614            if not p.endswith("/"):
1615                p += "/"
1616
1617            newPaths.append(p)
1618
1619        self.depotPaths = newPaths
1620
1621
1622        self.loadUserMapFromCache()
1623        self.labels = {}
1624        if self.detectLabels:
1625            self.getLabels();
1626
1627        if self.detectBranches:
1628            ## FIXME - what's a P4 projectName ?
1629            self.projectName = self.guessProjectName()
1630
1631            if self.hasOrigin:
1632                self.getBranchMappingFromGitBranches()
1633            else:
1634                self.getBranchMapping()
1635            if self.verbose:
1636                print "p4-git branches: %s" % self.p4BranchesInGit
1637                print "initial parents: %s" % self.initialParents
1638            for b in self.p4BranchesInGit:
1639                if b != "master":
1640
1641                    ## FIXME
1642                    b = b[len(self.projectName):]
1643                self.createdBranches.add(b)
1644
1645        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1646
1647        importProcess = subprocess.Popen(["git", "fast-import"],
1648                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1649                                         stderr=subprocess.PIPE);
1650        self.gitOutput = importProcess.stdout
1651        self.gitStream = importProcess.stdin
1652        self.gitError = importProcess.stderr
1653
1654        if revision:
1655            self.importHeadRevision(revision)
1656        else:
1657            changes = []
1658
1659            if len(self.changesFile) > 0:
1660                output = open(self.changesFile).readlines()
1661                changeSet = set()
1662                for line in output:
1663                    changeSet.add(int(line))
1664
1665                for change in changeSet:
1666                    changes.append(change)
1667
1668                changes.sort()
1669            else:
1670                if self.verbose:
1671                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1672                                                              self.changeRange)
1673                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1674
1675                if len(self.maxChanges) > 0:
1676                    changes = changes[:min(int(self.maxChanges), len(changes))]
1677
1678            if len(changes) == 0:
1679                if not self.silent:
1680                    print "No changes to import!"
1681                return True
1682
1683            if not self.silent and not self.detectBranches:
1684                print "Import destination: %s" % self.branch
1685
1686            self.updatedBranches = set()
1687
1688            self.importChanges(changes)
1689
1690            if not self.silent:
1691                print ""
1692                if len(self.updatedBranches) > 0:
1693                    sys.stdout.write("Updated branches: ")
1694                    for b in self.updatedBranches:
1695                        sys.stdout.write("%s " % b)
1696                    sys.stdout.write("\n")
1697
1698        self.gitStream.close()
1699        if importProcess.wait() != 0:
1700            die("fast-import failed: %s" % self.gitError.read())
1701        self.gitOutput.close()
1702        self.gitError.close()
1703
1704        return True
1705
1706class P4Rebase(Command):
1707    def __init__(self):
1708        Command.__init__(self)
1709        self.options = [ ]
1710        self.description = ("Fetches the latest revision from perforce and "
1711                            + "rebases the current work (branch) against it")
1712        self.verbose = False
1713
1714    def run(self, args):
1715        sync = P4Sync()
1716        sync.run([])
1717
1718        return self.rebase()
1719
1720    def rebase(self):
1721        if os.system("git update-index --refresh") != 0:
1722            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1723        if len(read_pipe("git diff-index HEAD --")) > 0:
1724            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1725
1726        [upstream, settings] = findUpstreamBranchPoint()
1727        if len(upstream) == 0:
1728            die("Cannot find upstream branchpoint for rebase")
1729
1730        # the branchpoint may be p4/foo~3, so strip off the parent
1731        upstream = re.sub("~[0-9]+$", "", upstream)
1732
1733        print "Rebasing the current branch onto %s" % upstream
1734        oldHead = read_pipe("git rev-parse HEAD").strip()
1735        system("git rebase %s" % upstream)
1736        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1737        return True
1738
1739class P4Clone(P4Sync):
1740    def __init__(self):
1741        P4Sync.__init__(self)
1742        self.description = "Creates a new git repository and imports from Perforce into it"
1743        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1744        self.options += [
1745            optparse.make_option("--destination", dest="cloneDestination",
1746                                 action='store', default=None,
1747                                 help="where to leave result of the clone"),
1748            optparse.make_option("-/", dest="cloneExclude",
1749                                 action="append", type="string",
1750                                 help="exclude depot path")
1751        ]
1752        self.cloneDestination = None
1753        self.needsGit = False
1754
1755    # This is required for the "append" cloneExclude action
1756    def ensure_value(self, attr, value):
1757        if not hasattr(self, attr) or getattr(self, attr) is None:
1758            setattr(self, attr, value)
1759        return getattr(self, attr)
1760
1761    def defaultDestination(self, args):
1762        ## TODO: use common prefix of args?
1763        depotPath = args[0]
1764        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1765        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1766        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1767        depotDir = re.sub(r"/$", "", depotDir)
1768        return os.path.split(depotDir)[1]
1769
1770    def run(self, args):
1771        if len(args) < 1:
1772            return False
1773
1774        if self.keepRepoPath and not self.cloneDestination:
1775            sys.stderr.write("Must specify destination for --keep-path\n")
1776            sys.exit(1)
1777
1778        depotPaths = args
1779
1780        if not self.cloneDestination and len(depotPaths) > 1:
1781            self.cloneDestination = depotPaths[-1]
1782            depotPaths = depotPaths[:-1]
1783
1784        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1785        for p in depotPaths:
1786            if not p.startswith("//"):
1787                return False
1788
1789        if not self.cloneDestination:
1790            self.cloneDestination = self.defaultDestination(args)
1791
1792        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1793        if not os.path.exists(self.cloneDestination):
1794            os.makedirs(self.cloneDestination)
1795        chdir(self.cloneDestination)
1796        system("git init")
1797        self.gitdir = os.getcwd() + "/.git"
1798        if not P4Sync.run(self, depotPaths):
1799            return False
1800        if self.branch != "master":
1801            if self.importIntoRemotes:
1802                masterbranch = "refs/remotes/p4/master"
1803            else:
1804                masterbranch = "refs/heads/p4/master"
1805            if gitBranchExists(masterbranch):
1806                system("git branch master %s" % masterbranch)
1807                system("git checkout -f")
1808            else:
1809                print "Could not detect main branch. No checkout/master branch created."
1810
1811        return True
1812
1813class P4Branches(Command):
1814    def __init__(self):
1815        Command.__init__(self)
1816        self.options = [ ]
1817        self.description = ("Shows the git branches that hold imports and their "
1818                            + "corresponding perforce depot paths")
1819        self.verbose = False
1820
1821    def run(self, args):
1822        if originP4BranchesExist():
1823            createOrUpdateBranchesFromOrigin()
1824
1825        cmdline = "git rev-parse --symbolic "
1826        cmdline += " --remotes"
1827
1828        for line in read_pipe_lines(cmdline):
1829            line = line.strip()
1830
1831            if not line.startswith('p4/') or line == "p4/HEAD":
1832                continue
1833            branch = line
1834
1835            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1836            settings = extractSettingsGitLog(log)
1837
1838            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1839        return True
1840
1841class HelpFormatter(optparse.IndentedHelpFormatter):
1842    def __init__(self):
1843        optparse.IndentedHelpFormatter.__init__(self)
1844
1845    def format_description(self, description):
1846        if description:
1847            return description + "\n"
1848        else:
1849            return ""
1850
1851def printUsage(commands):
1852    print "usage: %s <command> [options]" % sys.argv[0]
1853    print ""
1854    print "valid commands: %s" % ", ".join(commands)
1855    print ""
1856    print "Try %s <command> --help for command specific help." % sys.argv[0]
1857    print ""
1858
1859commands = {
1860    "debug" : P4Debug,
1861    "submit" : P4Submit,
1862    "commit" : P4Submit,
1863    "sync" : P4Sync,
1864    "rebase" : P4Rebase,
1865    "clone" : P4Clone,
1866    "rollback" : P4RollBack,
1867    "branches" : P4Branches
1868}
1869
1870
1871def main():
1872    if len(sys.argv[1:]) == 0:
1873        printUsage(commands.keys())
1874        sys.exit(2)
1875
1876    cmd = ""
1877    cmdName = sys.argv[1]
1878    try:
1879        klass = commands[cmdName]
1880        cmd = klass()
1881    except KeyError:
1882        print "unknown command %s" % cmdName
1883        print ""
1884        printUsage(commands.keys())
1885        sys.exit(2)
1886
1887    options = cmd.options
1888    cmd.gitdir = os.environ.get("GIT_DIR", None)
1889
1890    args = sys.argv[2:]
1891
1892    if len(options) > 0:
1893        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1894
1895        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1896                                       options,
1897                                       description = cmd.description,
1898                                       formatter = HelpFormatter())
1899
1900        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1901    global verbose
1902    verbose = cmd.verbose
1903    if cmd.needsGit:
1904        if cmd.gitdir == None:
1905            cmd.gitdir = os.path.abspath(".git")
1906            if not isValidGitDir(cmd.gitdir):
1907                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1908                if os.path.exists(cmd.gitdir):
1909                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1910                    if len(cdup) > 0:
1911                        chdir(cdup);
1912
1913        if not isValidGitDir(cmd.gitdir):
1914            if isValidGitDir(cmd.gitdir + "/.git"):
1915                cmd.gitdir += "/.git"
1916            else:
1917                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1918
1919        os.environ["GIT_DIR"] = cmd.gitdir
1920
1921    if not cmd.run(args):
1922        parser.print_help()
1923
1924
1925if __name__ == '__main__':
1926    main()