Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'jk/unwanted-advices' (dc1b0c0)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = p4_read_pipe("diff -du ...")
 710
 711            newdiff = ""
 712            for newFile in filesToAdd:
 713                newdiff += "==== new file ====\n"
 714                newdiff += "--- /dev/null\n"
 715                newdiff += "+++ %s\n" % newFile
 716                f = open(newFile, "r")
 717                for line in f.readlines():
 718                    newdiff += "+" + line
 719                f.close()
 720
 721            separatorLine = "######## everything below this line is just the diff #######\n"
 722
 723            [handle, fileName] = tempfile.mkstemp()
 724            tmpFile = os.fdopen(handle, "w+")
 725            if self.isWindows:
 726                submitTemplate = submitTemplate.replace("\n", "\r\n")
 727                separatorLine = separatorLine.replace("\n", "\r\n")
 728                newdiff = newdiff.replace("\n", "\r\n")
 729            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 730            tmpFile.close()
 731            mtime = os.stat(fileName).st_mtime
 732            defaultEditor = "vi"
 733            if platform.system() == "Windows":
 734                defaultEditor = "notepad"
 735            if os.environ.has_key("P4EDITOR"):
 736                editor = os.environ.get("P4EDITOR")
 737            else:
 738                editor = os.environ.get("EDITOR", defaultEditor);
 739            system(editor + " " + fileName)
 740
 741            response = "y"
 742            if os.stat(fileName).st_mtime <= mtime:
 743                response = "x"
 744                while response != "y" and response != "n":
 745                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 746
 747            if response == "y":
 748                tmpFile = open(fileName, "rb")
 749                message = tmpFile.read()
 750                tmpFile.close()
 751                submitTemplate = message[:message.index(separatorLine)]
 752                if self.isWindows:
 753                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 754                p4_write_pipe("submit -i", submitTemplate)
 755            else:
 756                for f in editedFiles:
 757                    p4_system("revert \"%s\"" % f);
 758                for f in filesToAdd:
 759                    p4_system("revert \"%s\"" % f);
 760                    system("rm %s" %f)
 761
 762            os.remove(fileName)
 763        else:
 764            fileName = "submit.txt"
 765            file = open(fileName, "w+")
 766            file.write(self.prepareLogMessage(template, logMessage))
 767            file.close()
 768            print ("Perforce submit template written as %s. "
 769                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 770                   % (fileName, fileName))
 771
 772    def run(self, args):
 773        if len(args) == 0:
 774            self.master = currentGitBranch()
 775            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 776                die("Detecting current git branch failed!")
 777        elif len(args) == 1:
 778            self.master = args[0]
 779        else:
 780            return False
 781
 782        allowSubmit = gitConfig("git-p4.allowSubmit")
 783        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 784            die("%s is not in git-p4.allowSubmit" % self.master)
 785
 786        [upstream, settings] = findUpstreamBranchPoint()
 787        self.depotPath = settings['depot-paths'][0]
 788        if len(self.origin) == 0:
 789            self.origin = upstream
 790
 791        if self.verbose:
 792            print "Origin branch is " + self.origin
 793
 794        if len(self.depotPath) == 0:
 795            print "Internal error: cannot locate perforce depot path from existing branches"
 796            sys.exit(128)
 797
 798        self.clientPath = p4Where(self.depotPath)
 799
 800        if len(self.clientPath) == 0:
 801            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 802            sys.exit(128)
 803
 804        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 805        self.oldWorkingDirectory = os.getcwd()
 806
 807        chdir(self.clientPath)
 808        print "Syncronizing p4 checkout..."
 809        p4_system("sync ...")
 810
 811        self.check()
 812
 813        commits = []
 814        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 815            commits.append(line.strip())
 816        commits.reverse()
 817
 818        while len(commits) > 0:
 819            commit = commits[0]
 820            commits = commits[1:]
 821            self.applyCommit(commit)
 822            if not self.interactive:
 823                break
 824
 825        if len(commits) == 0:
 826            print "All changes applied!"
 827            chdir(self.oldWorkingDirectory)
 828
 829            sync = P4Sync()
 830            sync.run([])
 831
 832            rebase = P4Rebase()
 833            rebase.rebase()
 834
 835        return True
 836
 837class P4Sync(Command):
 838    def __init__(self):
 839        Command.__init__(self)
 840        self.options = [
 841                optparse.make_option("--branch", dest="branch"),
 842                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 843                optparse.make_option("--changesfile", dest="changesFile"),
 844                optparse.make_option("--silent", dest="silent", action="store_true"),
 845                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 846                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 847                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 848                                     help="Import into refs/heads/ , not refs/remotes"),
 849                optparse.make_option("--max-changes", dest="maxChanges"),
 850                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 851                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 852                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 853                                     help="Only sync files that are included in the Perforce Client Spec")
 854        ]
 855        self.description = """Imports from Perforce into a git repository.\n
 856    example:
 857    //depot/my/project/ -- to import the current head
 858    //depot/my/project/@all -- to import everything
 859    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 860
 861    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 862
 863        self.usage += " //depot/path[@revRange]"
 864        self.silent = False
 865        self.createdBranches = set()
 866        self.committedChanges = set()
 867        self.branch = ""
 868        self.detectBranches = False
 869        self.detectLabels = False
 870        self.changesFile = ""
 871        self.syncWithOrigin = True
 872        self.verbose = False
 873        self.importIntoRemotes = True
 874        self.maxChanges = ""
 875        self.isWindows = (platform.system() == "Windows")
 876        self.keepRepoPath = False
 877        self.depotPaths = None
 878        self.p4BranchesInGit = []
 879        self.cloneExclude = []
 880        self.useClientSpec = False
 881        self.clientSpecDirs = []
 882
 883        if gitConfig("git-p4.syncFromOrigin") == "false":
 884            self.syncWithOrigin = False
 885
 886    def extractFilesFromCommit(self, commit):
 887        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 888                             for path in self.cloneExclude]
 889        files = []
 890        fnum = 0
 891        while commit.has_key("depotFile%s" % fnum):
 892            path =  commit["depotFile%s" % fnum]
 893
 894            if [p for p in self.cloneExclude
 895                if path.startswith (p)]:
 896                found = False
 897            else:
 898                found = [p for p in self.depotPaths
 899                         if path.startswith (p)]
 900            if not found:
 901                fnum = fnum + 1
 902                continue
 903
 904            file = {}
 905            file["path"] = path
 906            file["rev"] = commit["rev%s" % fnum]
 907            file["action"] = commit["action%s" % fnum]
 908            file["type"] = commit["type%s" % fnum]
 909            files.append(file)
 910            fnum = fnum + 1
 911        return files
 912
 913    def stripRepoPath(self, path, prefixes):
 914        if self.keepRepoPath:
 915            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 916
 917        for p in prefixes:
 918            if path.startswith(p):
 919                path = path[len(p):]
 920
 921        return path
 922
 923    def splitFilesIntoBranches(self, commit):
 924        branches = {}
 925        fnum = 0
 926        while commit.has_key("depotFile%s" % fnum):
 927            path =  commit["depotFile%s" % fnum]
 928            found = [p for p in self.depotPaths
 929                     if path.startswith (p)]
 930            if not found:
 931                fnum = fnum + 1
 932                continue
 933
 934            file = {}
 935            file["path"] = path
 936            file["rev"] = commit["rev%s" % fnum]
 937            file["action"] = commit["action%s" % fnum]
 938            file["type"] = commit["type%s" % fnum]
 939            fnum = fnum + 1
 940
 941            relPath = self.stripRepoPath(path, self.depotPaths)
 942
 943            for branch in self.knownBranches.keys():
 944
 945                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 946                if relPath.startswith(branch + "/"):
 947                    if branch not in branches:
 948                        branches[branch] = []
 949                    branches[branch].append(file)
 950                    break
 951
 952        return branches
 953
 954    # output one file from the P4 stream
 955    # - helper for streamP4Files
 956
 957    def streamOneP4File(self, file, contents):
 958        if file["type"] == "apple":
 959            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 960                file['depotFile']
 961            return
 962
 963        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 964        if verbose:
 965            sys.stderr.write("%s\n" % relPath)
 966
 967        mode = "644"
 968        if isP4Exec(file["type"]):
 969            mode = "755"
 970        elif file["type"] == "symlink":
 971            mode = "120000"
 972            # p4 print on a symlink contains "target\n", so strip it off
 973            last = contents.pop()
 974            last = last[:-1]
 975            contents.append(last)
 976
 977        if self.isWindows and file["type"].endswith("text"):
 978            mangled = []
 979            for data in contents:
 980                data = data.replace("\r\n", "\n")
 981                mangled.append(data)
 982            contents = mangled
 983
 984        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 985            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 986        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 987            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 988
 989        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990
 991        # total length...
 992        length = 0
 993        for d in contents:
 994            length = length + len(d)
 995
 996        self.gitStream.write("data %d\n" % length)
 997        for d in contents:
 998            self.gitStream.write(d)
 999        self.gitStream.write("\n")
1000
1001    def streamOneP4Deletion(self, file):
1002        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1003        if verbose:
1004            sys.stderr.write("delete %s\n" % relPath)
1005        self.gitStream.write("D %s\n" % relPath)
1006
1007    # handle another chunk of streaming data
1008    def streamP4FilesCb(self, marshalled):
1009
1010        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1011            # start of a new file - output the old one first
1012            self.streamOneP4File(self.stream_file, self.stream_contents)
1013            self.stream_file = {}
1014            self.stream_contents = []
1015            self.stream_have_file_info = False
1016
1017        # pick up the new file information... for the
1018        # 'data' field we need to append to our array
1019        for k in marshalled.keys():
1020            if k == 'data':
1021                self.stream_contents.append(marshalled['data'])
1022            else:
1023                self.stream_file[k] = marshalled[k]
1024
1025        self.stream_have_file_info = True
1026
1027    # Stream directly from "p4 files" into "git fast-import"
1028    def streamP4Files(self, files):
1029        filesForCommit = []
1030        filesToRead = []
1031        filesToDelete = []
1032
1033        for f in files:
1034            includeFile = True
1035            for val in self.clientSpecDirs:
1036                if f['path'].startswith(val[0]):
1037                    if val[1] <= 0:
1038                        includeFile = False
1039                    break
1040
1041            if includeFile:
1042                filesForCommit.append(f)
1043                if f['action'] not in ('delete', 'purge'):
1044                    filesToRead.append(f)
1045                else:
1046                    filesToDelete.append(f)
1047
1048        # deleted files...
1049        for f in filesToDelete:
1050            self.streamOneP4Deletion(f)
1051
1052        if len(filesToRead) > 0:
1053            self.stream_file = {}
1054            self.stream_contents = []
1055            self.stream_have_file_info = False
1056
1057            # curry self argument
1058            def streamP4FilesCbSelf(entry):
1059                self.streamP4FilesCb(entry)
1060
1061            p4CmdList("-x - print",
1062                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1063                                                  for f in filesToRead]),
1064                cb=streamP4FilesCbSelf)
1065
1066            # do the last chunk
1067            if self.stream_file.has_key('depotFile'):
1068                self.streamOneP4File(self.stream_file, self.stream_contents)
1069
1070    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1071        epoch = details["time"]
1072        author = details["user"]
1073        self.branchPrefixes = branchPrefixes
1074
1075        if self.verbose:
1076            print "commit into %s" % branch
1077
1078        # start with reading files; if that fails, we should not
1079        # create a commit.
1080        new_files = []
1081        for f in files:
1082            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1083                new_files.append (f)
1084            else:
1085                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1086
1087        self.gitStream.write("commit %s\n" % branch)
1088#        gitStream.write("mark :%s\n" % details["change"])
1089        self.committedChanges.add(int(details["change"]))
1090        committer = ""
1091        if author not in self.users:
1092            self.getUserMapFromPerforceServer()
1093        if author in self.users:
1094            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1095        else:
1096            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1097
1098        self.gitStream.write("committer %s\n" % committer)
1099
1100        self.gitStream.write("data <<EOT\n")
1101        self.gitStream.write(details["desc"])
1102        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1103                             % (','.join (branchPrefixes), details["change"]))
1104        if len(details['options']) > 0:
1105            self.gitStream.write(": options = %s" % details['options'])
1106        self.gitStream.write("]\nEOT\n\n")
1107
1108        if len(parent) > 0:
1109            if self.verbose:
1110                print "parent %s" % parent
1111            self.gitStream.write("from %s\n" % parent)
1112
1113        self.streamP4Files(new_files)
1114        self.gitStream.write("\n")
1115
1116        change = int(details["change"])
1117
1118        if self.labels.has_key(change):
1119            label = self.labels[change]
1120            labelDetails = label[0]
1121            labelRevisions = label[1]
1122            if self.verbose:
1123                print "Change %s is labelled %s" % (change, labelDetails)
1124
1125            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1126                                                    for p in branchPrefixes]))
1127
1128            if len(files) == len(labelRevisions):
1129
1130                cleanedFiles = {}
1131                for info in files:
1132                    if info["action"] in ("delete", "purge"):
1133                        continue
1134                    cleanedFiles[info["depotFile"]] = info["rev"]
1135
1136                if cleanedFiles == labelRevisions:
1137                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1138                    self.gitStream.write("from %s\n" % branch)
1139
1140                    owner = labelDetails["Owner"]
1141                    tagger = ""
1142                    if author in self.users:
1143                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1144                    else:
1145                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1146                    self.gitStream.write("tagger %s\n" % tagger)
1147                    self.gitStream.write("data <<EOT\n")
1148                    self.gitStream.write(labelDetails["Description"])
1149                    self.gitStream.write("EOT\n\n")
1150
1151                else:
1152                    if not self.silent:
1153                        print ("Tag %s does not match with change %s: files do not match."
1154                               % (labelDetails["label"], change))
1155
1156            else:
1157                if not self.silent:
1158                    print ("Tag %s does not match with change %s: file count is different."
1159                           % (labelDetails["label"], change))
1160
1161    def getUserCacheFilename(self):
1162        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1163        return home + "/.gitp4-usercache.txt"
1164
1165    def getUserMapFromPerforceServer(self):
1166        if self.userMapFromPerforceServer:
1167            return
1168        self.users = {}
1169
1170        for output in p4CmdList("users"):
1171            if not output.has_key("User"):
1172                continue
1173            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174
1175
1176        s = ''
1177        for (key, val) in self.users.items():
1178            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1179
1180        open(self.getUserCacheFilename(), "wb").write(s)
1181        self.userMapFromPerforceServer = True
1182
1183    def loadUserMapFromCache(self):
1184        self.users = {}
1185        self.userMapFromPerforceServer = False
1186        try:
1187            cache = open(self.getUserCacheFilename(), "rb")
1188            lines = cache.readlines()
1189            cache.close()
1190            for line in lines:
1191                entry = line.strip().split("\t")
1192                self.users[entry[0]] = entry[1]
1193        except IOError:
1194            self.getUserMapFromPerforceServer()
1195
1196    def getLabels(self):
1197        self.labels = {}
1198
1199        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1200        if len(l) > 0 and not self.silent:
1201            print "Finding files belonging to labels in %s" % `self.depotPaths`
1202
1203        for output in l:
1204            label = output["label"]
1205            revisions = {}
1206            newestChange = 0
1207            if self.verbose:
1208                print "Querying files for label %s" % label
1209            for file in p4CmdList("files "
1210                                  +  ' '.join (["%s...@%s" % (p, label)
1211                                                for p in self.depotPaths])):
1212                revisions[file["depotFile"]] = file["rev"]
1213                change = int(file["change"])
1214                if change > newestChange:
1215                    newestChange = change
1216
1217            self.labels[newestChange] = [output, revisions]
1218
1219        if self.verbose:
1220            print "Label changes: %s" % self.labels.keys()
1221
1222    def guessProjectName(self):
1223        for p in self.depotPaths:
1224            if p.endswith("/"):
1225                p = p[:-1]
1226            p = p[p.strip().rfind("/") + 1:]
1227            if not p.endswith("/"):
1228               p += "/"
1229            return p
1230
1231    def getBranchMapping(self):
1232        lostAndFoundBranches = set()
1233
1234        for info in p4CmdList("branches"):
1235            details = p4Cmd("branch -o %s" % info["branch"])
1236            viewIdx = 0
1237            while details.has_key("View%s" % viewIdx):
1238                paths = details["View%s" % viewIdx].split(" ")
1239                viewIdx = viewIdx + 1
1240                # require standard //depot/foo/... //depot/bar/... mapping
1241                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1242                    continue
1243                source = paths[0]
1244                destination = paths[1]
1245                ## HACK
1246                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1247                    source = source[len(self.depotPaths[0]):-4]
1248                    destination = destination[len(self.depotPaths[0]):-4]
1249
1250                    if destination in self.knownBranches:
1251                        if not self.silent:
1252                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1253                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1254                        continue
1255
1256                    self.knownBranches[destination] = source
1257
1258                    lostAndFoundBranches.discard(destination)
1259
1260                    if source not in self.knownBranches:
1261                        lostAndFoundBranches.add(source)
1262
1263
1264        for branch in lostAndFoundBranches:
1265            self.knownBranches[branch] = branch
1266
1267    def getBranchMappingFromGitBranches(self):
1268        branches = p4BranchesInGit(self.importIntoRemotes)
1269        for branch in branches.keys():
1270            if branch == "master":
1271                branch = "main"
1272            else:
1273                branch = branch[len(self.projectName):]
1274            self.knownBranches[branch] = branch
1275
1276    def listExistingP4GitBranches(self):
1277        # branches holds mapping from name to commit
1278        branches = p4BranchesInGit(self.importIntoRemotes)
1279        self.p4BranchesInGit = branches.keys()
1280        for branch in branches.keys():
1281            self.initialParents[self.refPrefix + branch] = branches[branch]
1282
1283    def updateOptionDict(self, d):
1284        option_keys = {}
1285        if self.keepRepoPath:
1286            option_keys['keepRepoPath'] = 1
1287
1288        d["options"] = ' '.join(sorted(option_keys.keys()))
1289
1290    def readOptions(self, d):
1291        self.keepRepoPath = (d.has_key('options')
1292                             and ('keepRepoPath' in d['options']))
1293
1294    def gitRefForBranch(self, branch):
1295        if branch == "main":
1296            return self.refPrefix + "master"
1297
1298        if len(branch) <= 0:
1299            return branch
1300
1301        return self.refPrefix + self.projectName + branch
1302
1303    def gitCommitByP4Change(self, ref, change):
1304        if self.verbose:
1305            print "looking in ref " + ref + " for change %s using bisect..." % change
1306
1307        earliestCommit = ""
1308        latestCommit = parseRevision(ref)
1309
1310        while True:
1311            if self.verbose:
1312                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1313            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1314            if len(next) == 0:
1315                if self.verbose:
1316                    print "argh"
1317                return ""
1318            log = extractLogMessageFromGitCommit(next)
1319            settings = extractSettingsGitLog(log)
1320            currentChange = int(settings['change'])
1321            if self.verbose:
1322                print "current change %s" % currentChange
1323
1324            if currentChange == change:
1325                if self.verbose:
1326                    print "found %s" % next
1327                return next
1328
1329            if currentChange < change:
1330                earliestCommit = "^%s" % next
1331            else:
1332                latestCommit = "%s" % next
1333
1334        return ""
1335
1336    def importNewBranch(self, branch, maxChange):
1337        # make fast-import flush all changes to disk and update the refs using the checkpoint
1338        # command so that we can try to find the branch parent in the git history
1339        self.gitStream.write("checkpoint\n\n");
1340        self.gitStream.flush();
1341        branchPrefix = self.depotPaths[0] + branch + "/"
1342        range = "@1,%s" % maxChange
1343        #print "prefix" + branchPrefix
1344        changes = p4ChangesForPaths([branchPrefix], range)
1345        if len(changes) <= 0:
1346            return False
1347        firstChange = changes[0]
1348        #print "first change in branch: %s" % firstChange
1349        sourceBranch = self.knownBranches[branch]
1350        sourceDepotPath = self.depotPaths[0] + sourceBranch
1351        sourceRef = self.gitRefForBranch(sourceBranch)
1352        #print "source " + sourceBranch
1353
1354        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1355        #print "branch parent: %s" % branchParentChange
1356        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1357        if len(gitParent) > 0:
1358            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1359            #print "parent git commit: %s" % gitParent
1360
1361        self.importChanges(changes)
1362        return True
1363
1364    def importChanges(self, changes):
1365        cnt = 1
1366        for change in changes:
1367            description = p4Cmd("describe %s" % change)
1368            self.updateOptionDict(description)
1369
1370            if not self.silent:
1371                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1372                sys.stdout.flush()
1373            cnt = cnt + 1
1374
1375            try:
1376                if self.detectBranches:
1377                    branches = self.splitFilesIntoBranches(description)
1378                    for branch in branches.keys():
1379                        ## HACK  --hwn
1380                        branchPrefix = self.depotPaths[0] + branch + "/"
1381
1382                        parent = ""
1383
1384                        filesForCommit = branches[branch]
1385
1386                        if self.verbose:
1387                            print "branch is %s" % branch
1388
1389                        self.updatedBranches.add(branch)
1390
1391                        if branch not in self.createdBranches:
1392                            self.createdBranches.add(branch)
1393                            parent = self.knownBranches[branch]
1394                            if parent == branch:
1395                                parent = ""
1396                            else:
1397                                fullBranch = self.projectName + branch
1398                                if fullBranch not in self.p4BranchesInGit:
1399                                    if not self.silent:
1400                                        print("\n    Importing new branch %s" % fullBranch);
1401                                    if self.importNewBranch(branch, change - 1):
1402                                        parent = ""
1403                                        self.p4BranchesInGit.append(fullBranch)
1404                                    if not self.silent:
1405                                        print("\n    Resuming with change %s" % change);
1406
1407                                if self.verbose:
1408                                    print "parent determined through known branches: %s" % parent
1409
1410                        branch = self.gitRefForBranch(branch)
1411                        parent = self.gitRefForBranch(parent)
1412
1413                        if self.verbose:
1414                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1415
1416                        if len(parent) == 0 and branch in self.initialParents:
1417                            parent = self.initialParents[branch]
1418                            del self.initialParents[branch]
1419
1420                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1421                else:
1422                    files = self.extractFilesFromCommit(description)
1423                    self.commit(description, files, self.branch, self.depotPaths,
1424                                self.initialParent)
1425                    self.initialParent = ""
1426            except IOError:
1427                print self.gitError.read()
1428                sys.exit(1)
1429
1430    def importHeadRevision(self, revision):
1431        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1432
1433        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1434        details["desc"] = ("Initial import of %s from the state at revision %s"
1435                           % (' '.join(self.depotPaths), revision))
1436        details["change"] = revision
1437        newestRevision = 0
1438
1439        fileCnt = 0
1440        for info in p4CmdList("files "
1441                              +  ' '.join(["%s...%s"
1442                                           % (p, revision)
1443                                           for p in self.depotPaths])):
1444
1445            if info['code'] == 'error':
1446                sys.stderr.write("p4 returned an error: %s\n"
1447                                 % info['data'])
1448                sys.exit(1)
1449
1450
1451            change = int(info["change"])
1452            if change > newestRevision:
1453                newestRevision = change
1454
1455            if info["action"] in ("delete", "purge"):
1456                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1457                #fileCnt = fileCnt + 1
1458                continue
1459
1460            for prop in ["depotFile", "rev", "action", "type" ]:
1461                details["%s%s" % (prop, fileCnt)] = info[prop]
1462
1463            fileCnt = fileCnt + 1
1464
1465        details["change"] = newestRevision
1466        self.updateOptionDict(details)
1467        try:
1468            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1469        except IOError:
1470            print "IO error with git fast-import. Is your git version recent enough?"
1471            print self.gitError.read()
1472
1473
1474    def getClientSpec(self):
1475        specList = p4CmdList( "client -o" )
1476        temp = {}
1477        for entry in specList:
1478            for k,v in entry.iteritems():
1479                if k.startswith("View"):
1480                    if v.startswith('"'):
1481                        start = 1
1482                    else:
1483                        start = 0
1484                    index = v.find("...")
1485                    v = v[start:index]
1486                    if v.startswith("-"):
1487                        v = v[1:]
1488                        temp[v] = -len(v)
1489                    else:
1490                        temp[v] = len(v)
1491        self.clientSpecDirs = temp.items()
1492        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1493
1494    def run(self, args):
1495        self.depotPaths = []
1496        self.changeRange = ""
1497        self.initialParent = ""
1498        self.previousDepotPaths = []
1499
1500        # map from branch depot path to parent branch
1501        self.knownBranches = {}
1502        self.initialParents = {}
1503        self.hasOrigin = originP4BranchesExist()
1504        if not self.syncWithOrigin:
1505            self.hasOrigin = False
1506
1507        if self.importIntoRemotes:
1508            self.refPrefix = "refs/remotes/p4/"
1509        else:
1510            self.refPrefix = "refs/heads/p4/"
1511
1512        if self.syncWithOrigin and self.hasOrigin:
1513            if not self.silent:
1514                print "Syncing with origin first by calling git fetch origin"
1515            system("git fetch origin")
1516
1517        if len(self.branch) == 0:
1518            self.branch = self.refPrefix + "master"
1519            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1520                system("git update-ref %s refs/heads/p4" % self.branch)
1521                system("git branch -D p4");
1522            # create it /after/ importing, when master exists
1523            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1524                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1525
1526        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1527            self.getClientSpec()
1528
1529        # TODO: should always look at previous commits,
1530        # merge with previous imports, if possible.
1531        if args == []:
1532            if self.hasOrigin:
1533                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1534            self.listExistingP4GitBranches()
1535
1536            if len(self.p4BranchesInGit) > 1:
1537                if not self.silent:
1538                    print "Importing from/into multiple branches"
1539                self.detectBranches = True
1540
1541            if self.verbose:
1542                print "branches: %s" % self.p4BranchesInGit
1543
1544            p4Change = 0
1545            for branch in self.p4BranchesInGit:
1546                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1547
1548                settings = extractSettingsGitLog(logMsg)
1549
1550                self.readOptions(settings)
1551                if (settings.has_key('depot-paths')
1552                    and settings.has_key ('change')):
1553                    change = int(settings['change']) + 1
1554                    p4Change = max(p4Change, change)
1555
1556                    depotPaths = sorted(settings['depot-paths'])
1557                    if self.previousDepotPaths == []:
1558                        self.previousDepotPaths = depotPaths
1559                    else:
1560                        paths = []
1561                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1562                            for i in range(0, min(len(cur), len(prev))):
1563                                if cur[i] <> prev[i]:
1564                                    i = i - 1
1565                                    break
1566
1567                            paths.append (cur[:i + 1])
1568
1569                        self.previousDepotPaths = paths
1570
1571            if p4Change > 0:
1572                self.depotPaths = sorted(self.previousDepotPaths)
1573                self.changeRange = "@%s,#head" % p4Change
1574                if not self.detectBranches:
1575                    self.initialParent = parseRevision(self.branch)
1576                if not self.silent and not self.detectBranches:
1577                    print "Performing incremental import into %s git branch" % self.branch
1578
1579        if not self.branch.startswith("refs/"):
1580            self.branch = "refs/heads/" + self.branch
1581
1582        if len(args) == 0 and self.depotPaths:
1583            if not self.silent:
1584                print "Depot paths: %s" % ' '.join(self.depotPaths)
1585        else:
1586            if self.depotPaths and self.depotPaths != args:
1587                print ("previous import used depot path %s and now %s was specified. "
1588                       "This doesn't work!" % (' '.join (self.depotPaths),
1589                                               ' '.join (args)))
1590                sys.exit(1)
1591
1592            self.depotPaths = sorted(args)
1593
1594        revision = ""
1595        self.users = {}
1596
1597        newPaths = []
1598        for p in self.depotPaths:
1599            if p.find("@") != -1:
1600                atIdx = p.index("@")
1601                self.changeRange = p[atIdx:]
1602                if self.changeRange == "@all":
1603                    self.changeRange = ""
1604                elif ',' not in self.changeRange:
1605                    revision = self.changeRange
1606                    self.changeRange = ""
1607                p = p[:atIdx]
1608            elif p.find("#") != -1:
1609                hashIdx = p.index("#")
1610                revision = p[hashIdx:]
1611                p = p[:hashIdx]
1612            elif self.previousDepotPaths == []:
1613                revision = "#head"
1614
1615            p = re.sub ("\.\.\.$", "", p)
1616            if not p.endswith("/"):
1617                p += "/"
1618
1619            newPaths.append(p)
1620
1621        self.depotPaths = newPaths
1622
1623
1624        self.loadUserMapFromCache()
1625        self.labels = {}
1626        if self.detectLabels:
1627            self.getLabels();
1628
1629        if self.detectBranches:
1630            ## FIXME - what's a P4 projectName ?
1631            self.projectName = self.guessProjectName()
1632
1633            if self.hasOrigin:
1634                self.getBranchMappingFromGitBranches()
1635            else:
1636                self.getBranchMapping()
1637            if self.verbose:
1638                print "p4-git branches: %s" % self.p4BranchesInGit
1639                print "initial parents: %s" % self.initialParents
1640            for b in self.p4BranchesInGit:
1641                if b != "master":
1642
1643                    ## FIXME
1644                    b = b[len(self.projectName):]
1645                self.createdBranches.add(b)
1646
1647        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1648
1649        importProcess = subprocess.Popen(["git", "fast-import"],
1650                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1651                                         stderr=subprocess.PIPE);
1652        self.gitOutput = importProcess.stdout
1653        self.gitStream = importProcess.stdin
1654        self.gitError = importProcess.stderr
1655
1656        if revision:
1657            self.importHeadRevision(revision)
1658        else:
1659            changes = []
1660
1661            if len(self.changesFile) > 0:
1662                output = open(self.changesFile).readlines()
1663                changeSet = set()
1664                for line in output:
1665                    changeSet.add(int(line))
1666
1667                for change in changeSet:
1668                    changes.append(change)
1669
1670                changes.sort()
1671            else:
1672                if self.verbose:
1673                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1674                                                              self.changeRange)
1675                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1676
1677                if len(self.maxChanges) > 0:
1678                    changes = changes[:min(int(self.maxChanges), len(changes))]
1679
1680            if len(changes) == 0:
1681                if not self.silent:
1682                    print "No changes to import!"
1683                return True
1684
1685            if not self.silent and not self.detectBranches:
1686                print "Import destination: %s" % self.branch
1687
1688            self.updatedBranches = set()
1689
1690            self.importChanges(changes)
1691
1692            if not self.silent:
1693                print ""
1694                if len(self.updatedBranches) > 0:
1695                    sys.stdout.write("Updated branches: ")
1696                    for b in self.updatedBranches:
1697                        sys.stdout.write("%s " % b)
1698                    sys.stdout.write("\n")
1699
1700        self.gitStream.close()
1701        if importProcess.wait() != 0:
1702            die("fast-import failed: %s" % self.gitError.read())
1703        self.gitOutput.close()
1704        self.gitError.close()
1705
1706        return True
1707
1708class P4Rebase(Command):
1709    def __init__(self):
1710        Command.__init__(self)
1711        self.options = [ ]
1712        self.description = ("Fetches the latest revision from perforce and "
1713                            + "rebases the current work (branch) against it")
1714        self.verbose = False
1715
1716    def run(self, args):
1717        sync = P4Sync()
1718        sync.run([])
1719
1720        return self.rebase()
1721
1722    def rebase(self):
1723        if os.system("git update-index --refresh") != 0:
1724            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1725        if len(read_pipe("git diff-index HEAD --")) > 0:
1726            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1727
1728        [upstream, settings] = findUpstreamBranchPoint()
1729        if len(upstream) == 0:
1730            die("Cannot find upstream branchpoint for rebase")
1731
1732        # the branchpoint may be p4/foo~3, so strip off the parent
1733        upstream = re.sub("~[0-9]+$", "", upstream)
1734
1735        print "Rebasing the current branch onto %s" % upstream
1736        oldHead = read_pipe("git rev-parse HEAD").strip()
1737        system("git rebase %s" % upstream)
1738        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1739        return True
1740
1741class P4Clone(P4Sync):
1742    def __init__(self):
1743        P4Sync.__init__(self)
1744        self.description = "Creates a new git repository and imports from Perforce into it"
1745        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1746        self.options += [
1747            optparse.make_option("--destination", dest="cloneDestination",
1748                                 action='store', default=None,
1749                                 help="where to leave result of the clone"),
1750            optparse.make_option("-/", dest="cloneExclude",
1751                                 action="append", type="string",
1752                                 help="exclude depot path")
1753        ]
1754        self.cloneDestination = None
1755        self.needsGit = False
1756
1757    # This is required for the "append" cloneExclude action
1758    def ensure_value(self, attr, value):
1759        if not hasattr(self, attr) or getattr(self, attr) is None:
1760            setattr(self, attr, value)
1761        return getattr(self, attr)
1762
1763    def defaultDestination(self, args):
1764        ## TODO: use common prefix of args?
1765        depotPath = args[0]
1766        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1767        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1768        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1769        depotDir = re.sub(r"/$", "", depotDir)
1770        return os.path.split(depotDir)[1]
1771
1772    def run(self, args):
1773        if len(args) < 1:
1774            return False
1775
1776        if self.keepRepoPath and not self.cloneDestination:
1777            sys.stderr.write("Must specify destination for --keep-path\n")
1778            sys.exit(1)
1779
1780        depotPaths = args
1781
1782        if not self.cloneDestination and len(depotPaths) > 1:
1783            self.cloneDestination = depotPaths[-1]
1784            depotPaths = depotPaths[:-1]
1785
1786        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1787        for p in depotPaths:
1788            if not p.startswith("//"):
1789                return False
1790
1791        if not self.cloneDestination:
1792            self.cloneDestination = self.defaultDestination(args)
1793
1794        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1795        if not os.path.exists(self.cloneDestination):
1796            os.makedirs(self.cloneDestination)
1797        chdir(self.cloneDestination)
1798        system("git init")
1799        self.gitdir = os.getcwd() + "/.git"
1800        if not P4Sync.run(self, depotPaths):
1801            return False
1802        if self.branch != "master":
1803            if self.importIntoRemotes:
1804                masterbranch = "refs/remotes/p4/master"
1805            else:
1806                masterbranch = "refs/heads/p4/master"
1807            if gitBranchExists(masterbranch):
1808                system("git branch master %s" % masterbranch)
1809                system("git checkout -f")
1810            else:
1811                print "Could not detect main branch. No checkout/master branch created."
1812
1813        return True
1814
1815class P4Branches(Command):
1816    def __init__(self):
1817        Command.__init__(self)
1818        self.options = [ ]
1819        self.description = ("Shows the git branches that hold imports and their "
1820                            + "corresponding perforce depot paths")
1821        self.verbose = False
1822
1823    def run(self, args):
1824        if originP4BranchesExist():
1825            createOrUpdateBranchesFromOrigin()
1826
1827        cmdline = "git rev-parse --symbolic "
1828        cmdline += " --remotes"
1829
1830        for line in read_pipe_lines(cmdline):
1831            line = line.strip()
1832
1833            if not line.startswith('p4/') or line == "p4/HEAD":
1834                continue
1835            branch = line
1836
1837            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1838            settings = extractSettingsGitLog(log)
1839
1840            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1841        return True
1842
1843class HelpFormatter(optparse.IndentedHelpFormatter):
1844    def __init__(self):
1845        optparse.IndentedHelpFormatter.__init__(self)
1846
1847    def format_description(self, description):
1848        if description:
1849            return description + "\n"
1850        else:
1851            return ""
1852
1853def printUsage(commands):
1854    print "usage: %s <command> [options]" % sys.argv[0]
1855    print ""
1856    print "valid commands: %s" % ", ".join(commands)
1857    print ""
1858    print "Try %s <command> --help for command specific help." % sys.argv[0]
1859    print ""
1860
1861commands = {
1862    "debug" : P4Debug,
1863    "submit" : P4Submit,
1864    "commit" : P4Submit,
1865    "sync" : P4Sync,
1866    "rebase" : P4Rebase,
1867    "clone" : P4Clone,
1868    "rollback" : P4RollBack,
1869    "branches" : P4Branches
1870}
1871
1872
1873def main():
1874    if len(sys.argv[1:]) == 0:
1875        printUsage(commands.keys())
1876        sys.exit(2)
1877
1878    cmd = ""
1879    cmdName = sys.argv[1]
1880    try:
1881        klass = commands[cmdName]
1882        cmd = klass()
1883    except KeyError:
1884        print "unknown command %s" % cmdName
1885        print ""
1886        printUsage(commands.keys())
1887        sys.exit(2)
1888
1889    options = cmd.options
1890    cmd.gitdir = os.environ.get("GIT_DIR", None)
1891
1892    args = sys.argv[2:]
1893
1894    if len(options) > 0:
1895        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1896
1897        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1898                                       options,
1899                                       description = cmd.description,
1900                                       formatter = HelpFormatter())
1901
1902        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1903    global verbose
1904    verbose = cmd.verbose
1905    if cmd.needsGit:
1906        if cmd.gitdir == None:
1907            cmd.gitdir = os.path.abspath(".git")
1908            if not isValidGitDir(cmd.gitdir):
1909                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1910                if os.path.exists(cmd.gitdir):
1911                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1912                    if len(cdup) > 0:
1913                        chdir(cdup);
1914
1915        if not isValidGitDir(cmd.gitdir):
1916            if isValidGitDir(cmd.gitdir + "/.git"):
1917                cmd.gitdir += "/.git"
1918            else:
1919                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1920
1921        os.environ["GIT_DIR"] = cmd.gitdir
1922
1923    if not cmd.run(args):
1924        parser.print_help()
1925
1926
1927if __name__ == '__main__':
1928    main()