Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: fix key error for p4 problem (68b2859)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    def __init__(self):
 838        Command.__init__(self)
 839        self.options = [
 840                optparse.make_option("--branch", dest="branch"),
 841                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 842                optparse.make_option("--changesfile", dest="changesFile"),
 843                optparse.make_option("--silent", dest="silent", action="store_true"),
 844                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 845                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 846                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 847                                     help="Import into refs/heads/ , not refs/remotes"),
 848                optparse.make_option("--max-changes", dest="maxChanges"),
 849                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 850                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 851                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 852                                     help="Only sync files that are included in the Perforce Client Spec")
 853        ]
 854        self.description = """Imports from Perforce into a git repository.\n
 855    example:
 856    //depot/my/project/ -- to import the current head
 857    //depot/my/project/@all -- to import everything
 858    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 859
 860    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 861
 862        self.usage += " //depot/path[@revRange]"
 863        self.silent = False
 864        self.createdBranches = set()
 865        self.committedChanges = set()
 866        self.branch = ""
 867        self.detectBranches = False
 868        self.detectLabels = False
 869        self.changesFile = ""
 870        self.syncWithOrigin = True
 871        self.verbose = False
 872        self.importIntoRemotes = True
 873        self.maxChanges = ""
 874        self.isWindows = (platform.system() == "Windows")
 875        self.keepRepoPath = False
 876        self.depotPaths = None
 877        self.p4BranchesInGit = []
 878        self.cloneExclude = []
 879        self.useClientSpec = False
 880        self.clientSpecDirs = []
 881
 882        if gitConfig("git-p4.syncFromOrigin") == "false":
 883            self.syncWithOrigin = False
 884
 885    def extractFilesFromCommit(self, commit):
 886        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 887                             for path in self.cloneExclude]
 888        files = []
 889        fnum = 0
 890        while commit.has_key("depotFile%s" % fnum):
 891            path =  commit["depotFile%s" % fnum]
 892
 893            if [p for p in self.cloneExclude
 894                if path.startswith (p)]:
 895                found = False
 896            else:
 897                found = [p for p in self.depotPaths
 898                         if path.startswith (p)]
 899            if not found:
 900                fnum = fnum + 1
 901                continue
 902
 903            file = {}
 904            file["path"] = path
 905            file["rev"] = commit["rev%s" % fnum]
 906            file["action"] = commit["action%s" % fnum]
 907            file["type"] = commit["type%s" % fnum]
 908            files.append(file)
 909            fnum = fnum + 1
 910        return files
 911
 912    def stripRepoPath(self, path, prefixes):
 913        if self.keepRepoPath:
 914            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 915
 916        for p in prefixes:
 917            if path.startswith(p):
 918                path = path[len(p):]
 919
 920        return path
 921
 922    def splitFilesIntoBranches(self, commit):
 923        branches = {}
 924        fnum = 0
 925        while commit.has_key("depotFile%s" % fnum):
 926            path =  commit["depotFile%s" % fnum]
 927            found = [p for p in self.depotPaths
 928                     if path.startswith (p)]
 929            if not found:
 930                fnum = fnum + 1
 931                continue
 932
 933            file = {}
 934            file["path"] = path
 935            file["rev"] = commit["rev%s" % fnum]
 936            file["action"] = commit["action%s" % fnum]
 937            file["type"] = commit["type%s" % fnum]
 938            fnum = fnum + 1
 939
 940            relPath = self.stripRepoPath(path, self.depotPaths)
 941
 942            for branch in self.knownBranches.keys():
 943
 944                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 945                if relPath.startswith(branch + "/"):
 946                    if branch not in branches:
 947                        branches[branch] = []
 948                    branches[branch].append(file)
 949                    break
 950
 951        return branches
 952
 953    # output one file from the P4 stream
 954    # - helper for streamP4Files
 955
 956    def streamOneP4File(self, file, contents):
 957        if file["type"] == "apple":
 958            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 959                file['depotFile']
 960            return
 961
 962        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 963        if verbose:
 964            sys.stderr.write("%s\n" % relPath)
 965
 966        mode = "644"
 967        if isP4Exec(file["type"]):
 968            mode = "755"
 969        elif file["type"] == "symlink":
 970            mode = "120000"
 971            # p4 print on a symlink contains "target\n", so strip it off
 972            data = ''.join(contents)
 973            contents = [data[:-1]]
 974
 975        if self.isWindows and file["type"].endswith("text"):
 976            mangled = []
 977            for data in contents:
 978                data = data.replace("\r\n", "\n")
 979                mangled.append(data)
 980            contents = mangled
 981
 982        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 983            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 984        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 985            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 986
 987        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 988
 989        # total length...
 990        length = 0
 991        for d in contents:
 992            length = length + len(d)
 993
 994        self.gitStream.write("data %d\n" % length)
 995        for d in contents:
 996            self.gitStream.write(d)
 997        self.gitStream.write("\n")
 998
 999    def streamOneP4Deletion(self, file):
1000        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1001        if verbose:
1002            sys.stderr.write("delete %s\n" % relPath)
1003        self.gitStream.write("D %s\n" % relPath)
1004
1005    # handle another chunk of streaming data
1006    def streamP4FilesCb(self, marshalled):
1007
1008        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1009            # start of a new file - output the old one first
1010            self.streamOneP4File(self.stream_file, self.stream_contents)
1011            self.stream_file = {}
1012            self.stream_contents = []
1013            self.stream_have_file_info = False
1014
1015        # pick up the new file information... for the
1016        # 'data' field we need to append to our array
1017        for k in marshalled.keys():
1018            if k == 'data':
1019                self.stream_contents.append(marshalled['data'])
1020            else:
1021                self.stream_file[k] = marshalled[k]
1022
1023        self.stream_have_file_info = True
1024
1025    # Stream directly from "p4 files" into "git fast-import"
1026    def streamP4Files(self, files):
1027        filesForCommit = []
1028        filesToRead = []
1029        filesToDelete = []
1030
1031        for f in files:
1032            includeFile = True
1033            for val in self.clientSpecDirs:
1034                if f['path'].startswith(val[0]):
1035                    if val[1] <= 0:
1036                        includeFile = False
1037                    break
1038
1039            if includeFile:
1040                filesForCommit.append(f)
1041                if f['action'] not in ('delete', 'move/delete', 'purge'):
1042                    filesToRead.append(f)
1043                else:
1044                    filesToDelete.append(f)
1045
1046        # deleted files...
1047        for f in filesToDelete:
1048            self.streamOneP4Deletion(f)
1049
1050        if len(filesToRead) > 0:
1051            self.stream_file = {}
1052            self.stream_contents = []
1053            self.stream_have_file_info = False
1054
1055            # curry self argument
1056            def streamP4FilesCbSelf(entry):
1057                self.streamP4FilesCb(entry)
1058
1059            p4CmdList("-x - print",
1060                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1061                                                  for f in filesToRead]),
1062                cb=streamP4FilesCbSelf)
1063
1064            # do the last chunk
1065            if self.stream_file.has_key('depotFile'):
1066                self.streamOneP4File(self.stream_file, self.stream_contents)
1067
1068    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1069        epoch = details["time"]
1070        author = details["user"]
1071        self.branchPrefixes = branchPrefixes
1072
1073        if self.verbose:
1074            print "commit into %s" % branch
1075
1076        # start with reading files; if that fails, we should not
1077        # create a commit.
1078        new_files = []
1079        for f in files:
1080            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1081                new_files.append (f)
1082            else:
1083                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1084
1085        self.gitStream.write("commit %s\n" % branch)
1086#        gitStream.write("mark :%s\n" % details["change"])
1087        self.committedChanges.add(int(details["change"]))
1088        committer = ""
1089        if author not in self.users:
1090            self.getUserMapFromPerforceServer()
1091        if author in self.users:
1092            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1093        else:
1094            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1095
1096        self.gitStream.write("committer %s\n" % committer)
1097
1098        self.gitStream.write("data <<EOT\n")
1099        self.gitStream.write(details["desc"])
1100        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1101                             % (','.join (branchPrefixes), details["change"]))
1102        if len(details['options']) > 0:
1103            self.gitStream.write(": options = %s" % details['options'])
1104        self.gitStream.write("]\nEOT\n\n")
1105
1106        if len(parent) > 0:
1107            if self.verbose:
1108                print "parent %s" % parent
1109            self.gitStream.write("from %s\n" % parent)
1110
1111        self.streamP4Files(new_files)
1112        self.gitStream.write("\n")
1113
1114        change = int(details["change"])
1115
1116        if self.labels.has_key(change):
1117            label = self.labels[change]
1118            labelDetails = label[0]
1119            labelRevisions = label[1]
1120            if self.verbose:
1121                print "Change %s is labelled %s" % (change, labelDetails)
1122
1123            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1124                                                    for p in branchPrefixes]))
1125
1126            if len(files) == len(labelRevisions):
1127
1128                cleanedFiles = {}
1129                for info in files:
1130                    if info["action"] in ("delete", "purge"):
1131                        continue
1132                    cleanedFiles[info["depotFile"]] = info["rev"]
1133
1134                if cleanedFiles == labelRevisions:
1135                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1136                    self.gitStream.write("from %s\n" % branch)
1137
1138                    owner = labelDetails["Owner"]
1139                    tagger = ""
1140                    if author in self.users:
1141                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1142                    else:
1143                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1144                    self.gitStream.write("tagger %s\n" % tagger)
1145                    self.gitStream.write("data <<EOT\n")
1146                    self.gitStream.write(labelDetails["Description"])
1147                    self.gitStream.write("EOT\n\n")
1148
1149                else:
1150                    if not self.silent:
1151                        print ("Tag %s does not match with change %s: files do not match."
1152                               % (labelDetails["label"], change))
1153
1154            else:
1155                if not self.silent:
1156                    print ("Tag %s does not match with change %s: file count is different."
1157                           % (labelDetails["label"], change))
1158
1159    def getUserCacheFilename(self):
1160        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1161        return home + "/.gitp4-usercache.txt"
1162
1163    def getUserMapFromPerforceServer(self):
1164        if self.userMapFromPerforceServer:
1165            return
1166        self.users = {}
1167
1168        for output in p4CmdList("users"):
1169            if not output.has_key("User"):
1170                continue
1171            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1172
1173
1174        s = ''
1175        for (key, val) in self.users.items():
1176            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1177
1178        open(self.getUserCacheFilename(), "wb").write(s)
1179        self.userMapFromPerforceServer = True
1180
1181    def loadUserMapFromCache(self):
1182        self.users = {}
1183        self.userMapFromPerforceServer = False
1184        try:
1185            cache = open(self.getUserCacheFilename(), "rb")
1186            lines = cache.readlines()
1187            cache.close()
1188            for line in lines:
1189                entry = line.strip().split("\t")
1190                self.users[entry[0]] = entry[1]
1191        except IOError:
1192            self.getUserMapFromPerforceServer()
1193
1194    def getLabels(self):
1195        self.labels = {}
1196
1197        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1198        if len(l) > 0 and not self.silent:
1199            print "Finding files belonging to labels in %s" % `self.depotPaths`
1200
1201        for output in l:
1202            label = output["label"]
1203            revisions = {}
1204            newestChange = 0
1205            if self.verbose:
1206                print "Querying files for label %s" % label
1207            for file in p4CmdList("files "
1208                                  +  ' '.join (["%s...@%s" % (p, label)
1209                                                for p in self.depotPaths])):
1210                revisions[file["depotFile"]] = file["rev"]
1211                change = int(file["change"])
1212                if change > newestChange:
1213                    newestChange = change
1214
1215            self.labels[newestChange] = [output, revisions]
1216
1217        if self.verbose:
1218            print "Label changes: %s" % self.labels.keys()
1219
1220    def guessProjectName(self):
1221        for p in self.depotPaths:
1222            if p.endswith("/"):
1223                p = p[:-1]
1224            p = p[p.strip().rfind("/") + 1:]
1225            if not p.endswith("/"):
1226               p += "/"
1227            return p
1228
1229    def getBranchMapping(self):
1230        lostAndFoundBranches = set()
1231
1232        for info in p4CmdList("branches"):
1233            details = p4Cmd("branch -o %s" % info["branch"])
1234            viewIdx = 0
1235            while details.has_key("View%s" % viewIdx):
1236                paths = details["View%s" % viewIdx].split(" ")
1237                viewIdx = viewIdx + 1
1238                # require standard //depot/foo/... //depot/bar/... mapping
1239                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1240                    continue
1241                source = paths[0]
1242                destination = paths[1]
1243                ## HACK
1244                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1245                    source = source[len(self.depotPaths[0]):-4]
1246                    destination = destination[len(self.depotPaths[0]):-4]
1247
1248                    if destination in self.knownBranches:
1249                        if not self.silent:
1250                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1251                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1252                        continue
1253
1254                    self.knownBranches[destination] = source
1255
1256                    lostAndFoundBranches.discard(destination)
1257
1258                    if source not in self.knownBranches:
1259                        lostAndFoundBranches.add(source)
1260
1261
1262        for branch in lostAndFoundBranches:
1263            self.knownBranches[branch] = branch
1264
1265    def getBranchMappingFromGitBranches(self):
1266        branches = p4BranchesInGit(self.importIntoRemotes)
1267        for branch in branches.keys():
1268            if branch == "master":
1269                branch = "main"
1270            else:
1271                branch = branch[len(self.projectName):]
1272            self.knownBranches[branch] = branch
1273
1274    def listExistingP4GitBranches(self):
1275        # branches holds mapping from name to commit
1276        branches = p4BranchesInGit(self.importIntoRemotes)
1277        self.p4BranchesInGit = branches.keys()
1278        for branch in branches.keys():
1279            self.initialParents[self.refPrefix + branch] = branches[branch]
1280
1281    def updateOptionDict(self, d):
1282        option_keys = {}
1283        if self.keepRepoPath:
1284            option_keys['keepRepoPath'] = 1
1285
1286        d["options"] = ' '.join(sorted(option_keys.keys()))
1287
1288    def readOptions(self, d):
1289        self.keepRepoPath = (d.has_key('options')
1290                             and ('keepRepoPath' in d['options']))
1291
1292    def gitRefForBranch(self, branch):
1293        if branch == "main":
1294            return self.refPrefix + "master"
1295
1296        if len(branch) <= 0:
1297            return branch
1298
1299        return self.refPrefix + self.projectName + branch
1300
1301    def gitCommitByP4Change(self, ref, change):
1302        if self.verbose:
1303            print "looking in ref " + ref + " for change %s using bisect..." % change
1304
1305        earliestCommit = ""
1306        latestCommit = parseRevision(ref)
1307
1308        while True:
1309            if self.verbose:
1310                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1311            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1312            if len(next) == 0:
1313                if self.verbose:
1314                    print "argh"
1315                return ""
1316            log = extractLogMessageFromGitCommit(next)
1317            settings = extractSettingsGitLog(log)
1318            currentChange = int(settings['change'])
1319            if self.verbose:
1320                print "current change %s" % currentChange
1321
1322            if currentChange == change:
1323                if self.verbose:
1324                    print "found %s" % next
1325                return next
1326
1327            if currentChange < change:
1328                earliestCommit = "^%s" % next
1329            else:
1330                latestCommit = "%s" % next
1331
1332        return ""
1333
1334    def importNewBranch(self, branch, maxChange):
1335        # make fast-import flush all changes to disk and update the refs using the checkpoint
1336        # command so that we can try to find the branch parent in the git history
1337        self.gitStream.write("checkpoint\n\n");
1338        self.gitStream.flush();
1339        branchPrefix = self.depotPaths[0] + branch + "/"
1340        range = "@1,%s" % maxChange
1341        #print "prefix" + branchPrefix
1342        changes = p4ChangesForPaths([branchPrefix], range)
1343        if len(changes) <= 0:
1344            return False
1345        firstChange = changes[0]
1346        #print "first change in branch: %s" % firstChange
1347        sourceBranch = self.knownBranches[branch]
1348        sourceDepotPath = self.depotPaths[0] + sourceBranch
1349        sourceRef = self.gitRefForBranch(sourceBranch)
1350        #print "source " + sourceBranch
1351
1352        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1353        #print "branch parent: %s" % branchParentChange
1354        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1355        if len(gitParent) > 0:
1356            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1357            #print "parent git commit: %s" % gitParent
1358
1359        self.importChanges(changes)
1360        return True
1361
1362    def importChanges(self, changes):
1363        cnt = 1
1364        for change in changes:
1365            description = p4Cmd("describe %s" % change)
1366            self.updateOptionDict(description)
1367
1368            if not self.silent:
1369                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1370                sys.stdout.flush()
1371            cnt = cnt + 1
1372
1373            try:
1374                if self.detectBranches:
1375                    branches = self.splitFilesIntoBranches(description)
1376                    for branch in branches.keys():
1377                        ## HACK  --hwn
1378                        branchPrefix = self.depotPaths[0] + branch + "/"
1379
1380                        parent = ""
1381
1382                        filesForCommit = branches[branch]
1383
1384                        if self.verbose:
1385                            print "branch is %s" % branch
1386
1387                        self.updatedBranches.add(branch)
1388
1389                        if branch not in self.createdBranches:
1390                            self.createdBranches.add(branch)
1391                            parent = self.knownBranches[branch]
1392                            if parent == branch:
1393                                parent = ""
1394                            else:
1395                                fullBranch = self.projectName + branch
1396                                if fullBranch not in self.p4BranchesInGit:
1397                                    if not self.silent:
1398                                        print("\n    Importing new branch %s" % fullBranch);
1399                                    if self.importNewBranch(branch, change - 1):
1400                                        parent = ""
1401                                        self.p4BranchesInGit.append(fullBranch)
1402                                    if not self.silent:
1403                                        print("\n    Resuming with change %s" % change);
1404
1405                                if self.verbose:
1406                                    print "parent determined through known branches: %s" % parent
1407
1408                        branch = self.gitRefForBranch(branch)
1409                        parent = self.gitRefForBranch(parent)
1410
1411                        if self.verbose:
1412                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1413
1414                        if len(parent) == 0 and branch in self.initialParents:
1415                            parent = self.initialParents[branch]
1416                            del self.initialParents[branch]
1417
1418                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1419                else:
1420                    files = self.extractFilesFromCommit(description)
1421                    self.commit(description, files, self.branch, self.depotPaths,
1422                                self.initialParent)
1423                    self.initialParent = ""
1424            except IOError:
1425                print self.gitError.read()
1426                sys.exit(1)
1427
1428    def importHeadRevision(self, revision):
1429        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1430
1431        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1432        details["desc"] = ("Initial import of %s from the state at revision %s"
1433                           % (' '.join(self.depotPaths), revision))
1434        details["change"] = revision
1435        newestRevision = 0
1436
1437        fileCnt = 0
1438        for info in p4CmdList("files "
1439                              +  ' '.join(["%s...%s"
1440                                           % (p, revision)
1441                                           for p in self.depotPaths])):
1442
1443            if 'code' in info and info['code'] == 'error':
1444                sys.stderr.write("p4 returned an error: %s\n"
1445                                 % info['data'])
1446                sys.exit(1)
1447            if 'p4ExitCode' in info:
1448                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1449                sys.exit(1)
1450
1451
1452            change = int(info["change"])
1453            if change > newestRevision:
1454                newestRevision = change
1455
1456            if info["action"] in ("delete", "purge"):
1457                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1458                #fileCnt = fileCnt + 1
1459                continue
1460
1461            for prop in ["depotFile", "rev", "action", "type" ]:
1462                details["%s%s" % (prop, fileCnt)] = info[prop]
1463
1464            fileCnt = fileCnt + 1
1465
1466        details["change"] = newestRevision
1467        self.updateOptionDict(details)
1468        try:
1469            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1470        except IOError:
1471            print "IO error with git fast-import. Is your git version recent enough?"
1472            print self.gitError.read()
1473
1474
1475    def getClientSpec(self):
1476        specList = p4CmdList( "client -o" )
1477        temp = {}
1478        for entry in specList:
1479            for k,v in entry.iteritems():
1480                if k.startswith("View"):
1481                    if v.startswith('"'):
1482                        start = 1
1483                    else:
1484                        start = 0
1485                    index = v.find("...")
1486                    v = v[start:index]
1487                    if v.startswith("-"):
1488                        v = v[1:]
1489                        temp[v] = -len(v)
1490                    else:
1491                        temp[v] = len(v)
1492        self.clientSpecDirs = temp.items()
1493        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1494
1495    def run(self, args):
1496        self.depotPaths = []
1497        self.changeRange = ""
1498        self.initialParent = ""
1499        self.previousDepotPaths = []
1500
1501        # map from branch depot path to parent branch
1502        self.knownBranches = {}
1503        self.initialParents = {}
1504        self.hasOrigin = originP4BranchesExist()
1505        if not self.syncWithOrigin:
1506            self.hasOrigin = False
1507
1508        if self.importIntoRemotes:
1509            self.refPrefix = "refs/remotes/p4/"
1510        else:
1511            self.refPrefix = "refs/heads/p4/"
1512
1513        if self.syncWithOrigin and self.hasOrigin:
1514            if not self.silent:
1515                print "Syncing with origin first by calling git fetch origin"
1516            system("git fetch origin")
1517
1518        if len(self.branch) == 0:
1519            self.branch = self.refPrefix + "master"
1520            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1521                system("git update-ref %s refs/heads/p4" % self.branch)
1522                system("git branch -D p4");
1523            # create it /after/ importing, when master exists
1524            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1525                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1526
1527        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1528            self.getClientSpec()
1529
1530        # TODO: should always look at previous commits,
1531        # merge with previous imports, if possible.
1532        if args == []:
1533            if self.hasOrigin:
1534                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1535            self.listExistingP4GitBranches()
1536
1537            if len(self.p4BranchesInGit) > 1:
1538                if not self.silent:
1539                    print "Importing from/into multiple branches"
1540                self.detectBranches = True
1541
1542            if self.verbose:
1543                print "branches: %s" % self.p4BranchesInGit
1544
1545            p4Change = 0
1546            for branch in self.p4BranchesInGit:
1547                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1548
1549                settings = extractSettingsGitLog(logMsg)
1550
1551                self.readOptions(settings)
1552                if (settings.has_key('depot-paths')
1553                    and settings.has_key ('change')):
1554                    change = int(settings['change']) + 1
1555                    p4Change = max(p4Change, change)
1556
1557                    depotPaths = sorted(settings['depot-paths'])
1558                    if self.previousDepotPaths == []:
1559                        self.previousDepotPaths = depotPaths
1560                    else:
1561                        paths = []
1562                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1563                            for i in range(0, min(len(cur), len(prev))):
1564                                if cur[i] <> prev[i]:
1565                                    i = i - 1
1566                                    break
1567
1568                            paths.append (cur[:i + 1])
1569
1570                        self.previousDepotPaths = paths
1571
1572            if p4Change > 0:
1573                self.depotPaths = sorted(self.previousDepotPaths)
1574                self.changeRange = "@%s,#head" % p4Change
1575                if not self.detectBranches:
1576                    self.initialParent = parseRevision(self.branch)
1577                if not self.silent and not self.detectBranches:
1578                    print "Performing incremental import into %s git branch" % self.branch
1579
1580        if not self.branch.startswith("refs/"):
1581            self.branch = "refs/heads/" + self.branch
1582
1583        if len(args) == 0 and self.depotPaths:
1584            if not self.silent:
1585                print "Depot paths: %s" % ' '.join(self.depotPaths)
1586        else:
1587            if self.depotPaths and self.depotPaths != args:
1588                print ("previous import used depot path %s and now %s was specified. "
1589                       "This doesn't work!" % (' '.join (self.depotPaths),
1590                                               ' '.join (args)))
1591                sys.exit(1)
1592
1593            self.depotPaths = sorted(args)
1594
1595        revision = ""
1596        self.users = {}
1597
1598        newPaths = []
1599        for p in self.depotPaths:
1600            if p.find("@") != -1:
1601                atIdx = p.index("@")
1602                self.changeRange = p[atIdx:]
1603                if self.changeRange == "@all":
1604                    self.changeRange = ""
1605                elif ',' not in self.changeRange:
1606                    revision = self.changeRange
1607                    self.changeRange = ""
1608                p = p[:atIdx]
1609            elif p.find("#") != -1:
1610                hashIdx = p.index("#")
1611                revision = p[hashIdx:]
1612                p = p[:hashIdx]
1613            elif self.previousDepotPaths == []:
1614                revision = "#head"
1615
1616            p = re.sub ("\.\.\.$", "", p)
1617            if not p.endswith("/"):
1618                p += "/"
1619
1620            newPaths.append(p)
1621
1622        self.depotPaths = newPaths
1623
1624
1625        self.loadUserMapFromCache()
1626        self.labels = {}
1627        if self.detectLabels:
1628            self.getLabels();
1629
1630        if self.detectBranches:
1631            ## FIXME - what's a P4 projectName ?
1632            self.projectName = self.guessProjectName()
1633
1634            if self.hasOrigin:
1635                self.getBranchMappingFromGitBranches()
1636            else:
1637                self.getBranchMapping()
1638            if self.verbose:
1639                print "p4-git branches: %s" % self.p4BranchesInGit
1640                print "initial parents: %s" % self.initialParents
1641            for b in self.p4BranchesInGit:
1642                if b != "master":
1643
1644                    ## FIXME
1645                    b = b[len(self.projectName):]
1646                self.createdBranches.add(b)
1647
1648        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1649
1650        importProcess = subprocess.Popen(["git", "fast-import"],
1651                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1652                                         stderr=subprocess.PIPE);
1653        self.gitOutput = importProcess.stdout
1654        self.gitStream = importProcess.stdin
1655        self.gitError = importProcess.stderr
1656
1657        if revision:
1658            self.importHeadRevision(revision)
1659        else:
1660            changes = []
1661
1662            if len(self.changesFile) > 0:
1663                output = open(self.changesFile).readlines()
1664                changeSet = set()
1665                for line in output:
1666                    changeSet.add(int(line))
1667
1668                for change in changeSet:
1669                    changes.append(change)
1670
1671                changes.sort()
1672            else:
1673                if self.verbose:
1674                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1675                                                              self.changeRange)
1676                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1677
1678                if len(self.maxChanges) > 0:
1679                    changes = changes[:min(int(self.maxChanges), len(changes))]
1680
1681            if len(changes) == 0:
1682                if not self.silent:
1683                    print "No changes to import!"
1684                return True
1685
1686            if not self.silent and not self.detectBranches:
1687                print "Import destination: %s" % self.branch
1688
1689            self.updatedBranches = set()
1690
1691            self.importChanges(changes)
1692
1693            if not self.silent:
1694                print ""
1695                if len(self.updatedBranches) > 0:
1696                    sys.stdout.write("Updated branches: ")
1697                    for b in self.updatedBranches:
1698                        sys.stdout.write("%s " % b)
1699                    sys.stdout.write("\n")
1700
1701        self.gitStream.close()
1702        if importProcess.wait() != 0:
1703            die("fast-import failed: %s" % self.gitError.read())
1704        self.gitOutput.close()
1705        self.gitError.close()
1706
1707        return True
1708
1709class P4Rebase(Command):
1710    def __init__(self):
1711        Command.__init__(self)
1712        self.options = [ ]
1713        self.description = ("Fetches the latest revision from perforce and "
1714                            + "rebases the current work (branch) against it")
1715        self.verbose = False
1716
1717    def run(self, args):
1718        sync = P4Sync()
1719        sync.run([])
1720
1721        return self.rebase()
1722
1723    def rebase(self):
1724        if os.system("git update-index --refresh") != 0:
1725            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1726        if len(read_pipe("git diff-index HEAD --")) > 0:
1727            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1728
1729        [upstream, settings] = findUpstreamBranchPoint()
1730        if len(upstream) == 0:
1731            die("Cannot find upstream branchpoint for rebase")
1732
1733        # the branchpoint may be p4/foo~3, so strip off the parent
1734        upstream = re.sub("~[0-9]+$", "", upstream)
1735
1736        print "Rebasing the current branch onto %s" % upstream
1737        oldHead = read_pipe("git rev-parse HEAD").strip()
1738        system("git rebase %s" % upstream)
1739        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1740        return True
1741
1742class P4Clone(P4Sync):
1743    def __init__(self):
1744        P4Sync.__init__(self)
1745        self.description = "Creates a new git repository and imports from Perforce into it"
1746        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1747        self.options += [
1748            optparse.make_option("--destination", dest="cloneDestination",
1749                                 action='store', default=None,
1750                                 help="where to leave result of the clone"),
1751            optparse.make_option("-/", dest="cloneExclude",
1752                                 action="append", type="string",
1753                                 help="exclude depot path")
1754        ]
1755        self.cloneDestination = None
1756        self.needsGit = False
1757
1758    # This is required for the "append" cloneExclude action
1759    def ensure_value(self, attr, value):
1760        if not hasattr(self, attr) or getattr(self, attr) is None:
1761            setattr(self, attr, value)
1762        return getattr(self, attr)
1763
1764    def defaultDestination(self, args):
1765        ## TODO: use common prefix of args?
1766        depotPath = args[0]
1767        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1768        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1769        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1770        depotDir = re.sub(r"/$", "", depotDir)
1771        return os.path.split(depotDir)[1]
1772
1773    def run(self, args):
1774        if len(args) < 1:
1775            return False
1776
1777        if self.keepRepoPath and not self.cloneDestination:
1778            sys.stderr.write("Must specify destination for --keep-path\n")
1779            sys.exit(1)
1780
1781        depotPaths = args
1782
1783        if not self.cloneDestination and len(depotPaths) > 1:
1784            self.cloneDestination = depotPaths[-1]
1785            depotPaths = depotPaths[:-1]
1786
1787        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1788        for p in depotPaths:
1789            if not p.startswith("//"):
1790                return False
1791
1792        if not self.cloneDestination:
1793            self.cloneDestination = self.defaultDestination(args)
1794
1795        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1796        if not os.path.exists(self.cloneDestination):
1797            os.makedirs(self.cloneDestination)
1798        chdir(self.cloneDestination)
1799        system("git init")
1800        self.gitdir = os.getcwd() + "/.git"
1801        if not P4Sync.run(self, depotPaths):
1802            return False
1803        if self.branch != "master":
1804            if self.importIntoRemotes:
1805                masterbranch = "refs/remotes/p4/master"
1806            else:
1807                masterbranch = "refs/heads/p4/master"
1808            if gitBranchExists(masterbranch):
1809                system("git branch master %s" % masterbranch)
1810                system("git checkout -f")
1811            else:
1812                print "Could not detect main branch. No checkout/master branch created."
1813
1814        return True
1815
1816class P4Branches(Command):
1817    def __init__(self):
1818        Command.__init__(self)
1819        self.options = [ ]
1820        self.description = ("Shows the git branches that hold imports and their "
1821                            + "corresponding perforce depot paths")
1822        self.verbose = False
1823
1824    def run(self, args):
1825        if originP4BranchesExist():
1826            createOrUpdateBranchesFromOrigin()
1827
1828        cmdline = "git rev-parse --symbolic "
1829        cmdline += " --remotes"
1830
1831        for line in read_pipe_lines(cmdline):
1832            line = line.strip()
1833
1834            if not line.startswith('p4/') or line == "p4/HEAD":
1835                continue
1836            branch = line
1837
1838            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1839            settings = extractSettingsGitLog(log)
1840
1841            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1842        return True
1843
1844class HelpFormatter(optparse.IndentedHelpFormatter):
1845    def __init__(self):
1846        optparse.IndentedHelpFormatter.__init__(self)
1847
1848    def format_description(self, description):
1849        if description:
1850            return description + "\n"
1851        else:
1852            return ""
1853
1854def printUsage(commands):
1855    print "usage: %s <command> [options]" % sys.argv[0]
1856    print ""
1857    print "valid commands: %s" % ", ".join(commands)
1858    print ""
1859    print "Try %s <command> --help for command specific help." % sys.argv[0]
1860    print ""
1861
1862commands = {
1863    "debug" : P4Debug,
1864    "submit" : P4Submit,
1865    "commit" : P4Submit,
1866    "sync" : P4Sync,
1867    "rebase" : P4Rebase,
1868    "clone" : P4Clone,
1869    "rollback" : P4RollBack,
1870    "branches" : P4Branches
1871}
1872
1873
1874def main():
1875    if len(sys.argv[1:]) == 0:
1876        printUsage(commands.keys())
1877        sys.exit(2)
1878
1879    cmd = ""
1880    cmdName = sys.argv[1]
1881    try:
1882        klass = commands[cmdName]
1883        cmd = klass()
1884    except KeyError:
1885        print "unknown command %s" % cmdName
1886        print ""
1887        printUsage(commands.keys())
1888        sys.exit(2)
1889
1890    options = cmd.options
1891    cmd.gitdir = os.environ.get("GIT_DIR", None)
1892
1893    args = sys.argv[2:]
1894
1895    if len(options) > 0:
1896        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1897
1898        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1899                                       options,
1900                                       description = cmd.description,
1901                                       formatter = HelpFormatter())
1902
1903        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1904    global verbose
1905    verbose = cmd.verbose
1906    if cmd.needsGit:
1907        if cmd.gitdir == None:
1908            cmd.gitdir = os.path.abspath(".git")
1909            if not isValidGitDir(cmd.gitdir):
1910                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1911                if os.path.exists(cmd.gitdir):
1912                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1913                    if len(cdup) > 0:
1914                        chdir(cdup);
1915
1916        if not isValidGitDir(cmd.gitdir):
1917            if isValidGitDir(cmd.gitdir + "/.git"):
1918                cmd.gitdir += "/.git"
1919            else:
1920                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1921
1922        os.environ["GIT_DIR"] = cmd.gitdir
1923
1924    if not cmd.run(args):
1925        parser.print_help()
1926
1927
1928if __name__ == '__main__':
1929    main()