Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: better message for "git-p4 sync" when not cloned (e32e00d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    delete_actions = ( "delete", "move/delete", "purge" )
 838
 839    def __init__(self):
 840        Command.__init__(self)
 841        self.options = [
 842                optparse.make_option("--branch", dest="branch"),
 843                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                optparse.make_option("--changesfile", dest="changesFile"),
 845                optparse.make_option("--silent", dest="silent", action="store_true"),
 846                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                     help="Import into refs/heads/ , not refs/remotes"),
 850                optparse.make_option("--max-changes", dest="maxChanges"),
 851                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                     help="Only sync files that are included in the Perforce Client Spec")
 855        ]
 856        self.description = """Imports from Perforce into a git repository.\n
 857    example:
 858    //depot/my/project/ -- to import the current head
 859    //depot/my/project/@all -- to import everything
 860    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864        self.usage += " //depot/path[@revRange]"
 865        self.silent = False
 866        self.createdBranches = set()
 867        self.committedChanges = set()
 868        self.branch = ""
 869        self.detectBranches = False
 870        self.detectLabels = False
 871        self.changesFile = ""
 872        self.syncWithOrigin = True
 873        self.verbose = False
 874        self.importIntoRemotes = True
 875        self.maxChanges = ""
 876        self.isWindows = (platform.system() == "Windows")
 877        self.keepRepoPath = False
 878        self.depotPaths = None
 879        self.p4BranchesInGit = []
 880        self.cloneExclude = []
 881        self.useClientSpec = False
 882        self.clientSpecDirs = []
 883
 884        if gitConfig("git-p4.syncFromOrigin") == "false":
 885            self.syncWithOrigin = False
 886
 887    def extractFilesFromCommit(self, commit):
 888        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 889                             for path in self.cloneExclude]
 890        files = []
 891        fnum = 0
 892        while commit.has_key("depotFile%s" % fnum):
 893            path =  commit["depotFile%s" % fnum]
 894
 895            if [p for p in self.cloneExclude
 896                if path.startswith (p)]:
 897                found = False
 898            else:
 899                found = [p for p in self.depotPaths
 900                         if path.startswith (p)]
 901            if not found:
 902                fnum = fnum + 1
 903                continue
 904
 905            file = {}
 906            file["path"] = path
 907            file["rev"] = commit["rev%s" % fnum]
 908            file["action"] = commit["action%s" % fnum]
 909            file["type"] = commit["type%s" % fnum]
 910            files.append(file)
 911            fnum = fnum + 1
 912        return files
 913
 914    def stripRepoPath(self, path, prefixes):
 915        if self.keepRepoPath:
 916            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 917
 918        for p in prefixes:
 919            if path.startswith(p):
 920                path = path[len(p):]
 921
 922        return path
 923
 924    def splitFilesIntoBranches(self, commit):
 925        branches = {}
 926        fnum = 0
 927        while commit.has_key("depotFile%s" % fnum):
 928            path =  commit["depotFile%s" % fnum]
 929            found = [p for p in self.depotPaths
 930                     if path.startswith (p)]
 931            if not found:
 932                fnum = fnum + 1
 933                continue
 934
 935            file = {}
 936            file["path"] = path
 937            file["rev"] = commit["rev%s" % fnum]
 938            file["action"] = commit["action%s" % fnum]
 939            file["type"] = commit["type%s" % fnum]
 940            fnum = fnum + 1
 941
 942            relPath = self.stripRepoPath(path, self.depotPaths)
 943
 944            for branch in self.knownBranches.keys():
 945
 946                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 947                if relPath.startswith(branch + "/"):
 948                    if branch not in branches:
 949                        branches[branch] = []
 950                    branches[branch].append(file)
 951                    break
 952
 953        return branches
 954
 955    # output one file from the P4 stream
 956    # - helper for streamP4Files
 957
 958    def streamOneP4File(self, file, contents):
 959        if file["type"] == "apple":
 960            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 961                file['depotFile']
 962            return
 963
 964        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 965        if verbose:
 966            sys.stderr.write("%s\n" % relPath)
 967
 968        mode = "644"
 969        if isP4Exec(file["type"]):
 970            mode = "755"
 971        elif file["type"] == "symlink":
 972            mode = "120000"
 973            # p4 print on a symlink contains "target\n", so strip it off
 974            data = ''.join(contents)
 975            contents = [data[:-1]]
 976
 977        if self.isWindows and file["type"].endswith("text"):
 978            mangled = []
 979            for data in contents:
 980                data = data.replace("\r\n", "\n")
 981                mangled.append(data)
 982            contents = mangled
 983
 984        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 985            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 986        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 987            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 988
 989        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990
 991        # total length...
 992        length = 0
 993        for d in contents:
 994            length = length + len(d)
 995
 996        self.gitStream.write("data %d\n" % length)
 997        for d in contents:
 998            self.gitStream.write(d)
 999        self.gitStream.write("\n")
1000
1001    def streamOneP4Deletion(self, file):
1002        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1003        if verbose:
1004            sys.stderr.write("delete %s\n" % relPath)
1005        self.gitStream.write("D %s\n" % relPath)
1006
1007    # handle another chunk of streaming data
1008    def streamP4FilesCb(self, marshalled):
1009
1010        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1011            # start of a new file - output the old one first
1012            self.streamOneP4File(self.stream_file, self.stream_contents)
1013            self.stream_file = {}
1014            self.stream_contents = []
1015            self.stream_have_file_info = False
1016
1017        # pick up the new file information... for the
1018        # 'data' field we need to append to our array
1019        for k in marshalled.keys():
1020            if k == 'data':
1021                self.stream_contents.append(marshalled['data'])
1022            else:
1023                self.stream_file[k] = marshalled[k]
1024
1025        self.stream_have_file_info = True
1026
1027    # Stream directly from "p4 files" into "git fast-import"
1028    def streamP4Files(self, files):
1029        filesForCommit = []
1030        filesToRead = []
1031        filesToDelete = []
1032
1033        for f in files:
1034            includeFile = True
1035            for val in self.clientSpecDirs:
1036                if f['path'].startswith(val[0]):
1037                    if val[1] <= 0:
1038                        includeFile = False
1039                    break
1040
1041            if includeFile:
1042                filesForCommit.append(f)
1043                if f['action'] in self.delete_actions:
1044                    filesToDelete.append(f)
1045                else:
1046                    filesToRead.append(f)
1047
1048        # deleted files...
1049        for f in filesToDelete:
1050            self.streamOneP4Deletion(f)
1051
1052        if len(filesToRead) > 0:
1053            self.stream_file = {}
1054            self.stream_contents = []
1055            self.stream_have_file_info = False
1056
1057            # curry self argument
1058            def streamP4FilesCbSelf(entry):
1059                self.streamP4FilesCb(entry)
1060
1061            p4CmdList("-x - print",
1062                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1063                                                  for f in filesToRead]),
1064                cb=streamP4FilesCbSelf)
1065
1066            # do the last chunk
1067            if self.stream_file.has_key('depotFile'):
1068                self.streamOneP4File(self.stream_file, self.stream_contents)
1069
1070    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1071        epoch = details["time"]
1072        author = details["user"]
1073        self.branchPrefixes = branchPrefixes
1074
1075        if self.verbose:
1076            print "commit into %s" % branch
1077
1078        # start with reading files; if that fails, we should not
1079        # create a commit.
1080        new_files = []
1081        for f in files:
1082            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1083                new_files.append (f)
1084            else:
1085                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1086
1087        self.gitStream.write("commit %s\n" % branch)
1088#        gitStream.write("mark :%s\n" % details["change"])
1089        self.committedChanges.add(int(details["change"]))
1090        committer = ""
1091        if author not in self.users:
1092            self.getUserMapFromPerforceServer()
1093        if author in self.users:
1094            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1095        else:
1096            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1097
1098        self.gitStream.write("committer %s\n" % committer)
1099
1100        self.gitStream.write("data <<EOT\n")
1101        self.gitStream.write(details["desc"])
1102        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1103                             % (','.join (branchPrefixes), details["change"]))
1104        if len(details['options']) > 0:
1105            self.gitStream.write(": options = %s" % details['options'])
1106        self.gitStream.write("]\nEOT\n\n")
1107
1108        if len(parent) > 0:
1109            if self.verbose:
1110                print "parent %s" % parent
1111            self.gitStream.write("from %s\n" % parent)
1112
1113        self.streamP4Files(new_files)
1114        self.gitStream.write("\n")
1115
1116        change = int(details["change"])
1117
1118        if self.labels.has_key(change):
1119            label = self.labels[change]
1120            labelDetails = label[0]
1121            labelRevisions = label[1]
1122            if self.verbose:
1123                print "Change %s is labelled %s" % (change, labelDetails)
1124
1125            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1126                                                    for p in branchPrefixes]))
1127
1128            if len(files) == len(labelRevisions):
1129
1130                cleanedFiles = {}
1131                for info in files:
1132                    if info["action"] in self.delete_actions:
1133                        continue
1134                    cleanedFiles[info["depotFile"]] = info["rev"]
1135
1136                if cleanedFiles == labelRevisions:
1137                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1138                    self.gitStream.write("from %s\n" % branch)
1139
1140                    owner = labelDetails["Owner"]
1141                    tagger = ""
1142                    if author in self.users:
1143                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1144                    else:
1145                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1146                    self.gitStream.write("tagger %s\n" % tagger)
1147                    self.gitStream.write("data <<EOT\n")
1148                    self.gitStream.write(labelDetails["Description"])
1149                    self.gitStream.write("EOT\n\n")
1150
1151                else:
1152                    if not self.silent:
1153                        print ("Tag %s does not match with change %s: files do not match."
1154                               % (labelDetails["label"], change))
1155
1156            else:
1157                if not self.silent:
1158                    print ("Tag %s does not match with change %s: file count is different."
1159                           % (labelDetails["label"], change))
1160
1161    def getUserCacheFilename(self):
1162        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1163        return home + "/.gitp4-usercache.txt"
1164
1165    def getUserMapFromPerforceServer(self):
1166        if self.userMapFromPerforceServer:
1167            return
1168        self.users = {}
1169
1170        for output in p4CmdList("users"):
1171            if not output.has_key("User"):
1172                continue
1173            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174
1175
1176        s = ''
1177        for (key, val) in self.users.items():
1178            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1179
1180        open(self.getUserCacheFilename(), "wb").write(s)
1181        self.userMapFromPerforceServer = True
1182
1183    def loadUserMapFromCache(self):
1184        self.users = {}
1185        self.userMapFromPerforceServer = False
1186        try:
1187            cache = open(self.getUserCacheFilename(), "rb")
1188            lines = cache.readlines()
1189            cache.close()
1190            for line in lines:
1191                entry = line.strip().split("\t")
1192                self.users[entry[0]] = entry[1]
1193        except IOError:
1194            self.getUserMapFromPerforceServer()
1195
1196    def getLabels(self):
1197        self.labels = {}
1198
1199        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1200        if len(l) > 0 and not self.silent:
1201            print "Finding files belonging to labels in %s" % `self.depotPaths`
1202
1203        for output in l:
1204            label = output["label"]
1205            revisions = {}
1206            newestChange = 0
1207            if self.verbose:
1208                print "Querying files for label %s" % label
1209            for file in p4CmdList("files "
1210                                  +  ' '.join (["%s...@%s" % (p, label)
1211                                                for p in self.depotPaths])):
1212                revisions[file["depotFile"]] = file["rev"]
1213                change = int(file["change"])
1214                if change > newestChange:
1215                    newestChange = change
1216
1217            self.labels[newestChange] = [output, revisions]
1218
1219        if self.verbose:
1220            print "Label changes: %s" % self.labels.keys()
1221
1222    def guessProjectName(self):
1223        for p in self.depotPaths:
1224            if p.endswith("/"):
1225                p = p[:-1]
1226            p = p[p.strip().rfind("/") + 1:]
1227            if not p.endswith("/"):
1228               p += "/"
1229            return p
1230
1231    def getBranchMapping(self):
1232        lostAndFoundBranches = set()
1233
1234        for info in p4CmdList("branches"):
1235            details = p4Cmd("branch -o %s" % info["branch"])
1236            viewIdx = 0
1237            while details.has_key("View%s" % viewIdx):
1238                paths = details["View%s" % viewIdx].split(" ")
1239                viewIdx = viewIdx + 1
1240                # require standard //depot/foo/... //depot/bar/... mapping
1241                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1242                    continue
1243                source = paths[0]
1244                destination = paths[1]
1245                ## HACK
1246                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1247                    source = source[len(self.depotPaths[0]):-4]
1248                    destination = destination[len(self.depotPaths[0]):-4]
1249
1250                    if destination in self.knownBranches:
1251                        if not self.silent:
1252                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1253                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1254                        continue
1255
1256                    self.knownBranches[destination] = source
1257
1258                    lostAndFoundBranches.discard(destination)
1259
1260                    if source not in self.knownBranches:
1261                        lostAndFoundBranches.add(source)
1262
1263
1264        for branch in lostAndFoundBranches:
1265            self.knownBranches[branch] = branch
1266
1267    def getBranchMappingFromGitBranches(self):
1268        branches = p4BranchesInGit(self.importIntoRemotes)
1269        for branch in branches.keys():
1270            if branch == "master":
1271                branch = "main"
1272            else:
1273                branch = branch[len(self.projectName):]
1274            self.knownBranches[branch] = branch
1275
1276    def listExistingP4GitBranches(self):
1277        # branches holds mapping from name to commit
1278        branches = p4BranchesInGit(self.importIntoRemotes)
1279        self.p4BranchesInGit = branches.keys()
1280        for branch in branches.keys():
1281            self.initialParents[self.refPrefix + branch] = branches[branch]
1282
1283    def updateOptionDict(self, d):
1284        option_keys = {}
1285        if self.keepRepoPath:
1286            option_keys['keepRepoPath'] = 1
1287
1288        d["options"] = ' '.join(sorted(option_keys.keys()))
1289
1290    def readOptions(self, d):
1291        self.keepRepoPath = (d.has_key('options')
1292                             and ('keepRepoPath' in d['options']))
1293
1294    def gitRefForBranch(self, branch):
1295        if branch == "main":
1296            return self.refPrefix + "master"
1297
1298        if len(branch) <= 0:
1299            return branch
1300
1301        return self.refPrefix + self.projectName + branch
1302
1303    def gitCommitByP4Change(self, ref, change):
1304        if self.verbose:
1305            print "looking in ref " + ref + " for change %s using bisect..." % change
1306
1307        earliestCommit = ""
1308        latestCommit = parseRevision(ref)
1309
1310        while True:
1311            if self.verbose:
1312                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1313            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1314            if len(next) == 0:
1315                if self.verbose:
1316                    print "argh"
1317                return ""
1318            log = extractLogMessageFromGitCommit(next)
1319            settings = extractSettingsGitLog(log)
1320            currentChange = int(settings['change'])
1321            if self.verbose:
1322                print "current change %s" % currentChange
1323
1324            if currentChange == change:
1325                if self.verbose:
1326                    print "found %s" % next
1327                return next
1328
1329            if currentChange < change:
1330                earliestCommit = "^%s" % next
1331            else:
1332                latestCommit = "%s" % next
1333
1334        return ""
1335
1336    def importNewBranch(self, branch, maxChange):
1337        # make fast-import flush all changes to disk and update the refs using the checkpoint
1338        # command so that we can try to find the branch parent in the git history
1339        self.gitStream.write("checkpoint\n\n");
1340        self.gitStream.flush();
1341        branchPrefix = self.depotPaths[0] + branch + "/"
1342        range = "@1,%s" % maxChange
1343        #print "prefix" + branchPrefix
1344        changes = p4ChangesForPaths([branchPrefix], range)
1345        if len(changes) <= 0:
1346            return False
1347        firstChange = changes[0]
1348        #print "first change in branch: %s" % firstChange
1349        sourceBranch = self.knownBranches[branch]
1350        sourceDepotPath = self.depotPaths[0] + sourceBranch
1351        sourceRef = self.gitRefForBranch(sourceBranch)
1352        #print "source " + sourceBranch
1353
1354        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1355        #print "branch parent: %s" % branchParentChange
1356        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1357        if len(gitParent) > 0:
1358            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1359            #print "parent git commit: %s" % gitParent
1360
1361        self.importChanges(changes)
1362        return True
1363
1364    def importChanges(self, changes):
1365        cnt = 1
1366        for change in changes:
1367            description = p4Cmd("describe %s" % change)
1368            self.updateOptionDict(description)
1369
1370            if not self.silent:
1371                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1372                sys.stdout.flush()
1373            cnt = cnt + 1
1374
1375            try:
1376                if self.detectBranches:
1377                    branches = self.splitFilesIntoBranches(description)
1378                    for branch in branches.keys():
1379                        ## HACK  --hwn
1380                        branchPrefix = self.depotPaths[0] + branch + "/"
1381
1382                        parent = ""
1383
1384                        filesForCommit = branches[branch]
1385
1386                        if self.verbose:
1387                            print "branch is %s" % branch
1388
1389                        self.updatedBranches.add(branch)
1390
1391                        if branch not in self.createdBranches:
1392                            self.createdBranches.add(branch)
1393                            parent = self.knownBranches[branch]
1394                            if parent == branch:
1395                                parent = ""
1396                            else:
1397                                fullBranch = self.projectName + branch
1398                                if fullBranch not in self.p4BranchesInGit:
1399                                    if not self.silent:
1400                                        print("\n    Importing new branch %s" % fullBranch);
1401                                    if self.importNewBranch(branch, change - 1):
1402                                        parent = ""
1403                                        self.p4BranchesInGit.append(fullBranch)
1404                                    if not self.silent:
1405                                        print("\n    Resuming with change %s" % change);
1406
1407                                if self.verbose:
1408                                    print "parent determined through known branches: %s" % parent
1409
1410                        branch = self.gitRefForBranch(branch)
1411                        parent = self.gitRefForBranch(parent)
1412
1413                        if self.verbose:
1414                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1415
1416                        if len(parent) == 0 and branch in self.initialParents:
1417                            parent = self.initialParents[branch]
1418                            del self.initialParents[branch]
1419
1420                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1421                else:
1422                    files = self.extractFilesFromCommit(description)
1423                    self.commit(description, files, self.branch, self.depotPaths,
1424                                self.initialParent)
1425                    self.initialParent = ""
1426            except IOError:
1427                print self.gitError.read()
1428                sys.exit(1)
1429
1430    def importHeadRevision(self, revision):
1431        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1432
1433        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1434        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1435                           % (' '.join(self.depotPaths), revision))
1436        details["change"] = revision
1437        newestRevision = 0
1438
1439        fileCnt = 0
1440        for info in p4CmdList("files "
1441                              +  ' '.join(["%s...%s"
1442                                           % (p, revision)
1443                                           for p in self.depotPaths])):
1444
1445            if 'code' in info and info['code'] == 'error':
1446                sys.stderr.write("p4 returned an error: %s\n"
1447                                 % info['data'])
1448                if info['data'].find("must refer to client") >= 0:
1449                    sys.stderr.write("This particular p4 error is misleading.\n")
1450                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
1451                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1452                sys.exit(1)
1453            if 'p4ExitCode' in info:
1454                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1455                sys.exit(1)
1456
1457
1458            change = int(info["change"])
1459            if change > newestRevision:
1460                newestRevision = change
1461
1462            if info["action"] in self.delete_actions:
1463                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1464                #fileCnt = fileCnt + 1
1465                continue
1466
1467            for prop in ["depotFile", "rev", "action", "type" ]:
1468                details["%s%s" % (prop, fileCnt)] = info[prop]
1469
1470            fileCnt = fileCnt + 1
1471
1472        details["change"] = newestRevision
1473        self.updateOptionDict(details)
1474        try:
1475            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1476        except IOError:
1477            print "IO error with git fast-import. Is your git version recent enough?"
1478            print self.gitError.read()
1479
1480
1481    def getClientSpec(self):
1482        specList = p4CmdList( "client -o" )
1483        temp = {}
1484        for entry in specList:
1485            for k,v in entry.iteritems():
1486                if k.startswith("View"):
1487                    if v.startswith('"'):
1488                        start = 1
1489                    else:
1490                        start = 0
1491                    index = v.find("...")
1492                    v = v[start:index]
1493                    if v.startswith("-"):
1494                        v = v[1:]
1495                        temp[v] = -len(v)
1496                    else:
1497                        temp[v] = len(v)
1498        self.clientSpecDirs = temp.items()
1499        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1500
1501    def run(self, args):
1502        self.depotPaths = []
1503        self.changeRange = ""
1504        self.initialParent = ""
1505        self.previousDepotPaths = []
1506
1507        # map from branch depot path to parent branch
1508        self.knownBranches = {}
1509        self.initialParents = {}
1510        self.hasOrigin = originP4BranchesExist()
1511        if not self.syncWithOrigin:
1512            self.hasOrigin = False
1513
1514        if self.importIntoRemotes:
1515            self.refPrefix = "refs/remotes/p4/"
1516        else:
1517            self.refPrefix = "refs/heads/p4/"
1518
1519        if self.syncWithOrigin and self.hasOrigin:
1520            if not self.silent:
1521                print "Syncing with origin first by calling git fetch origin"
1522            system("git fetch origin")
1523
1524        if len(self.branch) == 0:
1525            self.branch = self.refPrefix + "master"
1526            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1527                system("git update-ref %s refs/heads/p4" % self.branch)
1528                system("git branch -D p4");
1529            # create it /after/ importing, when master exists
1530            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1531                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1532
1533        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1534            self.getClientSpec()
1535
1536        # TODO: should always look at previous commits,
1537        # merge with previous imports, if possible.
1538        if args == []:
1539            if self.hasOrigin:
1540                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1541            self.listExistingP4GitBranches()
1542
1543            if len(self.p4BranchesInGit) > 1:
1544                if not self.silent:
1545                    print "Importing from/into multiple branches"
1546                self.detectBranches = True
1547
1548            if self.verbose:
1549                print "branches: %s" % self.p4BranchesInGit
1550
1551            p4Change = 0
1552            for branch in self.p4BranchesInGit:
1553                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1554
1555                settings = extractSettingsGitLog(logMsg)
1556
1557                self.readOptions(settings)
1558                if (settings.has_key('depot-paths')
1559                    and settings.has_key ('change')):
1560                    change = int(settings['change']) + 1
1561                    p4Change = max(p4Change, change)
1562
1563                    depotPaths = sorted(settings['depot-paths'])
1564                    if self.previousDepotPaths == []:
1565                        self.previousDepotPaths = depotPaths
1566                    else:
1567                        paths = []
1568                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1569                            for i in range(0, min(len(cur), len(prev))):
1570                                if cur[i] <> prev[i]:
1571                                    i = i - 1
1572                                    break
1573
1574                            paths.append (cur[:i + 1])
1575
1576                        self.previousDepotPaths = paths
1577
1578            if p4Change > 0:
1579                self.depotPaths = sorted(self.previousDepotPaths)
1580                self.changeRange = "@%s,#head" % p4Change
1581                if not self.detectBranches:
1582                    self.initialParent = parseRevision(self.branch)
1583                if not self.silent and not self.detectBranches:
1584                    print "Performing incremental import into %s git branch" % self.branch
1585
1586        if not self.branch.startswith("refs/"):
1587            self.branch = "refs/heads/" + self.branch
1588
1589        if len(args) == 0 and self.depotPaths:
1590            if not self.silent:
1591                print "Depot paths: %s" % ' '.join(self.depotPaths)
1592        else:
1593            if self.depotPaths and self.depotPaths != args:
1594                print ("previous import used depot path %s and now %s was specified. "
1595                       "This doesn't work!" % (' '.join (self.depotPaths),
1596                                               ' '.join (args)))
1597                sys.exit(1)
1598
1599            self.depotPaths = sorted(args)
1600
1601        revision = ""
1602        self.users = {}
1603
1604        newPaths = []
1605        for p in self.depotPaths:
1606            if p.find("@") != -1:
1607                atIdx = p.index("@")
1608                self.changeRange = p[atIdx:]
1609                if self.changeRange == "@all":
1610                    self.changeRange = ""
1611                elif ',' not in self.changeRange:
1612                    revision = self.changeRange
1613                    self.changeRange = ""
1614                p = p[:atIdx]
1615            elif p.find("#") != -1:
1616                hashIdx = p.index("#")
1617                revision = p[hashIdx:]
1618                p = p[:hashIdx]
1619            elif self.previousDepotPaths == []:
1620                revision = "#head"
1621
1622            p = re.sub ("\.\.\.$", "", p)
1623            if not p.endswith("/"):
1624                p += "/"
1625
1626            newPaths.append(p)
1627
1628        self.depotPaths = newPaths
1629
1630
1631        self.loadUserMapFromCache()
1632        self.labels = {}
1633        if self.detectLabels:
1634            self.getLabels();
1635
1636        if self.detectBranches:
1637            ## FIXME - what's a P4 projectName ?
1638            self.projectName = self.guessProjectName()
1639
1640            if self.hasOrigin:
1641                self.getBranchMappingFromGitBranches()
1642            else:
1643                self.getBranchMapping()
1644            if self.verbose:
1645                print "p4-git branches: %s" % self.p4BranchesInGit
1646                print "initial parents: %s" % self.initialParents
1647            for b in self.p4BranchesInGit:
1648                if b != "master":
1649
1650                    ## FIXME
1651                    b = b[len(self.projectName):]
1652                self.createdBranches.add(b)
1653
1654        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1655
1656        importProcess = subprocess.Popen(["git", "fast-import"],
1657                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1658                                         stderr=subprocess.PIPE);
1659        self.gitOutput = importProcess.stdout
1660        self.gitStream = importProcess.stdin
1661        self.gitError = importProcess.stderr
1662
1663        if revision:
1664            self.importHeadRevision(revision)
1665        else:
1666            changes = []
1667
1668            if len(self.changesFile) > 0:
1669                output = open(self.changesFile).readlines()
1670                changeSet = set()
1671                for line in output:
1672                    changeSet.add(int(line))
1673
1674                for change in changeSet:
1675                    changes.append(change)
1676
1677                changes.sort()
1678            else:
1679                if not self.p4BranchesInGit:
1680                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1681                if self.verbose:
1682                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1683                                                              self.changeRange)
1684                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1685
1686                if len(self.maxChanges) > 0:
1687                    changes = changes[:min(int(self.maxChanges), len(changes))]
1688
1689            if len(changes) == 0:
1690                if not self.silent:
1691                    print "No changes to import!"
1692                return True
1693
1694            if not self.silent and not self.detectBranches:
1695                print "Import destination: %s" % self.branch
1696
1697            self.updatedBranches = set()
1698
1699            self.importChanges(changes)
1700
1701            if not self.silent:
1702                print ""
1703                if len(self.updatedBranches) > 0:
1704                    sys.stdout.write("Updated branches: ")
1705                    for b in self.updatedBranches:
1706                        sys.stdout.write("%s " % b)
1707                    sys.stdout.write("\n")
1708
1709        self.gitStream.close()
1710        if importProcess.wait() != 0:
1711            die("fast-import failed: %s" % self.gitError.read())
1712        self.gitOutput.close()
1713        self.gitError.close()
1714
1715        return True
1716
1717class P4Rebase(Command):
1718    def __init__(self):
1719        Command.__init__(self)
1720        self.options = [ ]
1721        self.description = ("Fetches the latest revision from perforce and "
1722                            + "rebases the current work (branch) against it")
1723        self.verbose = False
1724
1725    def run(self, args):
1726        sync = P4Sync()
1727        sync.run([])
1728
1729        return self.rebase()
1730
1731    def rebase(self):
1732        if os.system("git update-index --refresh") != 0:
1733            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1734        if len(read_pipe("git diff-index HEAD --")) > 0:
1735            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1736
1737        [upstream, settings] = findUpstreamBranchPoint()
1738        if len(upstream) == 0:
1739            die("Cannot find upstream branchpoint for rebase")
1740
1741        # the branchpoint may be p4/foo~3, so strip off the parent
1742        upstream = re.sub("~[0-9]+$", "", upstream)
1743
1744        print "Rebasing the current branch onto %s" % upstream
1745        oldHead = read_pipe("git rev-parse HEAD").strip()
1746        system("git rebase %s" % upstream)
1747        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1748        return True
1749
1750class P4Clone(P4Sync):
1751    def __init__(self):
1752        P4Sync.__init__(self)
1753        self.description = "Creates a new git repository and imports from Perforce into it"
1754        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1755        self.options += [
1756            optparse.make_option("--destination", dest="cloneDestination",
1757                                 action='store', default=None,
1758                                 help="where to leave result of the clone"),
1759            optparse.make_option("-/", dest="cloneExclude",
1760                                 action="append", type="string",
1761                                 help="exclude depot path")
1762        ]
1763        self.cloneDestination = None
1764        self.needsGit = False
1765
1766    # This is required for the "append" cloneExclude action
1767    def ensure_value(self, attr, value):
1768        if not hasattr(self, attr) or getattr(self, attr) is None:
1769            setattr(self, attr, value)
1770        return getattr(self, attr)
1771
1772    def defaultDestination(self, args):
1773        ## TODO: use common prefix of args?
1774        depotPath = args[0]
1775        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1776        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1777        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1778        depotDir = re.sub(r"/$", "", depotDir)
1779        return os.path.split(depotDir)[1]
1780
1781    def run(self, args):
1782        if len(args) < 1:
1783            return False
1784
1785        if self.keepRepoPath and not self.cloneDestination:
1786            sys.stderr.write("Must specify destination for --keep-path\n")
1787            sys.exit(1)
1788
1789        depotPaths = args
1790
1791        if not self.cloneDestination and len(depotPaths) > 1:
1792            self.cloneDestination = depotPaths[-1]
1793            depotPaths = depotPaths[:-1]
1794
1795        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1796        for p in depotPaths:
1797            if not p.startswith("//"):
1798                return False
1799
1800        if not self.cloneDestination:
1801            self.cloneDestination = self.defaultDestination(args)
1802
1803        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1804        if not os.path.exists(self.cloneDestination):
1805            os.makedirs(self.cloneDestination)
1806        chdir(self.cloneDestination)
1807        system("git init")
1808        self.gitdir = os.getcwd() + "/.git"
1809        if not P4Sync.run(self, depotPaths):
1810            return False
1811        if self.branch != "master":
1812            if self.importIntoRemotes:
1813                masterbranch = "refs/remotes/p4/master"
1814            else:
1815                masterbranch = "refs/heads/p4/master"
1816            if gitBranchExists(masterbranch):
1817                system("git branch master %s" % masterbranch)
1818                system("git checkout -f")
1819            else:
1820                print "Could not detect main branch. No checkout/master branch created."
1821
1822        return True
1823
1824class P4Branches(Command):
1825    def __init__(self):
1826        Command.__init__(self)
1827        self.options = [ ]
1828        self.description = ("Shows the git branches that hold imports and their "
1829                            + "corresponding perforce depot paths")
1830        self.verbose = False
1831
1832    def run(self, args):
1833        if originP4BranchesExist():
1834            createOrUpdateBranchesFromOrigin()
1835
1836        cmdline = "git rev-parse --symbolic "
1837        cmdline += " --remotes"
1838
1839        for line in read_pipe_lines(cmdline):
1840            line = line.strip()
1841
1842            if not line.startswith('p4/') or line == "p4/HEAD":
1843                continue
1844            branch = line
1845
1846            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1847            settings = extractSettingsGitLog(log)
1848
1849            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1850        return True
1851
1852class HelpFormatter(optparse.IndentedHelpFormatter):
1853    def __init__(self):
1854        optparse.IndentedHelpFormatter.__init__(self)
1855
1856    def format_description(self, description):
1857        if description:
1858            return description + "\n"
1859        else:
1860            return ""
1861
1862def printUsage(commands):
1863    print "usage: %s <command> [options]" % sys.argv[0]
1864    print ""
1865    print "valid commands: %s" % ", ".join(commands)
1866    print ""
1867    print "Try %s <command> --help for command specific help." % sys.argv[0]
1868    print ""
1869
1870commands = {
1871    "debug" : P4Debug,
1872    "submit" : P4Submit,
1873    "commit" : P4Submit,
1874    "sync" : P4Sync,
1875    "rebase" : P4Rebase,
1876    "clone" : P4Clone,
1877    "rollback" : P4RollBack,
1878    "branches" : P4Branches
1879}
1880
1881
1882def main():
1883    if len(sys.argv[1:]) == 0:
1884        printUsage(commands.keys())
1885        sys.exit(2)
1886
1887    cmd = ""
1888    cmdName = sys.argv[1]
1889    try:
1890        klass = commands[cmdName]
1891        cmd = klass()
1892    except KeyError:
1893        print "unknown command %s" % cmdName
1894        print ""
1895        printUsage(commands.keys())
1896        sys.exit(2)
1897
1898    options = cmd.options
1899    cmd.gitdir = os.environ.get("GIT_DIR", None)
1900
1901    args = sys.argv[2:]
1902
1903    if len(options) > 0:
1904        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1905
1906        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1907                                       options,
1908                                       description = cmd.description,
1909                                       formatter = HelpFormatter())
1910
1911        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1912    global verbose
1913    verbose = cmd.verbose
1914    if cmd.needsGit:
1915        if cmd.gitdir == None:
1916            cmd.gitdir = os.path.abspath(".git")
1917            if not isValidGitDir(cmd.gitdir):
1918                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1919                if os.path.exists(cmd.gitdir):
1920                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1921                    if len(cdup) > 0:
1922                        chdir(cdup);
1923
1924        if not isValidGitDir(cmd.gitdir):
1925            if isValidGitDir(cmd.gitdir + "/.git"):
1926                cmd.gitdir += "/.git"
1927            else:
1928                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1929
1930        os.environ["GIT_DIR"] = cmd.gitdir
1931
1932    if not cmd.run(args):
1933        parser.print_help()
1934
1935
1936if __name__ == '__main__':
1937    main()