Andrew's git - gitweb.git/blob - contrib/fast-import/git-p4

   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = p4_read_pipe("diff -du ...")
 710
 711            newdiff = ""
 712            for newFile in filesToAdd:
 713                newdiff += "==== new file ====\n"
 714                newdiff += "--- /dev/null\n"
 715                newdiff += "+++ %s\n" % newFile
 716                f = open(newFile, "r")
 717                for line in f.readlines():
 718                    newdiff += "+" + line
 719                f.close()
 720
 721            separatorLine = "######## everything below this line is just the diff #######\n"
 722
 723            [handle, fileName] = tempfile.mkstemp()
 724            tmpFile = os.fdopen(handle, "w+")
 725            if self.isWindows:
 726                submitTemplate = submitTemplate.replace("\n", "\r\n")
 727                separatorLine = separatorLine.replace("\n", "\r\n")
 728                newdiff = newdiff.replace("\n", "\r\n")
 729            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 730            tmpFile.close()
 731            mtime = os.stat(fileName).st_mtime
 732            if os.environ.has_key("P4EDITOR"):
 733                editor = os.environ.get("P4EDITOR")
 734            else:
 735                editor = read_pipe("git var GIT_EDITOR")
 736            system(editor + " " + fileName)
 737
 738            response = "y"
 739            if os.stat(fileName).st_mtime <= mtime:
 740                response = "x"
 741                while response != "y" and response != "n":
 742                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 743
 744            if response == "y":
 745                tmpFile = open(fileName, "rb")
 746                message = tmpFile.read()
 747                tmpFile.close()
 748                submitTemplate = message[:message.index(separatorLine)]
 749                if self.isWindows:
 750                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 751                p4_write_pipe("submit -i", submitTemplate)
 752            else:
 753                for f in editedFiles:
 754                    p4_system("revert \"%s\"" % f);
 755                for f in filesToAdd:
 756                    p4_system("revert \"%s\"" % f);
 757                    system("rm %s" %f)
 758
 759            os.remove(fileName)
 760        else:
 761            fileName = "submit.txt"
 762            file = open(fileName, "w+")
 763            file.write(self.prepareLogMessage(template, logMessage))
 764            file.close()
 765            print ("Perforce submit template written as %s. "
 766                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 767                   % (fileName, fileName))
 768
 769    def run(self, args):
 770        if len(args) == 0:
 771            self.master = currentGitBranch()
 772            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 773                die("Detecting current git branch failed!")
 774        elif len(args) == 1:
 775            self.master = args[0]
 776        else:
 777            return False
 778
 779        allowSubmit = gitConfig("git-p4.allowSubmit")
 780        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 781            die("%s is not in git-p4.allowSubmit" % self.master)
 782
 783        [upstream, settings] = findUpstreamBranchPoint()
 784        self.depotPath = settings['depot-paths'][0]
 785        if len(self.origin) == 0:
 786            self.origin = upstream
 787
 788        if self.verbose:
 789            print "Origin branch is " + self.origin
 790
 791        if len(self.depotPath) == 0:
 792            print "Internal error: cannot locate perforce depot path from existing branches"
 793            sys.exit(128)
 794
 795        self.clientPath = p4Where(self.depotPath)
 796
 797        if len(self.clientPath) == 0:
 798            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 799            sys.exit(128)
 800
 801        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 802        self.oldWorkingDirectory = os.getcwd()
 803
 804        chdir(self.clientPath)
 805        print "Syncronizing p4 checkout..."
 806        p4_system("sync ...")
 807
 808        self.check()
 809
 810        commits = []
 811        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 812            commits.append(line.strip())
 813        commits.reverse()
 814
 815        while len(commits) > 0:
 816            commit = commits[0]
 817            commits = commits[1:]
 818            self.applyCommit(commit)
 819            if not self.interactive:
 820                break
 821
 822        if len(commits) == 0:
 823            print "All changes applied!"
 824            chdir(self.oldWorkingDirectory)
 825
 826            sync = P4Sync()
 827            sync.run([])
 828
 829            rebase = P4Rebase()
 830            rebase.rebase()
 831
 832        return True
 833
 834class P4Sync(Command):
 835    def __init__(self):
 836        Command.__init__(self)
 837        self.options = [
 838                optparse.make_option("--branch", dest="branch"),
 839                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 840                optparse.make_option("--changesfile", dest="changesFile"),
 841                optparse.make_option("--silent", dest="silent", action="store_true"),
 842                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 843                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 844                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 845                                     help="Import into refs/heads/ , not refs/remotes"),
 846                optparse.make_option("--max-changes", dest="maxChanges"),
 847                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 848                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 849                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 850                                     help="Only sync files that are included in the Perforce Client Spec")
 851        ]
 852        self.description = """Imports from Perforce into a git repository.\n
 853    example:
 854    //depot/my/project/ -- to import the current head
 855    //depot/my/project/@all -- to import everything
 856    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 857
 858    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 859
 860        self.usage += " //depot/path[@revRange]"
 861        self.silent = False
 862        self.createdBranches = set()
 863        self.committedChanges = set()
 864        self.branch = ""
 865        self.detectBranches = False
 866        self.detectLabels = False
 867        self.changesFile = ""
 868        self.syncWithOrigin = True
 869        self.verbose = False
 870        self.importIntoRemotes = True
 871        self.maxChanges = ""
 872        self.isWindows = (platform.system() == "Windows")
 873        self.keepRepoPath = False
 874        self.depotPaths = None
 875        self.p4BranchesInGit = []
 876        self.cloneExclude = []
 877        self.useClientSpec = False
 878        self.clientSpecDirs = []
 879
 880        if gitConfig("git-p4.syncFromOrigin") == "false":
 881            self.syncWithOrigin = False
 882
 883    def extractFilesFromCommit(self, commit):
 884        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 885                             for path in self.cloneExclude]
 886        files = []
 887        fnum = 0
 888        while commit.has_key("depotFile%s" % fnum):
 889            path =  commit["depotFile%s" % fnum]
 890
 891            if [p for p in self.cloneExclude
 892                if path.startswith (p)]:
 893                found = False
 894            else:
 895                found = [p for p in self.depotPaths
 896                         if path.startswith (p)]
 897            if not found:
 898                fnum = fnum + 1
 899                continue
 900
 901            file = {}
 902            file["path"] = path
 903            file["rev"] = commit["rev%s" % fnum]
 904            file["action"] = commit["action%s" % fnum]
 905            file["type"] = commit["type%s" % fnum]
 906            files.append(file)
 907            fnum = fnum + 1
 908        return files
 909
 910    def stripRepoPath(self, path, prefixes):
 911        if self.keepRepoPath:
 912            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 913
 914        for p in prefixes:
 915            if path.startswith(p):
 916                path = path[len(p):]
 917
 918        return path
 919
 920    def splitFilesIntoBranches(self, commit):
 921        branches = {}
 922        fnum = 0
 923        while commit.has_key("depotFile%s" % fnum):
 924            path =  commit["depotFile%s" % fnum]
 925            found = [p for p in self.depotPaths
 926                     if path.startswith (p)]
 927            if not found:
 928                fnum = fnum + 1
 929                continue
 930
 931            file = {}
 932            file["path"] = path
 933            file["rev"] = commit["rev%s" % fnum]
 934            file["action"] = commit["action%s" % fnum]
 935            file["type"] = commit["type%s" % fnum]
 936            fnum = fnum + 1
 937
 938            relPath = self.stripRepoPath(path, self.depotPaths)
 939
 940            for branch in self.knownBranches.keys():
 941
 942                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 943                if relPath.startswith(branch + "/"):
 944                    if branch not in branches:
 945                        branches[branch] = []
 946                    branches[branch].append(file)
 947                    break
 948
 949        return branches
 950
 951    # output one file from the P4 stream
 952    # - helper for streamP4Files
 953
 954    def streamOneP4File(self, file, contents):
 955        if file["type"] == "apple":
 956            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 957                file['depotFile']
 958            return
 959
 960        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 961        if verbose:
 962            sys.stderr.write("%s\n" % relPath)
 963
 964        mode = "644"
 965        if isP4Exec(file["type"]):
 966            mode = "755"
 967        elif file["type"] == "symlink":
 968            mode = "120000"
 969            # p4 print on a symlink contains "target\n", so strip it off
 970            last = contents.pop()
 971            last = last[:-1]
 972            contents.append(last)
 973
 974        if self.isWindows and file["type"].endswith("text"):
 975            mangled = []
 976            for data in contents:
 977                data = data.replace("\r\n", "\n")
 978                mangled.append(data)
 979            contents = mangled
 980
 981        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 982            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
 983        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 984            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
 985
 986        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 987
 988        # total length...
 989        length = 0
 990        for d in contents:
 991            length = length + len(d)
 992
 993        self.gitStream.write("data %d\n" % length)
 994        for d in contents:
 995            self.gitStream.write(d)
 996        self.gitStream.write("\n")
 997
 998    def streamOneP4Deletion(self, file):
 999        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1000        if verbose:
1001            sys.stderr.write("delete %s\n" % relPath)
1002        self.gitStream.write("D %s\n" % relPath)
1003
1004    # handle another chunk of streaming data
1005    def streamP4FilesCb(self, marshalled):
1006
1007        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1008            # start of a new file - output the old one first
1009            self.streamOneP4File(self.stream_file, self.stream_contents)
1010            self.stream_file = {}
1011            self.stream_contents = []
1012            self.stream_have_file_info = False
1013
1014        # pick up the new file information... for the
1015        # 'data' field we need to append to our array
1016        for k in marshalled.keys():
1017            if k == 'data':
1018                self.stream_contents.append(marshalled['data'])
1019            else:
1020                self.stream_file[k] = marshalled[k]
1021
1022        self.stream_have_file_info = True
1023
1024    # Stream directly from "p4 files" into "git fast-import"
1025    def streamP4Files(self, files):
1026        filesForCommit = []
1027        filesToRead = []
1028        filesToDelete = []
1029
1030        for f in files:
1031            includeFile = True
1032            for val in self.clientSpecDirs:
1033                if f['path'].startswith(val[0]):
1034                    if val[1] <= 0:
1035                        includeFile = False
1036                    break
1037
1038            if includeFile:
1039                filesForCommit.append(f)
1040                if f['action'] not in ('delete', 'purge'):
1041                    filesToRead.append(f)
1042                else:
1043                    filesToDelete.append(f)
1044
1045        # deleted files...
1046        for f in filesToDelete:
1047            self.streamOneP4Deletion(f)
1048
1049        if len(filesToRead) > 0:
1050            self.stream_file = {}
1051            self.stream_contents = []
1052            self.stream_have_file_info = False
1053
1054            # curry self argument
1055            def streamP4FilesCbSelf(entry):
1056                self.streamP4FilesCb(entry)
1057
1058            p4CmdList("-x - print",
1059                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1060                                                  for f in filesToRead]),
1061                cb=streamP4FilesCbSelf)
1062
1063            # do the last chunk
1064            if self.stream_file.has_key('depotFile'):
1065                self.streamOneP4File(self.stream_file, self.stream_contents)
1066
1067    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1068        epoch = details["time"]
1069        author = details["user"]
1070        self.branchPrefixes = branchPrefixes
1071
1072        if self.verbose:
1073            print "commit into %s" % branch
1074
1075        # start with reading files; if that fails, we should not
1076        # create a commit.
1077        new_files = []
1078        for f in files:
1079            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1080                new_files.append (f)
1081            else:
1082                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1083
1084        self.gitStream.write("commit %s\n" % branch)
1085#        gitStream.write("mark :%s\n" % details["change"])
1086        self.committedChanges.add(int(details["change"]))
1087        committer = ""
1088        if author not in self.users:
1089            self.getUserMapFromPerforceServer()
1090        if author in self.users:
1091            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1092        else:
1093            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1094
1095        self.gitStream.write("committer %s\n" % committer)
1096
1097        self.gitStream.write("data <<EOT\n")
1098        self.gitStream.write(details["desc"])
1099        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1100                             % (','.join (branchPrefixes), details["change"]))
1101        if len(details['options']) > 0:
1102            self.gitStream.write(": options = %s" % details['options'])
1103        self.gitStream.write("]\nEOT\n\n")
1104
1105        if len(parent) > 0:
1106            if self.verbose:
1107                print "parent %s" % parent
1108            self.gitStream.write("from %s\n" % parent)
1109
1110        self.streamP4Files(new_files)
1111        self.gitStream.write("\n")
1112
1113        change = int(details["change"])
1114
1115        if self.labels.has_key(change):
1116            label = self.labels[change]
1117            labelDetails = label[0]
1118            labelRevisions = label[1]
1119            if self.verbose:
1120                print "Change %s is labelled %s" % (change, labelDetails)
1121
1122            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1123                                                    for p in branchPrefixes]))
1124
1125            if len(files) == len(labelRevisions):
1126
1127                cleanedFiles = {}
1128                for info in files:
1129                    if info["action"] in ("delete", "purge"):
1130                        continue
1131                    cleanedFiles[info["depotFile"]] = info["rev"]
1132
1133                if cleanedFiles == labelRevisions:
1134                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1135                    self.gitStream.write("from %s\n" % branch)
1136
1137                    owner = labelDetails["Owner"]
1138                    tagger = ""
1139                    if author in self.users:
1140                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1141                    else:
1142                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1143                    self.gitStream.write("tagger %s\n" % tagger)
1144                    self.gitStream.write("data <<EOT\n")
1145                    self.gitStream.write(labelDetails["Description"])
1146                    self.gitStream.write("EOT\n\n")
1147
1148                else:
1149                    if not self.silent:
1150                        print ("Tag %s does not match with change %s: files do not match."
1151                               % (labelDetails["label"], change))
1152
1153            else:
1154                if not self.silent:
1155                    print ("Tag %s does not match with change %s: file count is different."
1156                           % (labelDetails["label"], change))
1157
1158    def getUserCacheFilename(self):
1159        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1160        return home + "/.gitp4-usercache.txt"
1161
1162    def getUserMapFromPerforceServer(self):
1163        if self.userMapFromPerforceServer:
1164            return
1165        self.users = {}
1166
1167        for output in p4CmdList("users"):
1168            if not output.has_key("User"):
1169                continue
1170            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1171
1172
1173        s = ''
1174        for (key, val) in self.users.items():
1175            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1176
1177        open(self.getUserCacheFilename(), "wb").write(s)
1178        self.userMapFromPerforceServer = True
1179
1180    def loadUserMapFromCache(self):
1181        self.users = {}
1182        self.userMapFromPerforceServer = False
1183        try:
1184            cache = open(self.getUserCacheFilename(), "rb")
1185            lines = cache.readlines()
1186            cache.close()
1187            for line in lines:
1188                entry = line.strip().split("\t")
1189                self.users[entry[0]] = entry[1]
1190        except IOError:
1191            self.getUserMapFromPerforceServer()
1192
1193    def getLabels(self):
1194        self.labels = {}
1195
1196        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1197        if len(l) > 0 and not self.silent:
1198            print "Finding files belonging to labels in %s" % `self.depotPaths`
1199
1200        for output in l:
1201            label = output["label"]
1202            revisions = {}
1203            newestChange = 0
1204            if self.verbose:
1205                print "Querying files for label %s" % label
1206            for file in p4CmdList("files "
1207                                  +  ' '.join (["%s...@%s" % (p, label)
1208                                                for p in self.depotPaths])):
1209                revisions[file["depotFile"]] = file["rev"]
1210                change = int(file["change"])
1211                if change > newestChange:
1212                    newestChange = change
1213
1214            self.labels[newestChange] = [output, revisions]
1215
1216        if self.verbose:
1217            print "Label changes: %s" % self.labels.keys()
1218
1219    def guessProjectName(self):
1220        for p in self.depotPaths:
1221            if p.endswith("/"):
1222                p = p[:-1]
1223            p = p[p.strip().rfind("/") + 1:]
1224            if not p.endswith("/"):
1225               p += "/"
1226            return p
1227
1228    def getBranchMapping(self):
1229        lostAndFoundBranches = set()
1230
1231        for info in p4CmdList("branches"):
1232            details = p4Cmd("branch -o %s" % info["branch"])
1233            viewIdx = 0
1234            while details.has_key("View%s" % viewIdx):
1235                paths = details["View%s" % viewIdx].split(" ")
1236                viewIdx = viewIdx + 1
1237                # require standard //depot/foo/... //depot/bar/... mapping
1238                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1239                    continue
1240                source = paths[0]
1241                destination = paths[1]
1242                ## HACK
1243                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1244                    source = source[len(self.depotPaths[0]):-4]
1245                    destination = destination[len(self.depotPaths[0]):-4]
1246
1247                    if destination in self.knownBranches:
1248                        if not self.silent:
1249                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1250                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1251                        continue
1252
1253                    self.knownBranches[destination] = source
1254
1255                    lostAndFoundBranches.discard(destination)
1256
1257                    if source not in self.knownBranches:
1258                        lostAndFoundBranches.add(source)
1259
1260
1261        for branch in lostAndFoundBranches:
1262            self.knownBranches[branch] = branch
1263
1264    def getBranchMappingFromGitBranches(self):
1265        branches = p4BranchesInGit(self.importIntoRemotes)
1266        for branch in branches.keys():
1267            if branch == "master":
1268                branch = "main"
1269            else:
1270                branch = branch[len(self.projectName):]
1271            self.knownBranches[branch] = branch
1272
1273    def listExistingP4GitBranches(self):
1274        # branches holds mapping from name to commit
1275        branches = p4BranchesInGit(self.importIntoRemotes)
1276        self.p4BranchesInGit = branches.keys()
1277        for branch in branches.keys():
1278            self.initialParents[self.refPrefix + branch] = branches[branch]
1279
1280    def updateOptionDict(self, d):
1281        option_keys = {}
1282        if self.keepRepoPath:
1283            option_keys['keepRepoPath'] = 1
1284
1285        d["options"] = ' '.join(sorted(option_keys.keys()))
1286
1287    def readOptions(self, d):
1288        self.keepRepoPath = (d.has_key('options')
1289                             and ('keepRepoPath' in d['options']))
1290
1291    def gitRefForBranch(self, branch):
1292        if branch == "main":
1293            return self.refPrefix + "master"
1294
1295        if len(branch) <= 0:
1296            return branch
1297
1298        return self.refPrefix + self.projectName + branch
1299
1300    def gitCommitByP4Change(self, ref, change):
1301        if self.verbose:
1302            print "looking in ref " + ref + " for change %s using bisect..." % change
1303
1304        earliestCommit = ""
1305        latestCommit = parseRevision(ref)
1306
1307        while True:
1308            if self.verbose:
1309                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1310            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1311            if len(next) == 0:
1312                if self.verbose:
1313                    print "argh"
1314                return ""
1315            log = extractLogMessageFromGitCommit(next)
1316            settings = extractSettingsGitLog(log)
1317            currentChange = int(settings['change'])
1318            if self.verbose:
1319                print "current change %s" % currentChange
1320
1321            if currentChange == change:
1322                if self.verbose:
1323                    print "found %s" % next
1324                return next
1325
1326            if currentChange < change:
1327                earliestCommit = "^%s" % next
1328            else:
1329                latestCommit = "%s" % next
1330
1331        return ""
1332
1333    def importNewBranch(self, branch, maxChange):
1334        # make fast-import flush all changes to disk and update the refs using the checkpoint
1335        # command so that we can try to find the branch parent in the git history
1336        self.gitStream.write("checkpoint\n\n");
1337        self.gitStream.flush();
1338        branchPrefix = self.depotPaths[0] + branch + "/"
1339        range = "@1,%s" % maxChange
1340        #print "prefix" + branchPrefix
1341        changes = p4ChangesForPaths([branchPrefix], range)
1342        if len(changes) <= 0:
1343            return False
1344        firstChange = changes[0]
1345        #print "first change in branch: %s" % firstChange
1346        sourceBranch = self.knownBranches[branch]
1347        sourceDepotPath = self.depotPaths[0] + sourceBranch
1348        sourceRef = self.gitRefForBranch(sourceBranch)
1349        #print "source " + sourceBranch
1350
1351        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1352        #print "branch parent: %s" % branchParentChange
1353        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1354        if len(gitParent) > 0:
1355            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1356            #print "parent git commit: %s" % gitParent
1357
1358        self.importChanges(changes)
1359        return True
1360
1361    def importChanges(self, changes):
1362        cnt = 1
1363        for change in changes:
1364            description = p4Cmd("describe %s" % change)
1365            self.updateOptionDict(description)
1366
1367            if not self.silent:
1368                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1369                sys.stdout.flush()
1370            cnt = cnt + 1
1371
1372            try:
1373                if self.detectBranches:
1374                    branches = self.splitFilesIntoBranches(description)
1375                    for branch in branches.keys():
1376                        ## HACK  --hwn
1377                        branchPrefix = self.depotPaths[0] + branch + "/"
1378
1379                        parent = ""
1380
1381                        filesForCommit = branches[branch]
1382
1383                        if self.verbose:
1384                            print "branch is %s" % branch
1385
1386                        self.updatedBranches.add(branch)
1387
1388                        if branch not in self.createdBranches:
1389                            self.createdBranches.add(branch)
1390                            parent = self.knownBranches[branch]
1391                            if parent == branch:
1392                                parent = ""
1393                            else:
1394                                fullBranch = self.projectName + branch
1395                                if fullBranch not in self.p4BranchesInGit:
1396                                    if not self.silent:
1397                                        print("\n    Importing new branch %s" % fullBranch);
1398                                    if self.importNewBranch(branch, change - 1):
1399                                        parent = ""
1400                                        self.p4BranchesInGit.append(fullBranch)
1401                                    if not self.silent:
1402                                        print("\n    Resuming with change %s" % change);
1403
1404                                if self.verbose:
1405                                    print "parent determined through known branches: %s" % parent
1406
1407                        branch = self.gitRefForBranch(branch)
1408                        parent = self.gitRefForBranch(parent)
1409
1410                        if self.verbose:
1411                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1412
1413                        if len(parent) == 0 and branch in self.initialParents:
1414                            parent = self.initialParents[branch]
1415                            del self.initialParents[branch]
1416
1417                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1418                else:
1419                    files = self.extractFilesFromCommit(description)
1420                    self.commit(description, files, self.branch, self.depotPaths,
1421                                self.initialParent)
1422                    self.initialParent = ""
1423            except IOError:
1424                print self.gitError.read()
1425                sys.exit(1)
1426
1427    def importHeadRevision(self, revision):
1428        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1429
1430        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1431        details["desc"] = ("Initial import of %s from the state at revision %s"
1432                           % (' '.join(self.depotPaths), revision))
1433        details["change"] = revision
1434        newestRevision = 0
1435
1436        fileCnt = 0
1437        for info in p4CmdList("files "
1438                              +  ' '.join(["%s...%s"
1439                                           % (p, revision)
1440                                           for p in self.depotPaths])):
1441
1442            if info['code'] == 'error':
1443                sys.stderr.write("p4 returned an error: %s\n"
1444                                 % info['data'])
1445                sys.exit(1)
1446
1447
1448            change = int(info["change"])
1449            if change > newestRevision:
1450                newestRevision = change
1451
1452            if info["action"] in ("delete", "purge"):
1453                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1454                #fileCnt = fileCnt + 1
1455                continue
1456
1457            for prop in ["depotFile", "rev", "action", "type" ]:
1458                details["%s%s" % (prop, fileCnt)] = info[prop]
1459
1460            fileCnt = fileCnt + 1
1461
1462        details["change"] = newestRevision
1463        self.updateOptionDict(details)
1464        try:
1465            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1466        except IOError:
1467            print "IO error with git fast-import. Is your git version recent enough?"
1468            print self.gitError.read()
1469
1470
1471    def getClientSpec(self):
1472        specList = p4CmdList( "client -o" )
1473        temp = {}
1474        for entry in specList:
1475            for k,v in entry.iteritems():
1476                if k.startswith("View"):
1477                    if v.startswith('"'):
1478                        start = 1
1479                    else:
1480                        start = 0
1481                    index = v.find("...")
1482                    v = v[start:index]
1483                    if v.startswith("-"):
1484                        v = v[1:]
1485                        temp[v] = -len(v)
1486                    else:
1487                        temp[v] = len(v)
1488        self.clientSpecDirs = temp.items()
1489        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1490
1491    def run(self, args):
1492        self.depotPaths = []
1493        self.changeRange = ""
1494        self.initialParent = ""
1495        self.previousDepotPaths = []
1496
1497        # map from branch depot path to parent branch
1498        self.knownBranches = {}
1499        self.initialParents = {}
1500        self.hasOrigin = originP4BranchesExist()
1501        if not self.syncWithOrigin:
1502            self.hasOrigin = False
1503
1504        if self.importIntoRemotes:
1505            self.refPrefix = "refs/remotes/p4/"
1506        else:
1507            self.refPrefix = "refs/heads/p4/"
1508
1509        if self.syncWithOrigin and self.hasOrigin:
1510            if not self.silent:
1511                print "Syncing with origin first by calling git fetch origin"
1512            system("git fetch origin")
1513
1514        if len(self.branch) == 0:
1515            self.branch = self.refPrefix + "master"
1516            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1517                system("git update-ref %s refs/heads/p4" % self.branch)
1518                system("git branch -D p4");
1519            # create it /after/ importing, when master exists
1520            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1521                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1522
1523        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1524            self.getClientSpec()
1525
1526        # TODO: should always look at previous commits,
1527        # merge with previous imports, if possible.
1528        if args == []:
1529            if self.hasOrigin:
1530                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1531            self.listExistingP4GitBranches()
1532
1533            if len(self.p4BranchesInGit) > 1:
1534                if not self.silent:
1535                    print "Importing from/into multiple branches"
1536                self.detectBranches = True
1537
1538            if self.verbose:
1539                print "branches: %s" % self.p4BranchesInGit
1540
1541            p4Change = 0
1542            for branch in self.p4BranchesInGit:
1543                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1544
1545                settings = extractSettingsGitLog(logMsg)
1546
1547                self.readOptions(settings)
1548                if (settings.has_key('depot-paths')
1549                    and settings.has_key ('change')):
1550                    change = int(settings['change']) + 1
1551                    p4Change = max(p4Change, change)
1552
1553                    depotPaths = sorted(settings['depot-paths'])
1554                    if self.previousDepotPaths == []:
1555                        self.previousDepotPaths = depotPaths
1556                    else:
1557                        paths = []
1558                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1559                            for i in range(0, min(len(cur), len(prev))):
1560                                if cur[i] <> prev[i]:
1561                                    i = i - 1
1562                                    break
1563
1564                            paths.append (cur[:i + 1])
1565
1566                        self.previousDepotPaths = paths
1567
1568            if p4Change > 0:
1569                self.depotPaths = sorted(self.previousDepotPaths)
1570                self.changeRange = "@%s,#head" % p4Change
1571                if not self.detectBranches:
1572                    self.initialParent = parseRevision(self.branch)
1573                if not self.silent and not self.detectBranches:
1574                    print "Performing incremental import into %s git branch" % self.branch
1575
1576        if not self.branch.startswith("refs/"):
1577            self.branch = "refs/heads/" + self.branch
1578
1579        if len(args) == 0 and self.depotPaths:
1580            if not self.silent:
1581                print "Depot paths: %s" % ' '.join(self.depotPaths)
1582        else:
1583            if self.depotPaths and self.depotPaths != args:
1584                print ("previous import used depot path %s and now %s was specified. "
1585                       "This doesn't work!" % (' '.join (self.depotPaths),
1586                                               ' '.join (args)))
1587                sys.exit(1)
1588
1589            self.depotPaths = sorted(args)
1590
1591        revision = ""
1592        self.users = {}
1593
1594        newPaths = []
1595        for p in self.depotPaths:
1596            if p.find("@") != -1:
1597                atIdx = p.index("@")
1598                self.changeRange = p[atIdx:]
1599                if self.changeRange == "@all":
1600                    self.changeRange = ""
1601                elif ',' not in self.changeRange:
1602                    revision = self.changeRange
1603                    self.changeRange = ""
1604                p = p[:atIdx]
1605            elif p.find("#") != -1:
1606                hashIdx = p.index("#")
1607                revision = p[hashIdx:]
1608                p = p[:hashIdx]
1609            elif self.previousDepotPaths == []:
1610                revision = "#head"
1611
1612            p = re.sub ("\.\.\.$", "", p)
1613            if not p.endswith("/"):
1614                p += "/"
1615
1616            newPaths.append(p)
1617
1618        self.depotPaths = newPaths
1619
1620
1621        self.loadUserMapFromCache()
1622        self.labels = {}
1623        if self.detectLabels:
1624            self.getLabels();
1625
1626        if self.detectBranches:
1627            ## FIXME - what's a P4 projectName ?
1628            self.projectName = self.guessProjectName()
1629
1630            if self.hasOrigin:
1631                self.getBranchMappingFromGitBranches()
1632            else:
1633                self.getBranchMapping()
1634            if self.verbose:
1635                print "p4-git branches: %s" % self.p4BranchesInGit
1636                print "initial parents: %s" % self.initialParents
1637            for b in self.p4BranchesInGit:
1638                if b != "master":
1639
1640                    ## FIXME
1641                    b = b[len(self.projectName):]
1642                self.createdBranches.add(b)
1643
1644        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1645
1646        importProcess = subprocess.Popen(["git", "fast-import"],
1647                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1648                                         stderr=subprocess.PIPE);
1649        self.gitOutput = importProcess.stdout
1650        self.gitStream = importProcess.stdin
1651        self.gitError = importProcess.stderr
1652
1653        if revision:
1654            self.importHeadRevision(revision)
1655        else:
1656            changes = []
1657
1658            if len(self.changesFile) > 0:
1659                output = open(self.changesFile).readlines()
1660                changeSet = set()
1661                for line in output:
1662                    changeSet.add(int(line))
1663
1664                for change in changeSet:
1665                    changes.append(change)
1666
1667                changes.sort()
1668            else:
1669                if self.verbose:
1670                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1671                                                              self.changeRange)
1672                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1673
1674                if len(self.maxChanges) > 0:
1675                    changes = changes[:min(int(self.maxChanges), len(changes))]
1676
1677            if len(changes) == 0:
1678                if not self.silent:
1679                    print "No changes to import!"
1680                return True
1681
1682            if not self.silent and not self.detectBranches:
1683                print "Import destination: %s" % self.branch
1684
1685            self.updatedBranches = set()
1686
1687            self.importChanges(changes)
1688
1689            if not self.silent:
1690                print ""
1691                if len(self.updatedBranches) > 0:
1692                    sys.stdout.write("Updated branches: ")
1693                    for b in self.updatedBranches:
1694                        sys.stdout.write("%s " % b)
1695                    sys.stdout.write("\n")
1696
1697        self.gitStream.close()
1698        if importProcess.wait() != 0:
1699            die("fast-import failed: %s" % self.gitError.read())
1700        self.gitOutput.close()
1701        self.gitError.close()
1702
1703        return True
1704
1705class P4Rebase(Command):
1706    def __init__(self):
1707        Command.__init__(self)
1708        self.options = [ ]
1709        self.description = ("Fetches the latest revision from perforce and "
1710                            + "rebases the current work (branch) against it")
1711        self.verbose = False
1712
1713    def run(self, args):
1714        sync = P4Sync()
1715        sync.run([])
1716
1717        return self.rebase()
1718
1719    def rebase(self):
1720        if os.system("git update-index --refresh") != 0:
1721            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1722        if len(read_pipe("git diff-index HEAD --")) > 0:
1723            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1724
1725        [upstream, settings] = findUpstreamBranchPoint()
1726        if len(upstream) == 0:
1727            die("Cannot find upstream branchpoint for rebase")
1728
1729        # the branchpoint may be p4/foo~3, so strip off the parent
1730        upstream = re.sub("~[0-9]+$", "", upstream)
1731
1732        print "Rebasing the current branch onto %s" % upstream
1733        oldHead = read_pipe("git rev-parse HEAD").strip()
1734        system("git rebase %s" % upstream)
1735        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1736        return True
1737
1738class P4Clone(P4Sync):
1739    def __init__(self):
1740        P4Sync.__init__(self)
1741        self.description = "Creates a new git repository and imports from Perforce into it"
1742        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1743        self.options += [
1744            optparse.make_option("--destination", dest="cloneDestination",
1745                                 action='store', default=None,
1746                                 help="where to leave result of the clone"),
1747            optparse.make_option("-/", dest="cloneExclude",
1748                                 action="append", type="string",
1749                                 help="exclude depot path")
1750        ]
1751        self.cloneDestination = None
1752        self.needsGit = False
1753
1754    # This is required for the "append" cloneExclude action
1755    def ensure_value(self, attr, value):
1756        if not hasattr(self, attr) or getattr(self, attr) is None:
1757            setattr(self, attr, value)
1758        return getattr(self, attr)
1759
1760    def defaultDestination(self, args):
1761        ## TODO: use common prefix of args?
1762        depotPath = args[0]
1763        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1764        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1765        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1766        depotDir = re.sub(r"/$", "", depotDir)
1767        return os.path.split(depotDir)[1]
1768
1769    def run(self, args):
1770        if len(args) < 1:
1771            return False
1772
1773        if self.keepRepoPath and not self.cloneDestination:
1774            sys.stderr.write("Must specify destination for --keep-path\n")
1775            sys.exit(1)
1776
1777        depotPaths = args
1778
1779        if not self.cloneDestination and len(depotPaths) > 1:
1780            self.cloneDestination = depotPaths[-1]
1781            depotPaths = depotPaths[:-1]
1782
1783        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1784        for p in depotPaths:
1785            if not p.startswith("//"):
1786                return False
1787
1788        if not self.cloneDestination:
1789            self.cloneDestination = self.defaultDestination(args)
1790
1791        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1792        if not os.path.exists(self.cloneDestination):
1793            os.makedirs(self.cloneDestination)
1794        chdir(self.cloneDestination)
1795        system("git init")
1796        self.gitdir = os.getcwd() + "/.git"
1797        if not P4Sync.run(self, depotPaths):
1798            return False
1799        if self.branch != "master":
1800            if self.importIntoRemotes:
1801                masterbranch = "refs/remotes/p4/master"
1802            else:
1803                masterbranch = "refs/heads/p4/master"
1804            if gitBranchExists(masterbranch):
1805                system("git branch master %s" % masterbranch)
1806                system("git checkout -f")
1807            else:
1808                print "Could not detect main branch. No checkout/master branch created."
1809
1810        return True
1811
1812class P4Branches(Command):
1813    def __init__(self):
1814        Command.__init__(self)
1815        self.options = [ ]
1816        self.description = ("Shows the git branches that hold imports and their "
1817                            + "corresponding perforce depot paths")
1818        self.verbose = False
1819
1820    def run(self, args):
1821        if originP4BranchesExist():
1822            createOrUpdateBranchesFromOrigin()
1823
1824        cmdline = "git rev-parse --symbolic "
1825        cmdline += " --remotes"
1826
1827        for line in read_pipe_lines(cmdline):
1828            line = line.strip()
1829
1830            if not line.startswith('p4/') or line == "p4/HEAD":
1831                continue
1832            branch = line
1833
1834            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1835            settings = extractSettingsGitLog(log)
1836
1837            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1838        return True
1839
1840class HelpFormatter(optparse.IndentedHelpFormatter):
1841    def __init__(self):
1842        optparse.IndentedHelpFormatter.__init__(self)
1843
1844    def format_description(self, description):
1845        if description:
1846            return description + "\n"
1847        else:
1848            return ""
1849
1850def printUsage(commands):
1851    print "usage: %s <command> [options]" % sys.argv[0]
1852    print ""
1853    print "valid commands: %s" % ", ".join(commands)
1854    print ""
1855    print "Try %s <command> --help for command specific help." % sys.argv[0]
1856    print ""
1857
1858commands = {
1859    "debug" : P4Debug,
1860    "submit" : P4Submit,
1861    "commit" : P4Submit,
1862    "sync" : P4Sync,
1863    "rebase" : P4Rebase,
1864    "clone" : P4Clone,
1865    "rollback" : P4RollBack,
1866    "branches" : P4Branches
1867}
1868
1869
1870def main():
1871    if len(sys.argv[1:]) == 0:
1872        printUsage(commands.keys())
1873        sys.exit(2)
1874
1875    cmd = ""
1876    cmdName = sys.argv[1]
1877    try:
1878        klass = commands[cmdName]
1879        cmd = klass()
1880    except KeyError:
1881        print "unknown command %s" % cmdName
1882        print ""
1883        printUsage(commands.keys())
1884        sys.exit(2)
1885
1886    options = cmd.options
1887    cmd.gitdir = os.environ.get("GIT_DIR", None)
1888
1889    args = sys.argv[2:]
1890
1891    if len(options) > 0:
1892        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1893
1894        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1895                                       options,
1896                                       description = cmd.description,
1897                                       formatter = HelpFormatter())
1898
1899        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1900    global verbose
1901    verbose = cmd.verbose
1902    if cmd.needsGit:
1903        if cmd.gitdir == None:
1904            cmd.gitdir = os.path.abspath(".git")
1905            if not isValidGitDir(cmd.gitdir):
1906                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1907                if os.path.exists(cmd.gitdir):
1908                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1909                    if len(cdup) > 0:
1910                        chdir(cdup);
1911
1912        if not isValidGitDir(cmd.gitdir):
1913            if isValidGitDir(cmd.gitdir + "/.git"):
1914                cmd.gitdir += "/.git"
1915            else:
1916                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1917
1918        os.environ["GIT_DIR"] = cmd.gitdir
1919
1920    if not cmd.run(args):
1921        parser.print_help()
1922
1923
1924if __name__ == '__main__':
1925    main()