Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: decode p4 wildcard characters (084f630)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRename", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRename = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616        diffOpts = ("", "-M")[self.detectRename]
 617        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618        filesToAdd = set()
 619        filesToDelete = set()
 620        editedFiles = set()
 621        filesToChangeExecBit = {}
 622        for line in diff:
 623            diff = parseDiffTreeEntry(line)
 624            modifier = diff['status']
 625            path = diff['src']
 626            if modifier == "M":
 627                p4_system("edit \"%s\"" % path)
 628                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                    filesToChangeExecBit[path] = diff['dst_mode']
 630                editedFiles.add(path)
 631            elif modifier == "A":
 632                filesToAdd.add(path)
 633                filesToChangeExecBit[path] = diff['dst_mode']
 634                if path in filesToDelete:
 635                    filesToDelete.remove(path)
 636            elif modifier == "D":
 637                filesToDelete.add(path)
 638                if path in filesToAdd:
 639                    filesToAdd.remove(path)
 640            elif modifier == "R":
 641                src, dest = diff['src'], diff['dst']
 642                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                p4_system("edit \"%s\"" % (dest))
 644                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                    filesToChangeExecBit[dest] = diff['dst_mode']
 646                os.unlink(dest)
 647                editedFiles.add(dest)
 648                filesToDelete.add(src)
 649            else:
 650                die("unknown modifier %s for %s" % (modifier, path))
 651
 652        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653        patchcmd = diffcmd + " | git apply "
 654        tryPatchCmd = patchcmd + "--check -"
 655        applyPatchCmd = patchcmd + "--check --apply -"
 656
 657        if os.system(tryPatchCmd) != 0:
 658            print "Unfortunately applying the change failed!"
 659            print "What do you want to do?"
 660            response = "x"
 661            while response != "s" and response != "a" and response != "w":
 662                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664            if response == "s":
 665                print "Skipping! Good luck with the next patches..."
 666                for f in editedFiles:
 667                    p4_system("revert \"%s\"" % f);
 668                for f in filesToAdd:
 669                    system("rm %s" %f)
 670                return
 671            elif response == "a":
 672                os.system(applyPatchCmd)
 673                if len(filesToAdd) > 0:
 674                    print "You may also want to call p4 add on the following files:"
 675                    print " ".join(filesToAdd)
 676                if len(filesToDelete):
 677                    print "The following files should be scheduled for deletion with p4 delete:"
 678                    print " ".join(filesToDelete)
 679                die("Please resolve and submit the conflict manually and "
 680                    + "continue afterwards with git-p4 submit --continue")
 681            elif response == "w":
 682                system(diffcmd + " > patch.txt")
 683                print "Patch saved to patch.txt in %s !" % self.clientPath
 684                die("Please resolve and submit the conflict manually and "
 685                    "continue afterwards with git-p4 submit --continue")
 686
 687        system(applyPatchCmd)
 688
 689        for f in filesToAdd:
 690            p4_system("add \"%s\"" % f)
 691        for f in filesToDelete:
 692            p4_system("revert \"%s\"" % f)
 693            p4_system("delete \"%s\"" % f)
 694
 695        # Set/clear executable bits
 696        for f in filesToChangeExecBit.keys():
 697            mode = filesToChangeExecBit[f]
 698            setP4ExecBit(f, mode)
 699
 700        logMessage = extractLogMessageFromGitCommit(id)
 701        logMessage = logMessage.strip()
 702
 703        template = self.prepareSubmitTemplate()
 704
 705        if self.interactive:
 706            submitTemplate = self.prepareLogMessage(template, logMessage)
 707            if os.environ.has_key("P4DIFF"):
 708                del(os.environ["P4DIFF"])
 709            diff = ""
 710            for editedFile in editedFiles:
 711                diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713            newdiff = ""
 714            for newFile in filesToAdd:
 715                newdiff += "==== new file ====\n"
 716                newdiff += "--- /dev/null\n"
 717                newdiff += "+++ %s\n" % newFile
 718                f = open(newFile, "r")
 719                for line in f.readlines():
 720                    newdiff += "+" + line
 721                f.close()
 722
 723            separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725            [handle, fileName] = tempfile.mkstemp()
 726            tmpFile = os.fdopen(handle, "w+")
 727            if self.isWindows:
 728                submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                separatorLine = separatorLine.replace("\n", "\r\n")
 730                newdiff = newdiff.replace("\n", "\r\n")
 731            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732            tmpFile.close()
 733            mtime = os.stat(fileName).st_mtime
 734            if os.environ.has_key("P4EDITOR"):
 735                editor = os.environ.get("P4EDITOR")
 736            else:
 737                editor = read_pipe("git var GIT_EDITOR").strip()
 738            system(editor + " " + fileName)
 739
 740            response = "y"
 741            if os.stat(fileName).st_mtime <= mtime:
 742                response = "x"
 743                while response != "y" and response != "n":
 744                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746            if response == "y":
 747                tmpFile = open(fileName, "rb")
 748                message = tmpFile.read()
 749                tmpFile.close()
 750                submitTemplate = message[:message.index(separatorLine)]
 751                if self.isWindows:
 752                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                p4_write_pipe("submit -i", submitTemplate)
 754            else:
 755                for f in editedFiles:
 756                    p4_system("revert \"%s\"" % f);
 757                for f in filesToAdd:
 758                    p4_system("revert \"%s\"" % f);
 759                    system("rm %s" %f)
 760
 761            os.remove(fileName)
 762        else:
 763            fileName = "submit.txt"
 764            file = open(fileName, "w+")
 765            file.write(self.prepareLogMessage(template, logMessage))
 766            file.close()
 767            print ("Perforce submit template written as %s. "
 768                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                   % (fileName, fileName))
 770
 771    def run(self, args):
 772        if len(args) == 0:
 773            self.master = currentGitBranch()
 774            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                die("Detecting current git branch failed!")
 776        elif len(args) == 1:
 777            self.master = args[0]
 778        else:
 779            return False
 780
 781        allowSubmit = gitConfig("git-p4.allowSubmit")
 782        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783            die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785        [upstream, settings] = findUpstreamBranchPoint()
 786        self.depotPath = settings['depot-paths'][0]
 787        if len(self.origin) == 0:
 788            self.origin = upstream
 789
 790        if self.verbose:
 791            print "Origin branch is " + self.origin
 792
 793        if len(self.depotPath) == 0:
 794            print "Internal error: cannot locate perforce depot path from existing branches"
 795            sys.exit(128)
 796
 797        self.clientPath = p4Where(self.depotPath)
 798
 799        if len(self.clientPath) == 0:
 800            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801            sys.exit(128)
 802
 803        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804        self.oldWorkingDirectory = os.getcwd()
 805
 806        chdir(self.clientPath)
 807        print "Synchronizing p4 checkout..."
 808        p4_system("sync ...")
 809
 810        self.check()
 811
 812        commits = []
 813        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814            commits.append(line.strip())
 815        commits.reverse()
 816
 817        while len(commits) > 0:
 818            commit = commits[0]
 819            commits = commits[1:]
 820            self.applyCommit(commit)
 821            if not self.interactive:
 822                break
 823
 824        if len(commits) == 0:
 825            print "All changes applied!"
 826            chdir(self.oldWorkingDirectory)
 827
 828            sync = P4Sync()
 829            sync.run([])
 830
 831            rebase = P4Rebase()
 832            rebase.rebase()
 833
 834        return True
 835
 836class P4Sync(Command):
 837    delete_actions = ( "delete", "move/delete", "purge" )
 838
 839    def __init__(self):
 840        Command.__init__(self)
 841        self.options = [
 842                optparse.make_option("--branch", dest="branch"),
 843                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                optparse.make_option("--changesfile", dest="changesFile"),
 845                optparse.make_option("--silent", dest="silent", action="store_true"),
 846                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                     help="Import into refs/heads/ , not refs/remotes"),
 850                optparse.make_option("--max-changes", dest="maxChanges"),
 851                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                     help="Only sync files that are included in the Perforce Client Spec")
 855        ]
 856        self.description = """Imports from Perforce into a git repository.\n
 857    example:
 858    //depot/my/project/ -- to import the current head
 859    //depot/my/project/@all -- to import everything
 860    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864        self.usage += " //depot/path[@revRange]"
 865        self.silent = False
 866        self.createdBranches = set()
 867        self.committedChanges = set()
 868        self.branch = ""
 869        self.detectBranches = False
 870        self.detectLabels = False
 871        self.changesFile = ""
 872        self.syncWithOrigin = True
 873        self.verbose = False
 874        self.importIntoRemotes = True
 875        self.maxChanges = ""
 876        self.isWindows = (platform.system() == "Windows")
 877        self.keepRepoPath = False
 878        self.depotPaths = None
 879        self.p4BranchesInGit = []
 880        self.cloneExclude = []
 881        self.useClientSpec = False
 882        self.clientSpecDirs = []
 883
 884        if gitConfig("git-p4.syncFromOrigin") == "false":
 885            self.syncWithOrigin = False
 886
 887    #
 888    # P4 wildcards are not allowed in filenames.  P4 complains
 889    # if you simply add them, but you can force it with "-f", in
 890    # which case it translates them into %xx encoding internally.
 891    # Search for and fix just these four characters.  Do % last so
 892    # that fixing it does not inadvertently create new %-escapes.
 893    #
 894    def wildcard_decode(self, path):
 895        # Cannot have * in a filename in windows; untested as to
 896        # what p4 would do in such a case.
 897        if not self.isWindows:
 898            path = path.replace("%2A", "*")
 899        path = path.replace("%23", "#") \
 900                   .replace("%40", "@") \
 901                   .replace("%25", "%")
 902        return path
 903
 904    def extractFilesFromCommit(self, commit):
 905        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 906                             for path in self.cloneExclude]
 907        files = []
 908        fnum = 0
 909        while commit.has_key("depotFile%s" % fnum):
 910            path =  commit["depotFile%s" % fnum]
 911
 912            if [p for p in self.cloneExclude
 913                if path.startswith (p)]:
 914                found = False
 915            else:
 916                found = [p for p in self.depotPaths
 917                         if path.startswith (p)]
 918            if not found:
 919                fnum = fnum + 1
 920                continue
 921
 922            file = {}
 923            file["path"] = path
 924            file["rev"] = commit["rev%s" % fnum]
 925            file["action"] = commit["action%s" % fnum]
 926            file["type"] = commit["type%s" % fnum]
 927            files.append(file)
 928            fnum = fnum + 1
 929        return files
 930
 931    def stripRepoPath(self, path, prefixes):
 932        if self.keepRepoPath:
 933            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 934
 935        for p in prefixes:
 936            if path.startswith(p):
 937                path = path[len(p):]
 938
 939        return path
 940
 941    def splitFilesIntoBranches(self, commit):
 942        branches = {}
 943        fnum = 0
 944        while commit.has_key("depotFile%s" % fnum):
 945            path =  commit["depotFile%s" % fnum]
 946            found = [p for p in self.depotPaths
 947                     if path.startswith (p)]
 948            if not found:
 949                fnum = fnum + 1
 950                continue
 951
 952            file = {}
 953            file["path"] = path
 954            file["rev"] = commit["rev%s" % fnum]
 955            file["action"] = commit["action%s" % fnum]
 956            file["type"] = commit["type%s" % fnum]
 957            fnum = fnum + 1
 958
 959            relPath = self.stripRepoPath(path, self.depotPaths)
 960
 961            for branch in self.knownBranches.keys():
 962
 963                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 964                if relPath.startswith(branch + "/"):
 965                    if branch not in branches:
 966                        branches[branch] = []
 967                    branches[branch].append(file)
 968                    break
 969
 970        return branches
 971
 972    # output one file from the P4 stream
 973    # - helper for streamP4Files
 974
 975    def streamOneP4File(self, file, contents):
 976        if file["type"] == "apple":
 977            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 978                file['depotFile']
 979            return
 980
 981        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 982        relPath = self.wildcard_decode(relPath)
 983        if verbose:
 984            sys.stderr.write("%s\n" % relPath)
 985
 986        mode = "644"
 987        if isP4Exec(file["type"]):
 988            mode = "755"
 989        elif file["type"] == "symlink":
 990            mode = "120000"
 991            # p4 print on a symlink contains "target\n", so strip it off
 992            data = ''.join(contents)
 993            contents = [data[:-1]]
 994
 995        if self.isWindows and file["type"].endswith("text"):
 996            mangled = []
 997            for data in contents:
 998                data = data.replace("\r\n", "\n")
 999                mangled.append(data)
1000            contents = mangled
1001
1002        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1003            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1004        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1005            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1006
1007        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1008
1009        # total length...
1010        length = 0
1011        for d in contents:
1012            length = length + len(d)
1013
1014        self.gitStream.write("data %d\n" % length)
1015        for d in contents:
1016            self.gitStream.write(d)
1017        self.gitStream.write("\n")
1018
1019    def streamOneP4Deletion(self, file):
1020        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1021        if verbose:
1022            sys.stderr.write("delete %s\n" % relPath)
1023        self.gitStream.write("D %s\n" % relPath)
1024
1025    # handle another chunk of streaming data
1026    def streamP4FilesCb(self, marshalled):
1027
1028        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1029            # start of a new file - output the old one first
1030            self.streamOneP4File(self.stream_file, self.stream_contents)
1031            self.stream_file = {}
1032            self.stream_contents = []
1033            self.stream_have_file_info = False
1034
1035        # pick up the new file information... for the
1036        # 'data' field we need to append to our array
1037        for k in marshalled.keys():
1038            if k == 'data':
1039                self.stream_contents.append(marshalled['data'])
1040            else:
1041                self.stream_file[k] = marshalled[k]
1042
1043        self.stream_have_file_info = True
1044
1045    # Stream directly from "p4 files" into "git fast-import"
1046    def streamP4Files(self, files):
1047        filesForCommit = []
1048        filesToRead = []
1049        filesToDelete = []
1050
1051        for f in files:
1052            includeFile = True
1053            for val in self.clientSpecDirs:
1054                if f['path'].startswith(val[0]):
1055                    if val[1] <= 0:
1056                        includeFile = False
1057                    break
1058
1059            if includeFile:
1060                filesForCommit.append(f)
1061                if f['action'] in self.delete_actions:
1062                    filesToDelete.append(f)
1063                else:
1064                    filesToRead.append(f)
1065
1066        # deleted files...
1067        for f in filesToDelete:
1068            self.streamOneP4Deletion(f)
1069
1070        if len(filesToRead) > 0:
1071            self.stream_file = {}
1072            self.stream_contents = []
1073            self.stream_have_file_info = False
1074
1075            # curry self argument
1076            def streamP4FilesCbSelf(entry):
1077                self.streamP4FilesCb(entry)
1078
1079            p4CmdList("-x - print",
1080                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1081                                                  for f in filesToRead]),
1082                cb=streamP4FilesCbSelf)
1083
1084            # do the last chunk
1085            if self.stream_file.has_key('depotFile'):
1086                self.streamOneP4File(self.stream_file, self.stream_contents)
1087
1088    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1089        epoch = details["time"]
1090        author = details["user"]
1091        self.branchPrefixes = branchPrefixes
1092
1093        if self.verbose:
1094            print "commit into %s" % branch
1095
1096        # start with reading files; if that fails, we should not
1097        # create a commit.
1098        new_files = []
1099        for f in files:
1100            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1101                new_files.append (f)
1102            else:
1103                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1104
1105        self.gitStream.write("commit %s\n" % branch)
1106#        gitStream.write("mark :%s\n" % details["change"])
1107        self.committedChanges.add(int(details["change"]))
1108        committer = ""
1109        if author not in self.users:
1110            self.getUserMapFromPerforceServer()
1111        if author in self.users:
1112            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1113        else:
1114            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1115
1116        self.gitStream.write("committer %s\n" % committer)
1117
1118        self.gitStream.write("data <<EOT\n")
1119        self.gitStream.write(details["desc"])
1120        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1121                             % (','.join (branchPrefixes), details["change"]))
1122        if len(details['options']) > 0:
1123            self.gitStream.write(": options = %s" % details['options'])
1124        self.gitStream.write("]\nEOT\n\n")
1125
1126        if len(parent) > 0:
1127            if self.verbose:
1128                print "parent %s" % parent
1129            self.gitStream.write("from %s\n" % parent)
1130
1131        self.streamP4Files(new_files)
1132        self.gitStream.write("\n")
1133
1134        change = int(details["change"])
1135
1136        if self.labels.has_key(change):
1137            label = self.labels[change]
1138            labelDetails = label[0]
1139            labelRevisions = label[1]
1140            if self.verbose:
1141                print "Change %s is labelled %s" % (change, labelDetails)
1142
1143            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1144                                                    for p in branchPrefixes]))
1145
1146            if len(files) == len(labelRevisions):
1147
1148                cleanedFiles = {}
1149                for info in files:
1150                    if info["action"] in self.delete_actions:
1151                        continue
1152                    cleanedFiles[info["depotFile"]] = info["rev"]
1153
1154                if cleanedFiles == labelRevisions:
1155                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1156                    self.gitStream.write("from %s\n" % branch)
1157
1158                    owner = labelDetails["Owner"]
1159                    tagger = ""
1160                    if author in self.users:
1161                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1162                    else:
1163                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1164                    self.gitStream.write("tagger %s\n" % tagger)
1165                    self.gitStream.write("data <<EOT\n")
1166                    self.gitStream.write(labelDetails["Description"])
1167                    self.gitStream.write("EOT\n\n")
1168
1169                else:
1170                    if not self.silent:
1171                        print ("Tag %s does not match with change %s: files do not match."
1172                               % (labelDetails["label"], change))
1173
1174            else:
1175                if not self.silent:
1176                    print ("Tag %s does not match with change %s: file count is different."
1177                           % (labelDetails["label"], change))
1178
1179    def getUserCacheFilename(self):
1180        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1181        return home + "/.gitp4-usercache.txt"
1182
1183    def getUserMapFromPerforceServer(self):
1184        if self.userMapFromPerforceServer:
1185            return
1186        self.users = {}
1187
1188        for output in p4CmdList("users"):
1189            if not output.has_key("User"):
1190                continue
1191            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1192
1193
1194        s = ''
1195        for (key, val) in self.users.items():
1196            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1197
1198        open(self.getUserCacheFilename(), "wb").write(s)
1199        self.userMapFromPerforceServer = True
1200
1201    def loadUserMapFromCache(self):
1202        self.users = {}
1203        self.userMapFromPerforceServer = False
1204        try:
1205            cache = open(self.getUserCacheFilename(), "rb")
1206            lines = cache.readlines()
1207            cache.close()
1208            for line in lines:
1209                entry = line.strip().split("\t")
1210                self.users[entry[0]] = entry[1]
1211        except IOError:
1212            self.getUserMapFromPerforceServer()
1213
1214    def getLabels(self):
1215        self.labels = {}
1216
1217        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1218        if len(l) > 0 and not self.silent:
1219            print "Finding files belonging to labels in %s" % `self.depotPaths`
1220
1221        for output in l:
1222            label = output["label"]
1223            revisions = {}
1224            newestChange = 0
1225            if self.verbose:
1226                print "Querying files for label %s" % label
1227            for file in p4CmdList("files "
1228                                  +  ' '.join (["%s...@%s" % (p, label)
1229                                                for p in self.depotPaths])):
1230                revisions[file["depotFile"]] = file["rev"]
1231                change = int(file["change"])
1232                if change > newestChange:
1233                    newestChange = change
1234
1235            self.labels[newestChange] = [output, revisions]
1236
1237        if self.verbose:
1238            print "Label changes: %s" % self.labels.keys()
1239
1240    def guessProjectName(self):
1241        for p in self.depotPaths:
1242            if p.endswith("/"):
1243                p = p[:-1]
1244            p = p[p.strip().rfind("/") + 1:]
1245            if not p.endswith("/"):
1246               p += "/"
1247            return p
1248
1249    def getBranchMapping(self):
1250        lostAndFoundBranches = set()
1251
1252        for info in p4CmdList("branches"):
1253            details = p4Cmd("branch -o %s" % info["branch"])
1254            viewIdx = 0
1255            while details.has_key("View%s" % viewIdx):
1256                paths = details["View%s" % viewIdx].split(" ")
1257                viewIdx = viewIdx + 1
1258                # require standard //depot/foo/... //depot/bar/... mapping
1259                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1260                    continue
1261                source = paths[0]
1262                destination = paths[1]
1263                ## HACK
1264                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1265                    source = source[len(self.depotPaths[0]):-4]
1266                    destination = destination[len(self.depotPaths[0]):-4]
1267
1268                    if destination in self.knownBranches:
1269                        if not self.silent:
1270                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1271                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1272                        continue
1273
1274                    self.knownBranches[destination] = source
1275
1276                    lostAndFoundBranches.discard(destination)
1277
1278                    if source not in self.knownBranches:
1279                        lostAndFoundBranches.add(source)
1280
1281
1282        for branch in lostAndFoundBranches:
1283            self.knownBranches[branch] = branch
1284
1285    def getBranchMappingFromGitBranches(self):
1286        branches = p4BranchesInGit(self.importIntoRemotes)
1287        for branch in branches.keys():
1288            if branch == "master":
1289                branch = "main"
1290            else:
1291                branch = branch[len(self.projectName):]
1292            self.knownBranches[branch] = branch
1293
1294    def listExistingP4GitBranches(self):
1295        # branches holds mapping from name to commit
1296        branches = p4BranchesInGit(self.importIntoRemotes)
1297        self.p4BranchesInGit = branches.keys()
1298        for branch in branches.keys():
1299            self.initialParents[self.refPrefix + branch] = branches[branch]
1300
1301    def updateOptionDict(self, d):
1302        option_keys = {}
1303        if self.keepRepoPath:
1304            option_keys['keepRepoPath'] = 1
1305
1306        d["options"] = ' '.join(sorted(option_keys.keys()))
1307
1308    def readOptions(self, d):
1309        self.keepRepoPath = (d.has_key('options')
1310                             and ('keepRepoPath' in d['options']))
1311
1312    def gitRefForBranch(self, branch):
1313        if branch == "main":
1314            return self.refPrefix + "master"
1315
1316        if len(branch) <= 0:
1317            return branch
1318
1319        return self.refPrefix + self.projectName + branch
1320
1321    def gitCommitByP4Change(self, ref, change):
1322        if self.verbose:
1323            print "looking in ref " + ref + " for change %s using bisect..." % change
1324
1325        earliestCommit = ""
1326        latestCommit = parseRevision(ref)
1327
1328        while True:
1329            if self.verbose:
1330                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1331            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1332            if len(next) == 0:
1333                if self.verbose:
1334                    print "argh"
1335                return ""
1336            log = extractLogMessageFromGitCommit(next)
1337            settings = extractSettingsGitLog(log)
1338            currentChange = int(settings['change'])
1339            if self.verbose:
1340                print "current change %s" % currentChange
1341
1342            if currentChange == change:
1343                if self.verbose:
1344                    print "found %s" % next
1345                return next
1346
1347            if currentChange < change:
1348                earliestCommit = "^%s" % next
1349            else:
1350                latestCommit = "%s" % next
1351
1352        return ""
1353
1354    def importNewBranch(self, branch, maxChange):
1355        # make fast-import flush all changes to disk and update the refs using the checkpoint
1356        # command so that we can try to find the branch parent in the git history
1357        self.gitStream.write("checkpoint\n\n");
1358        self.gitStream.flush();
1359        branchPrefix = self.depotPaths[0] + branch + "/"
1360        range = "@1,%s" % maxChange
1361        #print "prefix" + branchPrefix
1362        changes = p4ChangesForPaths([branchPrefix], range)
1363        if len(changes) <= 0:
1364            return False
1365        firstChange = changes[0]
1366        #print "first change in branch: %s" % firstChange
1367        sourceBranch = self.knownBranches[branch]
1368        sourceDepotPath = self.depotPaths[0] + sourceBranch
1369        sourceRef = self.gitRefForBranch(sourceBranch)
1370        #print "source " + sourceBranch
1371
1372        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1373        #print "branch parent: %s" % branchParentChange
1374        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1375        if len(gitParent) > 0:
1376            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1377            #print "parent git commit: %s" % gitParent
1378
1379        self.importChanges(changes)
1380        return True
1381
1382    def importChanges(self, changes):
1383        cnt = 1
1384        for change in changes:
1385            description = p4Cmd("describe %s" % change)
1386            self.updateOptionDict(description)
1387
1388            if not self.silent:
1389                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1390                sys.stdout.flush()
1391            cnt = cnt + 1
1392
1393            try:
1394                if self.detectBranches:
1395                    branches = self.splitFilesIntoBranches(description)
1396                    for branch in branches.keys():
1397                        ## HACK  --hwn
1398                        branchPrefix = self.depotPaths[0] + branch + "/"
1399
1400                        parent = ""
1401
1402                        filesForCommit = branches[branch]
1403
1404                        if self.verbose:
1405                            print "branch is %s" % branch
1406
1407                        self.updatedBranches.add(branch)
1408
1409                        if branch not in self.createdBranches:
1410                            self.createdBranches.add(branch)
1411                            parent = self.knownBranches[branch]
1412                            if parent == branch:
1413                                parent = ""
1414                            else:
1415                                fullBranch = self.projectName + branch
1416                                if fullBranch not in self.p4BranchesInGit:
1417                                    if not self.silent:
1418                                        print("\n    Importing new branch %s" % fullBranch);
1419                                    if self.importNewBranch(branch, change - 1):
1420                                        parent = ""
1421                                        self.p4BranchesInGit.append(fullBranch)
1422                                    if not self.silent:
1423                                        print("\n    Resuming with change %s" % change);
1424
1425                                if self.verbose:
1426                                    print "parent determined through known branches: %s" % parent
1427
1428                        branch = self.gitRefForBranch(branch)
1429                        parent = self.gitRefForBranch(parent)
1430
1431                        if self.verbose:
1432                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1433
1434                        if len(parent) == 0 and branch in self.initialParents:
1435                            parent = self.initialParents[branch]
1436                            del self.initialParents[branch]
1437
1438                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1439                else:
1440                    files = self.extractFilesFromCommit(description)
1441                    self.commit(description, files, self.branch, self.depotPaths,
1442                                self.initialParent)
1443                    self.initialParent = ""
1444            except IOError:
1445                print self.gitError.read()
1446                sys.exit(1)
1447
1448    def importHeadRevision(self, revision):
1449        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1450
1451        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1452        details["desc"] = ("Initial import of %s from the state at revision %s\n"
1453                           % (' '.join(self.depotPaths), revision))
1454        details["change"] = revision
1455        newestRevision = 0
1456
1457        fileCnt = 0
1458        for info in p4CmdList("files "
1459                              +  ' '.join(["%s...%s"
1460                                           % (p, revision)
1461                                           for p in self.depotPaths])):
1462
1463            if 'code' in info and info['code'] == 'error':
1464                sys.stderr.write("p4 returned an error: %s\n"
1465                                 % info['data'])
1466                if info['data'].find("must refer to client") >= 0:
1467                    sys.stderr.write("This particular p4 error is misleading.\n")
1468                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
1469                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1470                sys.exit(1)
1471            if 'p4ExitCode' in info:
1472                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1473                sys.exit(1)
1474
1475
1476            change = int(info["change"])
1477            if change > newestRevision:
1478                newestRevision = change
1479
1480            if info["action"] in self.delete_actions:
1481                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1482                #fileCnt = fileCnt + 1
1483                continue
1484
1485            for prop in ["depotFile", "rev", "action", "type" ]:
1486                details["%s%s" % (prop, fileCnt)] = info[prop]
1487
1488            fileCnt = fileCnt + 1
1489
1490        details["change"] = newestRevision
1491        self.updateOptionDict(details)
1492        try:
1493            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1494        except IOError:
1495            print "IO error with git fast-import. Is your git version recent enough?"
1496            print self.gitError.read()
1497
1498
1499    def getClientSpec(self):
1500        specList = p4CmdList( "client -o" )
1501        temp = {}
1502        for entry in specList:
1503            for k,v in entry.iteritems():
1504                if k.startswith("View"):
1505                    if v.startswith('"'):
1506                        start = 1
1507                    else:
1508                        start = 0
1509                    index = v.find("...")
1510                    v = v[start:index]
1511                    if v.startswith("-"):
1512                        v = v[1:]
1513                        temp[v] = -len(v)
1514                    else:
1515                        temp[v] = len(v)
1516        self.clientSpecDirs = temp.items()
1517        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1518
1519    def run(self, args):
1520        self.depotPaths = []
1521        self.changeRange = ""
1522        self.initialParent = ""
1523        self.previousDepotPaths = []
1524
1525        # map from branch depot path to parent branch
1526        self.knownBranches = {}
1527        self.initialParents = {}
1528        self.hasOrigin = originP4BranchesExist()
1529        if not self.syncWithOrigin:
1530            self.hasOrigin = False
1531
1532        if self.importIntoRemotes:
1533            self.refPrefix = "refs/remotes/p4/"
1534        else:
1535            self.refPrefix = "refs/heads/p4/"
1536
1537        if self.syncWithOrigin and self.hasOrigin:
1538            if not self.silent:
1539                print "Syncing with origin first by calling git fetch origin"
1540            system("git fetch origin")
1541
1542        if len(self.branch) == 0:
1543            self.branch = self.refPrefix + "master"
1544            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1545                system("git update-ref %s refs/heads/p4" % self.branch)
1546                system("git branch -D p4");
1547            # create it /after/ importing, when master exists
1548            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1549                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1550
1551        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1552            self.getClientSpec()
1553
1554        # TODO: should always look at previous commits,
1555        # merge with previous imports, if possible.
1556        if args == []:
1557            if self.hasOrigin:
1558                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1559            self.listExistingP4GitBranches()
1560
1561            if len(self.p4BranchesInGit) > 1:
1562                if not self.silent:
1563                    print "Importing from/into multiple branches"
1564                self.detectBranches = True
1565
1566            if self.verbose:
1567                print "branches: %s" % self.p4BranchesInGit
1568
1569            p4Change = 0
1570            for branch in self.p4BranchesInGit:
1571                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1572
1573                settings = extractSettingsGitLog(logMsg)
1574
1575                self.readOptions(settings)
1576                if (settings.has_key('depot-paths')
1577                    and settings.has_key ('change')):
1578                    change = int(settings['change']) + 1
1579                    p4Change = max(p4Change, change)
1580
1581                    depotPaths = sorted(settings['depot-paths'])
1582                    if self.previousDepotPaths == []:
1583                        self.previousDepotPaths = depotPaths
1584                    else:
1585                        paths = []
1586                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1587                            for i in range(0, min(len(cur), len(prev))):
1588                                if cur[i] <> prev[i]:
1589                                    i = i - 1
1590                                    break
1591
1592                            paths.append (cur[:i + 1])
1593
1594                        self.previousDepotPaths = paths
1595
1596            if p4Change > 0:
1597                self.depotPaths = sorted(self.previousDepotPaths)
1598                self.changeRange = "@%s,#head" % p4Change
1599                if not self.detectBranches:
1600                    self.initialParent = parseRevision(self.branch)
1601                if not self.silent and not self.detectBranches:
1602                    print "Performing incremental import into %s git branch" % self.branch
1603
1604        if not self.branch.startswith("refs/"):
1605            self.branch = "refs/heads/" + self.branch
1606
1607        if len(args) == 0 and self.depotPaths:
1608            if not self.silent:
1609                print "Depot paths: %s" % ' '.join(self.depotPaths)
1610        else:
1611            if self.depotPaths and self.depotPaths != args:
1612                print ("previous import used depot path %s and now %s was specified. "
1613                       "This doesn't work!" % (' '.join (self.depotPaths),
1614                                               ' '.join (args)))
1615                sys.exit(1)
1616
1617            self.depotPaths = sorted(args)
1618
1619        revision = ""
1620        self.users = {}
1621
1622        newPaths = []
1623        for p in self.depotPaths:
1624            if p.find("@") != -1:
1625                atIdx = p.index("@")
1626                self.changeRange = p[atIdx:]
1627                if self.changeRange == "@all":
1628                    self.changeRange = ""
1629                elif ',' not in self.changeRange:
1630                    revision = self.changeRange
1631                    self.changeRange = ""
1632                p = p[:atIdx]
1633            elif p.find("#") != -1:
1634                hashIdx = p.index("#")
1635                revision = p[hashIdx:]
1636                p = p[:hashIdx]
1637            elif self.previousDepotPaths == []:
1638                revision = "#head"
1639
1640            p = re.sub ("\.\.\.$", "", p)
1641            if not p.endswith("/"):
1642                p += "/"
1643
1644            newPaths.append(p)
1645
1646        self.depotPaths = newPaths
1647
1648
1649        self.loadUserMapFromCache()
1650        self.labels = {}
1651        if self.detectLabels:
1652            self.getLabels();
1653
1654        if self.detectBranches:
1655            ## FIXME - what's a P4 projectName ?
1656            self.projectName = self.guessProjectName()
1657
1658            if self.hasOrigin:
1659                self.getBranchMappingFromGitBranches()
1660            else:
1661                self.getBranchMapping()
1662            if self.verbose:
1663                print "p4-git branches: %s" % self.p4BranchesInGit
1664                print "initial parents: %s" % self.initialParents
1665            for b in self.p4BranchesInGit:
1666                if b != "master":
1667
1668                    ## FIXME
1669                    b = b[len(self.projectName):]
1670                self.createdBranches.add(b)
1671
1672        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1673
1674        importProcess = subprocess.Popen(["git", "fast-import"],
1675                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1676                                         stderr=subprocess.PIPE);
1677        self.gitOutput = importProcess.stdout
1678        self.gitStream = importProcess.stdin
1679        self.gitError = importProcess.stderr
1680
1681        if revision:
1682            self.importHeadRevision(revision)
1683        else:
1684            changes = []
1685
1686            if len(self.changesFile) > 0:
1687                output = open(self.changesFile).readlines()
1688                changeSet = set()
1689                for line in output:
1690                    changeSet.add(int(line))
1691
1692                for change in changeSet:
1693                    changes.append(change)
1694
1695                changes.sort()
1696            else:
1697                if not self.p4BranchesInGit:
1698                    die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1699                if self.verbose:
1700                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1701                                                              self.changeRange)
1702                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1703
1704                if len(self.maxChanges) > 0:
1705                    changes = changes[:min(int(self.maxChanges), len(changes))]
1706
1707            if len(changes) == 0:
1708                if not self.silent:
1709                    print "No changes to import!"
1710                return True
1711
1712            if not self.silent and not self.detectBranches:
1713                print "Import destination: %s" % self.branch
1714
1715            self.updatedBranches = set()
1716
1717            self.importChanges(changes)
1718
1719            if not self.silent:
1720                print ""
1721                if len(self.updatedBranches) > 0:
1722                    sys.stdout.write("Updated branches: ")
1723                    for b in self.updatedBranches:
1724                        sys.stdout.write("%s " % b)
1725                    sys.stdout.write("\n")
1726
1727        self.gitStream.close()
1728        if importProcess.wait() != 0:
1729            die("fast-import failed: %s" % self.gitError.read())
1730        self.gitOutput.close()
1731        self.gitError.close()
1732
1733        return True
1734
1735class P4Rebase(Command):
1736    def __init__(self):
1737        Command.__init__(self)
1738        self.options = [ ]
1739        self.description = ("Fetches the latest revision from perforce and "
1740                            + "rebases the current work (branch) against it")
1741        self.verbose = False
1742
1743    def run(self, args):
1744        sync = P4Sync()
1745        sync.run([])
1746
1747        return self.rebase()
1748
1749    def rebase(self):
1750        if os.system("git update-index --refresh") != 0:
1751            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1752        if len(read_pipe("git diff-index HEAD --")) > 0:
1753            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1754
1755        [upstream, settings] = findUpstreamBranchPoint()
1756        if len(upstream) == 0:
1757            die("Cannot find upstream branchpoint for rebase")
1758
1759        # the branchpoint may be p4/foo~3, so strip off the parent
1760        upstream = re.sub("~[0-9]+$", "", upstream)
1761
1762        print "Rebasing the current branch onto %s" % upstream
1763        oldHead = read_pipe("git rev-parse HEAD").strip()
1764        system("git rebase %s" % upstream)
1765        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1766        return True
1767
1768class P4Clone(P4Sync):
1769    def __init__(self):
1770        P4Sync.__init__(self)
1771        self.description = "Creates a new git repository and imports from Perforce into it"
1772        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1773        self.options += [
1774            optparse.make_option("--destination", dest="cloneDestination",
1775                                 action='store', default=None,
1776                                 help="where to leave result of the clone"),
1777            optparse.make_option("-/", dest="cloneExclude",
1778                                 action="append", type="string",
1779                                 help="exclude depot path")
1780        ]
1781        self.cloneDestination = None
1782        self.needsGit = False
1783
1784    # This is required for the "append" cloneExclude action
1785    def ensure_value(self, attr, value):
1786        if not hasattr(self, attr) or getattr(self, attr) is None:
1787            setattr(self, attr, value)
1788        return getattr(self, attr)
1789
1790    def defaultDestination(self, args):
1791        ## TODO: use common prefix of args?
1792        depotPath = args[0]
1793        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1794        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1795        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1796        depotDir = re.sub(r"/$", "", depotDir)
1797        return os.path.split(depotDir)[1]
1798
1799    def run(self, args):
1800        if len(args) < 1:
1801            return False
1802
1803        if self.keepRepoPath and not self.cloneDestination:
1804            sys.stderr.write("Must specify destination for --keep-path\n")
1805            sys.exit(1)
1806
1807        depotPaths = args
1808
1809        if not self.cloneDestination and len(depotPaths) > 1:
1810            self.cloneDestination = depotPaths[-1]
1811            depotPaths = depotPaths[:-1]
1812
1813        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1814        for p in depotPaths:
1815            if not p.startswith("//"):
1816                return False
1817
1818        if not self.cloneDestination:
1819            self.cloneDestination = self.defaultDestination(args)
1820
1821        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1822        if not os.path.exists(self.cloneDestination):
1823            os.makedirs(self.cloneDestination)
1824        chdir(self.cloneDestination)
1825        system("git init")
1826        self.gitdir = os.getcwd() + "/.git"
1827        if not P4Sync.run(self, depotPaths):
1828            return False
1829        if self.branch != "master":
1830            if self.importIntoRemotes:
1831                masterbranch = "refs/remotes/p4/master"
1832            else:
1833                masterbranch = "refs/heads/p4/master"
1834            if gitBranchExists(masterbranch):
1835                system("git branch master %s" % masterbranch)
1836                system("git checkout -f")
1837            else:
1838                print "Could not detect main branch. No checkout/master branch created."
1839
1840        return True
1841
1842class P4Branches(Command):
1843    def __init__(self):
1844        Command.__init__(self)
1845        self.options = [ ]
1846        self.description = ("Shows the git branches that hold imports and their "
1847                            + "corresponding perforce depot paths")
1848        self.verbose = False
1849
1850    def run(self, args):
1851        if originP4BranchesExist():
1852            createOrUpdateBranchesFromOrigin()
1853
1854        cmdline = "git rev-parse --symbolic "
1855        cmdline += " --remotes"
1856
1857        for line in read_pipe_lines(cmdline):
1858            line = line.strip()
1859
1860            if not line.startswith('p4/') or line == "p4/HEAD":
1861                continue
1862            branch = line
1863
1864            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1865            settings = extractSettingsGitLog(log)
1866
1867            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1868        return True
1869
1870class HelpFormatter(optparse.IndentedHelpFormatter):
1871    def __init__(self):
1872        optparse.IndentedHelpFormatter.__init__(self)
1873
1874    def format_description(self, description):
1875        if description:
1876            return description + "\n"
1877        else:
1878            return ""
1879
1880def printUsage(commands):
1881    print "usage: %s <command> [options]" % sys.argv[0]
1882    print ""
1883    print "valid commands: %s" % ", ".join(commands)
1884    print ""
1885    print "Try %s <command> --help for command specific help." % sys.argv[0]
1886    print ""
1887
1888commands = {
1889    "debug" : P4Debug,
1890    "submit" : P4Submit,
1891    "commit" : P4Submit,
1892    "sync" : P4Sync,
1893    "rebase" : P4Rebase,
1894    "clone" : P4Clone,
1895    "rollback" : P4RollBack,
1896    "branches" : P4Branches
1897}
1898
1899
1900def main():
1901    if len(sys.argv[1:]) == 0:
1902        printUsage(commands.keys())
1903        sys.exit(2)
1904
1905    cmd = ""
1906    cmdName = sys.argv[1]
1907    try:
1908        klass = commands[cmdName]
1909        cmd = klass()
1910    except KeyError:
1911        print "unknown command %s" % cmdName
1912        print ""
1913        printUsage(commands.keys())
1914        sys.exit(2)
1915
1916    options = cmd.options
1917    cmd.gitdir = os.environ.get("GIT_DIR", None)
1918
1919    args = sys.argv[2:]
1920
1921    if len(options) > 0:
1922        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1923
1924        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1925                                       options,
1926                                       description = cmd.description,
1927                                       formatter = HelpFormatter())
1928
1929        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1930    global verbose
1931    verbose = cmd.verbose
1932    if cmd.needsGit:
1933        if cmd.gitdir == None:
1934            cmd.gitdir = os.path.abspath(".git")
1935            if not isValidGitDir(cmd.gitdir):
1936                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1937                if os.path.exists(cmd.gitdir):
1938                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1939                    if len(cdup) > 0:
1940                        chdir(cdup);
1941
1942        if not isValidGitDir(cmd.gitdir):
1943            if isValidGitDir(cmd.gitdir + "/.git"):
1944                cmd.gitdir += "/.git"
1945            else:
1946                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1947
1948        os.environ["GIT_DIR"] = cmd.gitdir
1949
1950    if not cmd.run(args):
1951        parser.print_help()
1952
1953
1954if __name__ == '__main__':
1955    main()