Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit git-p4: Add copy detection support (4fddb41)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, subprocess, shelve
  12import tempfile, getopt, os.path, time, platform
  13import re
  14
  15verbose = False
  16
  17
  18def p4_build_cmd(cmd):
  19    """Build a suitable p4 command line.
  20
  21    This consolidates building and returning a p4 command line into one
  22    location. It means that hooking into the environment, or other configuration
  23    can be done more easily.
  24    """
  25    real_cmd = "%s " % "p4"
  26
  27    user = gitConfig("git-p4.user")
  28    if len(user) > 0:
  29        real_cmd += "-u %s " % user
  30
  31    password = gitConfig("git-p4.password")
  32    if len(password) > 0:
  33        real_cmd += "-P %s " % password
  34
  35    port = gitConfig("git-p4.port")
  36    if len(port) > 0:
  37        real_cmd += "-p %s " % port
  38
  39    host = gitConfig("git-p4.host")
  40    if len(host) > 0:
  41        real_cmd += "-h %s " % host
  42
  43    client = gitConfig("git-p4.client")
  44    if len(client) > 0:
  45        real_cmd += "-c %s " % client
  46
  47    real_cmd += "%s" % (cmd)
  48    if verbose:
  49        print real_cmd
  50    return real_cmd
  51
  52def chdir(dir):
  53    if os.name == 'nt':
  54        os.environ['PWD']=dir
  55    os.chdir(dir)
  56
  57def die(msg):
  58    if verbose:
  59        raise Exception(msg)
  60    else:
  61        sys.stderr.write(msg + "\n")
  62        sys.exit(1)
  63
  64def write_pipe(c, str):
  65    if verbose:
  66        sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68    pipe = os.popen(c, 'w')
  69    val = pipe.write(str)
  70    if pipe.close():
  71        die('Command failed: %s' % c)
  72
  73    return val
  74
  75def p4_write_pipe(c, str):
  76    real_cmd = p4_build_cmd(c)
  77    return write_pipe(real_cmd, str)
  78
  79def read_pipe(c, ignore_error=False):
  80    if verbose:
  81        sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83    pipe = os.popen(c, 'rb')
  84    val = pipe.read()
  85    if pipe.close() and not ignore_error:
  86        die('Command failed: %s' % c)
  87
  88    return val
  89
  90def p4_read_pipe(c, ignore_error=False):
  91    real_cmd = p4_build_cmd(c)
  92    return read_pipe(real_cmd, ignore_error)
  93
  94def read_pipe_lines(c):
  95    if verbose:
  96        sys.stderr.write('Reading pipe: %s\n' % c)
  97    ## todo: check return status
  98    pipe = os.popen(c, 'rb')
  99    val = pipe.readlines()
 100    if pipe.close():
 101        die('Command failed: %s' % c)
 102
 103    return val
 104
 105def p4_read_pipe_lines(c):
 106    """Specifically invoke p4 on the command supplied. """
 107    real_cmd = p4_build_cmd(c)
 108    return read_pipe_lines(real_cmd)
 109
 110def system(cmd):
 111    if verbose:
 112        sys.stderr.write("executing %s\n" % cmd)
 113    if os.system(cmd) != 0:
 114        die("command failed: %s" % cmd)
 115
 116def p4_system(cmd):
 117    """Specifically invoke p4 as the system command. """
 118    real_cmd = p4_build_cmd(cmd)
 119    return system(real_cmd)
 120
 121def isP4Exec(kind):
 122    """Determine if a Perforce 'kind' should have execute permission
 123
 124    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125    or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126    a plus sign, it is also executable"""
 127    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129def setP4ExecBit(file, mode):
 130    # Reopens an already open file and changes the execute bit to match
 131    # the execute bit setting in the passed in mode.
 132
 133    p4Type = "+x"
 134
 135    if not isModeExec(mode):
 136        p4Type = getP4OpenedType(file)
 137        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139        if p4Type[-1] == "+":
 140            p4Type = p4Type[0:-1]
 141
 142    p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144def getP4OpenedType(file):
 145    # Returns the perforce file type for the given file.
 146
 147    result = p4_read_pipe("opened %s" % file)
 148    match = re.match(".*\((.+)\)\r?$", result)
 149    if match:
 150        return match.group(1)
 151    else:
 152        die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154def diffTreePattern():
 155    # This is a simple generator for the diff tree regex pattern. This could be
 156    # a class variable if this and parseDiffTreeEntry were a part of a class.
 157    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158    while True:
 159        yield pattern
 160
 161def parseDiffTreeEntry(entry):
 162    """Parses a single diff tree entry into its component elements.
 163
 164    See git-diff-tree(1) manpage for details about the format of the diff
 165    output. This method returns a dictionary with the following elements:
 166
 167    src_mode - The mode of the source file
 168    dst_mode - The mode of the destination file
 169    src_sha1 - The sha1 for the source file
 170    dst_sha1 - The sha1 fr the destination file
 171    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172    status_score - The score for the status (applicable for 'C' and 'R'
 173                   statuses). This is None if there is no score.
 174    src - The path for the source file.
 175    dst - The path for the destination file. This is only present for
 176          copy or renames. If it is not present, this is None.
 177
 178    If the pattern is not matched, None is returned."""
 179
 180    match = diffTreePattern().next().match(entry)
 181    if match:
 182        return {
 183            'src_mode': match.group(1),
 184            'dst_mode': match.group(2),
 185            'src_sha1': match.group(3),
 186            'dst_sha1': match.group(4),
 187            'status': match.group(5),
 188            'status_score': match.group(6),
 189            'src': match.group(7),
 190            'dst': match.group(10)
 191        }
 192    return None
 193
 194def isModeExec(mode):
 195    # Returns True if the given git mode represents an executable file,
 196    # otherwise False.
 197    return mode[-3:] == "755"
 198
 199def isModeExecChanged(src_mode, dst_mode):
 200    return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203    cmd = p4_build_cmd("-G %s" % (cmd))
 204    if verbose:
 205        sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207    # Use a temporary file to avoid deadlocks without
 208    # subprocess.communicate(), which would put another copy
 209    # of stdout into memory.
 210    stdin_file = None
 211    if stdin is not None:
 212        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213        stdin_file.write(stdin)
 214        stdin_file.flush()
 215        stdin_file.seek(0)
 216
 217    p4 = subprocess.Popen(cmd, shell=True,
 218                          stdin=stdin_file,
 219                          stdout=subprocess.PIPE)
 220
 221    result = []
 222    try:
 223        while True:
 224            entry = marshal.load(p4.stdout)
 225            if cb is not None:
 226                cb(entry)
 227            else:
 228                result.append(entry)
 229    except EOFError:
 230        pass
 231    exitCode = p4.wait()
 232    if exitCode != 0:
 233        entry = {}
 234        entry["p4ExitCode"] = exitCode
 235        result.append(entry)
 236
 237    return result
 238
 239def p4Cmd(cmd):
 240    list = p4CmdList(cmd)
 241    result = {}
 242    for entry in list:
 243        result.update(entry)
 244    return result;
 245
 246def p4Where(depotPath):
 247    if not depotPath.endswith("/"):
 248        depotPath += "/"
 249    depotPath = depotPath + "..."
 250    outputList = p4CmdList("where %s" % depotPath)
 251    output = None
 252    for entry in outputList:
 253        if "depotFile" in entry:
 254            if entry["depotFile"] == depotPath:
 255                output = entry
 256                break
 257        elif "data" in entry:
 258            data = entry.get("data")
 259            space = data.find(" ")
 260            if data[:space] == depotPath:
 261                output = entry
 262                break
 263    if output == None:
 264        return ""
 265    if output["code"] == "error":
 266        return ""
 267    clientPath = ""
 268    if "path" in output:
 269        clientPath = output.get("path")
 270    elif "data" in output:
 271        data = output.get("data")
 272        lastSpace = data.rfind(" ")
 273        clientPath = data[lastSpace + 1:]
 274
 275    if clientPath.endswith("..."):
 276        clientPath = clientPath[:-3]
 277    return clientPath
 278
 279def currentGitBranch():
 280    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282def isValidGitDir(path):
 283    if (os.path.exists(path + "/HEAD")
 284        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285        return True;
 286    return False
 287
 288def parseRevision(ref):
 289    return read_pipe("git rev-parse %s" % ref).strip()
 290
 291def extractLogMessageFromGitCommit(commit):
 292    logMessage = ""
 293
 294    ## fixme: title is first line of commit, not 1st paragraph.
 295    foundTitle = False
 296    for log in read_pipe_lines("git cat-file commit %s" % commit):
 297       if not foundTitle:
 298           if len(log) == 1:
 299               foundTitle = True
 300           continue
 301
 302       logMessage += log
 303    return logMessage
 304
 305def extractSettingsGitLog(log):
 306    values = {}
 307    for line in log.split("\n"):
 308        line = line.strip()
 309        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310        if not m:
 311            continue
 312
 313        assignments = m.group(1).split (':')
 314        for a in assignments:
 315            vals = a.split ('=')
 316            key = vals[0].strip()
 317            val = ('='.join (vals[1:])).strip()
 318            if val.endswith ('\"') and val.startswith('"'):
 319                val = val[1:-1]
 320
 321            values[key] = val
 322
 323    paths = values.get("depot-paths")
 324    if not paths:
 325        paths = values.get("depot-path")
 326    if paths:
 327        values['depot-paths'] = paths.split(',')
 328    return values
 329
 330def gitBranchExists(branch):
 331    proc = subprocess.Popen(["git", "rev-parse", branch],
 332                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333    return proc.wait() == 0;
 334
 335_gitConfig = {}
 336def gitConfig(key):
 337    if not _gitConfig.has_key(key):
 338        _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339    return _gitConfig[key]
 340
 341def p4BranchesInGit(branchesAreInRemotes = True):
 342    branches = {}
 343
 344    cmdline = "git rev-parse --symbolic "
 345    if branchesAreInRemotes:
 346        cmdline += " --remotes"
 347    else:
 348        cmdline += " --branches"
 349
 350    for line in read_pipe_lines(cmdline):
 351        line = line.strip()
 352
 353        ## only import to p4/
 354        if not line.startswith('p4/') or line == "p4/HEAD":
 355            continue
 356        branch = line
 357
 358        # strip off p4
 359        branch = re.sub ("^p4/", "", line)
 360
 361        branches[branch] = parseRevision(line)
 362    return branches
 363
 364def findUpstreamBranchPoint(head = "HEAD"):
 365    branches = p4BranchesInGit()
 366    # map from depot-path to branch name
 367    branchByDepotPath = {}
 368    for branch in branches.keys():
 369        tip = branches[branch]
 370        log = extractLogMessageFromGitCommit(tip)
 371        settings = extractSettingsGitLog(log)
 372        if settings.has_key("depot-paths"):
 373            paths = ",".join(settings["depot-paths"])
 374            branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376    settings = None
 377    parent = 0
 378    while parent < 65535:
 379        commit = head + "~%s" % parent
 380        log = extractLogMessageFromGitCommit(commit)
 381        settings = extractSettingsGitLog(log)
 382        if settings.has_key("depot-paths"):
 383            paths = ",".join(settings["depot-paths"])
 384            if branchByDepotPath.has_key(paths):
 385                return [branchByDepotPath[paths], settings]
 386
 387        parent = parent + 1
 388
 389    return ["", settings]
 390
 391def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392    if not silent:
 393        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394               % localRefPrefix)
 395
 396    originPrefix = "origin/p4/"
 397
 398    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399        line = line.strip()
 400        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401            continue
 402
 403        headName = line[len(originPrefix):]
 404        remoteHead = localRefPrefix + headName
 405        originHead = line
 406
 407        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408        if (not original.has_key('depot-paths')
 409            or not original.has_key('change')):
 410            continue
 411
 412        update = False
 413        if not gitBranchExists(remoteHead):
 414            if verbose:
 415                print "creating %s" % remoteHead
 416            update = True
 417        else:
 418            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419            if settings.has_key('change') > 0:
 420                if settings['depot-paths'] == original['depot-paths']:
 421                    originP4Change = int(original['change'])
 422                    p4Change = int(settings['change'])
 423                    if originP4Change > p4Change:
 424                        print ("%s (%s) is newer than %s (%s). "
 425                               "Updating p4 branch from origin."
 426                               % (originHead, originP4Change,
 427                                  remoteHead, p4Change))
 428                        update = True
 429                else:
 430                    print ("Ignoring: %s was imported from %s while "
 431                           "%s was imported from %s"
 432                           % (originHead, ','.join(original['depot-paths']),
 433                              remoteHead, ','.join(settings['depot-paths'])))
 434
 435        if update:
 436            system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438def originP4BranchesExist():
 439        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441def p4ChangesForPaths(depotPaths, changeRange):
 442    assert depotPaths
 443    output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                        for p in depotPaths]))
 445
 446    changes = {}
 447    for line in output:
 448        changeNum = int(line.split(" ")[1])
 449        changes[changeNum] = True
 450
 451    changelist = changes.keys()
 452    changelist.sort()
 453    return changelist
 454
 455class Command:
 456    def __init__(self):
 457        self.usage = "usage: %prog [options]"
 458        self.needsGit = True
 459
 460class P4Debug(Command):
 461    def __init__(self):
 462        Command.__init__(self)
 463        self.options = [
 464            optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                 default=False),
 466            ]
 467        self.description = "A tool to debug the output of p4 -G."
 468        self.needsGit = False
 469        self.verbose = False
 470
 471    def run(self, args):
 472        j = 0
 473        for output in p4CmdList(" ".join(args)):
 474            print 'Element: %d' % j
 475            j += 1
 476            print output
 477        return True
 478
 479class P4RollBack(Command):
 480    def __init__(self):
 481        Command.__init__(self)
 482        self.options = [
 483            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485        ]
 486        self.description = "A tool to debug the multi-branch import. Don't use :)"
 487        self.verbose = False
 488        self.rollbackLocalBranches = False
 489
 490    def run(self, args):
 491        if len(args) != 1:
 492            return False
 493        maxChange = int(args[0])
 494
 495        if "p4ExitCode" in p4Cmd("changes -m 1"):
 496            die("Problems executing p4");
 497
 498        if self.rollbackLocalBranches:
 499            refPrefix = "refs/heads/"
 500            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501        else:
 502            refPrefix = "refs/remotes/"
 503            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505        for line in lines:
 506            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                line = line.strip()
 508                ref = refPrefix + line
 509                log = extractLogMessageFromGitCommit(ref)
 510                settings = extractSettingsGitLog(log)
 511
 512                depotPaths = settings['depot-paths']
 513                change = settings['change']
 514
 515                changed = False
 516
 517                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                           for p in depotPaths]))) == 0:
 519                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                    continue
 522
 523                while change and int(change) > maxChange:
 524                    changed = True
 525                    if self.verbose:
 526                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                    system("git update-ref %s \"%s^\"" % (ref, ref))
 528                    log = extractLogMessageFromGitCommit(ref)
 529                    settings =  extractSettingsGitLog(log)
 530
 531
 532                    depotPaths = settings['depot-paths']
 533                    change = settings['change']
 534
 535                if changed:
 536                    print "%s rewound to %s" % (ref, change)
 537
 538        return True
 539
 540class P4Submit(Command):
 541    def __init__(self):
 542        Command.__init__(self)
 543        self.options = [
 544                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                optparse.make_option("--origin", dest="origin"),
 546                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 547        ]
 548        self.description = "Submit changes from git to the perforce depot."
 549        self.usage += " [name of git branch to submit into perforce depot]"
 550        self.interactive = True
 551        self.origin = ""
 552        self.detectRenames = False
 553        self.verbose = False
 554        self.isWindows = (platform.system() == "Windows")
 555
 556    def check(self):
 557        if len(p4CmdList("opened ...")) > 0:
 558            die("You have files opened with perforce! Close them before starting the sync.")
 559
 560    # replaces everything between 'Description:' and the next P4 submit template field with the
 561    # commit message
 562    def prepareLogMessage(self, template, message):
 563        result = ""
 564
 565        inDescriptionSection = False
 566
 567        for line in template.split("\n"):
 568            if line.startswith("#"):
 569                result += line + "\n"
 570                continue
 571
 572            if inDescriptionSection:
 573                if line.startswith("Files:"):
 574                    inDescriptionSection = False
 575                else:
 576                    continue
 577            else:
 578                if line.startswith("Description:"):
 579                    inDescriptionSection = True
 580                    line += "\n"
 581                    for messageLine in message.split("\n"):
 582                        line += "\t" + messageLine + "\n"
 583
 584            result += line + "\n"
 585
 586        return result
 587
 588    def prepareSubmitTemplate(self):
 589        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590        template = ""
 591        inFilesSection = False
 592        for line in p4_read_pipe_lines("change -o"):
 593            if line.endswith("\r\n"):
 594                line = line[:-2] + "\n"
 595            if inFilesSection:
 596                if line.startswith("\t"):
 597                    # path starts and ends with a tab
 598                    path = line[1:]
 599                    lastTab = path.rfind("\t")
 600                    if lastTab != -1:
 601                        path = path[:lastTab]
 602                        if not path.startswith(self.depotPath):
 603                            continue
 604                else:
 605                    inFilesSection = False
 606            else:
 607                if line.startswith("Files:"):
 608                    inFilesSection = True
 609
 610            template += line
 611
 612        return template
 613
 614    def applyCommit(self, id):
 615        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616
 617        if not self.detectRenames:
 618            # If not explicitly set check the config variable
 619            self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true"
 620
 621        if self.detectRenames:
 622            diffOpts = "-M"
 623        else:
 624            diffOpts = ""
 625
 626        if gitConfig("git-p4.detectCopies").lower() == "true":
 627            diffOpts += " -C"
 628
 629        if gitConfig("git-p4.detectCopiesHarder").lower() == "true":
 630            diffOpts += " --find-copies-harder"
 631
 632        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 633        filesToAdd = set()
 634        filesToDelete = set()
 635        editedFiles = set()
 636        filesToChangeExecBit = {}
 637        for line in diff:
 638            diff = parseDiffTreeEntry(line)
 639            modifier = diff['status']
 640            path = diff['src']
 641            if modifier == "M":
 642                p4_system("edit \"%s\"" % path)
 643                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 644                    filesToChangeExecBit[path] = diff['dst_mode']
 645                editedFiles.add(path)
 646            elif modifier == "A":
 647                filesToAdd.add(path)
 648                filesToChangeExecBit[path] = diff['dst_mode']
 649                if path in filesToDelete:
 650                    filesToDelete.remove(path)
 651            elif modifier == "D":
 652                filesToDelete.add(path)
 653                if path in filesToAdd:
 654                    filesToAdd.remove(path)
 655            elif modifier == "C":
 656                src, dest = diff['src'], diff['dst']
 657                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 658                if diff['src_sha1'] != diff['dst_sha1']:
 659                    p4_system("edit \"%s\"" % (dest))
 660                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 661                    p4_system("edit \"%s\"" % (dest))
 662                    filesToChangeExecBit[dest] = diff['dst_mode']
 663                os.unlink(dest)
 664                editedFiles.add(dest)
 665            elif modifier == "R":
 666                src, dest = diff['src'], diff['dst']
 667                p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 668                if diff['src_sha1'] != diff['dst_sha1']:
 669                    p4_system("edit \"%s\"" % (dest))
 670                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 671                    p4_system("edit \"%s\"" % (dest))
 672                    filesToChangeExecBit[dest] = diff['dst_mode']
 673                os.unlink(dest)
 674                editedFiles.add(dest)
 675                filesToDelete.add(src)
 676            else:
 677                die("unknown modifier %s for %s" % (modifier, path))
 678
 679        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 680        patchcmd = diffcmd + " | git apply "
 681        tryPatchCmd = patchcmd + "--check -"
 682        applyPatchCmd = patchcmd + "--check --apply -"
 683
 684        if os.system(tryPatchCmd) != 0:
 685            print "Unfortunately applying the change failed!"
 686            print "What do you want to do?"
 687            response = "x"
 688            while response != "s" and response != "a" and response != "w":
 689                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 690                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 691            if response == "s":
 692                print "Skipping! Good luck with the next patches..."
 693                for f in editedFiles:
 694                    p4_system("revert \"%s\"" % f);
 695                for f in filesToAdd:
 696                    system("rm %s" %f)
 697                return
 698            elif response == "a":
 699                os.system(applyPatchCmd)
 700                if len(filesToAdd) > 0:
 701                    print "You may also want to call p4 add on the following files:"
 702                    print " ".join(filesToAdd)
 703                if len(filesToDelete):
 704                    print "The following files should be scheduled for deletion with p4 delete:"
 705                    print " ".join(filesToDelete)
 706                die("Please resolve and submit the conflict manually and "
 707                    + "continue afterwards with git-p4 submit --continue")
 708            elif response == "w":
 709                system(diffcmd + " > patch.txt")
 710                print "Patch saved to patch.txt in %s !" % self.clientPath
 711                die("Please resolve and submit the conflict manually and "
 712                    "continue afterwards with git-p4 submit --continue")
 713
 714        system(applyPatchCmd)
 715
 716        for f in filesToAdd:
 717            p4_system("add \"%s\"" % f)
 718        for f in filesToDelete:
 719            p4_system("revert \"%s\"" % f)
 720            p4_system("delete \"%s\"" % f)
 721
 722        # Set/clear executable bits
 723        for f in filesToChangeExecBit.keys():
 724            mode = filesToChangeExecBit[f]
 725            setP4ExecBit(f, mode)
 726
 727        logMessage = extractLogMessageFromGitCommit(id)
 728        logMessage = logMessage.strip()
 729
 730        template = self.prepareSubmitTemplate()
 731
 732        if self.interactive:
 733            submitTemplate = self.prepareLogMessage(template, logMessage)
 734            if os.environ.has_key("P4DIFF"):
 735                del(os.environ["P4DIFF"])
 736            diff = ""
 737            for editedFile in editedFiles:
 738                diff += p4_read_pipe("diff -du %r" % editedFile)
 739
 740            newdiff = ""
 741            for newFile in filesToAdd:
 742                newdiff += "==== new file ====\n"
 743                newdiff += "--- /dev/null\n"
 744                newdiff += "+++ %s\n" % newFile
 745                f = open(newFile, "r")
 746                for line in f.readlines():
 747                    newdiff += "+" + line
 748                f.close()
 749
 750            separatorLine = "######## everything below this line is just the diff #######\n"
 751
 752            [handle, fileName] = tempfile.mkstemp()
 753            tmpFile = os.fdopen(handle, "w+")
 754            if self.isWindows:
 755                submitTemplate = submitTemplate.replace("\n", "\r\n")
 756                separatorLine = separatorLine.replace("\n", "\r\n")
 757                newdiff = newdiff.replace("\n", "\r\n")
 758            tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 759            tmpFile.close()
 760            mtime = os.stat(fileName).st_mtime
 761            if os.environ.has_key("P4EDITOR"):
 762                editor = os.environ.get("P4EDITOR")
 763            else:
 764                editor = read_pipe("git var GIT_EDITOR").strip()
 765            system(editor + " " + fileName)
 766
 767            response = "y"
 768            if os.stat(fileName).st_mtime <= mtime:
 769                response = "x"
 770                while response != "y" and response != "n":
 771                    response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 772
 773            if response == "y":
 774                tmpFile = open(fileName, "rb")
 775                message = tmpFile.read()
 776                tmpFile.close()
 777                submitTemplate = message[:message.index(separatorLine)]
 778                if self.isWindows:
 779                    submitTemplate = submitTemplate.replace("\r\n", "\n")
 780                p4_write_pipe("submit -i", submitTemplate)
 781            else:
 782                for f in editedFiles:
 783                    p4_system("revert \"%s\"" % f);
 784                for f in filesToAdd:
 785                    p4_system("revert \"%s\"" % f);
 786                    system("rm %s" %f)
 787
 788            os.remove(fileName)
 789        else:
 790            fileName = "submit.txt"
 791            file = open(fileName, "w+")
 792            file.write(self.prepareLogMessage(template, logMessage))
 793            file.close()
 794            print ("Perforce submit template written as %s. "
 795                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 796                   % (fileName, fileName))
 797
 798    def run(self, args):
 799        if len(args) == 0:
 800            self.master = currentGitBranch()
 801            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 802                die("Detecting current git branch failed!")
 803        elif len(args) == 1:
 804            self.master = args[0]
 805        else:
 806            return False
 807
 808        allowSubmit = gitConfig("git-p4.allowSubmit")
 809        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 810            die("%s is not in git-p4.allowSubmit" % self.master)
 811
 812        [upstream, settings] = findUpstreamBranchPoint()
 813        self.depotPath = settings['depot-paths'][0]
 814        if len(self.origin) == 0:
 815            self.origin = upstream
 816
 817        if self.verbose:
 818            print "Origin branch is " + self.origin
 819
 820        if len(self.depotPath) == 0:
 821            print "Internal error: cannot locate perforce depot path from existing branches"
 822            sys.exit(128)
 823
 824        self.clientPath = p4Where(self.depotPath)
 825
 826        if len(self.clientPath) == 0:
 827            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 828            sys.exit(128)
 829
 830        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 831        self.oldWorkingDirectory = os.getcwd()
 832
 833        chdir(self.clientPath)
 834        print "Synchronizing p4 checkout..."
 835        p4_system("sync ...")
 836
 837        self.check()
 838
 839        commits = []
 840        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 841            commits.append(line.strip())
 842        commits.reverse()
 843
 844        while len(commits) > 0:
 845            commit = commits[0]
 846            commits = commits[1:]
 847            self.applyCommit(commit)
 848            if not self.interactive:
 849                break
 850
 851        if len(commits) == 0:
 852            print "All changes applied!"
 853            chdir(self.oldWorkingDirectory)
 854
 855            sync = P4Sync()
 856            sync.run([])
 857
 858            rebase = P4Rebase()
 859            rebase.rebase()
 860
 861        return True
 862
 863class P4Sync(Command):
 864    def __init__(self):
 865        Command.__init__(self)
 866        self.options = [
 867                optparse.make_option("--branch", dest="branch"),
 868                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 869                optparse.make_option("--changesfile", dest="changesFile"),
 870                optparse.make_option("--silent", dest="silent", action="store_true"),
 871                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 872                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 873                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 874                                     help="Import into refs/heads/ , not refs/remotes"),
 875                optparse.make_option("--max-changes", dest="maxChanges"),
 876                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 877                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 878                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 879                                     help="Only sync files that are included in the Perforce Client Spec")
 880        ]
 881        self.description = """Imports from Perforce into a git repository.\n
 882    example:
 883    //depot/my/project/ -- to import the current head
 884    //depot/my/project/@all -- to import everything
 885    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 886
 887    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 888
 889        self.usage += " //depot/path[@revRange]"
 890        self.silent = False
 891        self.createdBranches = set()
 892        self.committedChanges = set()
 893        self.branch = ""
 894        self.detectBranches = False
 895        self.detectLabels = False
 896        self.changesFile = ""
 897        self.syncWithOrigin = True
 898        self.verbose = False
 899        self.importIntoRemotes = True
 900        self.maxChanges = ""
 901        self.isWindows = (platform.system() == "Windows")
 902        self.keepRepoPath = False
 903        self.depotPaths = None
 904        self.p4BranchesInGit = []
 905        self.cloneExclude = []
 906        self.useClientSpec = False
 907        self.clientSpecDirs = []
 908
 909        if gitConfig("git-p4.syncFromOrigin") == "false":
 910            self.syncWithOrigin = False
 911
 912    def extractFilesFromCommit(self, commit):
 913        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 914                             for path in self.cloneExclude]
 915        files = []
 916        fnum = 0
 917        while commit.has_key("depotFile%s" % fnum):
 918            path =  commit["depotFile%s" % fnum]
 919
 920            if [p for p in self.cloneExclude
 921                if path.startswith (p)]:
 922                found = False
 923            else:
 924                found = [p for p in self.depotPaths
 925                         if path.startswith (p)]
 926            if not found:
 927                fnum = fnum + 1
 928                continue
 929
 930            file = {}
 931            file["path"] = path
 932            file["rev"] = commit["rev%s" % fnum]
 933            file["action"] = commit["action%s" % fnum]
 934            file["type"] = commit["type%s" % fnum]
 935            files.append(file)
 936            fnum = fnum + 1
 937        return files
 938
 939    def stripRepoPath(self, path, prefixes):
 940        if self.keepRepoPath:
 941            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 942
 943        for p in prefixes:
 944            if path.startswith(p):
 945                path = path[len(p):]
 946
 947        return path
 948
 949    def splitFilesIntoBranches(self, commit):
 950        branches = {}
 951        fnum = 0
 952        while commit.has_key("depotFile%s" % fnum):
 953            path =  commit["depotFile%s" % fnum]
 954            found = [p for p in self.depotPaths
 955                     if path.startswith (p)]
 956            if not found:
 957                fnum = fnum + 1
 958                continue
 959
 960            file = {}
 961            file["path"] = path
 962            file["rev"] = commit["rev%s" % fnum]
 963            file["action"] = commit["action%s" % fnum]
 964            file["type"] = commit["type%s" % fnum]
 965            fnum = fnum + 1
 966
 967            relPath = self.stripRepoPath(path, self.depotPaths)
 968
 969            for branch in self.knownBranches.keys():
 970
 971                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 972                if relPath.startswith(branch + "/"):
 973                    if branch not in branches:
 974                        branches[branch] = []
 975                    branches[branch].append(file)
 976                    break
 977
 978        return branches
 979
 980    # output one file from the P4 stream
 981    # - helper for streamP4Files
 982
 983    def streamOneP4File(self, file, contents):
 984        if file["type"] == "apple":
 985            print "\nfile %s is a strange apple file that forks. Ignoring" % \
 986                file['depotFile']
 987            return
 988
 989        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 990        if verbose:
 991            sys.stderr.write("%s\n" % relPath)
 992
 993        mode = "644"
 994        if isP4Exec(file["type"]):
 995            mode = "755"
 996        elif file["type"] == "symlink":
 997            mode = "120000"
 998            # p4 print on a symlink contains "target\n", so strip it off
 999            data = ''.join(contents)
1000            contents = [data[:-1]]
1001
1002        if self.isWindows and file["type"].endswith("text"):
1003            mangled = []
1004            for data in contents:
1005                data = data.replace("\r\n", "\n")
1006                mangled.append(data)
1007            contents = mangled
1008
1009        if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1010            contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1011        elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1012            contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1013
1014        self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1015
1016        # total length...
1017        length = 0
1018        for d in contents:
1019            length = length + len(d)
1020
1021        self.gitStream.write("data %d\n" % length)
1022        for d in contents:
1023            self.gitStream.write(d)
1024        self.gitStream.write("\n")
1025
1026    def streamOneP4Deletion(self, file):
1027        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1028        if verbose:
1029            sys.stderr.write("delete %s\n" % relPath)
1030        self.gitStream.write("D %s\n" % relPath)
1031
1032    # handle another chunk of streaming data
1033    def streamP4FilesCb(self, marshalled):
1034
1035        if marshalled.has_key('depotFile') and self.stream_have_file_info:
1036            # start of a new file - output the old one first
1037            self.streamOneP4File(self.stream_file, self.stream_contents)
1038            self.stream_file = {}
1039            self.stream_contents = []
1040            self.stream_have_file_info = False
1041
1042        # pick up the new file information... for the
1043        # 'data' field we need to append to our array
1044        for k in marshalled.keys():
1045            if k == 'data':
1046                self.stream_contents.append(marshalled['data'])
1047            else:
1048                self.stream_file[k] = marshalled[k]
1049
1050        self.stream_have_file_info = True
1051
1052    # Stream directly from "p4 files" into "git fast-import"
1053    def streamP4Files(self, files):
1054        filesForCommit = []
1055        filesToRead = []
1056        filesToDelete = []
1057
1058        for f in files:
1059            includeFile = True
1060            for val in self.clientSpecDirs:
1061                if f['path'].startswith(val[0]):
1062                    if val[1] <= 0:
1063                        includeFile = False
1064                    break
1065
1066            if includeFile:
1067                filesForCommit.append(f)
1068                if f['action'] not in ('delete', 'move/delete', 'purge'):
1069                    filesToRead.append(f)
1070                else:
1071                    filesToDelete.append(f)
1072
1073        # deleted files...
1074        for f in filesToDelete:
1075            self.streamOneP4Deletion(f)
1076
1077        if len(filesToRead) > 0:
1078            self.stream_file = {}
1079            self.stream_contents = []
1080            self.stream_have_file_info = False
1081
1082            # curry self argument
1083            def streamP4FilesCbSelf(entry):
1084                self.streamP4FilesCb(entry)
1085
1086            p4CmdList("-x - print",
1087                '\n'.join(['%s#%s' % (f['path'], f['rev'])
1088                                                  for f in filesToRead]),
1089                cb=streamP4FilesCbSelf)
1090
1091            # do the last chunk
1092            if self.stream_file.has_key('depotFile'):
1093                self.streamOneP4File(self.stream_file, self.stream_contents)
1094
1095    def commit(self, details, files, branch, branchPrefixes, parent = ""):
1096        epoch = details["time"]
1097        author = details["user"]
1098        self.branchPrefixes = branchPrefixes
1099
1100        if self.verbose:
1101            print "commit into %s" % branch
1102
1103        # start with reading files; if that fails, we should not
1104        # create a commit.
1105        new_files = []
1106        for f in files:
1107            if [p for p in branchPrefixes if f['path'].startswith(p)]:
1108                new_files.append (f)
1109            else:
1110                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1111
1112        self.gitStream.write("commit %s\n" % branch)
1113#        gitStream.write("mark :%s\n" % details["change"])
1114        self.committedChanges.add(int(details["change"]))
1115        committer = ""
1116        if author not in self.users:
1117            self.getUserMapFromPerforceServer()
1118        if author in self.users:
1119            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1120        else:
1121            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1122
1123        self.gitStream.write("committer %s\n" % committer)
1124
1125        self.gitStream.write("data <<EOT\n")
1126        self.gitStream.write(details["desc"])
1127        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1128                             % (','.join (branchPrefixes), details["change"]))
1129        if len(details['options']) > 0:
1130            self.gitStream.write(": options = %s" % details['options'])
1131        self.gitStream.write("]\nEOT\n\n")
1132
1133        if len(parent) > 0:
1134            if self.verbose:
1135                print "parent %s" % parent
1136            self.gitStream.write("from %s\n" % parent)
1137
1138        self.streamP4Files(new_files)
1139        self.gitStream.write("\n")
1140
1141        change = int(details["change"])
1142
1143        if self.labels.has_key(change):
1144            label = self.labels[change]
1145            labelDetails = label[0]
1146            labelRevisions = label[1]
1147            if self.verbose:
1148                print "Change %s is labelled %s" % (change, labelDetails)
1149
1150            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1151                                                    for p in branchPrefixes]))
1152
1153            if len(files) == len(labelRevisions):
1154
1155                cleanedFiles = {}
1156                for info in files:
1157                    if info["action"] in ("delete", "purge"):
1158                        continue
1159                    cleanedFiles[info["depotFile"]] = info["rev"]
1160
1161                if cleanedFiles == labelRevisions:
1162                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1163                    self.gitStream.write("from %s\n" % branch)
1164
1165                    owner = labelDetails["Owner"]
1166                    tagger = ""
1167                    if author in self.users:
1168                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1169                    else:
1170                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1171                    self.gitStream.write("tagger %s\n" % tagger)
1172                    self.gitStream.write("data <<EOT\n")
1173                    self.gitStream.write(labelDetails["Description"])
1174                    self.gitStream.write("EOT\n\n")
1175
1176                else:
1177                    if not self.silent:
1178                        print ("Tag %s does not match with change %s: files do not match."
1179                               % (labelDetails["label"], change))
1180
1181            else:
1182                if not self.silent:
1183                    print ("Tag %s does not match with change %s: file count is different."
1184                           % (labelDetails["label"], change))
1185
1186    def getUserCacheFilename(self):
1187        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1188        return home + "/.gitp4-usercache.txt"
1189
1190    def getUserMapFromPerforceServer(self):
1191        if self.userMapFromPerforceServer:
1192            return
1193        self.users = {}
1194
1195        for output in p4CmdList("users"):
1196            if not output.has_key("User"):
1197                continue
1198            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1199
1200
1201        s = ''
1202        for (key, val) in self.users.items():
1203            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1204
1205        open(self.getUserCacheFilename(), "wb").write(s)
1206        self.userMapFromPerforceServer = True
1207
1208    def loadUserMapFromCache(self):
1209        self.users = {}
1210        self.userMapFromPerforceServer = False
1211        try:
1212            cache = open(self.getUserCacheFilename(), "rb")
1213            lines = cache.readlines()
1214            cache.close()
1215            for line in lines:
1216                entry = line.strip().split("\t")
1217                self.users[entry[0]] = entry[1]
1218        except IOError:
1219            self.getUserMapFromPerforceServer()
1220
1221    def getLabels(self):
1222        self.labels = {}
1223
1224        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1225        if len(l) > 0 and not self.silent:
1226            print "Finding files belonging to labels in %s" % `self.depotPaths`
1227
1228        for output in l:
1229            label = output["label"]
1230            revisions = {}
1231            newestChange = 0
1232            if self.verbose:
1233                print "Querying files for label %s" % label
1234            for file in p4CmdList("files "
1235                                  +  ' '.join (["%s...@%s" % (p, label)
1236                                                for p in self.depotPaths])):
1237                revisions[file["depotFile"]] = file["rev"]
1238                change = int(file["change"])
1239                if change > newestChange:
1240                    newestChange = change
1241
1242            self.labels[newestChange] = [output, revisions]
1243
1244        if self.verbose:
1245            print "Label changes: %s" % self.labels.keys()
1246
1247    def guessProjectName(self):
1248        for p in self.depotPaths:
1249            if p.endswith("/"):
1250                p = p[:-1]
1251            p = p[p.strip().rfind("/") + 1:]
1252            if not p.endswith("/"):
1253               p += "/"
1254            return p
1255
1256    def getBranchMapping(self):
1257        lostAndFoundBranches = set()
1258
1259        for info in p4CmdList("branches"):
1260            details = p4Cmd("branch -o %s" % info["branch"])
1261            viewIdx = 0
1262            while details.has_key("View%s" % viewIdx):
1263                paths = details["View%s" % viewIdx].split(" ")
1264                viewIdx = viewIdx + 1
1265                # require standard //depot/foo/... //depot/bar/... mapping
1266                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1267                    continue
1268                source = paths[0]
1269                destination = paths[1]
1270                ## HACK
1271                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1272                    source = source[len(self.depotPaths[0]):-4]
1273                    destination = destination[len(self.depotPaths[0]):-4]
1274
1275                    if destination in self.knownBranches:
1276                        if not self.silent:
1277                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1278                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1279                        continue
1280
1281                    self.knownBranches[destination] = source
1282
1283                    lostAndFoundBranches.discard(destination)
1284
1285                    if source not in self.knownBranches:
1286                        lostAndFoundBranches.add(source)
1287
1288
1289        for branch in lostAndFoundBranches:
1290            self.knownBranches[branch] = branch
1291
1292    def getBranchMappingFromGitBranches(self):
1293        branches = p4BranchesInGit(self.importIntoRemotes)
1294        for branch in branches.keys():
1295            if branch == "master":
1296                branch = "main"
1297            else:
1298                branch = branch[len(self.projectName):]
1299            self.knownBranches[branch] = branch
1300
1301    def listExistingP4GitBranches(self):
1302        # branches holds mapping from name to commit
1303        branches = p4BranchesInGit(self.importIntoRemotes)
1304        self.p4BranchesInGit = branches.keys()
1305        for branch in branches.keys():
1306            self.initialParents[self.refPrefix + branch] = branches[branch]
1307
1308    def updateOptionDict(self, d):
1309        option_keys = {}
1310        if self.keepRepoPath:
1311            option_keys['keepRepoPath'] = 1
1312
1313        d["options"] = ' '.join(sorted(option_keys.keys()))
1314
1315    def readOptions(self, d):
1316        self.keepRepoPath = (d.has_key('options')
1317                             and ('keepRepoPath' in d['options']))
1318
1319    def gitRefForBranch(self, branch):
1320        if branch == "main":
1321            return self.refPrefix + "master"
1322
1323        if len(branch) <= 0:
1324            return branch
1325
1326        return self.refPrefix + self.projectName + branch
1327
1328    def gitCommitByP4Change(self, ref, change):
1329        if self.verbose:
1330            print "looking in ref " + ref + " for change %s using bisect..." % change
1331
1332        earliestCommit = ""
1333        latestCommit = parseRevision(ref)
1334
1335        while True:
1336            if self.verbose:
1337                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1338            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1339            if len(next) == 0:
1340                if self.verbose:
1341                    print "argh"
1342                return ""
1343            log = extractLogMessageFromGitCommit(next)
1344            settings = extractSettingsGitLog(log)
1345            currentChange = int(settings['change'])
1346            if self.verbose:
1347                print "current change %s" % currentChange
1348
1349            if currentChange == change:
1350                if self.verbose:
1351                    print "found %s" % next
1352                return next
1353
1354            if currentChange < change:
1355                earliestCommit = "^%s" % next
1356            else:
1357                latestCommit = "%s" % next
1358
1359        return ""
1360
1361    def importNewBranch(self, branch, maxChange):
1362        # make fast-import flush all changes to disk and update the refs using the checkpoint
1363        # command so that we can try to find the branch parent in the git history
1364        self.gitStream.write("checkpoint\n\n");
1365        self.gitStream.flush();
1366        branchPrefix = self.depotPaths[0] + branch + "/"
1367        range = "@1,%s" % maxChange
1368        #print "prefix" + branchPrefix
1369        changes = p4ChangesForPaths([branchPrefix], range)
1370        if len(changes) <= 0:
1371            return False
1372        firstChange = changes[0]
1373        #print "first change in branch: %s" % firstChange
1374        sourceBranch = self.knownBranches[branch]
1375        sourceDepotPath = self.depotPaths[0] + sourceBranch
1376        sourceRef = self.gitRefForBranch(sourceBranch)
1377        #print "source " + sourceBranch
1378
1379        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1380        #print "branch parent: %s" % branchParentChange
1381        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1382        if len(gitParent) > 0:
1383            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1384            #print "parent git commit: %s" % gitParent
1385
1386        self.importChanges(changes)
1387        return True
1388
1389    def importChanges(self, changes):
1390        cnt = 1
1391        for change in changes:
1392            description = p4Cmd("describe %s" % change)
1393            self.updateOptionDict(description)
1394
1395            if not self.silent:
1396                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1397                sys.stdout.flush()
1398            cnt = cnt + 1
1399
1400            try:
1401                if self.detectBranches:
1402                    branches = self.splitFilesIntoBranches(description)
1403                    for branch in branches.keys():
1404                        ## HACK  --hwn
1405                        branchPrefix = self.depotPaths[0] + branch + "/"
1406
1407                        parent = ""
1408
1409                        filesForCommit = branches[branch]
1410
1411                        if self.verbose:
1412                            print "branch is %s" % branch
1413
1414                        self.updatedBranches.add(branch)
1415
1416                        if branch not in self.createdBranches:
1417                            self.createdBranches.add(branch)
1418                            parent = self.knownBranches[branch]
1419                            if parent == branch:
1420                                parent = ""
1421                            else:
1422                                fullBranch = self.projectName + branch
1423                                if fullBranch not in self.p4BranchesInGit:
1424                                    if not self.silent:
1425                                        print("\n    Importing new branch %s" % fullBranch);
1426                                    if self.importNewBranch(branch, change - 1):
1427                                        parent = ""
1428                                        self.p4BranchesInGit.append(fullBranch)
1429                                    if not self.silent:
1430                                        print("\n    Resuming with change %s" % change);
1431
1432                                if self.verbose:
1433                                    print "parent determined through known branches: %s" % parent
1434
1435                        branch = self.gitRefForBranch(branch)
1436                        parent = self.gitRefForBranch(parent)
1437
1438                        if self.verbose:
1439                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1440
1441                        if len(parent) == 0 and branch in self.initialParents:
1442                            parent = self.initialParents[branch]
1443                            del self.initialParents[branch]
1444
1445                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1446                else:
1447                    files = self.extractFilesFromCommit(description)
1448                    self.commit(description, files, self.branch, self.depotPaths,
1449                                self.initialParent)
1450                    self.initialParent = ""
1451            except IOError:
1452                print self.gitError.read()
1453                sys.exit(1)
1454
1455    def importHeadRevision(self, revision):
1456        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1457
1458        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1459        details["desc"] = ("Initial import of %s from the state at revision %s"
1460                           % (' '.join(self.depotPaths), revision))
1461        details["change"] = revision
1462        newestRevision = 0
1463
1464        fileCnt = 0
1465        for info in p4CmdList("files "
1466                              +  ' '.join(["%s...%s"
1467                                           % (p, revision)
1468                                           for p in self.depotPaths])):
1469
1470            if info['code'] == 'error':
1471                sys.stderr.write("p4 returned an error: %s\n"
1472                                 % info['data'])
1473                sys.exit(1)
1474
1475
1476            change = int(info["change"])
1477            if change > newestRevision:
1478                newestRevision = change
1479
1480            if info["action"] in ("delete", "purge"):
1481                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1482                #fileCnt = fileCnt + 1
1483                continue
1484
1485            for prop in ["depotFile", "rev", "action", "type" ]:
1486                details["%s%s" % (prop, fileCnt)] = info[prop]
1487
1488            fileCnt = fileCnt + 1
1489
1490        details["change"] = newestRevision
1491        self.updateOptionDict(details)
1492        try:
1493            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1494        except IOError:
1495            print "IO error with git fast-import. Is your git version recent enough?"
1496            print self.gitError.read()
1497
1498
1499    def getClientSpec(self):
1500        specList = p4CmdList( "client -o" )
1501        temp = {}
1502        for entry in specList:
1503            for k,v in entry.iteritems():
1504                if k.startswith("View"):
1505                    if v.startswith('"'):
1506                        start = 1
1507                    else:
1508                        start = 0
1509                    index = v.find("...")
1510                    v = v[start:index]
1511                    if v.startswith("-"):
1512                        v = v[1:]
1513                        temp[v] = -len(v)
1514                    else:
1515                        temp[v] = len(v)
1516        self.clientSpecDirs = temp.items()
1517        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1518
1519    def run(self, args):
1520        self.depotPaths = []
1521        self.changeRange = ""
1522        self.initialParent = ""
1523        self.previousDepotPaths = []
1524
1525        # map from branch depot path to parent branch
1526        self.knownBranches = {}
1527        self.initialParents = {}
1528        self.hasOrigin = originP4BranchesExist()
1529        if not self.syncWithOrigin:
1530            self.hasOrigin = False
1531
1532        if self.importIntoRemotes:
1533            self.refPrefix = "refs/remotes/p4/"
1534        else:
1535            self.refPrefix = "refs/heads/p4/"
1536
1537        if self.syncWithOrigin and self.hasOrigin:
1538            if not self.silent:
1539                print "Syncing with origin first by calling git fetch origin"
1540            system("git fetch origin")
1541
1542        if len(self.branch) == 0:
1543            self.branch = self.refPrefix + "master"
1544            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1545                system("git update-ref %s refs/heads/p4" % self.branch)
1546                system("git branch -D p4");
1547            # create it /after/ importing, when master exists
1548            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1549                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1550
1551        if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1552            self.getClientSpec()
1553
1554        # TODO: should always look at previous commits,
1555        # merge with previous imports, if possible.
1556        if args == []:
1557            if self.hasOrigin:
1558                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1559            self.listExistingP4GitBranches()
1560
1561            if len(self.p4BranchesInGit) > 1:
1562                if not self.silent:
1563                    print "Importing from/into multiple branches"
1564                self.detectBranches = True
1565
1566            if self.verbose:
1567                print "branches: %s" % self.p4BranchesInGit
1568
1569            p4Change = 0
1570            for branch in self.p4BranchesInGit:
1571                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1572
1573                settings = extractSettingsGitLog(logMsg)
1574
1575                self.readOptions(settings)
1576                if (settings.has_key('depot-paths')
1577                    and settings.has_key ('change')):
1578                    change = int(settings['change']) + 1
1579                    p4Change = max(p4Change, change)
1580
1581                    depotPaths = sorted(settings['depot-paths'])
1582                    if self.previousDepotPaths == []:
1583                        self.previousDepotPaths = depotPaths
1584                    else:
1585                        paths = []
1586                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1587                            for i in range(0, min(len(cur), len(prev))):
1588                                if cur[i] <> prev[i]:
1589                                    i = i - 1
1590                                    break
1591
1592                            paths.append (cur[:i + 1])
1593
1594                        self.previousDepotPaths = paths
1595
1596            if p4Change > 0:
1597                self.depotPaths = sorted(self.previousDepotPaths)
1598                self.changeRange = "@%s,#head" % p4Change
1599                if not self.detectBranches:
1600                    self.initialParent = parseRevision(self.branch)
1601                if not self.silent and not self.detectBranches:
1602                    print "Performing incremental import into %s git branch" % self.branch
1603
1604        if not self.branch.startswith("refs/"):
1605            self.branch = "refs/heads/" + self.branch
1606
1607        if len(args) == 0 and self.depotPaths:
1608            if not self.silent:
1609                print "Depot paths: %s" % ' '.join(self.depotPaths)
1610        else:
1611            if self.depotPaths and self.depotPaths != args:
1612                print ("previous import used depot path %s and now %s was specified. "
1613                       "This doesn't work!" % (' '.join (self.depotPaths),
1614                                               ' '.join (args)))
1615                sys.exit(1)
1616
1617            self.depotPaths = sorted(args)
1618
1619        revision = ""
1620        self.users = {}
1621
1622        newPaths = []
1623        for p in self.depotPaths:
1624            if p.find("@") != -1:
1625                atIdx = p.index("@")
1626                self.changeRange = p[atIdx:]
1627                if self.changeRange == "@all":
1628                    self.changeRange = ""
1629                elif ',' not in self.changeRange:
1630                    revision = self.changeRange
1631                    self.changeRange = ""
1632                p = p[:atIdx]
1633            elif p.find("#") != -1:
1634                hashIdx = p.index("#")
1635                revision = p[hashIdx:]
1636                p = p[:hashIdx]
1637            elif self.previousDepotPaths == []:
1638                revision = "#head"
1639
1640            p = re.sub ("\.\.\.$", "", p)
1641            if not p.endswith("/"):
1642                p += "/"
1643
1644            newPaths.append(p)
1645
1646        self.depotPaths = newPaths
1647
1648
1649        self.loadUserMapFromCache()
1650        self.labels = {}
1651        if self.detectLabels:
1652            self.getLabels();
1653
1654        if self.detectBranches:
1655            ## FIXME - what's a P4 projectName ?
1656            self.projectName = self.guessProjectName()
1657
1658            if self.hasOrigin:
1659                self.getBranchMappingFromGitBranches()
1660            else:
1661                self.getBranchMapping()
1662            if self.verbose:
1663                print "p4-git branches: %s" % self.p4BranchesInGit
1664                print "initial parents: %s" % self.initialParents
1665            for b in self.p4BranchesInGit:
1666                if b != "master":
1667
1668                    ## FIXME
1669                    b = b[len(self.projectName):]
1670                self.createdBranches.add(b)
1671
1672        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1673
1674        importProcess = subprocess.Popen(["git", "fast-import"],
1675                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1676                                         stderr=subprocess.PIPE);
1677        self.gitOutput = importProcess.stdout
1678        self.gitStream = importProcess.stdin
1679        self.gitError = importProcess.stderr
1680
1681        if revision:
1682            self.importHeadRevision(revision)
1683        else:
1684            changes = []
1685
1686            if len(self.changesFile) > 0:
1687                output = open(self.changesFile).readlines()
1688                changeSet = set()
1689                for line in output:
1690                    changeSet.add(int(line))
1691
1692                for change in changeSet:
1693                    changes.append(change)
1694
1695                changes.sort()
1696            else:
1697                if self.verbose:
1698                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1699                                                              self.changeRange)
1700                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1701
1702                if len(self.maxChanges) > 0:
1703                    changes = changes[:min(int(self.maxChanges), len(changes))]
1704
1705            if len(changes) == 0:
1706                if not self.silent:
1707                    print "No changes to import!"
1708                return True
1709
1710            if not self.silent and not self.detectBranches:
1711                print "Import destination: %s" % self.branch
1712
1713            self.updatedBranches = set()
1714
1715            self.importChanges(changes)
1716
1717            if not self.silent:
1718                print ""
1719                if len(self.updatedBranches) > 0:
1720                    sys.stdout.write("Updated branches: ")
1721                    for b in self.updatedBranches:
1722                        sys.stdout.write("%s " % b)
1723                    sys.stdout.write("\n")
1724
1725        self.gitStream.close()
1726        if importProcess.wait() != 0:
1727            die("fast-import failed: %s" % self.gitError.read())
1728        self.gitOutput.close()
1729        self.gitError.close()
1730
1731        return True
1732
1733class P4Rebase(Command):
1734    def __init__(self):
1735        Command.__init__(self)
1736        self.options = [ ]
1737        self.description = ("Fetches the latest revision from perforce and "
1738                            + "rebases the current work (branch) against it")
1739        self.verbose = False
1740
1741    def run(self, args):
1742        sync = P4Sync()
1743        sync.run([])
1744
1745        return self.rebase()
1746
1747    def rebase(self):
1748        if os.system("git update-index --refresh") != 0:
1749            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1750        if len(read_pipe("git diff-index HEAD --")) > 0:
1751            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1752
1753        [upstream, settings] = findUpstreamBranchPoint()
1754        if len(upstream) == 0:
1755            die("Cannot find upstream branchpoint for rebase")
1756
1757        # the branchpoint may be p4/foo~3, so strip off the parent
1758        upstream = re.sub("~[0-9]+$", "", upstream)
1759
1760        print "Rebasing the current branch onto %s" % upstream
1761        oldHead = read_pipe("git rev-parse HEAD").strip()
1762        system("git rebase %s" % upstream)
1763        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1764        return True
1765
1766class P4Clone(P4Sync):
1767    def __init__(self):
1768        P4Sync.__init__(self)
1769        self.description = "Creates a new git repository and imports from Perforce into it"
1770        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1771        self.options += [
1772            optparse.make_option("--destination", dest="cloneDestination",
1773                                 action='store', default=None,
1774                                 help="where to leave result of the clone"),
1775            optparse.make_option("-/", dest="cloneExclude",
1776                                 action="append", type="string",
1777                                 help="exclude depot path")
1778        ]
1779        self.cloneDestination = None
1780        self.needsGit = False
1781
1782    # This is required for the "append" cloneExclude action
1783    def ensure_value(self, attr, value):
1784        if not hasattr(self, attr) or getattr(self, attr) is None:
1785            setattr(self, attr, value)
1786        return getattr(self, attr)
1787
1788    def defaultDestination(self, args):
1789        ## TODO: use common prefix of args?
1790        depotPath = args[0]
1791        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1792        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1793        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1794        depotDir = re.sub(r"/$", "", depotDir)
1795        return os.path.split(depotDir)[1]
1796
1797    def run(self, args):
1798        if len(args) < 1:
1799            return False
1800
1801        if self.keepRepoPath and not self.cloneDestination:
1802            sys.stderr.write("Must specify destination for --keep-path\n")
1803            sys.exit(1)
1804
1805        depotPaths = args
1806
1807        if not self.cloneDestination and len(depotPaths) > 1:
1808            self.cloneDestination = depotPaths[-1]
1809            depotPaths = depotPaths[:-1]
1810
1811        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1812        for p in depotPaths:
1813            if not p.startswith("//"):
1814                return False
1815
1816        if not self.cloneDestination:
1817            self.cloneDestination = self.defaultDestination(args)
1818
1819        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1820        if not os.path.exists(self.cloneDestination):
1821            os.makedirs(self.cloneDestination)
1822        chdir(self.cloneDestination)
1823        system("git init")
1824        self.gitdir = os.getcwd() + "/.git"
1825        if not P4Sync.run(self, depotPaths):
1826            return False
1827        if self.branch != "master":
1828            if self.importIntoRemotes:
1829                masterbranch = "refs/remotes/p4/master"
1830            else:
1831                masterbranch = "refs/heads/p4/master"
1832            if gitBranchExists(masterbranch):
1833                system("git branch master %s" % masterbranch)
1834                system("git checkout -f")
1835            else:
1836                print "Could not detect main branch. No checkout/master branch created."
1837
1838        return True
1839
1840class P4Branches(Command):
1841    def __init__(self):
1842        Command.__init__(self)
1843        self.options = [ ]
1844        self.description = ("Shows the git branches that hold imports and their "
1845                            + "corresponding perforce depot paths")
1846        self.verbose = False
1847
1848    def run(self, args):
1849        if originP4BranchesExist():
1850            createOrUpdateBranchesFromOrigin()
1851
1852        cmdline = "git rev-parse --symbolic "
1853        cmdline += " --remotes"
1854
1855        for line in read_pipe_lines(cmdline):
1856            line = line.strip()
1857
1858            if not line.startswith('p4/') or line == "p4/HEAD":
1859                continue
1860            branch = line
1861
1862            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1863            settings = extractSettingsGitLog(log)
1864
1865            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1866        return True
1867
1868class HelpFormatter(optparse.IndentedHelpFormatter):
1869    def __init__(self):
1870        optparse.IndentedHelpFormatter.__init__(self)
1871
1872    def format_description(self, description):
1873        if description:
1874            return description + "\n"
1875        else:
1876            return ""
1877
1878def printUsage(commands):
1879    print "usage: %s <command> [options]" % sys.argv[0]
1880    print ""
1881    print "valid commands: %s" % ", ".join(commands)
1882    print ""
1883    print "Try %s <command> --help for command specific help." % sys.argv[0]
1884    print ""
1885
1886commands = {
1887    "debug" : P4Debug,
1888    "submit" : P4Submit,
1889    "commit" : P4Submit,
1890    "sync" : P4Sync,
1891    "rebase" : P4Rebase,
1892    "clone" : P4Clone,
1893    "rollback" : P4RollBack,
1894    "branches" : P4Branches
1895}
1896
1897
1898def main():
1899    if len(sys.argv[1:]) == 0:
1900        printUsage(commands.keys())
1901        sys.exit(2)
1902
1903    cmd = ""
1904    cmdName = sys.argv[1]
1905    try:
1906        klass = commands[cmdName]
1907        cmd = klass()
1908    except KeyError:
1909        print "unknown command %s" % cmdName
1910        print ""
1911        printUsage(commands.keys())
1912        sys.exit(2)
1913
1914    options = cmd.options
1915    cmd.gitdir = os.environ.get("GIT_DIR", None)
1916
1917    args = sys.argv[2:]
1918
1919    if len(options) > 0:
1920        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1921
1922        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1923                                       options,
1924                                       description = cmd.description,
1925                                       formatter = HelpFormatter())
1926
1927        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1928    global verbose
1929    verbose = cmd.verbose
1930    if cmd.needsGit:
1931        if cmd.gitdir == None:
1932            cmd.gitdir = os.path.abspath(".git")
1933            if not isValidGitDir(cmd.gitdir):
1934                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1935                if os.path.exists(cmd.gitdir):
1936                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1937                    if len(cdup) > 0:
1938                        chdir(cdup);
1939
1940        if not isValidGitDir(cmd.gitdir):
1941            if isValidGitDir(cmd.gitdir + "/.git"):
1942                cmd.gitdir += "/.git"
1943            else:
1944                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1945
1946        os.environ["GIT_DIR"] = cmd.gitdir
1947
1948    if not cmd.run(args):
1949        parser.print_help()
1950
1951
1952if __name__ == '__main__':
1953    main()