Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Merge branch 'kb/maint-filter-branch-disappear' (832d586)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13import re
  14
  15from sets import Set;
  16
  17verbose = False
  18
  19def die(msg):
  20    if verbose:
  21        raise Exception(msg)
  22    else:
  23        sys.stderr.write(msg + "\n")
  24        sys.exit(1)
  25
  26def write_pipe(c, str):
  27    if verbose:
  28        sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30    pipe = os.popen(c, 'w')
  31    val = pipe.write(str)
  32    if pipe.close():
  33        die('Command failed: %s' % c)
  34
  35    return val
  36
  37def read_pipe(c, ignore_error=False):
  38    if verbose:
  39        sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41    pipe = os.popen(c, 'rb')
  42    val = pipe.read()
  43    if pipe.close() and not ignore_error:
  44        die('Command failed: %s' % c)
  45
  46    return val
  47
  48
  49def read_pipe_lines(c):
  50    if verbose:
  51        sys.stderr.write('Reading pipe: %s\n' % c)
  52    ## todo: check return status
  53    pipe = os.popen(c, 'rb')
  54    val = pipe.readlines()
  55    if pipe.close():
  56        die('Command failed: %s' % c)
  57
  58    return val
  59
  60def system(cmd):
  61    if verbose:
  62        sys.stderr.write("executing %s\n" % cmd)
  63    if os.system(cmd) != 0:
  64        die("command failed: %s" % cmd)
  65
  66def isP4Exec(kind):
  67    """Determine if a Perforce 'kind' should have execute permission
  68
  69    'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70    or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71    a plus sign, it is also executable"""
  72    return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74def setP4ExecBit(file, mode):
  75    # Reopens an already open file and changes the execute bit to match
  76    # the execute bit setting in the passed in mode.
  77
  78    p4Type = "+x"
  79
  80    if not isModeExec(mode):
  81        p4Type = getP4OpenedType(file)
  82        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84        if p4Type[-1] == "+":
  85            p4Type = p4Type[0:-1]
  86
  87    system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89def getP4OpenedType(file):
  90    # Returns the perforce file type for the given file.
  91
  92    result = read_pipe("p4 opened %s" % file)
  93    match = re.match(".*\((.+)\)$", result)
  94    if match:
  95        return match.group(1)
  96    else:
  97        die("Could not determine file type for %s" % file)
  98
  99def diffTreePattern():
 100    # This is a simple generator for the diff tree regex pattern. This could be
 101    # a class variable if this and parseDiffTreeEntry were a part of a class.
 102    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103    while True:
 104        yield pattern
 105
 106def parseDiffTreeEntry(entry):
 107    """Parses a single diff tree entry into its component elements.
 108
 109    See git-diff-tree(1) manpage for details about the format of the diff
 110    output. This method returns a dictionary with the following elements:
 111
 112    src_mode - The mode of the source file
 113    dst_mode - The mode of the destination file
 114    src_sha1 - The sha1 for the source file
 115    dst_sha1 - The sha1 fr the destination file
 116    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117    status_score - The score for the status (applicable for 'C' and 'R'
 118                   statuses). This is None if there is no score.
 119    src - The path for the source file.
 120    dst - The path for the destination file. This is only present for
 121          copy or renames. If it is not present, this is None.
 122
 123    If the pattern is not matched, None is returned."""
 124
 125    match = diffTreePattern().next().match(entry)
 126    if match:
 127        return {
 128            'src_mode': match.group(1),
 129            'dst_mode': match.group(2),
 130            'src_sha1': match.group(3),
 131            'dst_sha1': match.group(4),
 132            'status': match.group(5),
 133            'status_score': match.group(6),
 134            'src': match.group(7),
 135            'dst': match.group(10)
 136        }
 137    return None
 138
 139def isModeExec(mode):
 140    # Returns True if the given git mode represents an executable file,
 141    # otherwise False.
 142    return mode[-3:] == "755"
 143
 144def isModeExecChanged(src_mode, dst_mode):
 145    return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148    cmd = "p4 -G %s" % cmd
 149    if verbose:
 150        sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152    # Use a temporary file to avoid deadlocks without
 153    # subprocess.communicate(), which would put another copy
 154    # of stdout into memory.
 155    stdin_file = None
 156    if stdin is not None:
 157        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158        stdin_file.write(stdin)
 159        stdin_file.flush()
 160        stdin_file.seek(0)
 161
 162    p4 = subprocess.Popen(cmd, shell=True,
 163                          stdin=stdin_file,
 164                          stdout=subprocess.PIPE)
 165
 166    result = []
 167    try:
 168        while True:
 169            entry = marshal.load(p4.stdout)
 170            result.append(entry)
 171    except EOFError:
 172        pass
 173    exitCode = p4.wait()
 174    if exitCode != 0:
 175        entry = {}
 176        entry["p4ExitCode"] = exitCode
 177        result.append(entry)
 178
 179    return result
 180
 181def p4Cmd(cmd):
 182    list = p4CmdList(cmd)
 183    result = {}
 184    for entry in list:
 185        result.update(entry)
 186    return result;
 187
 188def p4Where(depotPath):
 189    if not depotPath.endswith("/"):
 190        depotPath += "/"
 191    output = p4Cmd("where %s..." % depotPath)
 192    if output["code"] == "error":
 193        return ""
 194    clientPath = ""
 195    if "path" in output:
 196        clientPath = output.get("path")
 197    elif "data" in output:
 198        data = output.get("data")
 199        lastSpace = data.rfind(" ")
 200        clientPath = data[lastSpace + 1:]
 201
 202    if clientPath.endswith("..."):
 203        clientPath = clientPath[:-3]
 204    return clientPath
 205
 206def currentGitBranch():
 207    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209def isValidGitDir(path):
 210    if (os.path.exists(path + "/HEAD")
 211        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212        return True;
 213    return False
 214
 215def parseRevision(ref):
 216    return read_pipe("git rev-parse %s" % ref).strip()
 217
 218def extractLogMessageFromGitCommit(commit):
 219    logMessage = ""
 220
 221    ## fixme: title is first line of commit, not 1st paragraph.
 222    foundTitle = False
 223    for log in read_pipe_lines("git cat-file commit %s" % commit):
 224       if not foundTitle:
 225           if len(log) == 1:
 226               foundTitle = True
 227           continue
 228
 229       logMessage += log
 230    return logMessage
 231
 232def extractSettingsGitLog(log):
 233    values = {}
 234    for line in log.split("\n"):
 235        line = line.strip()
 236        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237        if not m:
 238            continue
 239
 240        assignments = m.group(1).split (':')
 241        for a in assignments:
 242            vals = a.split ('=')
 243            key = vals[0].strip()
 244            val = ('='.join (vals[1:])).strip()
 245            if val.endswith ('\"') and val.startswith('"'):
 246                val = val[1:-1]
 247
 248            values[key] = val
 249
 250    paths = values.get("depot-paths")
 251    if not paths:
 252        paths = values.get("depot-path")
 253    if paths:
 254        values['depot-paths'] = paths.split(',')
 255    return values
 256
 257def gitBranchExists(branch):
 258    proc = subprocess.Popen(["git", "rev-parse", branch],
 259                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260    return proc.wait() == 0;
 261
 262def gitConfig(key):
 263    return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265def p4BranchesInGit(branchesAreInRemotes = True):
 266    branches = {}
 267
 268    cmdline = "git rev-parse --symbolic "
 269    if branchesAreInRemotes:
 270        cmdline += " --remotes"
 271    else:
 272        cmdline += " --branches"
 273
 274    for line in read_pipe_lines(cmdline):
 275        line = line.strip()
 276
 277        ## only import to p4/
 278        if not line.startswith('p4/') or line == "p4/HEAD":
 279            continue
 280        branch = line
 281
 282        # strip off p4
 283        branch = re.sub ("^p4/", "", line)
 284
 285        branches[branch] = parseRevision(line)
 286    return branches
 287
 288def findUpstreamBranchPoint(head = "HEAD"):
 289    branches = p4BranchesInGit()
 290    # map from depot-path to branch name
 291    branchByDepotPath = {}
 292    for branch in branches.keys():
 293        tip = branches[branch]
 294        log = extractLogMessageFromGitCommit(tip)
 295        settings = extractSettingsGitLog(log)
 296        if settings.has_key("depot-paths"):
 297            paths = ",".join(settings["depot-paths"])
 298            branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300    settings = None
 301    parent = 0
 302    while parent < 65535:
 303        commit = head + "~%s" % parent
 304        log = extractLogMessageFromGitCommit(commit)
 305        settings = extractSettingsGitLog(log)
 306        if settings.has_key("depot-paths"):
 307            paths = ",".join(settings["depot-paths"])
 308            if branchByDepotPath.has_key(paths):
 309                return [branchByDepotPath[paths], settings]
 310
 311        parent = parent + 1
 312
 313    return ["", settings]
 314
 315def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316    if not silent:
 317        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318               % localRefPrefix)
 319
 320    originPrefix = "origin/p4/"
 321
 322    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323        line = line.strip()
 324        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325            continue
 326
 327        headName = line[len(originPrefix):]
 328        remoteHead = localRefPrefix + headName
 329        originHead = line
 330
 331        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332        if (not original.has_key('depot-paths')
 333            or not original.has_key('change')):
 334            continue
 335
 336        update = False
 337        if not gitBranchExists(remoteHead):
 338            if verbose:
 339                print "creating %s" % remoteHead
 340            update = True
 341        else:
 342            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343            if settings.has_key('change') > 0:
 344                if settings['depot-paths'] == original['depot-paths']:
 345                    originP4Change = int(original['change'])
 346                    p4Change = int(settings['change'])
 347                    if originP4Change > p4Change:
 348                        print ("%s (%s) is newer than %s (%s). "
 349                               "Updating p4 branch from origin."
 350                               % (originHead, originP4Change,
 351                                  remoteHead, p4Change))
 352                        update = True
 353                else:
 354                    print ("Ignoring: %s was imported from %s while "
 355                           "%s was imported from %s"
 356                           % (originHead, ','.join(original['depot-paths']),
 357                              remoteHead, ','.join(settings['depot-paths'])))
 358
 359        if update:
 360            system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362def originP4BranchesExist():
 363        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365def p4ChangesForPaths(depotPaths, changeRange):
 366    assert depotPaths
 367    output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                        for p in depotPaths]))
 369
 370    changes = []
 371    for line in output:
 372        changeNum = line.split(" ")[1]
 373        changes.append(int(changeNum))
 374
 375    changes.sort()
 376    return changes
 377
 378class Command:
 379    def __init__(self):
 380        self.usage = "usage: %prog [options]"
 381        self.needsGit = True
 382
 383class P4Debug(Command):
 384    def __init__(self):
 385        Command.__init__(self)
 386        self.options = [
 387            optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                 default=False),
 389            ]
 390        self.description = "A tool to debug the output of p4 -G."
 391        self.needsGit = False
 392        self.verbose = False
 393
 394    def run(self, args):
 395        j = 0
 396        for output in p4CmdList(" ".join(args)):
 397            print 'Element: %d' % j
 398            j += 1
 399            print output
 400        return True
 401
 402class P4RollBack(Command):
 403    def __init__(self):
 404        Command.__init__(self)
 405        self.options = [
 406            optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408        ]
 409        self.description = "A tool to debug the multi-branch import. Don't use :)"
 410        self.verbose = False
 411        self.rollbackLocalBranches = False
 412
 413    def run(self, args):
 414        if len(args) != 1:
 415            return False
 416        maxChange = int(args[0])
 417
 418        if "p4ExitCode" in p4Cmd("changes -m 1"):
 419            die("Problems executing p4");
 420
 421        if self.rollbackLocalBranches:
 422            refPrefix = "refs/heads/"
 423            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424        else:
 425            refPrefix = "refs/remotes/"
 426            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428        for line in lines:
 429            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                line = line.strip()
 431                ref = refPrefix + line
 432                log = extractLogMessageFromGitCommit(ref)
 433                settings = extractSettingsGitLog(log)
 434
 435                depotPaths = settings['depot-paths']
 436                change = settings['change']
 437
 438                changed = False
 439
 440                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                           for p in depotPaths]))) == 0:
 442                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                    continue
 445
 446                while change and int(change) > maxChange:
 447                    changed = True
 448                    if self.verbose:
 449                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                    system("git update-ref %s \"%s^\"" % (ref, ref))
 451                    log = extractLogMessageFromGitCommit(ref)
 452                    settings =  extractSettingsGitLog(log)
 453
 454
 455                    depotPaths = settings['depot-paths']
 456                    change = settings['change']
 457
 458                if changed:
 459                    print "%s rewound to %s" % (ref, change)
 460
 461        return True
 462
 463class P4Submit(Command):
 464    def __init__(self):
 465        Command.__init__(self)
 466        self.options = [
 467                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                optparse.make_option("--origin", dest="origin"),
 469                optparse.make_option("-M", dest="detectRename", action="store_true"),
 470        ]
 471        self.description = "Submit changes from git to the perforce depot."
 472        self.usage += " [name of git branch to submit into perforce depot]"
 473        self.interactive = True
 474        self.origin = ""
 475        self.detectRename = False
 476        self.verbose = False
 477        self.isWindows = (platform.system() == "Windows")
 478
 479    def check(self):
 480        if len(p4CmdList("opened ...")) > 0:
 481            die("You have files opened with perforce! Close them before starting the sync.")
 482
 483    # replaces everything between 'Description:' and the next P4 submit template field with the
 484    # commit message
 485    def prepareLogMessage(self, template, message):
 486        result = ""
 487
 488        inDescriptionSection = False
 489
 490        for line in template.split("\n"):
 491            if line.startswith("#"):
 492                result += line + "\n"
 493                continue
 494
 495            if inDescriptionSection:
 496                if line.startswith("Files:"):
 497                    inDescriptionSection = False
 498                else:
 499                    continue
 500            else:
 501                if line.startswith("Description:"):
 502                    inDescriptionSection = True
 503                    line += "\n"
 504                    for messageLine in message.split("\n"):
 505                        line += "\t" + messageLine + "\n"
 506
 507            result += line + "\n"
 508
 509        return result
 510
 511    def prepareSubmitTemplate(self):
 512        # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513        template = ""
 514        inFilesSection = False
 515        for line in read_pipe_lines("p4 change -o"):
 516            if inFilesSection:
 517                if line.startswith("\t"):
 518                    # path starts and ends with a tab
 519                    path = line[1:]
 520                    lastTab = path.rfind("\t")
 521                    if lastTab != -1:
 522                        path = path[:lastTab]
 523                        if not path.startswith(self.depotPath):
 524                            continue
 525                else:
 526                    inFilesSection = False
 527            else:
 528                if line.startswith("Files:"):
 529                    inFilesSection = True
 530
 531            template += line
 532
 533        return template
 534
 535    def applyCommit(self, id):
 536        print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537        diffOpts = ("", "-M")[self.detectRename]
 538        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539        filesToAdd = set()
 540        filesToDelete = set()
 541        editedFiles = set()
 542        filesToChangeExecBit = {}
 543        for line in diff:
 544            diff = parseDiffTreeEntry(line)
 545            modifier = diff['status']
 546            path = diff['src']
 547            if modifier == "M":
 548                system("p4 edit \"%s\"" % path)
 549                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                    filesToChangeExecBit[path] = diff['dst_mode']
 551                editedFiles.add(path)
 552            elif modifier == "A":
 553                filesToAdd.add(path)
 554                filesToChangeExecBit[path] = diff['dst_mode']
 555                if path in filesToDelete:
 556                    filesToDelete.remove(path)
 557            elif modifier == "D":
 558                filesToDelete.add(path)
 559                if path in filesToAdd:
 560                    filesToAdd.remove(path)
 561            elif modifier == "R":
 562                src, dest = diff['src'], diff['dst']
 563                system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                system("p4 edit \"%s\"" % (dest))
 565                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                    filesToChangeExecBit[dest] = diff['dst_mode']
 567                os.unlink(dest)
 568                editedFiles.add(dest)
 569                filesToDelete.add(src)
 570            else:
 571                die("unknown modifier %s for %s" % (modifier, path))
 572
 573        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574        patchcmd = diffcmd + " | git apply "
 575        tryPatchCmd = patchcmd + "--check -"
 576        applyPatchCmd = patchcmd + "--check --apply -"
 577
 578        if os.system(tryPatchCmd) != 0:
 579            print "Unfortunately applying the change failed!"
 580            print "What do you want to do?"
 581            response = "x"
 582            while response != "s" and response != "a" and response != "w":
 583                response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                     "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585            if response == "s":
 586                print "Skipping! Good luck with the next patches..."
 587                for f in editedFiles:
 588                    system("p4 revert \"%s\"" % f);
 589                for f in filesToAdd:
 590                    system("rm %s" %f)
 591                return
 592            elif response == "a":
 593                os.system(applyPatchCmd)
 594                if len(filesToAdd) > 0:
 595                    print "You may also want to call p4 add on the following files:"
 596                    print " ".join(filesToAdd)
 597                if len(filesToDelete):
 598                    print "The following files should be scheduled for deletion with p4 delete:"
 599                    print " ".join(filesToDelete)
 600                die("Please resolve and submit the conflict manually and "
 601                    + "continue afterwards with git-p4 submit --continue")
 602            elif response == "w":
 603                system(diffcmd + " > patch.txt")
 604                print "Patch saved to patch.txt in %s !" % self.clientPath
 605                die("Please resolve and submit the conflict manually and "
 606                    "continue afterwards with git-p4 submit --continue")
 607
 608        system(applyPatchCmd)
 609
 610        for f in filesToAdd:
 611            system("p4 add \"%s\"" % f)
 612        for f in filesToDelete:
 613            system("p4 revert \"%s\"" % f)
 614            system("p4 delete \"%s\"" % f)
 615
 616        # Set/clear executable bits
 617        for f in filesToChangeExecBit.keys():
 618            mode = filesToChangeExecBit[f]
 619            setP4ExecBit(f, mode)
 620
 621        logMessage = extractLogMessageFromGitCommit(id)
 622        if self.isWindows:
 623            logMessage = logMessage.replace("\n", "\r\n")
 624        logMessage = logMessage.strip()
 625
 626        template = self.prepareSubmitTemplate()
 627
 628        if self.interactive:
 629            submitTemplate = self.prepareLogMessage(template, logMessage)
 630            diff = read_pipe("p4 diff -du ...")
 631
 632            for newFile in filesToAdd:
 633                diff += "==== new file ====\n"
 634                diff += "--- /dev/null\n"
 635                diff += "+++ %s\n" % newFile
 636                f = open(newFile, "r")
 637                for line in f.readlines():
 638                    diff += "+" + line
 639                f.close()
 640
 641            separatorLine = "######## everything below this line is just the diff #######"
 642            if platform.system() == "Windows":
 643                separatorLine += "\r"
 644            separatorLine += "\n"
 645
 646            [handle, fileName] = tempfile.mkstemp()
 647            tmpFile = os.fdopen(handle, "w+")
 648            tmpFile.write(submitTemplate + separatorLine + diff)
 649            tmpFile.close()
 650            defaultEditor = "vi"
 651            if platform.system() == "Windows":
 652                defaultEditor = "notepad"
 653            editor = os.environ.get("EDITOR", defaultEditor);
 654            system(editor + " " + fileName)
 655            tmpFile = open(fileName, "rb")
 656            message = tmpFile.read()
 657            tmpFile.close()
 658            os.remove(fileName)
 659            submitTemplate = message[:message.index(separatorLine)]
 660            if self.isWindows:
 661                submitTemplate = submitTemplate.replace("\r\n", "\n")
 662
 663            write_pipe("p4 submit -i", submitTemplate)
 664        else:
 665            fileName = "submit.txt"
 666            file = open(fileName, "w+")
 667            file.write(self.prepareLogMessage(template, logMessage))
 668            file.close()
 669            print ("Perforce submit template written as %s. "
 670                   + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 671                   % (fileName, fileName))
 672
 673    def run(self, args):
 674        if len(args) == 0:
 675            self.master = currentGitBranch()
 676            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 677                die("Detecting current git branch failed!")
 678        elif len(args) == 1:
 679            self.master = args[0]
 680        else:
 681            return False
 682
 683        [upstream, settings] = findUpstreamBranchPoint()
 684        self.depotPath = settings['depot-paths'][0]
 685        if len(self.origin) == 0:
 686            self.origin = upstream
 687
 688        if self.verbose:
 689            print "Origin branch is " + self.origin
 690
 691        if len(self.depotPath) == 0:
 692            print "Internal error: cannot locate perforce depot path from existing branches"
 693            sys.exit(128)
 694
 695        self.clientPath = p4Where(self.depotPath)
 696
 697        if len(self.clientPath) == 0:
 698            print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 699            sys.exit(128)
 700
 701        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 702        self.oldWorkingDirectory = os.getcwd()
 703
 704        os.chdir(self.clientPath)
 705        print "Syncronizing p4 checkout..."
 706        system("p4 sync ...")
 707
 708        self.check()
 709
 710        commits = []
 711        for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 712            commits.append(line.strip())
 713        commits.reverse()
 714
 715        while len(commits) > 0:
 716            commit = commits[0]
 717            commits = commits[1:]
 718            self.applyCommit(commit)
 719            if not self.interactive:
 720                break
 721
 722        if len(commits) == 0:
 723            print "All changes applied!"
 724            os.chdir(self.oldWorkingDirectory)
 725
 726            sync = P4Sync()
 727            sync.run([])
 728
 729            rebase = P4Rebase()
 730            rebase.rebase()
 731
 732        return True
 733
 734class P4Sync(Command):
 735    def __init__(self):
 736        Command.__init__(self)
 737        self.options = [
 738                optparse.make_option("--branch", dest="branch"),
 739                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 740                optparse.make_option("--changesfile", dest="changesFile"),
 741                optparse.make_option("--silent", dest="silent", action="store_true"),
 742                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 743                optparse.make_option("--verbose", dest="verbose", action="store_true"),
 744                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 745                                     help="Import into refs/heads/ , not refs/remotes"),
 746                optparse.make_option("--max-changes", dest="maxChanges"),
 747                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 748                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 749                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 750                                     help="Only sync files that are included in the Perforce Client Spec")
 751        ]
 752        self.description = """Imports from Perforce into a git repository.\n
 753    example:
 754    //depot/my/project/ -- to import the current head
 755    //depot/my/project/@all -- to import everything
 756    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 757
 758    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 759
 760        self.usage += " //depot/path[@revRange]"
 761        self.silent = False
 762        self.createdBranches = Set()
 763        self.committedChanges = Set()
 764        self.branch = ""
 765        self.detectBranches = False
 766        self.detectLabels = False
 767        self.changesFile = ""
 768        self.syncWithOrigin = True
 769        self.verbose = False
 770        self.importIntoRemotes = True
 771        self.maxChanges = ""
 772        self.isWindows = (platform.system() == "Windows")
 773        self.keepRepoPath = False
 774        self.depotPaths = None
 775        self.p4BranchesInGit = []
 776        self.cloneExclude = []
 777        self.useClientSpec = False
 778        self.clientSpecDirs = []
 779
 780        if gitConfig("git-p4.syncFromOrigin") == "false":
 781            self.syncWithOrigin = False
 782
 783    def extractFilesFromCommit(self, commit):
 784        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 785                             for path in self.cloneExclude]
 786        files = []
 787        fnum = 0
 788        while commit.has_key("depotFile%s" % fnum):
 789            path =  commit["depotFile%s" % fnum]
 790
 791            if [p for p in self.cloneExclude
 792                if path.startswith (p)]:
 793                found = False
 794            else:
 795                found = [p for p in self.depotPaths
 796                         if path.startswith (p)]
 797            if not found:
 798                fnum = fnum + 1
 799                continue
 800
 801            file = {}
 802            file["path"] = path
 803            file["rev"] = commit["rev%s" % fnum]
 804            file["action"] = commit["action%s" % fnum]
 805            file["type"] = commit["type%s" % fnum]
 806            files.append(file)
 807            fnum = fnum + 1
 808        return files
 809
 810    def stripRepoPath(self, path, prefixes):
 811        if self.keepRepoPath:
 812            prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 813
 814        for p in prefixes:
 815            if path.startswith(p):
 816                path = path[len(p):]
 817
 818        return path
 819
 820    def splitFilesIntoBranches(self, commit):
 821        branches = {}
 822        fnum = 0
 823        while commit.has_key("depotFile%s" % fnum):
 824            path =  commit["depotFile%s" % fnum]
 825            found = [p for p in self.depotPaths
 826                     if path.startswith (p)]
 827            if not found:
 828                fnum = fnum + 1
 829                continue
 830
 831            file = {}
 832            file["path"] = path
 833            file["rev"] = commit["rev%s" % fnum]
 834            file["action"] = commit["action%s" % fnum]
 835            file["type"] = commit["type%s" % fnum]
 836            fnum = fnum + 1
 837
 838            relPath = self.stripRepoPath(path, self.depotPaths)
 839
 840            for branch in self.knownBranches.keys():
 841
 842                # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 843                if relPath.startswith(branch + "/"):
 844                    if branch not in branches:
 845                        branches[branch] = []
 846                    branches[branch].append(file)
 847                    break
 848
 849        return branches
 850
 851    ## Should move this out, doesn't use SELF.
 852    def readP4Files(self, files):
 853        filesForCommit = []
 854        filesToRead = []
 855
 856        for f in files:
 857            includeFile = True
 858            for val in self.clientSpecDirs:
 859                if f['path'].startswith(val[0]):
 860                    if val[1] <= 0:
 861                        includeFile = False
 862                    break
 863
 864            if includeFile:
 865                filesForCommit.append(f)
 866                if f['action'] != 'delete':
 867                    filesToRead.append(f)
 868
 869        filedata = []
 870        if len(filesToRead) > 0:
 871            filedata = p4CmdList('-x - print',
 872                                 stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 873                                                  for f in filesToRead]),
 874                                 stdin_mode='w+')
 875
 876            if "p4ExitCode" in filedata[0]:
 877                die("Problems executing p4. Error: [%d]."
 878                    % (filedata[0]['p4ExitCode']));
 879
 880        j = 0;
 881        contents = {}
 882        while j < len(filedata):
 883            stat = filedata[j]
 884            j += 1
 885            text = ''
 886            while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 887                tmp = filedata[j]['data']
 888                if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 889                    tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 890                elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 891                    tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 892                text += tmp
 893                j += 1
 894
 895
 896            if not stat.has_key('depotFile'):
 897                sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 898                continue
 899
 900            contents[stat['depotFile']] = text
 901
 902        for f in filesForCommit:
 903            path = f['path']
 904            if contents.has_key(path):
 905                f['data'] = contents[path]
 906
 907        return filesForCommit
 908
 909    def commit(self, details, files, branch, branchPrefixes, parent = ""):
 910        epoch = details["time"]
 911        author = details["user"]
 912
 913        if self.verbose:
 914            print "commit into %s" % branch
 915
 916        # start with reading files; if that fails, we should not
 917        # create a commit.
 918        new_files = []
 919        for f in files:
 920            if [p for p in branchPrefixes if f['path'].startswith(p)]:
 921                new_files.append (f)
 922            else:
 923                sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 924        files = self.readP4Files(new_files)
 925
 926        self.gitStream.write("commit %s\n" % branch)
 927#        gitStream.write("mark :%s\n" % details["change"])
 928        self.committedChanges.add(int(details["change"]))
 929        committer = ""
 930        if author not in self.users:
 931            self.getUserMapFromPerforceServer()
 932        if author in self.users:
 933            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 934        else:
 935            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 936
 937        self.gitStream.write("committer %s\n" % committer)
 938
 939        self.gitStream.write("data <<EOT\n")
 940        self.gitStream.write(details["desc"])
 941        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 942                             % (','.join (branchPrefixes), details["change"]))
 943        if len(details['options']) > 0:
 944            self.gitStream.write(": options = %s" % details['options'])
 945        self.gitStream.write("]\nEOT\n\n")
 946
 947        if len(parent) > 0:
 948            if self.verbose:
 949                print "parent %s" % parent
 950            self.gitStream.write("from %s\n" % parent)
 951
 952        for file in files:
 953            if file["type"] == "apple":
 954                print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 955                continue
 956
 957            relPath = self.stripRepoPath(file['path'], branchPrefixes)
 958            if file["action"] == "delete":
 959                self.gitStream.write("D %s\n" % relPath)
 960            else:
 961                data = file['data']
 962
 963                mode = "644"
 964                if isP4Exec(file["type"]):
 965                    mode = "755"
 966                elif file["type"] == "symlink":
 967                    mode = "120000"
 968                    # p4 print on a symlink contains "target\n", so strip it off
 969                    data = data[:-1]
 970
 971                if self.isWindows and file["type"].endswith("text"):
 972                    data = data.replace("\r\n", "\n")
 973
 974                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 975                self.gitStream.write("data %s\n" % len(data))
 976                self.gitStream.write(data)
 977                self.gitStream.write("\n")
 978
 979        self.gitStream.write("\n")
 980
 981        change = int(details["change"])
 982
 983        if self.labels.has_key(change):
 984            label = self.labels[change]
 985            labelDetails = label[0]
 986            labelRevisions = label[1]
 987            if self.verbose:
 988                print "Change %s is labelled %s" % (change, labelDetails)
 989
 990            files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 991                                                    for p in branchPrefixes]))
 992
 993            if len(files) == len(labelRevisions):
 994
 995                cleanedFiles = {}
 996                for info in files:
 997                    if info["action"] == "delete":
 998                        continue
 999                    cleanedFiles[info["depotFile"]] = info["rev"]
1000
1001                if cleanedFiles == labelRevisions:
1002                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1003                    self.gitStream.write("from %s\n" % branch)
1004
1005                    owner = labelDetails["Owner"]
1006                    tagger = ""
1007                    if author in self.users:
1008                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1009                    else:
1010                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1011                    self.gitStream.write("tagger %s\n" % tagger)
1012                    self.gitStream.write("data <<EOT\n")
1013                    self.gitStream.write(labelDetails["Description"])
1014                    self.gitStream.write("EOT\n\n")
1015
1016                else:
1017                    if not self.silent:
1018                        print ("Tag %s does not match with change %s: files do not match."
1019                               % (labelDetails["label"], change))
1020
1021            else:
1022                if not self.silent:
1023                    print ("Tag %s does not match with change %s: file count is different."
1024                           % (labelDetails["label"], change))
1025
1026    def getUserCacheFilename(self):
1027        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1028        return home + "/.gitp4-usercache.txt"
1029
1030    def getUserMapFromPerforceServer(self):
1031        if self.userMapFromPerforceServer:
1032            return
1033        self.users = {}
1034
1035        for output in p4CmdList("users"):
1036            if not output.has_key("User"):
1037                continue
1038            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1039
1040
1041        s = ''
1042        for (key, val) in self.users.items():
1043            s += "%s\t%s\n" % (key, val)
1044
1045        open(self.getUserCacheFilename(), "wb").write(s)
1046        self.userMapFromPerforceServer = True
1047
1048    def loadUserMapFromCache(self):
1049        self.users = {}
1050        self.userMapFromPerforceServer = False
1051        try:
1052            cache = open(self.getUserCacheFilename(), "rb")
1053            lines = cache.readlines()
1054            cache.close()
1055            for line in lines:
1056                entry = line.strip().split("\t")
1057                self.users[entry[0]] = entry[1]
1058        except IOError:
1059            self.getUserMapFromPerforceServer()
1060
1061    def getLabels(self):
1062        self.labels = {}
1063
1064        l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1065        if len(l) > 0 and not self.silent:
1066            print "Finding files belonging to labels in %s" % `self.depotPaths`
1067
1068        for output in l:
1069            label = output["label"]
1070            revisions = {}
1071            newestChange = 0
1072            if self.verbose:
1073                print "Querying files for label %s" % label
1074            for file in p4CmdList("files "
1075                                  +  ' '.join (["%s...@%s" % (p, label)
1076                                                for p in self.depotPaths])):
1077                revisions[file["depotFile"]] = file["rev"]
1078                change = int(file["change"])
1079                if change > newestChange:
1080                    newestChange = change
1081
1082            self.labels[newestChange] = [output, revisions]
1083
1084        if self.verbose:
1085            print "Label changes: %s" % self.labels.keys()
1086
1087    def guessProjectName(self):
1088        for p in self.depotPaths:
1089            if p.endswith("/"):
1090                p = p[:-1]
1091            p = p[p.strip().rfind("/") + 1:]
1092            if not p.endswith("/"):
1093               p += "/"
1094            return p
1095
1096    def getBranchMapping(self):
1097        lostAndFoundBranches = set()
1098
1099        for info in p4CmdList("branches"):
1100            details = p4Cmd("branch -o %s" % info["branch"])
1101            viewIdx = 0
1102            while details.has_key("View%s" % viewIdx):
1103                paths = details["View%s" % viewIdx].split(" ")
1104                viewIdx = viewIdx + 1
1105                # require standard //depot/foo/... //depot/bar/... mapping
1106                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1107                    continue
1108                source = paths[0]
1109                destination = paths[1]
1110                ## HACK
1111                if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1112                    source = source[len(self.depotPaths[0]):-4]
1113                    destination = destination[len(self.depotPaths[0]):-4]
1114
1115                    if destination in self.knownBranches:
1116                        if not self.silent:
1117                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1118                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1119                        continue
1120
1121                    self.knownBranches[destination] = source
1122
1123                    lostAndFoundBranches.discard(destination)
1124
1125                    if source not in self.knownBranches:
1126                        lostAndFoundBranches.add(source)
1127
1128
1129        for branch in lostAndFoundBranches:
1130            self.knownBranches[branch] = branch
1131
1132    def getBranchMappingFromGitBranches(self):
1133        branches = p4BranchesInGit(self.importIntoRemotes)
1134        for branch in branches.keys():
1135            if branch == "master":
1136                branch = "main"
1137            else:
1138                branch = branch[len(self.projectName):]
1139            self.knownBranches[branch] = branch
1140
1141    def listExistingP4GitBranches(self):
1142        # branches holds mapping from name to commit
1143        branches = p4BranchesInGit(self.importIntoRemotes)
1144        self.p4BranchesInGit = branches.keys()
1145        for branch in branches.keys():
1146            self.initialParents[self.refPrefix + branch] = branches[branch]
1147
1148    def updateOptionDict(self, d):
1149        option_keys = {}
1150        if self.keepRepoPath:
1151            option_keys['keepRepoPath'] = 1
1152
1153        d["options"] = ' '.join(sorted(option_keys.keys()))
1154
1155    def readOptions(self, d):
1156        self.keepRepoPath = (d.has_key('options')
1157                             and ('keepRepoPath' in d['options']))
1158
1159    def gitRefForBranch(self, branch):
1160        if branch == "main":
1161            return self.refPrefix + "master"
1162
1163        if len(branch) <= 0:
1164            return branch
1165
1166        return self.refPrefix + self.projectName + branch
1167
1168    def gitCommitByP4Change(self, ref, change):
1169        if self.verbose:
1170            print "looking in ref " + ref + " for change %s using bisect..." % change
1171
1172        earliestCommit = ""
1173        latestCommit = parseRevision(ref)
1174
1175        while True:
1176            if self.verbose:
1177                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1178            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1179            if len(next) == 0:
1180                if self.verbose:
1181                    print "argh"
1182                return ""
1183            log = extractLogMessageFromGitCommit(next)
1184            settings = extractSettingsGitLog(log)
1185            currentChange = int(settings['change'])
1186            if self.verbose:
1187                print "current change %s" % currentChange
1188
1189            if currentChange == change:
1190                if self.verbose:
1191                    print "found %s" % next
1192                return next
1193
1194            if currentChange < change:
1195                earliestCommit = "^%s" % next
1196            else:
1197                latestCommit = "%s" % next
1198
1199        return ""
1200
1201    def importNewBranch(self, branch, maxChange):
1202        # make fast-import flush all changes to disk and update the refs using the checkpoint
1203        # command so that we can try to find the branch parent in the git history
1204        self.gitStream.write("checkpoint\n\n");
1205        self.gitStream.flush();
1206        branchPrefix = self.depotPaths[0] + branch + "/"
1207        range = "@1,%s" % maxChange
1208        #print "prefix" + branchPrefix
1209        changes = p4ChangesForPaths([branchPrefix], range)
1210        if len(changes) <= 0:
1211            return False
1212        firstChange = changes[0]
1213        #print "first change in branch: %s" % firstChange
1214        sourceBranch = self.knownBranches[branch]
1215        sourceDepotPath = self.depotPaths[0] + sourceBranch
1216        sourceRef = self.gitRefForBranch(sourceBranch)
1217        #print "source " + sourceBranch
1218
1219        branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1220        #print "branch parent: %s" % branchParentChange
1221        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1222        if len(gitParent) > 0:
1223            self.initialParents[self.gitRefForBranch(branch)] = gitParent
1224            #print "parent git commit: %s" % gitParent
1225
1226        self.importChanges(changes)
1227        return True
1228
1229    def importChanges(self, changes):
1230        cnt = 1
1231        for change in changes:
1232            description = p4Cmd("describe %s" % change)
1233            self.updateOptionDict(description)
1234
1235            if not self.silent:
1236                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1237                sys.stdout.flush()
1238            cnt = cnt + 1
1239
1240            try:
1241                if self.detectBranches:
1242                    branches = self.splitFilesIntoBranches(description)
1243                    for branch in branches.keys():
1244                        ## HACK  --hwn
1245                        branchPrefix = self.depotPaths[0] + branch + "/"
1246
1247                        parent = ""
1248
1249                        filesForCommit = branches[branch]
1250
1251                        if self.verbose:
1252                            print "branch is %s" % branch
1253
1254                        self.updatedBranches.add(branch)
1255
1256                        if branch not in self.createdBranches:
1257                            self.createdBranches.add(branch)
1258                            parent = self.knownBranches[branch]
1259                            if parent == branch:
1260                                parent = ""
1261                            else:
1262                                fullBranch = self.projectName + branch
1263                                if fullBranch not in self.p4BranchesInGit:
1264                                    if not self.silent:
1265                                        print("\n    Importing new branch %s" % fullBranch);
1266                                    if self.importNewBranch(branch, change - 1):
1267                                        parent = ""
1268                                        self.p4BranchesInGit.append(fullBranch)
1269                                    if not self.silent:
1270                                        print("\n    Resuming with change %s" % change);
1271
1272                                if self.verbose:
1273                                    print "parent determined through known branches: %s" % parent
1274
1275                        branch = self.gitRefForBranch(branch)
1276                        parent = self.gitRefForBranch(parent)
1277
1278                        if self.verbose:
1279                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1280
1281                        if len(parent) == 0 and branch in self.initialParents:
1282                            parent = self.initialParents[branch]
1283                            del self.initialParents[branch]
1284
1285                        self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1286                else:
1287                    files = self.extractFilesFromCommit(description)
1288                    self.commit(description, files, self.branch, self.depotPaths,
1289                                self.initialParent)
1290                    self.initialParent = ""
1291            except IOError:
1292                print self.gitError.read()
1293                sys.exit(1)
1294
1295    def importHeadRevision(self, revision):
1296        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1297
1298        details = { "user" : "git perforce import user", "time" : int(time.time()) }
1299        details["desc"] = ("Initial import of %s from the state at revision %s"
1300                           % (' '.join(self.depotPaths), revision))
1301        details["change"] = revision
1302        newestRevision = 0
1303
1304        fileCnt = 0
1305        for info in p4CmdList("files "
1306                              +  ' '.join(["%s...%s"
1307                                           % (p, revision)
1308                                           for p in self.depotPaths])):
1309
1310            if info['code'] == 'error':
1311                sys.stderr.write("p4 returned an error: %s\n"
1312                                 % info['data'])
1313                sys.exit(1)
1314
1315
1316            change = int(info["change"])
1317            if change > newestRevision:
1318                newestRevision = change
1319
1320            if info["action"] == "delete":
1321                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1322                #fileCnt = fileCnt + 1
1323                continue
1324
1325            for prop in ["depotFile", "rev", "action", "type" ]:
1326                details["%s%s" % (prop, fileCnt)] = info[prop]
1327
1328            fileCnt = fileCnt + 1
1329
1330        details["change"] = newestRevision
1331        self.updateOptionDict(details)
1332        try:
1333            self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1334        except IOError:
1335            print "IO error with git fast-import. Is your git version recent enough?"
1336            print self.gitError.read()
1337
1338
1339    def getClientSpec(self):
1340        specList = p4CmdList( "client -o" )
1341        temp = {}
1342        for entry in specList:
1343            for k,v in entry.iteritems():
1344                if k.startswith("View"):
1345                    if v.startswith('"'):
1346                        start = 1
1347                    else:
1348                        start = 0
1349                    index = v.find("...")
1350                    v = v[start:index]
1351                    if v.startswith("-"):
1352                        v = v[1:]
1353                        temp[v] = -len(v)
1354                    else:
1355                        temp[v] = len(v)
1356        self.clientSpecDirs = temp.items()
1357        self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1358
1359    def run(self, args):
1360        self.depotPaths = []
1361        self.changeRange = ""
1362        self.initialParent = ""
1363        self.previousDepotPaths = []
1364
1365        # map from branch depot path to parent branch
1366        self.knownBranches = {}
1367        self.initialParents = {}
1368        self.hasOrigin = originP4BranchesExist()
1369        if not self.syncWithOrigin:
1370            self.hasOrigin = False
1371
1372        if self.importIntoRemotes:
1373            self.refPrefix = "refs/remotes/p4/"
1374        else:
1375            self.refPrefix = "refs/heads/p4/"
1376
1377        if self.syncWithOrigin and self.hasOrigin:
1378            if not self.silent:
1379                print "Syncing with origin first by calling git fetch origin"
1380            system("git fetch origin")
1381
1382        if len(self.branch) == 0:
1383            self.branch = self.refPrefix + "master"
1384            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1385                system("git update-ref %s refs/heads/p4" % self.branch)
1386                system("git branch -D p4");
1387            # create it /after/ importing, when master exists
1388            if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1389                system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1390
1391        if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1392            self.getClientSpec()
1393
1394        # TODO: should always look at previous commits,
1395        # merge with previous imports, if possible.
1396        if args == []:
1397            if self.hasOrigin:
1398                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1399            self.listExistingP4GitBranches()
1400
1401            if len(self.p4BranchesInGit) > 1:
1402                if not self.silent:
1403                    print "Importing from/into multiple branches"
1404                self.detectBranches = True
1405
1406            if self.verbose:
1407                print "branches: %s" % self.p4BranchesInGit
1408
1409            p4Change = 0
1410            for branch in self.p4BranchesInGit:
1411                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1412
1413                settings = extractSettingsGitLog(logMsg)
1414
1415                self.readOptions(settings)
1416                if (settings.has_key('depot-paths')
1417                    and settings.has_key ('change')):
1418                    change = int(settings['change']) + 1
1419                    p4Change = max(p4Change, change)
1420
1421                    depotPaths = sorted(settings['depot-paths'])
1422                    if self.previousDepotPaths == []:
1423                        self.previousDepotPaths = depotPaths
1424                    else:
1425                        paths = []
1426                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1427                            for i in range(0, min(len(cur), len(prev))):
1428                                if cur[i] <> prev[i]:
1429                                    i = i - 1
1430                                    break
1431
1432                            paths.append (cur[:i + 1])
1433
1434                        self.previousDepotPaths = paths
1435
1436            if p4Change > 0:
1437                self.depotPaths = sorted(self.previousDepotPaths)
1438                self.changeRange = "@%s,#head" % p4Change
1439                if not self.detectBranches:
1440                    self.initialParent = parseRevision(self.branch)
1441                if not self.silent and not self.detectBranches:
1442                    print "Performing incremental import into %s git branch" % self.branch
1443
1444        if not self.branch.startswith("refs/"):
1445            self.branch = "refs/heads/" + self.branch
1446
1447        if len(args) == 0 and self.depotPaths:
1448            if not self.silent:
1449                print "Depot paths: %s" % ' '.join(self.depotPaths)
1450        else:
1451            if self.depotPaths and self.depotPaths != args:
1452                print ("previous import used depot path %s and now %s was specified. "
1453                       "This doesn't work!" % (' '.join (self.depotPaths),
1454                                               ' '.join (args)))
1455                sys.exit(1)
1456
1457            self.depotPaths = sorted(args)
1458
1459        revision = ""
1460        self.users = {}
1461
1462        newPaths = []
1463        for p in self.depotPaths:
1464            if p.find("@") != -1:
1465                atIdx = p.index("@")
1466                self.changeRange = p[atIdx:]
1467                if self.changeRange == "@all":
1468                    self.changeRange = ""
1469                elif ',' not in self.changeRange:
1470                    revision = self.changeRange
1471                    self.changeRange = ""
1472                p = p[:atIdx]
1473            elif p.find("#") != -1:
1474                hashIdx = p.index("#")
1475                revision = p[hashIdx:]
1476                p = p[:hashIdx]
1477            elif self.previousDepotPaths == []:
1478                revision = "#head"
1479
1480            p = re.sub ("\.\.\.$", "", p)
1481            if not p.endswith("/"):
1482                p += "/"
1483
1484            newPaths.append(p)
1485
1486        self.depotPaths = newPaths
1487
1488
1489        self.loadUserMapFromCache()
1490        self.labels = {}
1491        if self.detectLabels:
1492            self.getLabels();
1493
1494        if self.detectBranches:
1495            ## FIXME - what's a P4 projectName ?
1496            self.projectName = self.guessProjectName()
1497
1498            if self.hasOrigin:
1499                self.getBranchMappingFromGitBranches()
1500            else:
1501                self.getBranchMapping()
1502            if self.verbose:
1503                print "p4-git branches: %s" % self.p4BranchesInGit
1504                print "initial parents: %s" % self.initialParents
1505            for b in self.p4BranchesInGit:
1506                if b != "master":
1507
1508                    ## FIXME
1509                    b = b[len(self.projectName):]
1510                self.createdBranches.add(b)
1511
1512        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1513
1514        importProcess = subprocess.Popen(["git", "fast-import"],
1515                                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1516                                         stderr=subprocess.PIPE);
1517        self.gitOutput = importProcess.stdout
1518        self.gitStream = importProcess.stdin
1519        self.gitError = importProcess.stderr
1520
1521        if revision:
1522            self.importHeadRevision(revision)
1523        else:
1524            changes = []
1525
1526            if len(self.changesFile) > 0:
1527                output = open(self.changesFile).readlines()
1528                changeSet = Set()
1529                for line in output:
1530                    changeSet.add(int(line))
1531
1532                for change in changeSet:
1533                    changes.append(change)
1534
1535                changes.sort()
1536            else:
1537                if self.verbose:
1538                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1539                                                              self.changeRange)
1540                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1541
1542                if len(self.maxChanges) > 0:
1543                    changes = changes[:min(int(self.maxChanges), len(changes))]
1544
1545            if len(changes) == 0:
1546                if not self.silent:
1547                    print "No changes to import!"
1548                return True
1549
1550            if not self.silent and not self.detectBranches:
1551                print "Import destination: %s" % self.branch
1552
1553            self.updatedBranches = set()
1554
1555            self.importChanges(changes)
1556
1557            if not self.silent:
1558                print ""
1559                if len(self.updatedBranches) > 0:
1560                    sys.stdout.write("Updated branches: ")
1561                    for b in self.updatedBranches:
1562                        sys.stdout.write("%s " % b)
1563                    sys.stdout.write("\n")
1564
1565        self.gitStream.close()
1566        if importProcess.wait() != 0:
1567            die("fast-import failed: %s" % self.gitError.read())
1568        self.gitOutput.close()
1569        self.gitError.close()
1570
1571        return True
1572
1573class P4Rebase(Command):
1574    def __init__(self):
1575        Command.__init__(self)
1576        self.options = [ ]
1577        self.description = ("Fetches the latest revision from perforce and "
1578                            + "rebases the current work (branch) against it")
1579        self.verbose = False
1580
1581    def run(self, args):
1582        sync = P4Sync()
1583        sync.run([])
1584
1585        return self.rebase()
1586
1587    def rebase(self):
1588        if os.system("git update-index --refresh") != 0:
1589            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1590        if len(read_pipe("git diff-index HEAD --")) > 0:
1591            die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1592
1593        [upstream, settings] = findUpstreamBranchPoint()
1594        if len(upstream) == 0:
1595            die("Cannot find upstream branchpoint for rebase")
1596
1597        # the branchpoint may be p4/foo~3, so strip off the parent
1598        upstream = re.sub("~[0-9]+$", "", upstream)
1599
1600        print "Rebasing the current branch onto %s" % upstream
1601        oldHead = read_pipe("git rev-parse HEAD").strip()
1602        system("git rebase %s" % upstream)
1603        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1604        return True
1605
1606class P4Clone(P4Sync):
1607    def __init__(self):
1608        P4Sync.__init__(self)
1609        self.description = "Creates a new git repository and imports from Perforce into it"
1610        self.usage = "usage: %prog [options] //depot/path[@revRange]"
1611        self.options += [
1612            optparse.make_option("--destination", dest="cloneDestination",
1613                                 action='store', default=None,
1614                                 help="where to leave result of the clone"),
1615            optparse.make_option("-/", dest="cloneExclude",
1616                                 action="append", type="string",
1617                                 help="exclude depot path")
1618        ]
1619        self.cloneDestination = None
1620        self.needsGit = False
1621
1622    # This is required for the "append" cloneExclude action
1623    def ensure_value(self, attr, value):
1624        if not hasattr(self, attr) or getattr(self, attr) is None:
1625            setattr(self, attr, value)
1626        return getattr(self, attr)
1627
1628    def defaultDestination(self, args):
1629        ## TODO: use common prefix of args?
1630        depotPath = args[0]
1631        depotDir = re.sub("(@[^@]*)$", "", depotPath)
1632        depotDir = re.sub("(#[^#]*)$", "", depotDir)
1633        depotDir = re.sub(r"\.\.\.$", "", depotDir)
1634        depotDir = re.sub(r"/$", "", depotDir)
1635        return os.path.split(depotDir)[1]
1636
1637    def run(self, args):
1638        if len(args) < 1:
1639            return False
1640
1641        if self.keepRepoPath and not self.cloneDestination:
1642            sys.stderr.write("Must specify destination for --keep-path\n")
1643            sys.exit(1)
1644
1645        depotPaths = args
1646
1647        if not self.cloneDestination and len(depotPaths) > 1:
1648            self.cloneDestination = depotPaths[-1]
1649            depotPaths = depotPaths[:-1]
1650
1651        self.cloneExclude = ["/"+p for p in self.cloneExclude]
1652        for p in depotPaths:
1653            if not p.startswith("//"):
1654                return False
1655
1656        if not self.cloneDestination:
1657            self.cloneDestination = self.defaultDestination(args)
1658
1659        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1660        if not os.path.exists(self.cloneDestination):
1661            os.makedirs(self.cloneDestination)
1662        os.chdir(self.cloneDestination)
1663        system("git init")
1664        self.gitdir = os.getcwd() + "/.git"
1665        if not P4Sync.run(self, depotPaths):
1666            return False
1667        if self.branch != "master":
1668            if gitBranchExists("refs/remotes/p4/master"):
1669                system("git branch master refs/remotes/p4/master")
1670                system("git checkout -f")
1671            else:
1672                print "Could not detect main branch. No checkout/master branch created."
1673
1674        return True
1675
1676class P4Branches(Command):
1677    def __init__(self):
1678        Command.__init__(self)
1679        self.options = [ ]
1680        self.description = ("Shows the git branches that hold imports and their "
1681                            + "corresponding perforce depot paths")
1682        self.verbose = False
1683
1684    def run(self, args):
1685        if originP4BranchesExist():
1686            createOrUpdateBranchesFromOrigin()
1687
1688        cmdline = "git rev-parse --symbolic "
1689        cmdline += " --remotes"
1690
1691        for line in read_pipe_lines(cmdline):
1692            line = line.strip()
1693
1694            if not line.startswith('p4/') or line == "p4/HEAD":
1695                continue
1696            branch = line
1697
1698            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1699            settings = extractSettingsGitLog(log)
1700
1701            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1702        return True
1703
1704class HelpFormatter(optparse.IndentedHelpFormatter):
1705    def __init__(self):
1706        optparse.IndentedHelpFormatter.__init__(self)
1707
1708    def format_description(self, description):
1709        if description:
1710            return description + "\n"
1711        else:
1712            return ""
1713
1714def printUsage(commands):
1715    print "usage: %s <command> [options]" % sys.argv[0]
1716    print ""
1717    print "valid commands: %s" % ", ".join(commands)
1718    print ""
1719    print "Try %s <command> --help for command specific help." % sys.argv[0]
1720    print ""
1721
1722commands = {
1723    "debug" : P4Debug,
1724    "submit" : P4Submit,
1725    "commit" : P4Submit,
1726    "sync" : P4Sync,
1727    "rebase" : P4Rebase,
1728    "clone" : P4Clone,
1729    "rollback" : P4RollBack,
1730    "branches" : P4Branches
1731}
1732
1733
1734def main():
1735    if len(sys.argv[1:]) == 0:
1736        printUsage(commands.keys())
1737        sys.exit(2)
1738
1739    cmd = ""
1740    cmdName = sys.argv[1]
1741    try:
1742        klass = commands[cmdName]
1743        cmd = klass()
1744    except KeyError:
1745        print "unknown command %s" % cmdName
1746        print ""
1747        printUsage(commands.keys())
1748        sys.exit(2)
1749
1750    options = cmd.options
1751    cmd.gitdir = os.environ.get("GIT_DIR", None)
1752
1753    args = sys.argv[2:]
1754
1755    if len(options) > 0:
1756        options.append(optparse.make_option("--git-dir", dest="gitdir"))
1757
1758        parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1759                                       options,
1760                                       description = cmd.description,
1761                                       formatter = HelpFormatter())
1762
1763        (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1764    global verbose
1765    verbose = cmd.verbose
1766    if cmd.needsGit:
1767        if cmd.gitdir == None:
1768            cmd.gitdir = os.path.abspath(".git")
1769            if not isValidGitDir(cmd.gitdir):
1770                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1771                if os.path.exists(cmd.gitdir):
1772                    cdup = read_pipe("git rev-parse --show-cdup").strip()
1773                    if len(cdup) > 0:
1774                        os.chdir(cdup);
1775
1776        if not isValidGitDir(cmd.gitdir):
1777            if isValidGitDir(cmd.gitdir + "/.git"):
1778                cmd.gitdir += "/.git"
1779            else:
1780                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1781
1782        os.environ["GIT_DIR"] = cmd.gitdir
1783
1784    if not cmd.run(args):
1785        parser.print_help()
1786
1787
1788if __name__ == '__main__':
1789    main()