contrib / fast-import / git-p4on commit Clean up code duplication for revision parsing and fix previous commit to not (463e8af)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <hausmann@kde.org>
   6# Copyright: 2007 Simon Hausmann <hausmann@kde.org>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10# TODO: Add an option to sync/rebase to fetch and rebase from origin first.
  11#
  12
  13import optparse, sys, os, marshal, popen2, subprocess, shelve
  14import tempfile, getopt, sha, os.path, time, platform
  15from sets import Set;
  16
  17gitdir = os.environ.get("GIT_DIR", "")
  18
  19def mypopen(command):
  20    return os.popen(command, "rb");
  21
  22def p4CmdList(cmd):
  23    cmd = "p4 -G %s" % cmd
  24    pipe = os.popen(cmd, "rb")
  25
  26    result = []
  27    try:
  28        while True:
  29            entry = marshal.load(pipe)
  30            result.append(entry)
  31    except EOFError:
  32        pass
  33    pipe.close()
  34
  35    return result
  36
  37def p4Cmd(cmd):
  38    list = p4CmdList(cmd)
  39    result = {}
  40    for entry in list:
  41        result.update(entry)
  42    return result;
  43
  44def p4Where(depotPath):
  45    if not depotPath.endswith("/"):
  46        depotPath += "/"
  47    output = p4Cmd("where %s..." % depotPath)
  48    clientPath = ""
  49    if "path" in output:
  50        clientPath = output.get("path")
  51    elif "data" in output:
  52        data = output.get("data")
  53        lastSpace = data.rfind(" ")
  54        clientPath = data[lastSpace + 1:]
  55
  56    if clientPath.endswith("..."):
  57        clientPath = clientPath[:-3]
  58    return clientPath
  59
  60def die(msg):
  61    sys.stderr.write(msg + "\n")
  62    sys.exit(1)
  63
  64def currentGitBranch():
  65    return mypopen("git name-rev HEAD").read().split(" ")[1][:-1]
  66
  67def isValidGitDir(path):
  68    if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
  69        return True;
  70    return False
  71
  72def parseRevision(ref):
  73    return mypopen("git rev-parse %s" % ref).read()[:-1]
  74
  75def system(cmd):
  76    if os.system(cmd) != 0:
  77        die("command failed: %s" % cmd)
  78
  79def extractLogMessageFromGitCommit(commit):
  80    logMessage = ""
  81    foundTitle = False
  82    for log in mypopen("git cat-file commit %s" % commit).readlines():
  83       if not foundTitle:
  84           if len(log) == 1:
  85               foundTitle = True
  86           continue
  87
  88       logMessage += log
  89    return logMessage
  90
  91def extractDepotPathAndChangeFromGitLog(log):
  92    values = {}
  93    for line in log.split("\n"):
  94        line = line.strip()
  95        if line.startswith("[git-p4:") and line.endswith("]"):
  96            line = line[8:-1].strip()
  97            for assignment in line.split(":"):
  98                variable = assignment.strip()
  99                value = ""
 100                equalPos = assignment.find("=")
 101                if equalPos != -1:
 102                    variable = assignment[:equalPos].strip()
 103                    value = assignment[equalPos + 1:].strip()
 104                    if value.startswith("\"") and value.endswith("\""):
 105                        value = value[1:-1]
 106                values[variable] = value
 107
 108    return values.get("depot-path"), values.get("change")
 109
 110def gitBranchExists(branch):
 111    proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 112    return proc.wait() == 0;
 113
 114class Command:
 115    def __init__(self):
 116        self.usage = "usage: %prog [options]"
 117        self.needsGit = True
 118
 119class P4Debug(Command):
 120    def __init__(self):
 121        Command.__init__(self)
 122        self.options = [
 123        ]
 124        self.description = "A tool to debug the output of p4 -G."
 125        self.needsGit = False
 126
 127    def run(self, args):
 128        for output in p4CmdList(" ".join(args)):
 129            print output
 130        return True
 131
 132class P4CleanTags(Command):
 133    def __init__(self):
 134        Command.__init__(self)
 135        self.options = [
 136#                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
 137        ]
 138        self.description = "A tool to remove stale unused tags from incremental perforce imports."
 139    def run(self, args):
 140        branch = currentGitBranch()
 141        print "Cleaning out stale p4 import tags..."
 142        sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch)
 143        output = sout.read()
 144        try:
 145            tagIdx = output.index(" tags/p4/")
 146        except:
 147            print "Cannot find any p4/* tag. Nothing to do."
 148            sys.exit(0)
 149
 150        try:
 151            caretIdx = output.index("^")
 152        except:
 153            caretIdx = len(output) - 1
 154        rev = int(output[tagIdx + 9 : caretIdx])
 155
 156        allTags = mypopen("git tag -l p4/").readlines()
 157        for i in range(len(allTags)):
 158            allTags[i] = int(allTags[i][3:-1])
 159
 160        allTags.sort()
 161
 162        allTags.remove(rev)
 163
 164        for rev in allTags:
 165            print mypopen("git tag -d p4/%s" % rev).read()
 166
 167        print "%s tags removed." % len(allTags)
 168        return True
 169
 170class P4Submit(Command):
 171    def __init__(self):
 172        Command.__init__(self)
 173        self.options = [
 174                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 175                optparse.make_option("--origin", dest="origin"),
 176                optparse.make_option("--reset", action="store_true", dest="reset"),
 177                optparse.make_option("--log-substitutions", dest="substFile"),
 178                optparse.make_option("--noninteractive", action="store_false"),
 179                optparse.make_option("--dry-run", action="store_true"),
 180        ]
 181        self.description = "Submit changes from git to the perforce depot."
 182        self.usage += " [name of git branch to submit into perforce depot]"
 183        self.firstTime = True
 184        self.reset = False
 185        self.interactive = True
 186        self.dryRun = False
 187        self.substFile = ""
 188        self.firstTime = True
 189        self.origin = ""
 190
 191        self.logSubstitutions = {}
 192        self.logSubstitutions["<enter description here>"] = "%log%"
 193        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 194
 195    def check(self):
 196        if len(p4CmdList("opened ...")) > 0:
 197            die("You have files opened with perforce! Close them before starting the sync.")
 198
 199    def start(self):
 200        if len(self.config) > 0 and not self.reset:
 201            die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile)
 202
 203        commits = []
 204        for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
 205            commits.append(line[:-1])
 206        commits.reverse()
 207
 208        self.config["commits"] = commits
 209
 210    def prepareLogMessage(self, template, message):
 211        result = ""
 212
 213        for line in template.split("\n"):
 214            if line.startswith("#"):
 215                result += line + "\n"
 216                continue
 217
 218            substituted = False
 219            for key in self.logSubstitutions.keys():
 220                if line.find(key) != -1:
 221                    value = self.logSubstitutions[key]
 222                    value = value.replace("%log%", message)
 223                    if value != "@remove@":
 224                        result += line.replace(key, value) + "\n"
 225                    substituted = True
 226                    break
 227
 228            if not substituted:
 229                result += line + "\n"
 230
 231        return result
 232
 233    def apply(self, id):
 234        print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read())
 235        diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
 236        filesToAdd = set()
 237        filesToDelete = set()
 238        editedFiles = set()
 239        for line in diff:
 240            modifier = line[0]
 241            path = line[1:].strip()
 242            if modifier == "M":
 243                system("p4 edit \"%s\"" % path)
 244                editedFiles.add(path)
 245            elif modifier == "A":
 246                filesToAdd.add(path)
 247                if path in filesToDelete:
 248                    filesToDelete.remove(path)
 249            elif modifier == "D":
 250                filesToDelete.add(path)
 251                if path in filesToAdd:
 252                    filesToAdd.remove(path)
 253            else:
 254                die("unknown modifier %s for %s" % (modifier, path))
 255
 256        diffcmd = "git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\"" % (id, id)
 257        patchcmd = diffcmd + " | patch -p1"
 258
 259        if os.system(patchcmd + " --dry-run --silent") != 0:
 260            print "Unfortunately applying the change failed!"
 261            print "What do you want to do?"
 262            response = "x"
 263            while response != "s" and response != "a" and response != "w":
 264                response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ")
 265            if response == "s":
 266                print "Skipping! Good luck with the next patches..."
 267                return
 268            elif response == "a":
 269                os.system(patchcmd)
 270                if len(filesToAdd) > 0:
 271                    print "You may also want to call p4 add on the following files:"
 272                    print " ".join(filesToAdd)
 273                if len(filesToDelete):
 274                    print "The following files should be scheduled for deletion with p4 delete:"
 275                    print " ".join(filesToDelete)
 276                die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 277            elif response == "w":
 278                system(diffcmd + " > patch.txt")
 279                print "Patch saved to patch.txt in %s !" % self.clientPath
 280                die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 281
 282        system(patchcmd)
 283
 284        for f in filesToAdd:
 285            system("p4 add %s" % f)
 286        for f in filesToDelete:
 287            system("p4 revert %s" % f)
 288            system("p4 delete %s" % f)
 289
 290        logMessage = extractLogMessageFromGitCommit(id)
 291        logMessage = logMessage.replace("\n", "\n\t")
 292        logMessage = logMessage[:-1]
 293
 294        template = mypopen("p4 change -o").read()
 295
 296        if self.interactive:
 297            submitTemplate = self.prepareLogMessage(template, logMessage)
 298            diff = mypopen("p4 diff -du ...").read()
 299
 300            for newFile in filesToAdd:
 301                diff += "==== new file ====\n"
 302                diff += "--- /dev/null\n"
 303                diff += "+++ %s\n" % newFile
 304                f = open(newFile, "r")
 305                for line in f.readlines():
 306                    diff += "+" + line
 307                f.close()
 308
 309            separatorLine = "######## everything below this line is just the diff #######"
 310            if platform.system() == "Windows":
 311                separatorLine += "\r"
 312            separatorLine += "\n"
 313
 314            response = "e"
 315            firstIteration = True
 316            while response == "e":
 317                if not firstIteration:
 318                    response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 319                firstIteration = False
 320                if response == "e":
 321                    [handle, fileName] = tempfile.mkstemp()
 322                    tmpFile = os.fdopen(handle, "w+")
 323                    tmpFile.write(submitTemplate + separatorLine + diff)
 324                    tmpFile.close()
 325                    defaultEditor = "vi"
 326                    if platform.system() == "Windows":
 327                        defaultEditor = "notepad"
 328                    editor = os.environ.get("EDITOR", defaultEditor);
 329                    system(editor + " " + fileName)
 330                    tmpFile = open(fileName, "rb")
 331                    message = tmpFile.read()
 332                    tmpFile.close()
 333                    os.remove(fileName)
 334                    submitTemplate = message[:message.index(separatorLine)]
 335
 336            if response == "y" or response == "yes":
 337               if self.dryRun:
 338                   print submitTemplate
 339                   raw_input("Press return to continue...")
 340               else:
 341                    pipe = os.popen("p4 submit -i", "wb")
 342                    pipe.write(submitTemplate)
 343                    pipe.close()
 344            elif response == "s":
 345                for f in editedFiles:
 346                    system("p4 revert \"%s\"" % f);
 347                for f in filesToAdd:
 348                    system("p4 revert \"%s\"" % f);
 349                    system("rm %s" %f)
 350                for f in filesToDelete:
 351                    system("p4 delete \"%s\"" % f);
 352                return
 353            else:
 354                print "Not submitting!"
 355                self.interactive = False
 356        else:
 357            fileName = "submit.txt"
 358            file = open(fileName, "w+")
 359            file.write(self.prepareLogMessage(template, logMessage))
 360            file.close()
 361            print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
 362
 363    def run(self, args):
 364        global gitdir
 365        # make gitdir absolute so we can cd out into the perforce checkout
 366        gitdir = os.path.abspath(gitdir)
 367        os.environ["GIT_DIR"] = gitdir
 368
 369        if len(args) == 0:
 370            self.master = currentGitBranch()
 371            if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
 372                die("Detecting current git branch failed!")
 373        elif len(args) == 1:
 374            self.master = args[0]
 375        else:
 376            return False
 377
 378        depotPath = ""
 379        if gitBranchExists("p4"):
 380            [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 381        if len(depotPath) == 0 and gitBranchExists("origin"):
 382            [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 383
 384        if len(depotPath) == 0:
 385            print "Internal error: cannot locate perforce depot path from existing branches"
 386            sys.exit(128)
 387
 388        self.clientPath = p4Where(depotPath)
 389
 390        if len(self.clientPath) == 0:
 391            print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
 392            sys.exit(128)
 393
 394        print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath)
 395        oldWorkingDirectory = os.getcwd()
 396        os.chdir(self.clientPath)
 397        response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath)
 398        if response == "y" or response == "yes":
 399            system("p4 sync ...")
 400
 401        if len(self.origin) == 0:
 402            if gitBranchExists("p4"):
 403                self.origin = "p4"
 404            else:
 405                self.origin = "origin"
 406
 407        if self.reset:
 408            self.firstTime = True
 409
 410        if len(self.substFile) > 0:
 411            for line in open(self.substFile, "r").readlines():
 412                tokens = line[:-1].split("=")
 413                self.logSubstitutions[tokens[0]] = tokens[1]
 414
 415        self.check()
 416        self.configFile = gitdir + "/p4-git-sync.cfg"
 417        self.config = shelve.open(self.configFile, writeback=True)
 418
 419        if self.firstTime:
 420            self.start()
 421
 422        commits = self.config.get("commits", [])
 423
 424        while len(commits) > 0:
 425            self.firstTime = False
 426            commit = commits[0]
 427            commits = commits[1:]
 428            self.config["commits"] = commits
 429            self.apply(commit)
 430            if not self.interactive:
 431                break
 432
 433        self.config.close()
 434
 435        if len(commits) == 0:
 436            if self.firstTime:
 437                print "No changes found to apply between %s and current HEAD" % self.origin
 438            else:
 439                print "All changes applied!"
 440                response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ")
 441                if response == "y" or response == "yes":
 442                    os.chdir(oldWorkingDirectory)
 443                    rebase = P4Rebase()
 444                    rebase.run([])
 445            os.remove(self.configFile)
 446
 447        return True
 448
 449class P4Sync(Command):
 450    def __init__(self):
 451        Command.__init__(self)
 452        self.options = [
 453                optparse.make_option("--branch", dest="branch"),
 454                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 455                optparse.make_option("--changesfile", dest="changesFile"),
 456                optparse.make_option("--silent", dest="silent", action="store_true"),
 457                optparse.make_option("--known-branches", dest="knownBranches"),
 458                optparse.make_option("--data-cache", dest="dataCache", action="store_true"),
 459                optparse.make_option("--command-cache", dest="commandCache", action="store_true"),
 460                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true")
 461        ]
 462        self.description = """Imports from Perforce into a git repository.\n
 463    example:
 464    //depot/my/project/ -- to import the current head
 465    //depot/my/project/@all -- to import everything
 466    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 467
 468    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 469
 470        self.usage += " //depot/path[@revRange]"
 471
 472        self.dataCache = False
 473        self.commandCache = False
 474        self.silent = False
 475        self.knownBranches = Set()
 476        self.createdBranches = Set()
 477        self.committedChanges = Set()
 478        self.branch = ""
 479        self.detectBranches = False
 480        self.detectLabels = False
 481        self.changesFile = ""
 482
 483    def p4File(self, depotPath):
 484        return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 485
 486    def extractFilesFromCommit(self, commit):
 487        files = []
 488        fnum = 0
 489        while commit.has_key("depotFile%s" % fnum):
 490            path =  commit["depotFile%s" % fnum]
 491            if not path.startswith(self.depotPath):
 492    #            if not self.silent:
 493    #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change)
 494                fnum = fnum + 1
 495                continue
 496
 497            file = {}
 498            file["path"] = path
 499            file["rev"] = commit["rev%s" % fnum]
 500            file["action"] = commit["action%s" % fnum]
 501            file["type"] = commit["type%s" % fnum]
 502            files.append(file)
 503            fnum = fnum + 1
 504        return files
 505
 506    def isSubPathOf(self, first, second):
 507        if not first.startswith(second):
 508            return False
 509        if first == second:
 510            return True
 511        return first[len(second)] == "/"
 512
 513    def branchesForCommit(self, files):
 514        branches = Set()
 515
 516        for file in files:
 517            relativePath = file["path"][len(self.depotPath):]
 518            # strip off the filename
 519            relativePath = relativePath[0:relativePath.rfind("/")]
 520
 521    #        if len(branches) == 0:
 522    #            branches.add(relativePath)
 523    #            knownBranches.add(relativePath)
 524    #            continue
 525
 526            ###### this needs more testing :)
 527            knownBranch = False
 528            for branch in branches:
 529                if relativePath == branch:
 530                    knownBranch = True
 531                    break
 532    #            if relativePath.startswith(branch):
 533                if self.isSubPathOf(relativePath, branch):
 534                    knownBranch = True
 535                    break
 536    #            if branch.startswith(relativePath):
 537                if self.isSubPathOf(branch, relativePath):
 538                    branches.remove(branch)
 539                    break
 540
 541            if knownBranch:
 542                continue
 543
 544            for branch in self.knownBranches:
 545                #if relativePath.startswith(branch):
 546                if self.isSubPathOf(relativePath, branch):
 547                    if len(branches) == 0:
 548                        relativePath = branch
 549                    else:
 550                        knownBranch = True
 551                    break
 552
 553            if knownBranch:
 554                continue
 555
 556            branches.add(relativePath)
 557            self.knownBranches.add(relativePath)
 558
 559        return branches
 560
 561    def findBranchParent(self, branchPrefix, files):
 562        for file in files:
 563            path = file["path"]
 564            if not path.startswith(branchPrefix):
 565                continue
 566            action = file["action"]
 567            if action != "integrate" and action != "branch":
 568                continue
 569            rev = file["rev"]
 570            depotPath = path + "#" + rev
 571
 572            log = p4CmdList("filelog \"%s\"" % depotPath)
 573            if len(log) != 1:
 574                print "eek! I got confused by the filelog of %s" % depotPath
 575                sys.exit(1);
 576
 577            log = log[0]
 578            if log["action0"] != action:
 579                print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 580                sys.exit(1);
 581
 582            branchAction = log["how0,0"]
 583    #        if branchAction == "branch into" or branchAction == "ignored":
 584    #            continue # ignore for branching
 585
 586            if not branchAction.endswith(" from"):
 587                continue # ignore for branching
 588    #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 589    #            sys.exit(1);
 590
 591            source = log["file0,0"]
 592            if source.startswith(branchPrefix):
 593                continue
 594
 595            lastSourceRev = log["erev0,0"]
 596
 597            sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 598            if len(sourceLog) != 1:
 599                print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 600                sys.exit(1);
 601            sourceLog = sourceLog[0]
 602
 603            relPath = source[len(self.depotPath):]
 604            # strip off the filename
 605            relPath = relPath[0:relPath.rfind("/")]
 606
 607            for branch in self.knownBranches:
 608                if self.isSubPathOf(relPath, branch):
 609    #                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 610                    return branch
 611    #            else:
 612    #                print "%s is not a subpath of branch %s" % (relPath, branch)
 613
 614        return ""
 615
 616    def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""):
 617        epoch = details["time"]
 618        author = details["user"]
 619
 620        self.gitStream.write("commit %s\n" % branch)
 621    #    gitStream.write("mark :%s\n" % details["change"])
 622        self.committedChanges.add(int(details["change"]))
 623        committer = ""
 624        if author in self.users:
 625            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 626        else:
 627            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 628
 629        self.gitStream.write("committer %s\n" % committer)
 630
 631        self.gitStream.write("data <<EOT\n")
 632        self.gitStream.write(details["desc"])
 633        self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
 634        self.gitStream.write("EOT\n\n")
 635
 636        if len(parent) > 0:
 637            self.gitStream.write("from %s\n" % parent)
 638
 639        if len(merged) > 0:
 640            self.gitStream.write("merge %s\n" % merged)
 641
 642        for file in files:
 643            path = file["path"]
 644            if not path.startswith(branchPrefix):
 645    #            if not silent:
 646    #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 647                continue
 648            rev = file["rev"]
 649            depotPath = path + "#" + rev
 650            relPath = path[len(branchPrefix):]
 651            action = file["action"]
 652
 653            if file["type"] == "apple":
 654                print "\nfile %s is a strange apple file that forks. Ignoring!" % path
 655                continue
 656
 657            if action == "delete":
 658                self.gitStream.write("D %s\n" % relPath)
 659            else:
 660                mode = 644
 661                if file["type"].startswith("x"):
 662                    mode = 755
 663
 664                data = self.p4File(depotPath)
 665
 666                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 667                self.gitStream.write("data %s\n" % len(data))
 668                self.gitStream.write(data)
 669                self.gitStream.write("\n")
 670
 671        self.gitStream.write("\n")
 672
 673        change = int(details["change"])
 674
 675        self.lastChange = change
 676
 677        if change in self.labels:
 678            label = self.labels[change]
 679            labelDetails = label[0]
 680            labelRevisions = label[1]
 681
 682            files = p4CmdList("files %s...@%s" % (branchPrefix, change))
 683
 684            if len(files) == len(labelRevisions):
 685
 686                cleanedFiles = {}
 687                for info in files:
 688                    if info["action"] == "delete":
 689                        continue
 690                    cleanedFiles[info["depotFile"]] = info["rev"]
 691
 692                if cleanedFiles == labelRevisions:
 693                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 694                    self.gitStream.write("from %s\n" % branch)
 695
 696                    owner = labelDetails["Owner"]
 697                    tagger = ""
 698                    if author in self.users:
 699                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
 700                    else:
 701                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
 702                    self.gitStream.write("tagger %s\n" % tagger)
 703                    self.gitStream.write("data <<EOT\n")
 704                    self.gitStream.write(labelDetails["Description"])
 705                    self.gitStream.write("EOT\n\n")
 706
 707                else:
 708                    if not self.silent:
 709                        print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change)
 710
 711            else:
 712                if not self.silent:
 713                    print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)
 714
 715    def extractFilesInCommitToBranch(self, files, branchPrefix):
 716        newFiles = []
 717
 718        for file in files:
 719            path = file["path"]
 720            if path.startswith(branchPrefix):
 721                newFiles.append(file)
 722
 723        return newFiles
 724
 725    def findBranchSourceHeuristic(self, files, branch, branchPrefix):
 726        for file in files:
 727            action = file["action"]
 728            if action != "integrate" and action != "branch":
 729                continue
 730            path = file["path"]
 731            rev = file["rev"]
 732            depotPath = path + "#" + rev
 733
 734            log = p4CmdList("filelog \"%s\"" % depotPath)
 735            if len(log) != 1:
 736                print "eek! I got confused by the filelog of %s" % depotPath
 737                sys.exit(1);
 738
 739            log = log[0]
 740            if log["action0"] != action:
 741                print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 742                sys.exit(1);
 743
 744            branchAction = log["how0,0"]
 745
 746            if not branchAction.endswith(" from"):
 747                continue # ignore for branching
 748    #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 749    #            sys.exit(1);
 750
 751            source = log["file0,0"]
 752            if source.startswith(branchPrefix):
 753                continue
 754
 755            lastSourceRev = log["erev0,0"]
 756
 757            sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 758            if len(sourceLog) != 1:
 759                print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 760                sys.exit(1);
 761            sourceLog = sourceLog[0]
 762
 763            relPath = source[len(self.depotPath):]
 764            # strip off the filename
 765            relPath = relPath[0:relPath.rfind("/")]
 766
 767            for candidate in self.knownBranches:
 768                if self.isSubPathOf(relPath, candidate) and candidate != branch:
 769                    return candidate
 770
 771        return ""
 772
 773    def changeIsBranchMerge(self, sourceBranch, destinationBranch, change):
 774        sourceFiles = {}
 775        for file in p4CmdList("files %s...@%s" % (self.depotPath + sourceBranch + "/", change)):
 776            if file["action"] == "delete":
 777                continue
 778            sourceFiles[file["depotFile"]] = file
 779
 780        destinationFiles = {}
 781        for file in p4CmdList("files %s...@%s" % (self.depotPath + destinationBranch + "/", change)):
 782            destinationFiles[file["depotFile"]] = file
 783
 784        for fileName in sourceFiles.keys():
 785            integrations = []
 786            deleted = False
 787            integrationCount = 0
 788            for integration in p4CmdList("integrated \"%s\"" % fileName):
 789                toFile = integration["fromFile"] # yes, it's true, it's fromFile
 790                if not toFile in destinationFiles:
 791                    continue
 792                destFile = destinationFiles[toFile]
 793                if destFile["action"] == "delete":
 794    #                print "file %s has been deleted in %s" % (fileName, toFile)
 795                    deleted = True
 796                    break
 797                integrationCount += 1
 798                if integration["how"] == "branch from":
 799                    continue
 800
 801                if int(integration["change"]) == change:
 802                    integrations.append(integration)
 803                    continue
 804                if int(integration["change"]) > change:
 805                    continue
 806
 807                destRev = int(destFile["rev"])
 808
 809                startRev = integration["startFromRev"][1:]
 810                if startRev == "none":
 811                    startRev = 0
 812                else:
 813                    startRev = int(startRev)
 814
 815                endRev = integration["endFromRev"][1:]
 816                if endRev == "none":
 817                    endRev = 0
 818                else:
 819                    endRev = int(endRev)
 820
 821                initialBranch = (destRev == 1 and integration["how"] != "branch into")
 822                inRange = (destRev >= startRev and destRev <= endRev)
 823                newer = (destRev > startRev and destRev > endRev)
 824
 825                if initialBranch or inRange or newer:
 826                    integrations.append(integration)
 827
 828            if deleted:
 829                continue
 830
 831            if len(integrations) == 0 and integrationCount > 1:
 832                print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch)
 833                return False
 834
 835        return True
 836
 837    def getUserMap(self):
 838        self.users = {}
 839
 840        for output in p4CmdList("users"):
 841            if not output.has_key("User"):
 842                continue
 843            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 844
 845    def getLabels(self):
 846        self.labels = {}
 847
 848        l = p4CmdList("labels %s..." % self.depotPath)
 849        if len(l) > 0 and not self.silent:
 850            print "Finding files belonging to labels in %s" % self.depotPath
 851
 852        for output in l:
 853            label = output["label"]
 854            revisions = {}
 855            newestChange = 0
 856            for file in p4CmdList("files //...@%s" % label):
 857                revisions[file["depotFile"]] = file["rev"]
 858                change = int(file["change"])
 859                if change > newestChange:
 860                    newestChange = change
 861
 862            self.labels[newestChange] = [output, revisions]
 863
 864    def run(self, args):
 865        self.depotPath = ""
 866        self.changeRange = ""
 867        self.initialParent = ""
 868        self.previousDepotPath = ""
 869
 870        if len(self.branch) == 0:
 871            self.branch = "p4"
 872
 873        if len(args) == 0:
 874            if not gitBranchExists(self.branch) and gitBranchExists("origin"):
 875                if not self.silent:
 876                    print "Creating %s branch in git repository based on origin" % self.branch
 877                system("git branch %s origin" % self.branch)
 878
 879            [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch))
 880            if len(self.previousDepotPath) > 0 and len(p4Change) > 0:
 881                p4Change = int(p4Change) + 1
 882                self.depotPath = self.previousDepotPath
 883                self.changeRange = "@%s,#head" % p4Change
 884                self.initialParent = parseRevision(self.branch)
 885                if not self.silent:
 886                    print "Performing incremental import into %s git branch" % self.branch
 887
 888        if not self.branch.startswith("refs/"):
 889            self.branch = "refs/heads/" + self.branch
 890
 891        if len(self.depotPath) != 0:
 892            self.depotPath = self.depotPath[:-1]
 893
 894        if len(args) == 0 and len(self.depotPath) != 0:
 895            if not self.silent:
 896                print "Depot path: %s" % self.depotPath
 897        elif len(args) != 1:
 898            return False
 899        else:
 900            if len(self.depotPath) != 0 and self.depotPath != args[0]:
 901                print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0])
 902                sys.exit(1)
 903            self.depotPath = args[0]
 904
 905        self.revision = ""
 906        self.users = {}
 907        self.lastChange = 0
 908
 909        if self.depotPath.find("@") != -1:
 910            atIdx = self.depotPath.index("@")
 911            self.changeRange = self.depotPath[atIdx:]
 912            if self.changeRange == "@all":
 913                self.changeRange = ""
 914            elif self.changeRange.find(",") == -1:
 915                self.revision = self.changeRange
 916                self.changeRange = ""
 917            self.depotPath = self.depotPath[0:atIdx]
 918        elif self.depotPath.find("#") != -1:
 919            hashIdx = self.depotPath.index("#")
 920            self.revision = self.depotPath[hashIdx:]
 921            self.depotPath = self.depotPath[0:hashIdx]
 922        elif len(self.previousDepotPath) == 0:
 923            self.revision = "#head"
 924
 925        if self.depotPath.endswith("..."):
 926            self.depotPath = self.depotPath[:-3]
 927
 928        if not self.depotPath.endswith("/"):
 929            self.depotPath += "/"
 930
 931        self.getUserMap()
 932        self.labels = {}
 933        if self.detectLabels:
 934            self.getLabels();
 935
 936        if len(self.changeRange) == 0:
 937            try:
 938                sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch)
 939                output = sout.read()
 940                if output.endswith("\n"):
 941                    output = output[:-1]
 942                tagIdx = output.index(" tags/p4/")
 943                caretIdx = output.find("^")
 944                endPos = len(output)
 945                if caretIdx != -1:
 946                    endPos = caretIdx
 947                self.rev = int(output[tagIdx + 9 : endPos]) + 1
 948                self.changeRange = "@%s,#head" % self.rev
 949                self.initialParent = parseRevision(self.branch)
 950            except:
 951                pass
 952
 953        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
 954
 955        importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
 956        self.gitOutput = importProcess.stdout
 957        self.gitStream = importProcess.stdin
 958        self.gitError = importProcess.stderr
 959
 960        if len(self.revision) > 0:
 961            print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision)
 962
 963            details = { "user" : "git perforce import user", "time" : int(time.time()) }
 964            details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision)
 965            details["change"] = self.revision
 966            newestRevision = 0
 967
 968            fileCnt = 0
 969            for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)):
 970                change = int(info["change"])
 971                if change > newestRevision:
 972                    newestRevision = change
 973
 974                if info["action"] == "delete":
 975                    # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
 976                    #fileCnt = fileCnt + 1
 977                    continue
 978
 979                for prop in [ "depotFile", "rev", "action", "type" ]:
 980                    details["%s%s" % (prop, fileCnt)] = info[prop]
 981
 982                fileCnt = fileCnt + 1
 983
 984            details["change"] = newestRevision
 985
 986            try:
 987                self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath)
 988            except IOError:
 989                print "IO error with git fast-import. Is your git version recent enough?"
 990                print self.gitError.read()
 991
 992        else:
 993            changes = []
 994
 995            if len(self.changesFile) > 0:
 996                output = open(self.changesFile).readlines()
 997                changeSet = Set()
 998                for line in output:
 999                    changeSet.add(int(line))
1000
1001                for change in changeSet:
1002                    changes.append(change)
1003
1004                changes.sort()
1005            else:
1006                output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines()
1007
1008                for line in output:
1009                    changeNum = line.split(" ")[1]
1010                    changes.append(changeNum)
1011
1012                changes.reverse()
1013
1014            if len(changes) == 0:
1015                if not self.silent:
1016                    print "no changes to import!"
1017                return True
1018
1019            cnt = 1
1020            for change in changes:
1021                description = p4Cmd("describe %s" % change)
1022
1023                if not self.silent:
1024                    sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1025                    sys.stdout.flush()
1026                cnt = cnt + 1
1027
1028                try:
1029                    files = self.extractFilesFromCommit(description)
1030                    if self.detectBranches:
1031                        for branch in self.branchesForCommit(files):
1032                            self.knownBranches.add(branch)
1033                            branchPrefix = self.depotPath + branch + "/"
1034
1035                            filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix)
1036
1037                            merged = ""
1038                            parent = ""
1039                            ########### remove cnt!!!
1040                            if branch not in self.createdBranches and cnt > 2:
1041                                self.createdBranches.add(branch)
1042                                parent = self.findBranchParent(branchPrefix, files)
1043                                if parent == branch:
1044                                    parent = ""
1045            #                    elif len(parent) > 0:
1046            #                        print "%s branched off of %s" % (branch, parent)
1047
1048                            if len(parent) == 0:
1049                                merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix)
1050                                if len(merged) > 0:
1051                                    print "change %s could be a merge from %s into %s" % (description["change"], merged, branch)
1052                                    if not self.changeIsBranchMerge(merged, branch, int(description["change"])):
1053                                        merged = ""
1054
1055                            branch = "refs/heads/" + branch
1056                            if len(parent) > 0:
1057                                parent = "refs/heads/" + parent
1058                            if len(merged) > 0:
1059                                merged = "refs/heads/" + merged
1060                            self.commit(description, files, branch, branchPrefix, parent, merged)
1061                    else:
1062                        self.commit(description, files, self.branch, self.depotPath, self.initialParent)
1063                        self.initialParent = ""
1064                except IOError:
1065                    print self.gitError.read()
1066                    sys.exit(1)
1067
1068        if not self.silent:
1069            print ""
1070
1071
1072        self.gitStream.close()
1073        self.gitOutput.close()
1074        self.gitError.close()
1075        importProcess.wait()
1076
1077        return True
1078
1079class P4Rebase(Command):
1080    def __init__(self):
1081        Command.__init__(self)
1082        self.options = [ ]
1083        self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it"
1084
1085    def run(self, args):
1086        sync = P4Sync()
1087        sync.run([])
1088        print "Rebasing the current branch"
1089        oldHead = mypopen("git rev-parse HEAD").read()[:-1]
1090        system("git rebase p4")
1091        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1092        return True
1093
1094class P4Clone(P4Sync):
1095    def __init__(self):
1096        P4Sync.__init__(self)
1097        self.description = "Creates a new git repository and imports from Perforce into it"
1098        self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]"
1099        self.needsGit = False
1100
1101    def run(self, args):
1102        if len(args) < 1:
1103            return False
1104        depotPath = args[0]
1105        dir = ""
1106        if len(args) == 2:
1107            dir = args[1]
1108        elif len(args) > 2:
1109            return False
1110
1111        if not depotPath.startswith("//"):
1112            return False
1113
1114        if len(dir) == 0:
1115            dir = depotPath
1116            atPos = dir.rfind("@")
1117            if atPos != -1:
1118                dir = dir[0:atPos]
1119            hashPos = dir.rfind("#")
1120            if hashPos != -1:
1121                dir = dir[0:hashPos]
1122
1123            if dir.endswith("..."):
1124                dir = dir[:-3]
1125
1126            if dir.endswith("/"):
1127               dir = dir[:-1]
1128
1129            slashPos = dir.rfind("/")
1130            if slashPos != -1:
1131                dir = dir[slashPos + 1:]
1132
1133        print "Importing from %s into %s" % (depotPath, dir)
1134        os.makedirs(dir)
1135        os.chdir(dir)
1136        system("git init")
1137        if not P4Sync.run(self, [depotPath]):
1138            return False
1139        if self.branch != "master":
1140            system("git branch master p4")
1141            system("git checkout -f")
1142        return True
1143
1144class HelpFormatter(optparse.IndentedHelpFormatter):
1145    def __init__(self):
1146        optparse.IndentedHelpFormatter.__init__(self)
1147
1148    def format_description(self, description):
1149        if description:
1150            return description + "\n"
1151        else:
1152            return ""
1153
1154def printUsage(commands):
1155    print "usage: %s <command> [options]" % sys.argv[0]
1156    print ""
1157    print "valid commands: %s" % ", ".join(commands)
1158    print ""
1159    print "Try %s <command> --help for command specific help." % sys.argv[0]
1160    print ""
1161
1162commands = {
1163    "debug" : P4Debug(),
1164    "clean-tags" : P4CleanTags(),
1165    "submit" : P4Submit(),
1166    "sync" : P4Sync(),
1167    "rebase" : P4Rebase(),
1168    "clone" : P4Clone()
1169}
1170
1171if len(sys.argv[1:]) == 0:
1172    printUsage(commands.keys())
1173    sys.exit(2)
1174
1175cmd = ""
1176cmdName = sys.argv[1]
1177try:
1178    cmd = commands[cmdName]
1179except KeyError:
1180    print "unknown command %s" % cmdName
1181    print ""
1182    printUsage(commands.keys())
1183    sys.exit(2)
1184
1185options = cmd.options
1186cmd.gitdir = gitdir
1187
1188args = sys.argv[2:]
1189
1190if len(options) > 0:
1191    options.append(optparse.make_option("--git-dir", dest="gitdir"))
1192
1193    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1194                                   options,
1195                                   description = cmd.description,
1196                                   formatter = HelpFormatter())
1197
1198    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1199
1200if cmd.needsGit:
1201    gitdir = cmd.gitdir
1202    if len(gitdir) == 0:
1203        gitdir = ".git"
1204        if not isValidGitDir(gitdir):
1205            gitdir = mypopen("git rev-parse --git-dir").read()[:-1]
1206            if os.path.exists(gitdir):
1207                cdup = mypopen("git rev-parse --show-cdup").read()[:-1];
1208                if len(cdup) > 0:
1209                    os.chdir(cdup);
1210
1211    if not isValidGitDir(gitdir):
1212        if isValidGitDir(gitdir + "/.git"):
1213            gitdir += "/.git"
1214        else:
1215            die("fatal: cannot locate git repository at %s" % gitdir)
1216
1217    os.environ["GIT_DIR"] = gitdir
1218
1219if not cmd.run(args):
1220    parser.print_help()
1221