Andrew's git / gitweb.git / contrib/fast-import/git-p4
?
contrib / fast-import / git-p4on commit Fix git-p4 clone //depot/project (head import) (cd6cc0d)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <hausmann@kde.org>
   6# Copyright: 2007 Simon Hausmann <hausmann@kde.org>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10
  11import optparse, sys, os, marshal, popen2, subprocess, shelve
  12import tempfile, getopt, sha, os.path, time, platform
  13from sets import Set;
  14
  15gitdir = os.environ.get("GIT_DIR", "")
  16
  17def mypopen(command):
  18    return os.popen(command, "rb");
  19
  20def p4CmdList(cmd):
  21    cmd = "p4 -G %s" % cmd
  22    pipe = os.popen(cmd, "rb")
  23
  24    result = []
  25    try:
  26        while True:
  27            entry = marshal.load(pipe)
  28            result.append(entry)
  29    except EOFError:
  30        pass
  31    pipe.close()
  32
  33    return result
  34
  35def p4Cmd(cmd):
  36    list = p4CmdList(cmd)
  37    result = {}
  38    for entry in list:
  39        result.update(entry)
  40    return result;
  41
  42def p4Where(depotPath):
  43    if not depotPath.endswith("/"):
  44        depotPath += "/"
  45    output = p4Cmd("where %s..." % depotPath)
  46    clientPath = ""
  47    if "path" in output:
  48        clientPath = output.get("path")
  49    elif "data" in output:
  50        data = output.get("data")
  51        lastSpace = data.rfind(" ")
  52        clientPath = data[lastSpace + 1:]
  53
  54    if clientPath.endswith("..."):
  55        clientPath = clientPath[:-3]
  56    return clientPath
  57
  58def die(msg):
  59    sys.stderr.write(msg + "\n")
  60    sys.exit(1)
  61
  62def currentGitBranch():
  63    return mypopen("git name-rev HEAD").read().split(" ")[1][:-1]
  64
  65def isValidGitDir(path):
  66    if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
  67        return True;
  68    return False
  69
  70def system(cmd):
  71    if os.system(cmd) != 0:
  72        die("command failed: %s" % cmd)
  73
  74def extractLogMessageFromGitCommit(commit):
  75    logMessage = ""
  76    foundTitle = False
  77    for log in mypopen("git cat-file commit %s" % commit).readlines():
  78       if not foundTitle:
  79           if len(log) == 1:
  80               foundTitle = True
  81           continue
  82
  83       logMessage += log
  84    return logMessage
  85
  86def extractDepotPathAndChangeFromGitLog(log):
  87    values = {}
  88    for line in log.split("\n"):
  89        line = line.strip()
  90        if line.startswith("[git-p4:") and line.endswith("]"):
  91            line = line[8:-1].strip()
  92            for assignment in line.split(":"):
  93                variable = assignment.strip()
  94                value = ""
  95                equalPos = assignment.find("=")
  96                if equalPos != -1:
  97                    variable = assignment[:equalPos].strip()
  98                    value = assignment[equalPos + 1:].strip()
  99                    if value.startswith("\"") and value.endswith("\""):
 100                        value = value[1:-1]
 101                values[variable] = value
 102
 103    return values.get("depot-path"), values.get("change")
 104
 105def gitBranchExists(branch):
 106    proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 107    return proc.wait() == 0;
 108
 109class Command:
 110    def __init__(self):
 111        self.usage = "usage: %prog [options]"
 112        self.needsGit = True
 113
 114class P4Debug(Command):
 115    def __init__(self):
 116        Command.__init__(self)
 117        self.options = [
 118        ]
 119        self.description = "A tool to debug the output of p4 -G."
 120        self.needsGit = False
 121
 122    def run(self, args):
 123        for output in p4CmdList(" ".join(args)):
 124            print output
 125        return True
 126
 127class P4CleanTags(Command):
 128    def __init__(self):
 129        Command.__init__(self)
 130        self.options = [
 131#                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
 132        ]
 133        self.description = "A tool to remove stale unused tags from incremental perforce imports."
 134    def run(self, args):
 135        branch = currentGitBranch()
 136        print "Cleaning out stale p4 import tags..."
 137        sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch)
 138        output = sout.read()
 139        try:
 140            tagIdx = output.index(" tags/p4/")
 141        except:
 142            print "Cannot find any p4/* tag. Nothing to do."
 143            sys.exit(0)
 144
 145        try:
 146            caretIdx = output.index("^")
 147        except:
 148            caretIdx = len(output) - 1
 149        rev = int(output[tagIdx + 9 : caretIdx])
 150
 151        allTags = mypopen("git tag -l p4/").readlines()
 152        for i in range(len(allTags)):
 153            allTags[i] = int(allTags[i][3:-1])
 154
 155        allTags.sort()
 156
 157        allTags.remove(rev)
 158
 159        for rev in allTags:
 160            print mypopen("git tag -d p4/%s" % rev).read()
 161
 162        print "%s tags removed." % len(allTags)
 163        return True
 164
 165class P4Submit(Command):
 166    def __init__(self):
 167        Command.__init__(self)
 168        self.options = [
 169                optparse.make_option("--continue", action="store_false", dest="firstTime"),
 170                optparse.make_option("--origin", dest="origin"),
 171                optparse.make_option("--reset", action="store_true", dest="reset"),
 172                optparse.make_option("--log-substitutions", dest="substFile"),
 173                optparse.make_option("--noninteractive", action="store_false"),
 174                optparse.make_option("--dry-run", action="store_true"),
 175        ]
 176        self.description = "Submit changes from git to the perforce depot."
 177        self.usage += " [name of git branch to submit into perforce depot]"
 178        self.firstTime = True
 179        self.reset = False
 180        self.interactive = True
 181        self.dryRun = False
 182        self.substFile = ""
 183        self.firstTime = True
 184        self.origin = ""
 185
 186        self.logSubstitutions = {}
 187        self.logSubstitutions["<enter description here>"] = "%log%"
 188        self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 189
 190    def check(self):
 191        if len(p4CmdList("opened ...")) > 0:
 192            die("You have files opened with perforce! Close them before starting the sync.")
 193
 194    def start(self):
 195        if len(self.config) > 0 and not self.reset:
 196            die("Cannot start sync. Previous sync config found at %s" % self.configFile)
 197
 198        commits = []
 199        for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
 200            commits.append(line[:-1])
 201        commits.reverse()
 202
 203        self.config["commits"] = commits
 204
 205    def prepareLogMessage(self, template, message):
 206        result = ""
 207
 208        for line in template.split("\n"):
 209            if line.startswith("#"):
 210                result += line + "\n"
 211                continue
 212
 213            substituted = False
 214            for key in self.logSubstitutions.keys():
 215                if line.find(key) != -1:
 216                    value = self.logSubstitutions[key]
 217                    value = value.replace("%log%", message)
 218                    if value != "@remove@":
 219                        result += line.replace(key, value) + "\n"
 220                    substituted = True
 221                    break
 222
 223            if not substituted:
 224                result += line + "\n"
 225
 226        return result
 227
 228    def apply(self, id):
 229        print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read())
 230        diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
 231        filesToAdd = set()
 232        filesToDelete = set()
 233        for line in diff:
 234            modifier = line[0]
 235            path = line[1:].strip()
 236            if modifier == "M":
 237                system("p4 edit %s" % path)
 238            elif modifier == "A":
 239                filesToAdd.add(path)
 240                if path in filesToDelete:
 241                    filesToDelete.remove(path)
 242            elif modifier == "D":
 243                filesToDelete.add(path)
 244                if path in filesToAdd:
 245                    filesToAdd.remove(path)
 246            else:
 247                die("unknown modifier %s for %s" % (modifier, path))
 248
 249        diffcmd = "git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\"" % (id, id)
 250        patchcmd = diffcmd + " | patch -p1"
 251
 252        if os.system(patchcmd + " --dry-run --silent") != 0:
 253            print "Unfortunately applying the change failed!"
 254            print "What do you want to do?"
 255            response = "x"
 256            while response != "s" and response != "a" and response != "w":
 257                response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ")
 258            if response == "s":
 259                print "Skipping! Good luck with the next patches..."
 260                return
 261            elif response == "a":
 262                os.system(patchcmd)
 263                if len(filesToAdd) > 0:
 264                    print "You may also want to call p4 add on the following files:"
 265                    print " ".join(filesToAdd)
 266                if len(filesToDelete):
 267                    print "The following files should be scheduled for deletion with p4 delete:"
 268                    print " ".join(filesToDelete)
 269                die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 270            elif response == "w":
 271                system(diffcmd + " > patch.txt")
 272                print "Patch saved to patch.txt in %s !" % self.clientPath
 273                die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 274
 275        system(patchcmd)
 276
 277        for f in filesToAdd:
 278            system("p4 add %s" % f)
 279        for f in filesToDelete:
 280            system("p4 revert %s" % f)
 281            system("p4 delete %s" % f)
 282
 283        logMessage = extractLogMessageFromGitCommit(id)
 284        logMessage = logMessage.replace("\n", "\n\t")
 285        logMessage = logMessage[:-1]
 286
 287        template = mypopen("p4 change -o").read()
 288
 289        if self.interactive:
 290            submitTemplate = self.prepareLogMessage(template, logMessage)
 291            diff = mypopen("p4 diff -du ...").read()
 292
 293            for newFile in filesToAdd:
 294                diff += "==== new file ====\n"
 295                diff += "--- /dev/null\n"
 296                diff += "+++ %s\n" % newFile
 297                f = open(newFile, "r")
 298                for line in f.readlines():
 299                    diff += "+" + line
 300                f.close()
 301
 302            separatorLine = "######## everything below this line is just the diff #######"
 303            if platform.system() == "Windows":
 304                separatorLine += "\r"
 305            separatorLine += "\n"
 306
 307            response = "e"
 308            firstIteration = True
 309            while response == "e":
 310                if not firstIteration:
 311                    response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o  ")
 312                firstIteration = False
 313                if response == "e":
 314                    [handle, fileName] = tempfile.mkstemp()
 315                    tmpFile = os.fdopen(handle, "w+")
 316                    tmpFile.write(submitTemplate + separatorLine + diff)
 317                    tmpFile.close()
 318                    defaultEditor = "vi"
 319                    if platform.system() == "Windows":
 320                        defaultEditor = "notepad"
 321                    editor = os.environ.get("EDITOR", defaultEditor);
 322                    system(editor + " " + fileName)
 323                    tmpFile = open(fileName, "rb")
 324                    message = tmpFile.read()
 325                    tmpFile.close()
 326                    os.remove(fileName)
 327                    submitTemplate = message[:message.index(separatorLine)]
 328
 329            if response == "y" or response == "yes":
 330               if self.dryRun:
 331                   print submitTemplate
 332                   raw_input("Press return to continue...")
 333               else:
 334                    pipe = os.popen("p4 submit -i", "wb")
 335                    pipe.write(submitTemplate)
 336                    pipe.close()
 337            else:
 338                print "Not submitting!"
 339                self.interactive = False
 340        else:
 341            fileName = "submit.txt"
 342            file = open(fileName, "w+")
 343            file.write(self.prepareLogMessage(template, logMessage))
 344            file.close()
 345            print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
 346
 347    def run(self, args):
 348        global gitdir
 349        # make gitdir absolute so we can cd out into the perforce checkout
 350        gitdir = os.path.abspath(gitdir)
 351        os.environ["GIT_DIR"] = gitdir
 352
 353        if len(args) == 0:
 354            self.master = currentGitBranch()
 355            if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
 356                die("Detecting current git branch failed!")
 357        elif len(args) == 1:
 358            self.master = args[0]
 359        else:
 360            return False
 361
 362        depotPath = ""
 363        if gitBranchExists("p4"):
 364            [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 365        if len(depotPath) == 0 and gitBranchExists("origin"):
 366            [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 367
 368        if len(depotPath) == 0:
 369            print "Internal error: cannot locate perforce depot path from existing branches"
 370            sys.exit(128)
 371
 372        self.clientPath = p4Where(depotPath)
 373
 374        if len(self.clientPath) == 0:
 375            print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
 376            sys.exit(128)
 377
 378        print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath)
 379        oldWorkingDirectory = os.getcwd()
 380        os.chdir(self.clientPath)
 381        response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath)
 382        if response == "y" or response == "yes":
 383            system("p4 sync ...")
 384
 385        if len(self.origin) == 0:
 386            if gitBranchExists("p4"):
 387                self.origin = "p4"
 388            else:
 389                self.origin = "origin"
 390
 391        if self.reset:
 392            self.firstTime = True
 393
 394        if len(self.substFile) > 0:
 395            for line in open(self.substFile, "r").readlines():
 396                tokens = line[:-1].split("=")
 397                self.logSubstitutions[tokens[0]] = tokens[1]
 398
 399        self.check()
 400        self.configFile = gitdir + "/p4-git-sync.cfg"
 401        self.config = shelve.open(self.configFile, writeback=True)
 402
 403        if self.firstTime:
 404            self.start()
 405
 406        commits = self.config.get("commits", [])
 407
 408        while len(commits) > 0:
 409            self.firstTime = False
 410            commit = commits[0]
 411            commits = commits[1:]
 412            self.config["commits"] = commits
 413            self.apply(commit)
 414            if not self.interactive:
 415                break
 416
 417        self.config.close()
 418
 419        if len(commits) == 0:
 420            if self.firstTime:
 421                print "No changes found to apply between %s and current HEAD" % self.origin
 422            else:
 423                print "All changes applied!"
 424                response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ")
 425                if response == "y" or response == "yes":
 426                    os.chdir(oldWorkingDirectory)
 427                    rebase = P4Rebase()
 428                    rebase.run([])
 429            os.remove(self.configFile)
 430
 431        return True
 432
 433class P4Sync(Command):
 434    def __init__(self):
 435        Command.__init__(self)
 436        self.options = [
 437                optparse.make_option("--branch", dest="branch"),
 438                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 439                optparse.make_option("--changesfile", dest="changesFile"),
 440                optparse.make_option("--silent", dest="silent", action="store_true"),
 441                optparse.make_option("--known-branches", dest="knownBranches"),
 442                optparse.make_option("--data-cache", dest="dataCache", action="store_true"),
 443                optparse.make_option("--command-cache", dest="commandCache", action="store_true"),
 444                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true")
 445        ]
 446        self.description = """Imports from Perforce into a git repository.\n
 447    example:
 448    //depot/my/project/ -- to import the current head
 449    //depot/my/project/@all -- to import everything
 450    //depot/my/project/@1,6 -- to import only from revision 1 to 6
 451
 452    (a ... is not needed in the path p4 specification, it's added implicitly)"""
 453
 454        self.usage += " //depot/path[@revRange]"
 455
 456        self.dataCache = False
 457        self.commandCache = False
 458        self.silent = False
 459        self.knownBranches = Set()
 460        self.createdBranches = Set()
 461        self.committedChanges = Set()
 462        self.branch = ""
 463        self.detectBranches = False
 464        self.detectLabels = False
 465        self.changesFile = ""
 466
 467    def p4File(self, depotPath):
 468        return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 469
 470    def extractFilesFromCommit(self, commit):
 471        files = []
 472        fnum = 0
 473        while commit.has_key("depotFile%s" % fnum):
 474            path =  commit["depotFile%s" % fnum]
 475            if not path.startswith(self.depotPath):
 476    #            if not self.silent:
 477    #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change)
 478                fnum = fnum + 1
 479                continue
 480
 481            file = {}
 482            file["path"] = path
 483            file["rev"] = commit["rev%s" % fnum]
 484            file["action"] = commit["action%s" % fnum]
 485            file["type"] = commit["type%s" % fnum]
 486            files.append(file)
 487            fnum = fnum + 1
 488        return files
 489
 490    def isSubPathOf(self, first, second):
 491        if not first.startswith(second):
 492            return False
 493        if first == second:
 494            return True
 495        return first[len(second)] == "/"
 496
 497    def branchesForCommit(self, files):
 498        branches = Set()
 499
 500        for file in files:
 501            relativePath = file["path"][len(self.depotPath):]
 502            # strip off the filename
 503            relativePath = relativePath[0:relativePath.rfind("/")]
 504
 505    #        if len(branches) == 0:
 506    #            branches.add(relativePath)
 507    #            knownBranches.add(relativePath)
 508    #            continue
 509
 510            ###### this needs more testing :)
 511            knownBranch = False
 512            for branch in branches:
 513                if relativePath == branch:
 514                    knownBranch = True
 515                    break
 516    #            if relativePath.startswith(branch):
 517                if self.isSubPathOf(relativePath, branch):
 518                    knownBranch = True
 519                    break
 520    #            if branch.startswith(relativePath):
 521                if self.isSubPathOf(branch, relativePath):
 522                    branches.remove(branch)
 523                    break
 524
 525            if knownBranch:
 526                continue
 527
 528            for branch in self.knownBranches:
 529                #if relativePath.startswith(branch):
 530                if self.isSubPathOf(relativePath, branch):
 531                    if len(branches) == 0:
 532                        relativePath = branch
 533                    else:
 534                        knownBranch = True
 535                    break
 536
 537            if knownBranch:
 538                continue
 539
 540            branches.add(relativePath)
 541            self.knownBranches.add(relativePath)
 542
 543        return branches
 544
 545    def findBranchParent(self, branchPrefix, files):
 546        for file in files:
 547            path = file["path"]
 548            if not path.startswith(branchPrefix):
 549                continue
 550            action = file["action"]
 551            if action != "integrate" and action != "branch":
 552                continue
 553            rev = file["rev"]
 554            depotPath = path + "#" + rev
 555
 556            log = p4CmdList("filelog \"%s\"" % depotPath)
 557            if len(log) != 1:
 558                print "eek! I got confused by the filelog of %s" % depotPath
 559                sys.exit(1);
 560
 561            log = log[0]
 562            if log["action0"] != action:
 563                print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 564                sys.exit(1);
 565
 566            branchAction = log["how0,0"]
 567    #        if branchAction == "branch into" or branchAction == "ignored":
 568    #            continue # ignore for branching
 569
 570            if not branchAction.endswith(" from"):
 571                continue # ignore for branching
 572    #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 573    #            sys.exit(1);
 574
 575            source = log["file0,0"]
 576            if source.startswith(branchPrefix):
 577                continue
 578
 579            lastSourceRev = log["erev0,0"]
 580
 581            sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 582            if len(sourceLog) != 1:
 583                print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 584                sys.exit(1);
 585            sourceLog = sourceLog[0]
 586
 587            relPath = source[len(self.depotPath):]
 588            # strip off the filename
 589            relPath = relPath[0:relPath.rfind("/")]
 590
 591            for branch in self.knownBranches:
 592                if self.isSubPathOf(relPath, branch):
 593    #                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 594                    return branch
 595    #            else:
 596    #                print "%s is not a subpath of branch %s" % (relPath, branch)
 597
 598        return ""
 599
 600    def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""):
 601        epoch = details["time"]
 602        author = details["user"]
 603
 604        self.gitStream.write("commit %s\n" % branch)
 605    #    gitStream.write("mark :%s\n" % details["change"])
 606        self.committedChanges.add(int(details["change"]))
 607        committer = ""
 608        if author in self.users:
 609            committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 610        else:
 611            committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 612
 613        self.gitStream.write("committer %s\n" % committer)
 614
 615        self.gitStream.write("data <<EOT\n")
 616        self.gitStream.write(details["desc"])
 617        self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
 618        self.gitStream.write("EOT\n\n")
 619
 620        if len(parent) > 0:
 621            self.gitStream.write("from %s\n" % parent)
 622
 623        if len(merged) > 0:
 624            self.gitStream.write("merge %s\n" % merged)
 625
 626        for file in files:
 627            path = file["path"]
 628            if not path.startswith(branchPrefix):
 629    #            if not silent:
 630    #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 631                continue
 632            rev = file["rev"]
 633            depotPath = path + "#" + rev
 634            relPath = path[len(branchPrefix):]
 635            action = file["action"]
 636
 637            if file["type"] == "apple":
 638                print "\nfile %s is a strange apple file that forks. Ignoring!" % path
 639                continue
 640
 641            if action == "delete":
 642                self.gitStream.write("D %s\n" % relPath)
 643            else:
 644                mode = 644
 645                if file["type"].startswith("x"):
 646                    mode = 755
 647
 648                data = self.p4File(depotPath)
 649
 650                self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 651                self.gitStream.write("data %s\n" % len(data))
 652                self.gitStream.write(data)
 653                self.gitStream.write("\n")
 654
 655        self.gitStream.write("\n")
 656
 657        change = int(details["change"])
 658
 659        self.lastChange = change
 660
 661        if change in self.labels:
 662            label = self.labels[change]
 663            labelDetails = label[0]
 664            labelRevisions = label[1]
 665
 666            files = p4CmdList("files %s...@%s" % (branchPrefix, change))
 667
 668            if len(files) == len(labelRevisions):
 669
 670                cleanedFiles = {}
 671                for info in files:
 672                    if info["action"] == "delete":
 673                        continue
 674                    cleanedFiles[info["depotFile"]] = info["rev"]
 675
 676                if cleanedFiles == labelRevisions:
 677                    self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 678                    self.gitStream.write("from %s\n" % branch)
 679
 680                    owner = labelDetails["Owner"]
 681                    tagger = ""
 682                    if author in self.users:
 683                        tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
 684                    else:
 685                        tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
 686                    self.gitStream.write("tagger %s\n" % tagger)
 687                    self.gitStream.write("data <<EOT\n")
 688                    self.gitStream.write(labelDetails["Description"])
 689                    self.gitStream.write("EOT\n\n")
 690
 691                else:
 692                    if not self.silent:
 693                        print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change)
 694
 695            else:
 696                if not self.silent:
 697                    print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)
 698
 699    def extractFilesInCommitToBranch(self, files, branchPrefix):
 700        newFiles = []
 701
 702        for file in files:
 703            path = file["path"]
 704            if path.startswith(branchPrefix):
 705                newFiles.append(file)
 706
 707        return newFiles
 708
 709    def findBranchSourceHeuristic(self, files, branch, branchPrefix):
 710        for file in files:
 711            action = file["action"]
 712            if action != "integrate" and action != "branch":
 713                continue
 714            path = file["path"]
 715            rev = file["rev"]
 716            depotPath = path + "#" + rev
 717
 718            log = p4CmdList("filelog \"%s\"" % depotPath)
 719            if len(log) != 1:
 720                print "eek! I got confused by the filelog of %s" % depotPath
 721                sys.exit(1);
 722
 723            log = log[0]
 724            if log["action0"] != action:
 725                print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 726                sys.exit(1);
 727
 728            branchAction = log["how0,0"]
 729
 730            if not branchAction.endswith(" from"):
 731                continue # ignore for branching
 732    #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 733    #            sys.exit(1);
 734
 735            source = log["file0,0"]
 736            if source.startswith(branchPrefix):
 737                continue
 738
 739            lastSourceRev = log["erev0,0"]
 740
 741            sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 742            if len(sourceLog) != 1:
 743                print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 744                sys.exit(1);
 745            sourceLog = sourceLog[0]
 746
 747            relPath = source[len(self.depotPath):]
 748            # strip off the filename
 749            relPath = relPath[0:relPath.rfind("/")]
 750
 751            for candidate in self.knownBranches:
 752                if self.isSubPathOf(relPath, candidate) and candidate != branch:
 753                    return candidate
 754
 755        return ""
 756
 757    def changeIsBranchMerge(self, sourceBranch, destinationBranch, change):
 758        sourceFiles = {}
 759        for file in p4CmdList("files %s...@%s" % (self.depotPath + sourceBranch + "/", change)):
 760            if file["action"] == "delete":
 761                continue
 762            sourceFiles[file["depotFile"]] = file
 763
 764        destinationFiles = {}
 765        for file in p4CmdList("files %s...@%s" % (self.depotPath + destinationBranch + "/", change)):
 766            destinationFiles[file["depotFile"]] = file
 767
 768        for fileName in sourceFiles.keys():
 769            integrations = []
 770            deleted = False
 771            integrationCount = 0
 772            for integration in p4CmdList("integrated \"%s\"" % fileName):
 773                toFile = integration["fromFile"] # yes, it's true, it's fromFile
 774                if not toFile in destinationFiles:
 775                    continue
 776                destFile = destinationFiles[toFile]
 777                if destFile["action"] == "delete":
 778    #                print "file %s has been deleted in %s" % (fileName, toFile)
 779                    deleted = True
 780                    break
 781                integrationCount += 1
 782                if integration["how"] == "branch from":
 783                    continue
 784
 785                if int(integration["change"]) == change:
 786                    integrations.append(integration)
 787                    continue
 788                if int(integration["change"]) > change:
 789                    continue
 790
 791                destRev = int(destFile["rev"])
 792
 793                startRev = integration["startFromRev"][1:]
 794                if startRev == "none":
 795                    startRev = 0
 796                else:
 797                    startRev = int(startRev)
 798
 799                endRev = integration["endFromRev"][1:]
 800                if endRev == "none":
 801                    endRev = 0
 802                else:
 803                    endRev = int(endRev)
 804
 805                initialBranch = (destRev == 1 and integration["how"] != "branch into")
 806                inRange = (destRev >= startRev and destRev <= endRev)
 807                newer = (destRev > startRev and destRev > endRev)
 808
 809                if initialBranch or inRange or newer:
 810                    integrations.append(integration)
 811
 812            if deleted:
 813                continue
 814
 815            if len(integrations) == 0 and integrationCount > 1:
 816                print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch)
 817                return False
 818
 819        return True
 820
 821    def getUserMap(self):
 822        self.users = {}
 823
 824        for output in p4CmdList("users"):
 825            if not output.has_key("User"):
 826                continue
 827            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 828
 829    def getLabels(self):
 830        self.labels = {}
 831
 832        l = p4CmdList("labels %s..." % self.depotPath)
 833        if len(l) > 0 and not self.silent:
 834            print "Finding files belonging to labels in %s" % self.depotPath
 835
 836        for output in l:
 837            label = output["label"]
 838            revisions = {}
 839            newestChange = 0
 840            for file in p4CmdList("files //...@%s" % label):
 841                revisions[file["depotFile"]] = file["rev"]
 842                change = int(file["change"])
 843                if change > newestChange:
 844                    newestChange = change
 845
 846            self.labels[newestChange] = [output, revisions]
 847
 848    def run(self, args):
 849        self.depotPath = ""
 850        self.changeRange = ""
 851        self.initialParent = ""
 852        self.previousDepotPath = ""
 853
 854        if len(self.branch) == 0:
 855            self.branch = "p4"
 856
 857        if len(args) == 0:
 858            if not gitBranchExists(self.branch) and gitBranchExists("origin"):
 859                if not self.silent:
 860                    print "Creating %s branch in git repository based on origin" % self.branch
 861                system("git branch %s origin" % self.branch)
 862
 863            [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch))
 864            if len(self.previousDepotPath) > 0 and len(p4Change) > 0:
 865                p4Change = int(p4Change) + 1
 866                self.depotPath = self.previousDepotPath
 867                self.changeRange = "@%s,#head" % p4Change
 868                self.initialParent = self.branch
 869                if not self.silent:
 870                    print "Performing incremental import into %s git branch" % self.branch
 871
 872        self.branch = "refs/heads/" + self.branch
 873
 874        if len(self.depotPath) != 0:
 875            self.depotPath = self.depotPath[:-1]
 876
 877        if len(args) == 0 and len(self.depotPath) != 0:
 878            if not self.silent:
 879                print "Depot path: %s" % self.depotPath
 880        elif len(args) != 1:
 881            return False
 882        else:
 883            if len(self.depotPath) != 0 and self.depotPath != args[0]:
 884                print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0])
 885                sys.exit(1)
 886            self.depotPath = args[0]
 887
 888        self.revision = ""
 889        self.users = {}
 890        self.lastChange = 0
 891
 892        if self.depotPath.find("@") != -1:
 893            atIdx = self.depotPath.index("@")
 894            self.changeRange = self.depotPath[atIdx:]
 895            if self.changeRange == "@all":
 896                self.changeRange = ""
 897            elif self.changeRange.find(",") == -1:
 898                self.revision = self.changeRange
 899                self.changeRange = ""
 900            self.depotPath = self.depotPath[0:atIdx]
 901        elif self.depotPath.find("#") != -1:
 902            hashIdx = self.depotPath.index("#")
 903            self.revision = self.depotPath[hashIdx:]
 904            self.depotPath = self.depotPath[0:hashIdx]
 905        elif len(self.previousDepotPath) == 0:
 906            self.revision = "#head"
 907
 908        if self.depotPath.endswith("..."):
 909            self.depotPath = self.depotPath[:-3]
 910
 911        if not self.depotPath.endswith("/"):
 912            self.depotPath += "/"
 913
 914        self.getUserMap()
 915        self.labels = {}
 916        if self.detectLabels:
 917            self.getLabels();
 918
 919        if len(self.changeRange) == 0:
 920            try:
 921                sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch)
 922                output = sout.read()
 923                if output.endswith("\n"):
 924                    output = output[:-1]
 925                tagIdx = output.index(" tags/p4/")
 926                caretIdx = output.find("^")
 927                endPos = len(output)
 928                if caretIdx != -1:
 929                    endPos = caretIdx
 930                self.rev = int(output[tagIdx + 9 : endPos]) + 1
 931                self.changeRange = "@%s,#head" % self.rev
 932                self.initialParent = mypopen("git rev-parse %s" % self.branch).read()[:-1]
 933            except:
 934                pass
 935
 936        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
 937
 938        importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
 939        self.gitOutput = importProcess.stdout
 940        self.gitStream = importProcess.stdin
 941        self.gitError = importProcess.stderr
 942
 943        if len(self.revision) > 0:
 944            print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision)
 945
 946            details = { "user" : "git perforce import user", "time" : int(time.time()) }
 947            details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision)
 948            details["change"] = self.revision
 949            newestRevision = 0
 950
 951            fileCnt = 0
 952            for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)):
 953                change = int(info["change"])
 954                if change > newestRevision:
 955                    newestRevision = change
 956
 957                if info["action"] == "delete":
 958                    # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
 959                    #fileCnt = fileCnt + 1
 960                    continue
 961
 962                for prop in [ "depotFile", "rev", "action", "type" ]:
 963                    details["%s%s" % (prop, fileCnt)] = info[prop]
 964
 965                fileCnt = fileCnt + 1
 966
 967            details["change"] = newestRevision
 968
 969            try:
 970                self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath)
 971            except IOError:
 972                print "IO error with git fast-import. Is your git version recent enough?"
 973                print self.gitError.read()
 974
 975        else:
 976            changes = []
 977
 978            if len(self.changesFile) > 0:
 979                output = open(self.changesFile).readlines()
 980                changeSet = Set()
 981                for line in output:
 982                    changeSet.add(int(line))
 983
 984                for change in changeSet:
 985                    changes.append(change)
 986
 987                changes.sort()
 988            else:
 989                output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines()
 990
 991                for line in output:
 992                    changeNum = line.split(" ")[1]
 993                    changes.append(changeNum)
 994
 995                changes.reverse()
 996
 997            if len(changes) == 0:
 998                if not self.silent:
 999                    print "no changes to import!"
1000                return True
1001
1002            cnt = 1
1003            for change in changes:
1004                description = p4Cmd("describe %s" % change)
1005
1006                if not self.silent:
1007                    sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1008                    sys.stdout.flush()
1009                cnt = cnt + 1
1010
1011                try:
1012                    files = self.extractFilesFromCommit(description)
1013                    if self.detectBranches:
1014                        for branch in self.branchesForCommit(files):
1015                            self.knownBranches.add(branch)
1016                            branchPrefix = self.depotPath + branch + "/"
1017
1018                            filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix)
1019
1020                            merged = ""
1021                            parent = ""
1022                            ########### remove cnt!!!
1023                            if branch not in self.createdBranches and cnt > 2:
1024                                self.createdBranches.add(branch)
1025                                parent = self.findBranchParent(branchPrefix, files)
1026                                if parent == branch:
1027                                    parent = ""
1028            #                    elif len(parent) > 0:
1029            #                        print "%s branched off of %s" % (branch, parent)
1030
1031                            if len(parent) == 0:
1032                                merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix)
1033                                if len(merged) > 0:
1034                                    print "change %s could be a merge from %s into %s" % (description["change"], merged, branch)
1035                                    if not self.changeIsBranchMerge(merged, branch, int(description["change"])):
1036                                        merged = ""
1037
1038                            branch = "refs/heads/" + branch
1039                            if len(parent) > 0:
1040                                parent = "refs/heads/" + parent
1041                            if len(merged) > 0:
1042                                merged = "refs/heads/" + merged
1043                            self.commit(description, files, branch, branchPrefix, parent, merged)
1044                    else:
1045                        self.commit(description, files, self.branch, self.depotPath, self.initialParent)
1046                        self.initialParent = ""
1047                except IOError:
1048                    print self.gitError.read()
1049                    sys.exit(1)
1050
1051        if not self.silent:
1052            print ""
1053
1054
1055        self.gitStream.close()
1056        self.gitOutput.close()
1057        self.gitError.close()
1058        importProcess.wait()
1059
1060        return True
1061
1062class P4Rebase(Command):
1063    def __init__(self):
1064        Command.__init__(self)
1065        self.options = [ ]
1066        self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it"
1067
1068    def run(self, args):
1069        sync = P4Sync()
1070        sync.run([])
1071        print "Rebasing the current branch"
1072        oldHead = mypopen("git rev-parse HEAD").read()[:-1]
1073        system("git rebase p4")
1074        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1075        return True
1076
1077class P4Clone(P4Sync):
1078    def __init__(self):
1079        P4Sync.__init__(self)
1080        self.description = "Creates a new git repository and imports from Perforce into it"
1081        self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]"
1082        self.needsGit = False
1083
1084    def run(self, args):
1085        if len(args) < 1:
1086            return False
1087        depotPath = args[0]
1088        dir = ""
1089        if len(args) == 2:
1090            dir = args[1]
1091        elif len(args) > 2:
1092            return False
1093
1094        if not depotPath.startswith("//"):
1095            return False
1096
1097        if len(dir) == 0:
1098            dir = depotPath
1099            atPos = dir.rfind("@")
1100            if atPos != -1:
1101                dir = dir[0:atPos]
1102            hashPos = dir.rfind("#")
1103            if hashPos != -1:
1104                dir = dir[0:hashPos]
1105
1106            if dir.endswith("..."):
1107                dir = dir[:-3]
1108
1109            if dir.endswith("/"):
1110               dir = dir[:-1]
1111
1112            slashPos = dir.rfind("/")
1113            if slashPos != -1:
1114                dir = dir[slashPos + 1:]
1115
1116        print "Importing from %s into %s" % (depotPath, dir)
1117        os.makedirs(dir)
1118        os.chdir(dir)
1119        system("git init")
1120        if not P4Sync.run(self, [depotPath]):
1121            return False
1122        if self.branch != "master":
1123            system("git branch master p4")
1124            system("git checkout -f")
1125        return True
1126
1127class HelpFormatter(optparse.IndentedHelpFormatter):
1128    def __init__(self):
1129        optparse.IndentedHelpFormatter.__init__(self)
1130
1131    def format_description(self, description):
1132        if description:
1133            return description + "\n"
1134        else:
1135            return ""
1136
1137def printUsage(commands):
1138    print "usage: %s <command> [options]" % sys.argv[0]
1139    print ""
1140    print "valid commands: %s" % ", ".join(commands)
1141    print ""
1142    print "Try %s <command> --help for command specific help." % sys.argv[0]
1143    print ""
1144
1145commands = {
1146    "debug" : P4Debug(),
1147    "clean-tags" : P4CleanTags(),
1148    "submit" : P4Submit(),
1149    "sync" : P4Sync(),
1150    "rebase" : P4Rebase(),
1151    "clone" : P4Clone()
1152}
1153
1154if len(sys.argv[1:]) == 0:
1155    printUsage(commands.keys())
1156    sys.exit(2)
1157
1158cmd = ""
1159cmdName = sys.argv[1]
1160try:
1161    cmd = commands[cmdName]
1162except KeyError:
1163    print "unknown command %s" % cmdName
1164    print ""
1165    printUsage(commands.keys())
1166    sys.exit(2)
1167
1168options = cmd.options
1169cmd.gitdir = gitdir
1170
1171args = sys.argv[2:]
1172
1173if len(options) > 0:
1174    options.append(optparse.make_option("--git-dir", dest="gitdir"))
1175
1176    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1177                                   options,
1178                                   description = cmd.description,
1179                                   formatter = HelpFormatter())
1180
1181    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1182
1183if cmd.needsGit:
1184    gitdir = cmd.gitdir
1185    if len(gitdir) == 0:
1186        gitdir = ".git"
1187        if not isValidGitDir(gitdir):
1188            cdup = mypopen("git rev-parse --show-cdup").read()[:-1]
1189            if isValidGitDir(cdup + "/" + gitdir):
1190                os.chdir(cdup)
1191
1192    if not isValidGitDir(gitdir):
1193        if isValidGitDir(gitdir + "/.git"):
1194            gitdir += "/.git"
1195        else:
1196            die("fatal: cannot locate git repository at %s" % gitdir)
1197
1198    os.environ["GIT_DIR"] = gitdir
1199
1200if not cmd.run(args):
1201    parser.print_help()
1202