Andrew's git - gitweb.git/blob

b39548917a611de6980dabff84a60e35ac7bc72d
   1#!/usr/bin/python
   2#
   3# p4-fast-export.py
   4#
   5# Author: Simon Hausmann <hausmann@kde.org>
   6# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   7#
   8# TODO:
   9#       - support integrations (at least p4i)
  10#       - support p4 submit (hah!)
  11#       - emulate p4's delete behavior: if a directory becomes empty delete it. continue
  12#         with parent dir until non-empty dir is found.
  13#
  14import os, string, sys, time
  15import marshal, popen2, getopt
  16from sets import Set;
  17
  18silent = False
  19knownBranches = Set()
  20committedChanges = Set()
  21branch = "refs/heads/master"
  22globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
  23detectBranches = False
  24changesFile = ""
  25if len(globalPrefix) != 0:
  26    globalPrefix = globalPrefix[:-1]
  27
  28try:
  29    opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent" ])
  30except getopt.GetoptError:
  31    print "fixme, syntax error"
  32    sys.exit(1)
  33
  34for o, a in opts:
  35    if o == "--branch":
  36        branch = "refs/heads/" + a
  37    elif o == "--detect-branches":
  38        detectBranches = True
  39    elif o == "--changesfile":
  40        changesFile = a
  41    elif o == "--silent":
  42        silent= True
  43
  44if len(args) == 0 and len(globalPrefix) != 0:
  45    print "[using previously specified depot path %s]" % globalPrefix
  46elif len(args) != 1:
  47    print "usage: %s //depot/path[@revRange]" % sys.argv[0]
  48    print "\n    example:"
  49    print "    %s //depot/my/project/ -- to import the current head"
  50    print "    %s //depot/my/project/@all -- to import everything"
  51    print "    %s //depot/my/project/@1,6 -- to import only from revision 1 to 6"
  52    print ""
  53    print "    (a ... is not needed in the path p4 specification, it's added implicitly)"
  54    print ""
  55    sys.exit(1)
  56else:
  57    if len(globalPrefix) != 0 and globalPrefix != args[0]:
  58        print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0])
  59        sys.exit(1)
  60    globalPrefix = args[0]
  61
  62changeRange = ""
  63revision = ""
  64users = {}
  65initialParent = ""
  66lastChange = 0
  67initialTag = ""
  68
  69if globalPrefix.find("@") != -1:
  70    atIdx = globalPrefix.index("@")
  71    changeRange = globalPrefix[atIdx:]
  72    if changeRange == "@all":
  73        changeRange = ""
  74    elif changeRange.find(",") == -1:
  75        revision = changeRange
  76        changeRange = ""
  77    globalPrefix = globalPrefix[0:atIdx]
  78elif globalPrefix.find("#") != -1:
  79    hashIdx = globalPrefix.index("#")
  80    revision = globalPrefix[hashIdx:]
  81    globalPrefix = globalPrefix[0:hashIdx]
  82elif len(previousDepotPath) == 0:
  83    revision = "#head"
  84
  85if globalPrefix.endswith("..."):
  86    globalPrefix = globalPrefix[:-3]
  87
  88if not globalPrefix.endswith("/"):
  89    globalPrefix += "/"
  90
  91def p4CmdList(cmd):
  92    pipe = os.popen("p4 -G %s" % cmd, "rb")
  93    result = []
  94    try:
  95        while True:
  96            entry = marshal.load(pipe)
  97            result.append(entry)
  98    except EOFError:
  99        pass
 100    pipe.close()
 101    return result
 102
 103def p4Cmd(cmd):
 104    list = p4CmdList(cmd)
 105    result = {}
 106    for entry in list:
 107        result.update(entry)
 108    return result;
 109
 110def extractFilesFromCommit(commit):
 111    files = []
 112    fnum = 0
 113    while commit.has_key("depotFile%s" % fnum):
 114        path =  commit["depotFile%s" % fnum]
 115        if not path.startswith(globalPrefix):
 116            if not silent:
 117                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change)
 118            fnum = fnum + 1
 119            continue
 120
 121        file = {}
 122        file["path"] = path
 123        file["rev"] = commit["rev%s" % fnum]
 124        file["action"] = commit["action%s" % fnum]
 125        file["type"] = commit["type%s" % fnum]
 126        files.append(file)
 127        fnum = fnum + 1
 128    return files
 129
 130def isSubPathOf(first, second):
 131    if not first.startswith(second):
 132        return False
 133    if first == second:
 134        return True
 135    return first[len(second)] == "/"
 136
 137def branchesForCommit(files):
 138    global knownBranches
 139    branches = Set()
 140
 141    for file in files:
 142        relativePath = file["path"][len(globalPrefix):]
 143        # strip off the filename
 144        relativePath = relativePath[0:relativePath.rfind("/")]
 145
 146#        if len(branches) == 0:
 147#            branches.add(relativePath)
 148#            knownBranches.add(relativePath)
 149#            continue
 150
 151        ###### this needs more testing :)
 152        knownBranch = False
 153        for branch in branches:
 154            if relativePath == branch:
 155                knownBranch = True
 156                break
 157#            if relativePath.startswith(branch):
 158            if isSubPathOf(relativePath, branch):
 159                knownBranch = True
 160                break
 161#            if branch.startswith(relativePath):
 162            if isSubPathOf(branch, relativePath):
 163                branches.remove(branch)
 164                break
 165
 166        if knownBranch:
 167            continue
 168
 169        for branch in knownBranches:
 170            #if relativePath.startswith(branch):
 171            if isSubPathOf(relativePath, branch):
 172                if len(branches) == 0:
 173                    relativePath = branch
 174                else:
 175                    knownBranch = True
 176                break
 177
 178        if knownBranch:
 179            continue
 180
 181        branches.add(relativePath)
 182        knownBranches.add(relativePath)
 183
 184    return branches
 185
 186def commit(details, files, branch, branchPrefix):
 187    global initialParent
 188    global users
 189    global lastChange
 190    global committedChanges
 191
 192    epoch = details["time"]
 193    author = details["user"]
 194
 195    gitStream.write("commit %s\n" % branch)
 196    gitStream.write("mark :%s\n" % details["change"])
 197    committedChanges.add(int(details["change"]))
 198    committer = ""
 199    if author in users:
 200        committer = "%s %s %s" % (users[author], epoch, tz)
 201    else:
 202        committer = "%s <a@b> %s %s" % (author, epoch, tz)
 203
 204    gitStream.write("committer %s\n" % committer)
 205
 206    gitStream.write("data <<EOT\n")
 207    gitStream.write(details["desc"])
 208    gitStream.write("\n[ imported from %s; change %s ]\n" % (branchPrefix, details["change"]))
 209    gitStream.write("EOT\n\n")
 210
 211    if len(initialParent) > 0:
 212        gitStream.write("from %s\n" % initialParent)
 213        initialParent = ""
 214
 215    #mergedBranches = Set()
 216    merges = Set()
 217
 218    for file in files:
 219        if lastChange == 0 or not detectBranches:
 220            continue
 221        path = file["path"]
 222        if not path.startswith(branchPrefix):
 223            continue
 224        action = file["action"]
 225        if action != "integrate" and action != "branch":
 226            continue
 227        rev = file["rev"]
 228        depotPath = path + "#" + rev
 229
 230        log = p4CmdList("filelog \"%s\"" % depotPath)
 231        if len(log) != 1:
 232            print "eek! I got confused by the filelog of %s" % depotPath
 233            sys.exit(1);
 234
 235        log = log[0]
 236        if log["action0"] != action:
 237            print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 238            sys.exit(1);
 239
 240        branchAction = log["how0,0"]
 241#        if branchAction == "branch into" or branchAction == "ignored":
 242#            continue # ignore for branching
 243
 244        if not branchAction.endswith(" from"):
 245            continue # ignore for branching
 246#            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 247#            sys.exit(1);
 248
 249        source = log["file0,0"]
 250        if source.startswith(branchPrefix):
 251            continue
 252
 253        lastSourceRev = log["erev0,0"]
 254
 255        sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 256        if len(sourceLog) != 1:
 257            print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 258            sys.exit(1);
 259        sourceLog = sourceLog[0]
 260
 261        change = int(sourceLog["change0"])
 262        merges.add(change)
 263
 264#        relPath = source[len(globalPrefix):]
 265#
 266#        for branch in knownBranches:
 267#            if relPath.startswith(branch) and branch not in mergedBranches:
 268#                gitStream.write("merge refs/heads/%s\n" % branch)
 269#                mergedBranches.add(branch)
 270#                break
 271
 272    for merge in merges:
 273        if merge in committedChanges:
 274            gitStream.write("merge :%s\n" % merge)
 275
 276    for file in files:
 277        path = file["path"]
 278        if not path.startswith(branchPrefix):
 279            if not silent:
 280                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 281            continue
 282        rev = file["rev"]
 283        depotPath = path + "#" + rev
 284        relPath = path[len(branchPrefix):]
 285        action = file["action"]
 286
 287        if action == "delete":
 288            gitStream.write("D %s\n" % relPath)
 289        else:
 290            mode = 644
 291            if file["type"].startswith("x"):
 292                mode = 755
 293
 294            data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 295
 296            gitStream.write("M %s inline %s\n" % (mode, relPath))
 297            gitStream.write("data %s\n" % len(data))
 298            gitStream.write(data)
 299            gitStream.write("\n")
 300
 301    gitStream.write("\n")
 302
 303    lastChange = int(details["change"])
 304
 305def getUserMap():
 306    users = {}
 307
 308    for output in p4CmdList("users"):
 309        if not output.has_key("User"):
 310            continue
 311        users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 312    return users
 313
 314users = getUserMap()
 315
 316if len(changeRange) == 0:
 317    try:
 318        sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 319        output = sout.read()
 320        if output.endswith("\n"):
 321            output = output[:-1]
 322        tagIdx = output.index(" tags/p4/")
 323        caretIdx = output.find("^")
 324        endPos = len(output)
 325        if caretIdx != -1:
 326            endPos = caretIdx
 327        rev = int(output[tagIdx + 9 : endPos]) + 1
 328        changeRange = "@%s,#head" % rev
 329        initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1]
 330        initialTag = "p4/%s" % (int(rev) - 1)
 331    except:
 332        pass
 333
 334sys.stderr.write("\n")
 335
 336tz = - time.timezone / 36
 337tzsign = ("%s" % tz)[0]
 338if tzsign != '+' and tzsign != '-':
 339    tz = "+" + ("%s" % tz)
 340
 341gitOutput, gitStream, gitError = popen2.popen3("git-fast-import")
 342
 343if len(revision) > 0:
 344    print "Doing initial import of %s from revision %s" % (globalPrefix, revision)
 345
 346    details = { "user" : "git perforce import user", "time" : int(time.time()) }
 347    details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision)
 348    details["change"] = revision
 349    newestRevision = 0
 350
 351    fileCnt = 0
 352    for info in p4CmdList("files %s...%s" % (globalPrefix, revision)):
 353        change = int(info["change"])
 354        if change > newestRevision:
 355            newestRevision = change
 356
 357        if info["action"] == "delete":
 358            continue
 359
 360        for prop in [ "depotFile", "rev", "action", "type" ]:
 361            details["%s%s" % (prop, fileCnt)] = info[prop]
 362
 363        fileCnt = fileCnt + 1
 364
 365    details["change"] = newestRevision
 366
 367    try:
 368        commit(details, extractFilesFromCommit(details), branch, globalPrefix)
 369    except:
 370        print gitError.read()
 371
 372else:
 373    changes = []
 374
 375    if len(changesFile) > 0:
 376        output = open(changesFile).readlines()
 377        changeSet = Set()
 378        for line in output:
 379            changeSet.add(int(line))
 380
 381        for change in changeSet:
 382            changes.append(change)
 383
 384        changes.sort()
 385    else:
 386        output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines()
 387
 388        for line in output:
 389            changeNum = line.split(" ")[1]
 390            changes.append(changeNum)
 391
 392        changes.reverse()
 393
 394    if len(changes) == 0:
 395        if not silent:
 396            print "no changes to import!"
 397        sys.exit(1)
 398
 399    cnt = 1
 400    for change in changes:
 401        description = p4Cmd("describe %s" % change)
 402
 403        if not silent:
 404            sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 405            sys.stdout.flush()
 406        cnt = cnt + 1
 407
 408#        try:
 409        files = extractFilesFromCommit(description)
 410        if detectBranches:
 411            for branch in branchesForCommit(files):
 412                knownBranches.add(branch)
 413                branchPrefix = globalPrefix + branch + "/"
 414                branch = "refs/heads/" + branch
 415                commit(description, files, branch, branchPrefix)
 416        else:
 417            commit(description, files, branch, globalPrefix)
 418#        except:
 419#            print gitError.read()
 420#            sys.exit(1)
 421
 422if not silent:
 423    print ""
 424
 425gitStream.write("reset refs/tags/p4/%s\n" % lastChange)
 426gitStream.write("from %s\n\n" % branch);
 427
 428
 429gitStream.close()
 430gitOutput.close()
 431gitError.close()
 432
 433os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read()
 434if len(initialTag) > 0:
 435    os.popen("git tag -d %s" % initialTag).read()
 436
 437sys.exit(0)