Andrew's git - gitweb.git/blob - contrib/fast-import/p4-fast-export.py

   1#!/usr/bin/python
   2#
   3# p4-fast-export.py
   4#
   5# Author: Simon Hausmann <hausmann@kde.org>
   6# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   7#
   8# TODO:
   9#       - support integrations (at least p4i)
  10#       - support p4 submit (hah!)
  11#
  12import os, string, sys, time
  13import marshal, popen2, getopt
  14from sets import Set;
  15
  16knownBranches = Set()
  17committedChanges = Set()
  18branch = "refs/heads/master"
  19globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
  20detectBranches = False
  21changesFile = ""
  22if len(globalPrefix) != 0:
  23    globalPrefix = globalPrefix[:-1]
  24
  25try:
  26    opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=" ])
  27except getopt.GetoptError:
  28    print "fixme, syntax error"
  29    sys.exit(1)
  30
  31for o, a in opts:
  32    if o == "--branch":
  33        branch = "refs/heads/" + a
  34    elif o == "--detect-branches":
  35        detectBranches = True
  36    elif o == "--changesfile":
  37        changesFile = a
  38
  39if len(args) == 0 and len(globalPrefix) != 0:
  40    print "[using previously specified depot path %s]" % globalPrefix
  41elif len(args) != 1:
  42    print "usage: %s //depot/path[@revRange]" % sys.argv[0]
  43    print "\n    example:"
  44    print "    %s //depot/my/project/ -- to import the current head"
  45    print "    %s //depot/my/project/@all -- to import everything"
  46    print "    %s //depot/my/project/@1,6 -- to import only from revision 1 to 6"
  47    print ""
  48    print "    (a ... is not needed in the path p4 specification, it's added implicitly)"
  49    print ""
  50    sys.exit(1)
  51else:
  52    if len(globalPrefix) != 0 and globalPrefix != args[0]:
  53        print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0])
  54        sys.exit(1)
  55    globalPrefix = args[0]
  56
  57changeRange = ""
  58revision = ""
  59users = {}
  60initialParent = ""
  61lastChange = 0
  62initialTag = ""
  63
  64if globalPrefix.find("@") != -1:
  65    atIdx = globalPrefix.index("@")
  66    changeRange = globalPrefix[atIdx:]
  67    if changeRange == "@all":
  68        changeRange = ""
  69    elif changeRange.find(",") == -1:
  70        revision = changeRange
  71        changeRange = ""
  72    globalPrefix = globalPrefix[0:atIdx]
  73elif globalPrefix.find("#") != -1:
  74    hashIdx = globalPrefix.index("#")
  75    revision = globalPrefix[hashIdx:]
  76    globalPrefix = globalPrefix[0:hashIdx]
  77elif len(previousDepotPath) == 0:
  78    revision = "#head"
  79
  80if globalPrefix.endswith("..."):
  81    globalPrefix = globalPrefix[:-3]
  82
  83if not globalPrefix.endswith("/"):
  84    globalPrefix += "/"
  85
  86def p4CmdList(cmd):
  87    pipe = os.popen("p4 -G %s" % cmd, "rb")
  88    result = []
  89    try:
  90        while True:
  91            entry = marshal.load(pipe)
  92            result.append(entry)
  93    except EOFError:
  94        pass
  95    pipe.close()
  96    return result
  97
  98def p4Cmd(cmd):
  99    list = p4CmdList(cmd)
 100    result = {}
 101    for entry in list:
 102        result.update(entry)
 103    return result;
 104
 105def extractFilesFromCommit(commit):
 106    files = []
 107    fnum = 0
 108    while commit.has_key("depotFile%s" % fnum):
 109        path =  commit["depotFile%s" % fnum]
 110        if not path.startswith(globalPrefix):
 111            print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change)
 112            fnum = fnum + 1
 113            continue
 114
 115        file = {}
 116        file["path"] = path
 117        file["rev"] = commit["rev%s" % fnum]
 118        file["action"] = commit["action%s" % fnum]
 119        file["type"] = commit["type%s" % fnum]
 120        files.append(file)
 121        fnum = fnum + 1
 122    return files
 123
 124def isSubPathOf(first, second):
 125    if not first.startswith(second):
 126        return False
 127    if first == second:
 128        return True
 129    return first[len(second)] == "/"
 130
 131def branchesForCommit(files):
 132    global knownBranches
 133    branches = Set()
 134
 135    for file in files:
 136        relativePath = file["path"][len(globalPrefix):]
 137        # strip off the filename
 138        relativePath = relativePath[0:relativePath.rfind("/")]
 139
 140#        if len(branches) == 0:
 141#            branches.add(relativePath)
 142#            knownBranches.add(relativePath)
 143#            continue
 144
 145        ###### this needs more testing :)
 146        knownBranch = False
 147        for branch in branches:
 148            if relativePath == branch:
 149                knownBranch = True
 150                break
 151#            if relativePath.startswith(branch):
 152            if isSubPathOf(relativePath, branch):
 153                knownBranch = True
 154                break
 155#            if branch.startswith(relativePath):
 156            if isSubPathOf(branch, relativePath):
 157                branches.remove(branch)
 158                break
 159
 160        if knownBranch:
 161            continue
 162
 163        for branch in knownBranches:
 164            #if relativePath.startswith(branch):
 165            if isSubPathOf(relativePath, branch):
 166                if len(branches) == 0:
 167                    relativePath = branch
 168                else:
 169                    knownBranch = True
 170                break
 171
 172        if knownBranch:
 173            continue
 174
 175        branches.add(relativePath)
 176        knownBranches.add(relativePath)
 177
 178    return branches
 179
 180def commit(details, files, branch, branchPrefix):
 181    global initialParent
 182    global users
 183    global lastChange
 184    global committedChanges
 185
 186    epoch = details["time"]
 187    author = details["user"]
 188
 189    gitStream.write("commit %s\n" % branch)
 190    gitStream.write("mark :%s\n" % details["change"])
 191    committedChanges.add(int(details["change"]))
 192    committer = ""
 193    if author in users:
 194        committer = "%s %s %s" % (users[author], epoch, tz)
 195    else:
 196        committer = "%s <a@b> %s %s" % (author, epoch, tz)
 197
 198    gitStream.write("committer %s\n" % committer)
 199
 200    gitStream.write("data <<EOT\n")
 201    gitStream.write(details["desc"])
 202    gitStream.write("\n[ imported from %s; change %s ]\n" % (branchPrefix, details["change"]))
 203    gitStream.write("EOT\n\n")
 204
 205    if len(initialParent) > 0:
 206        gitStream.write("from %s\n" % initialParent)
 207        initialParent = ""
 208
 209    #mergedBranches = Set()
 210    merges = Set()
 211
 212    for file in files:
 213        if lastChange == 0 or not detectBranches:
 214            continue
 215        path = file["path"]
 216        if not path.startswith(branchPrefix):
 217            continue
 218        action = file["action"]
 219        if action != "integrate" and action != "branch":
 220            continue
 221        rev = file["rev"]
 222        depotPath = path + "#" + rev
 223
 224        log = p4CmdList("filelog \"%s\"" % depotPath)
 225        if len(log) != 1:
 226            print "eek! I got confused by the filelog of %s" % depotPath
 227            sys.exit(1);
 228
 229        log = log[0]
 230        if log["action0"] != action:
 231            print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 232            sys.exit(1);
 233
 234        branchAction = log["how0,0"]
 235#        if branchAction == "branch into" or branchAction == "ignored":
 236#            continue # ignore for branching
 237
 238        if not branchAction.endswith(" from"):
 239            continue # ignore for branching
 240#            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 241#            sys.exit(1);
 242
 243        source = log["file0,0"]
 244        if source.startswith(branchPrefix):
 245            continue
 246
 247        lastSourceRev = log["erev0,0"]
 248
 249        sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 250        if len(sourceLog) != 1:
 251            print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 252            sys.exit(1);
 253        sourceLog = sourceLog[0]
 254
 255        change = int(sourceLog["change0"])
 256        merges.add(change)
 257
 258#        relPath = source[len(globalPrefix):]
 259#
 260#        for branch in knownBranches:
 261#            if relPath.startswith(branch) and branch not in mergedBranches:
 262#                gitStream.write("merge refs/heads/%s\n" % branch)
 263#                mergedBranches.add(branch)
 264#                break
 265
 266    for merge in merges:
 267        if merge in committedChanges:
 268            gitStream.write("merge :%s\n" % merge)
 269
 270    for file in files:
 271        path = file["path"]
 272        if not path.startswith(branchPrefix):
 273            print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 274            continue
 275        rev = file["rev"]
 276        depotPath = path + "#" + rev
 277        relPath = path[len(branchPrefix):]
 278        action = file["action"]
 279
 280        if action == "delete":
 281            gitStream.write("D %s\n" % relPath)
 282        else:
 283            mode = 644
 284            if file["type"].startswith("x"):
 285                mode = 755
 286
 287            data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 288
 289            gitStream.write("M %s inline %s\n" % (mode, relPath))
 290            gitStream.write("data %s\n" % len(data))
 291            gitStream.write(data)
 292            gitStream.write("\n")
 293
 294    gitStream.write("\n")
 295
 296    lastChange = int(details["change"])
 297
 298def getUserMap():
 299    users = {}
 300
 301    for output in p4CmdList("users"):
 302        if not output.has_key("User"):
 303            continue
 304        users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 305    return users
 306
 307users = getUserMap()
 308
 309if len(changeRange) == 0:
 310    try:
 311        sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 312        output = sout.read()
 313        if output.endswith("\n"):
 314            output = output[:-1]
 315        tagIdx = output.index(" tags/p4/")
 316        caretIdx = output.find("^")
 317        endPos = len(output)
 318        if caretIdx != -1:
 319            endPos = caretIdx
 320        rev = int(output[tagIdx + 9 : endPos]) + 1
 321        changeRange = "@%s,#head" % rev
 322        initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1]
 323        initialTag = "p4/%s" % (int(rev) - 1)
 324    except:
 325        pass
 326
 327sys.stderr.write("\n")
 328
 329tz = - time.timezone / 36
 330tzsign = ("%s" % tz)[0]
 331if tzsign != '+' and tzsign != '-':
 332    tz = "+" + ("%s" % tz)
 333
 334gitOutput, gitStream, gitError = popen2.popen3("git-fast-import")
 335
 336if len(revision) > 0:
 337    print "Doing initial import of %s from revision %s" % (globalPrefix, revision)
 338
 339    details = { "user" : "git perforce import user", "time" : int(time.time()) }
 340    details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision)
 341    details["change"] = revision
 342    newestRevision = 0
 343
 344    fileCnt = 0
 345    for info in p4CmdList("files %s...%s" % (globalPrefix, revision)):
 346        change = int(info["change"])
 347        if change > newestRevision:
 348            newestRevision = change
 349
 350        if info["action"] == "delete":
 351            continue
 352
 353        for prop in [ "depotFile", "rev", "action", "type" ]:
 354            details["%s%s" % (prop, fileCnt)] = info[prop]
 355
 356        fileCnt = fileCnt + 1
 357
 358    details["change"] = newestRevision
 359
 360    try:
 361        commit(details, extractFilesFromCommit(details), branch, globalPrefix)
 362    except:
 363        print gitError.read()
 364
 365else:
 366    changes = []
 367
 368    if len(changesFile) > 0:
 369        output = open(changesFile).readlines()
 370        changeSet = Set()
 371        for line in output:
 372            changeSet.add(int(line))
 373
 374        for change in changeSet:
 375            changes.append(change)
 376
 377        changes.sort()
 378    else:
 379        output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines()
 380
 381        for line in output:
 382            changeNum = line.split(" ")[1]
 383            changes.append(changeNum)
 384
 385        changes.reverse()
 386
 387    if len(changes) == 0:
 388        print "no changes to import!"
 389        sys.exit(1)
 390
 391    cnt = 1
 392    for change in changes:
 393        description = p4Cmd("describe %s" % change)
 394
 395        sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 396        sys.stdout.flush()
 397        cnt = cnt + 1
 398
 399#        try:
 400        files = extractFilesFromCommit(description)
 401        if detectBranches:
 402            for branch in branchesForCommit(files):
 403                knownBranches.add(branch)
 404                branchPrefix = globalPrefix + branch + "/"
 405                branch = "refs/heads/" + branch
 406                commit(description, files, branch, branchPrefix)
 407        else:
 408            commit(description, files, branch, globalPrefix)
 409#        except:
 410#            print gitError.read()
 411#            sys.exit(1)
 412
 413print ""
 414
 415gitStream.write("reset refs/tags/p4/%s\n" % lastChange)
 416gitStream.write("from %s\n\n" % branch);
 417
 418
 419gitStream.close()
 420gitOutput.close()
 421gitError.close()
 422
 423os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read()
 424if len(initialTag) > 0:
 425    os.popen("git tag -d %s" % initialTag).read()
 426
 427sys.exit(0)