Andrew's git - gitweb.git/blob - contrib/fast-import/p4-fast-export.py

   1#!/usr/bin/python
   2#
   3# p4-fast-export.py
   4#
   5# Author: Simon Hausmann <hausmann@kde.org>
   6# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   7#
   8# TODO:
   9#       - support integrations (at least p4i)
  10#       - support p4 submit (hah!)
  11#       - emulate p4's delete behavior: if a directory becomes empty delete it. continue
  12#         with parent dir until non-empty dir is found.
  13#
  14import os, string, sys, time, os.path
  15import marshal, popen2, getopt, sha
  16from sets import Set;
  17
  18cacheDebug = False
  19
  20silent = False
  21knownBranches = Set()
  22createdBranches = Set()
  23committedChanges = Set()
  24branch = "refs/heads/master"
  25globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
  26detectBranches = False
  27changesFile = ""
  28if len(globalPrefix) != 0:
  29    globalPrefix = globalPrefix[:-1]
  30
  31try:
  32    opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=" ])
  33except getopt.GetoptError:
  34    print "fixme, syntax error"
  35    sys.exit(1)
  36
  37for o, a in opts:
  38    if o == "--branch":
  39        branch = "refs/heads/" + a
  40    elif o == "--detect-branches":
  41        detectBranches = True
  42    elif o == "--changesfile":
  43        changesFile = a
  44    elif o == "--silent":
  45        silent= True
  46    elif o == "--known-branches":
  47        for branch in open(a).readlines():
  48            knownBranches.add(branch[:-1])
  49
  50if len(args) == 0 and len(globalPrefix) != 0:
  51    if not silent:
  52        print "[using previously specified depot path %s]" % globalPrefix
  53elif len(args) != 1:
  54    print "usage: %s //depot/path[@revRange]" % sys.argv[0]
  55    print "\n    example:"
  56    print "    %s //depot/my/project/ -- to import the current head"
  57    print "    %s //depot/my/project/@all -- to import everything"
  58    print "    %s //depot/my/project/@1,6 -- to import only from revision 1 to 6"
  59    print ""
  60    print "    (a ... is not needed in the path p4 specification, it's added implicitly)"
  61    print ""
  62    sys.exit(1)
  63else:
  64    if len(globalPrefix) != 0 and globalPrefix != args[0]:
  65        print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0])
  66        sys.exit(1)
  67    globalPrefix = args[0]
  68
  69changeRange = ""
  70revision = ""
  71users = {}
  72initialParent = ""
  73lastChange = 0
  74initialTag = ""
  75
  76if globalPrefix.find("@") != -1:
  77    atIdx = globalPrefix.index("@")
  78    changeRange = globalPrefix[atIdx:]
  79    if changeRange == "@all":
  80        changeRange = ""
  81    elif changeRange.find(",") == -1:
  82        revision = changeRange
  83        changeRange = ""
  84    globalPrefix = globalPrefix[0:atIdx]
  85elif globalPrefix.find("#") != -1:
  86    hashIdx = globalPrefix.index("#")
  87    revision = globalPrefix[hashIdx:]
  88    globalPrefix = globalPrefix[0:hashIdx]
  89elif len(previousDepotPath) == 0:
  90    revision = "#head"
  91
  92if globalPrefix.endswith("..."):
  93    globalPrefix = globalPrefix[:-3]
  94
  95if not globalPrefix.endswith("/"):
  96    globalPrefix += "/"
  97
  98def p4File(depotPath):
  99    cacheKey = "/tmp/p4cache/data-" + sha.new(depotPath).hexdigest()
 100
 101    data = 0
 102    try:
 103        if not cacheDebug:
 104            raise
 105        data = open(cacheKey, "rb").read()
 106    except:
 107        data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 108        if cacheDebug:
 109            open(cacheKey, "wb").write(data)
 110
 111    return data
 112
 113def p4CmdList(cmd):
 114    fullCmd = "p4 -G %s" % cmd;
 115
 116    cacheKey = sha.new(fullCmd).hexdigest()
 117    cacheKey = "/tmp/p4cache/cmd-" + cacheKey
 118
 119    cached = True
 120    pipe = 0
 121    try:
 122        if not cacheDebug:
 123            raise
 124        pipe = open(cacheKey, "rb")
 125    except:
 126        cached = False
 127        pipe = os.popen(fullCmd, "rb")
 128
 129    result = []
 130    try:
 131        while True:
 132            entry = marshal.load(pipe)
 133            result.append(entry)
 134    except EOFError:
 135        pass
 136    pipe.close()
 137
 138    if not cached and cacheDebug:
 139        pipe = open(cacheKey, "wb")
 140        for r in result:
 141            marshal.dump(r, pipe)
 142        pipe.close()
 143
 144    return result
 145
 146def p4Cmd(cmd):
 147    list = p4CmdList(cmd)
 148    result = {}
 149    for entry in list:
 150        result.update(entry)
 151    return result;
 152
 153def extractFilesFromCommit(commit):
 154    files = []
 155    fnum = 0
 156    while commit.has_key("depotFile%s" % fnum):
 157        path =  commit["depotFile%s" % fnum]
 158        if not path.startswith(globalPrefix):
 159#            if not silent:
 160#                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change)
 161            fnum = fnum + 1
 162            continue
 163
 164        file = {}
 165        file["path"] = path
 166        file["rev"] = commit["rev%s" % fnum]
 167        file["action"] = commit["action%s" % fnum]
 168        file["type"] = commit["type%s" % fnum]
 169        files.append(file)
 170        fnum = fnum + 1
 171    return files
 172
 173def isSubPathOf(first, second):
 174    if not first.startswith(second):
 175        return False
 176    if first == second:
 177        return True
 178    return first[len(second)] == "/"
 179
 180def branchesForCommit(files):
 181    global knownBranches
 182    branches = Set()
 183
 184    for file in files:
 185        relativePath = file["path"][len(globalPrefix):]
 186        # strip off the filename
 187        relativePath = relativePath[0:relativePath.rfind("/")]
 188
 189#        if len(branches) == 0:
 190#            branches.add(relativePath)
 191#            knownBranches.add(relativePath)
 192#            continue
 193
 194        ###### this needs more testing :)
 195        knownBranch = False
 196        for branch in branches:
 197            if relativePath == branch:
 198                knownBranch = True
 199                break
 200#            if relativePath.startswith(branch):
 201            if isSubPathOf(relativePath, branch):
 202                knownBranch = True
 203                break
 204#            if branch.startswith(relativePath):
 205            if isSubPathOf(branch, relativePath):
 206                branches.remove(branch)
 207                break
 208
 209        if knownBranch:
 210            continue
 211
 212        for branch in knownBranches:
 213            #if relativePath.startswith(branch):
 214            if isSubPathOf(relativePath, branch):
 215                if len(branches) == 0:
 216                    relativePath = branch
 217                else:
 218                    knownBranch = True
 219                break
 220
 221        if knownBranch:
 222            continue
 223
 224        branches.add(relativePath)
 225        knownBranches.add(relativePath)
 226
 227    return branches
 228
 229def findBranchParent(branchPrefix, files):
 230    for file in files:
 231        path = file["path"]
 232        if not path.startswith(branchPrefix):
 233            continue
 234        action = file["action"]
 235        if action != "integrate" and action != "branch":
 236            continue
 237        rev = file["rev"]
 238        depotPath = path + "#" + rev
 239
 240        log = p4CmdList("filelog \"%s\"" % depotPath)
 241        if len(log) != 1:
 242            print "eek! I got confused by the filelog of %s" % depotPath
 243            sys.exit(1);
 244
 245        log = log[0]
 246        if log["action0"] != action:
 247            print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 248            sys.exit(1);
 249
 250        branchAction = log["how0,0"]
 251#        if branchAction == "branch into" or branchAction == "ignored":
 252#            continue # ignore for branching
 253
 254        if not branchAction.endswith(" from"):
 255            continue # ignore for branching
 256#            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 257#            sys.exit(1);
 258
 259        source = log["file0,0"]
 260        if source.startswith(branchPrefix):
 261            continue
 262
 263        lastSourceRev = log["erev0,0"]
 264
 265        sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 266        if len(sourceLog) != 1:
 267            print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 268            sys.exit(1);
 269        sourceLog = sourceLog[0]
 270
 271        relPath = source[len(globalPrefix):]
 272        # strip off the filename
 273        relPath = relPath[0:relPath.rfind("/")]
 274
 275        for branch in knownBranches:
 276            if isSubPathOf(relPath, branch):
 277#                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 278                return "refs/heads/%s" % branch
 279#            else:
 280#                print "%s is not a subpath of branch %s" % (relPath, branch)
 281
 282    return ""
 283
 284def commit(details, files, branch, branchPrefix, parent):
 285    global users
 286    global lastChange
 287    global committedChanges
 288
 289    epoch = details["time"]
 290    author = details["user"]
 291
 292    gitStream.write("commit %s\n" % branch)
 293    gitStream.write("mark :%s\n" % details["change"])
 294    committedChanges.add(int(details["change"]))
 295    committer = ""
 296    if author in users:
 297        committer = "%s %s %s" % (users[author], epoch, tz)
 298    else:
 299        committer = "%s <a@b> %s %s" % (author, epoch, tz)
 300
 301    gitStream.write("committer %s\n" % committer)
 302
 303    gitStream.write("data <<EOT\n")
 304    gitStream.write(details["desc"])
 305    gitStream.write("\n[ imported from %s; change %s ]\n" % (branchPrefix, details["change"]))
 306    gitStream.write("EOT\n\n")
 307
 308    if len(parent) > 0:
 309        gitStream.write("from %s\n" % parent)
 310
 311    for file in files:
 312        path = file["path"]
 313        if not path.startswith(branchPrefix):
 314#            if not silent:
 315#                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 316            continue
 317        rev = file["rev"]
 318        depotPath = path + "#" + rev
 319        relPath = path[len(branchPrefix):]
 320        action = file["action"]
 321
 322        if action == "delete":
 323            gitStream.write("D %s\n" % relPath)
 324        else:
 325            mode = 644
 326            if file["type"].startswith("x"):
 327                mode = 755
 328
 329            data = p4File(depotPath)
 330
 331            gitStream.write("M %s inline %s\n" % (mode, relPath))
 332            gitStream.write("data %s\n" % len(data))
 333            gitStream.write(data)
 334            gitStream.write("\n")
 335
 336    gitStream.write("\n")
 337
 338    lastChange = int(details["change"])
 339
 340def getUserMap():
 341    users = {}
 342
 343    for output in p4CmdList("users"):
 344        if not output.has_key("User"):
 345            continue
 346        users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 347    return users
 348
 349users = getUserMap()
 350
 351if len(changeRange) == 0:
 352    try:
 353        sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 354        output = sout.read()
 355        if output.endswith("\n"):
 356            output = output[:-1]
 357        tagIdx = output.index(" tags/p4/")
 358        caretIdx = output.find("^")
 359        endPos = len(output)
 360        if caretIdx != -1:
 361            endPos = caretIdx
 362        rev = int(output[tagIdx + 9 : endPos]) + 1
 363        changeRange = "@%s,#head" % rev
 364        initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1]
 365        initialTag = "p4/%s" % (int(rev) - 1)
 366    except:
 367        pass
 368
 369tz = - time.timezone / 36
 370tzsign = ("%s" % tz)[0]
 371if tzsign != '+' and tzsign != '-':
 372    tz = "+" + ("%s" % tz)
 373
 374gitOutput, gitStream, gitError = popen2.popen3("git-fast-import")
 375
 376if len(revision) > 0:
 377    print "Doing initial import of %s from revision %s" % (globalPrefix, revision)
 378
 379    details = { "user" : "git perforce import user", "time" : int(time.time()) }
 380    details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision)
 381    details["change"] = revision
 382    newestRevision = 0
 383
 384    fileCnt = 0
 385    for info in p4CmdList("files %s...%s" % (globalPrefix, revision)):
 386        change = int(info["change"])
 387        if change > newestRevision:
 388            newestRevision = change
 389
 390        if info["action"] == "delete":
 391            continue
 392
 393        for prop in [ "depotFile", "rev", "action", "type" ]:
 394            details["%s%s" % (prop, fileCnt)] = info[prop]
 395
 396        fileCnt = fileCnt + 1
 397
 398    details["change"] = newestRevision
 399
 400    try:
 401        commit(details, extractFilesFromCommit(details), branch, globalPrefix)
 402    except:
 403        print gitError.read()
 404
 405else:
 406    changes = []
 407
 408    if len(changesFile) > 0:
 409        output = open(changesFile).readlines()
 410        changeSet = Set()
 411        for line in output:
 412            changeSet.add(int(line))
 413
 414        for change in changeSet:
 415            changes.append(change)
 416
 417        changes.sort()
 418    else:
 419        output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines()
 420
 421        for line in output:
 422            changeNum = line.split(" ")[1]
 423            changes.append(changeNum)
 424
 425        changes.reverse()
 426
 427    if len(changes) == 0:
 428        if not silent:
 429            print "no changes to import!"
 430        sys.exit(1)
 431
 432    cnt = 1
 433    for change in changes:
 434        description = p4Cmd("describe %s" % change)
 435
 436        if not silent:
 437            sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 438            sys.stdout.flush()
 439        cnt = cnt + 1
 440
 441#        try:
 442        files = extractFilesFromCommit(description)
 443        if detectBranches:
 444            for branch in branchesForCommit(files):
 445                knownBranches.add(branch)
 446                branchPrefix = globalPrefix + branch + "/"
 447
 448                parent = ""
 449                ########### remove cnt!!!
 450                if branch not in createdBranches and cnt > 2:
 451                    createdBranches.add(branch)
 452                    parent = findBranchParent(branchPrefix, files)
 453                    if parent == branch:
 454                        parent = ""
 455#                    elif len(parent) > 0:
 456#                        print "%s branched off of %s" % (branch, parent)
 457
 458                branch = "refs/heads/" + branch
 459                commit(description, files, branch, branchPrefix, parent)
 460        else:
 461            commit(description, files, branch, globalPrefix, initialParent)
 462            initialParent = ""
 463#        except:
 464#            print gitError.read()
 465#            sys.exit(1)
 466
 467if not silent:
 468    print ""
 469
 470gitStream.write("reset refs/tags/p4/%s\n" % lastChange)
 471gitStream.write("from %s\n\n" % branch);
 472
 473
 474gitStream.close()
 475gitOutput.close()
 476gitError.close()
 477
 478os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read()
 479if len(initialTag) > 0:
 480    os.popen("git tag -d %s" % initialTag).read()
 481
 482sys.exit(0)