Andrew's git - gitweb.git/blob - contrib/remote-helpers/git-remote-hg

   1#!/usr/bin/env python
   2#
   3# Copyright (c) 2012 Felipe Contreras
   4#
   5
   6# Inspired by Rocco Rutte's hg-fast-export
   7
   8# Just copy to your ~/bin, or anywhere in your $PATH.
   9# Then you can clone with:
  10# git clone hg::/path/to/mercurial/repo/
  11
  12from mercurial import hg, ui, bookmarks, context
  13
  14import re
  15import sys
  16import os
  17import json
  18
  19NAME_RE = re.compile('^([^<>]+)')
  20AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]+)>$')
  21RAW_AUTHOR_RE = re.compile('^(\w+) (?:(.+)? )?<(.+)> (\d+) ([+-]\d+)')
  22
  23def die(msg, *args):
  24    sys.stderr.write('ERROR: %s\n' % (msg % args))
  25    sys.exit(1)
  26
  27def warn(msg, *args):
  28    sys.stderr.write('WARNING: %s\n' % (msg % args))
  29
  30def gitmode(flags):
  31    return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
  32
  33def gittz(tz):
  34    return '%+03d%02d' % (-tz / 3600, -tz % 3600 / 60)
  35
  36def hgmode(mode):
  37    m = { '0100755': 'x', '0120000': 'l' }
  38    return m.get(mode, '')
  39
  40class Marks:
  41
  42    def __init__(self, path):
  43        self.path = path
  44        self.tips = {}
  45        self.marks = {}
  46        self.rev_marks = {}
  47        self.last_mark = 0
  48
  49        self.load()
  50
  51    def load(self):
  52        if not os.path.exists(self.path):
  53            return
  54
  55        tmp = json.load(open(self.path))
  56
  57        self.tips = tmp['tips']
  58        self.marks = tmp['marks']
  59        self.last_mark = tmp['last-mark']
  60
  61        for rev, mark in self.marks.iteritems():
  62            self.rev_marks[mark] = int(rev)
  63
  64    def dict(self):
  65        return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  66
  67    def store(self):
  68        json.dump(self.dict(), open(self.path, 'w'))
  69
  70    def __str__(self):
  71        return str(self.dict())
  72
  73    def from_rev(self, rev):
  74        return self.marks[str(rev)]
  75
  76    def to_rev(self, mark):
  77        return self.rev_marks[mark]
  78
  79    def get_mark(self, rev):
  80        self.last_mark += 1
  81        self.marks[str(rev)] = self.last_mark
  82        return self.last_mark
  83
  84    def new_mark(self, rev, mark):
  85        self.marks[str(rev)] = mark
  86        self.rev_marks[mark] = rev
  87        self.last_mark = mark
  88
  89    def is_marked(self, rev):
  90        return self.marks.has_key(str(rev))
  91
  92    def get_tip(self, branch):
  93        return self.tips.get(branch, 0)
  94
  95    def set_tip(self, branch, tip):
  96        self.tips[branch] = tip
  97
  98class Parser:
  99
 100    def __init__(self, repo):
 101        self.repo = repo
 102        self.line = self.get_line()
 103
 104    def get_line(self):
 105        return sys.stdin.readline().strip()
 106
 107    def __getitem__(self, i):
 108        return self.line.split()[i]
 109
 110    def check(self, word):
 111        return self.line.startswith(word)
 112
 113    def each_block(self, separator):
 114        while self.line != separator:
 115            yield self.line
 116            self.line = self.get_line()
 117
 118    def __iter__(self):
 119        return self.each_block('')
 120
 121    def next(self):
 122        self.line = self.get_line()
 123        if self.line == 'done':
 124            self.line = None
 125
 126    def get_mark(self):
 127        i = self.line.index(':') + 1
 128        return int(self.line[i:])
 129
 130    def get_data(self):
 131        if not self.check('data'):
 132            return None
 133        i = self.line.index(' ') + 1
 134        size = int(self.line[i:])
 135        return sys.stdin.read(size)
 136
 137    def get_author(self):
 138        m = RAW_AUTHOR_RE.match(self.line)
 139        if not m:
 140            return None
 141        _, name, email, date, tz = m.groups()
 142
 143        if email != 'unknown':
 144            if name:
 145                user = '%s <%s>' % (name, email)
 146            else:
 147                user = '<%s>' % (email)
 148        else:
 149            user = name
 150
 151        tz = int(tz)
 152        tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 153        return (user, int(date), -tz)
 154
 155def export_file(fc):
 156    d = fc.data()
 157    print "M %s inline %s" % (gitmode(fc.flags()), fc.path())
 158    print "data %d" % len(d)
 159    print d
 160
 161def get_filechanges(repo, ctx, parent):
 162    modified = set()
 163    added = set()
 164    removed = set()
 165
 166    cur = ctx.manifest()
 167    prev = repo[parent].manifest().copy()
 168
 169    for fn in cur:
 170        if fn in prev:
 171            if (cur.flags(fn) != prev.flags(fn) or cur[fn] != prev[fn]):
 172                modified.add(fn)
 173            del prev[fn]
 174        else:
 175            added.add(fn)
 176    removed |= set(prev.keys())
 177
 178    return added | modified, removed
 179
 180def fixup_user(user):
 181    user = user.replace('"', '')
 182    name = mail = None
 183    m = AUTHOR_RE.match(user)
 184    if m:
 185        name = m.group(1)
 186        mail = m.group(2).strip()
 187    else:
 188        m = NAME_RE.match(user)
 189        if m:
 190            name = m.group(1).strip()
 191
 192    if not name:
 193        name = 'Unknown'
 194    if not mail:
 195        mail = 'unknown'
 196
 197    return '%s <%s>' % (name, mail)
 198
 199def get_repo(url, alias):
 200    global dirname, peer
 201
 202    myui = ui.ui()
 203    myui.setconfig('ui', 'interactive', 'off')
 204
 205    if hg.islocal(url):
 206        repo = hg.repository(myui, url)
 207    else:
 208        local_path = os.path.join(dirname, 'clone')
 209        if not os.path.exists(local_path):
 210            peer, dstpeer = hg.clone(myui, {}, url, local_path, update=False, pull=True)
 211            repo = dstpeer.local()
 212        else:
 213            repo = hg.repository(myui, local_path)
 214            peer = hg.peer(myui, {}, url)
 215            repo.pull(peer, heads=None, force=True)
 216
 217    return repo
 218
 219def rev_to_mark(rev):
 220    global marks
 221    return marks.from_rev(rev)
 222
 223def mark_to_rev(mark):
 224    global marks
 225    return marks.to_rev(mark)
 226
 227def export_ref(repo, name, kind, head):
 228    global prefix, marks
 229
 230    ename = '%s/%s' % (kind, name)
 231    tip = marks.get_tip(ename)
 232
 233    # mercurial takes too much time checking this
 234    if tip and tip == head.rev():
 235        # nothing to do
 236        return
 237    revs = repo.revs('%u:%u' % (tip, head))
 238    count = 0
 239
 240    revs = [rev for rev in revs if not marks.is_marked(rev)]
 241
 242    for rev in revs:
 243
 244        c = repo[rev]
 245        (manifest, user, (time, tz), files, desc, extra) = repo.changelog.read(c.node())
 246        rev_branch = extra['branch']
 247
 248        author = "%s %d %s" % (fixup_user(user), time, gittz(tz))
 249        if 'committer' in extra:
 250            user, time, tz = extra['committer'].rsplit(' ', 2)
 251            committer = "%s %s %s" % (user, time, gittz(int(tz)))
 252        else:
 253            committer = author
 254
 255        parents = [p for p in repo.changelog.parentrevs(rev) if p >= 0]
 256
 257        if len(parents) == 0:
 258            modified = c.manifest().keys()
 259            removed = []
 260        else:
 261            modified, removed = get_filechanges(repo, c, parents[0])
 262
 263        if len(parents) == 0 and rev:
 264            print 'reset %s/%s' % (prefix, ename)
 265
 266        print "commit %s/%s" % (prefix, ename)
 267        print "mark :%d" % (marks.get_mark(rev))
 268        print "author %s" % (author)
 269        print "committer %s" % (committer)
 270        print "data %d" % (len(desc))
 271        print desc
 272
 273        if len(parents) > 0:
 274            print "from :%s" % (rev_to_mark(parents[0]))
 275            if len(parents) > 1:
 276                print "merge :%s" % (rev_to_mark(parents[1]))
 277
 278        for f in modified:
 279            export_file(c.filectx(f))
 280        for f in removed:
 281            print "D %s" % (f)
 282        print
 283
 284        count += 1
 285        if (count % 100 == 0):
 286            print "progress revision %d '%s' (%d/%d)" % (rev, name, count, len(revs))
 287            print "#############################################################"
 288
 289    # make sure the ref is updated
 290    print "reset %s/%s" % (prefix, ename)
 291    print "from :%u" % rev_to_mark(rev)
 292    print
 293
 294    marks.set_tip(ename, rev)
 295
 296def export_tag(repo, tag):
 297    export_ref(repo, tag, 'tags', repo[tag])
 298
 299def export_bookmark(repo, bmark):
 300    head = bmarks[bmark]
 301    export_ref(repo, bmark, 'bookmarks', head)
 302
 303def export_branch(repo, branch):
 304    tip = get_branch_tip(repo, branch)
 305    head = repo[tip]
 306    export_ref(repo, branch, 'branches', head)
 307
 308def export_head(repo):
 309    global g_head
 310    export_ref(repo, g_head[0], 'bookmarks', g_head[1])
 311
 312def do_capabilities(parser):
 313    global prefix, dirname
 314
 315    print "import"
 316    print "export"
 317    print "refspec refs/heads/branches/*:%s/branches/*" % prefix
 318    print "refspec refs/heads/*:%s/bookmarks/*" % prefix
 319    print "refspec refs/tags/*:%s/tags/*" % prefix
 320
 321    path = os.path.join(dirname, 'marks-git')
 322
 323    if os.path.exists(path):
 324        print "*import-marks %s" % path
 325    print "*export-marks %s" % path
 326
 327    print
 328
 329def get_branch_tip(repo, branch):
 330    global branches
 331
 332    heads = branches.get(branch, None)
 333    if not heads:
 334        return None
 335
 336    # verify there's only one head
 337    if (len(heads) > 1):
 338        warn("Branch '%s' has more than one head, consider merging" % branch)
 339        # older versions of mercurial don't have this
 340        if hasattr(repo, "branchtip"):
 341            return repo.branchtip(branch)
 342
 343    return heads[0]
 344
 345def list_head(repo, cur):
 346    global g_head
 347
 348    head = bookmarks.readcurrent(repo)
 349    if not head:
 350        return
 351    node = repo[head]
 352    print "@refs/heads/%s HEAD" % head
 353    g_head = (head, node)
 354
 355def do_list(parser):
 356    global branches, bmarks
 357
 358    repo = parser.repo
 359    for branch in repo.branchmap():
 360        heads = repo.branchheads(branch)
 361        if len(heads):
 362            branches[branch] = heads
 363
 364    for bmark, node in bookmarks.listbookmarks(repo).iteritems():
 365        bmarks[bmark] = repo[node]
 366
 367    cur = repo.dirstate.branch()
 368
 369    list_head(repo, cur)
 370    for branch in branches:
 371        print "? refs/heads/branches/%s" % branch
 372    for bmark in bmarks:
 373        print "? refs/heads/%s" % bmark
 374
 375    for tag, node in repo.tagslist():
 376        if tag == 'tip':
 377            continue
 378        print "? refs/tags/%s" % tag
 379
 380    print
 381
 382def do_import(parser):
 383    repo = parser.repo
 384
 385    path = os.path.join(dirname, 'marks-git')
 386
 387    print "feature done"
 388    if os.path.exists(path):
 389        print "feature import-marks=%s" % path
 390    print "feature export-marks=%s" % path
 391    sys.stdout.flush()
 392
 393    # lets get all the import lines
 394    while parser.check('import'):
 395        ref = parser[1]
 396
 397        if (ref == 'HEAD'):
 398            export_head(repo)
 399        elif ref.startswith('refs/heads/branches/'):
 400            branch = ref[len('refs/heads/branches/'):]
 401            export_branch(repo, branch)
 402        elif ref.startswith('refs/heads/'):
 403            bmark = ref[len('refs/heads/'):]
 404            export_bookmark(repo, bmark)
 405        elif ref.startswith('refs/tags/'):
 406            tag = ref[len('refs/tags/'):]
 407            export_tag(repo, tag)
 408
 409        parser.next()
 410
 411    print 'done'
 412
 413def parse_blob(parser):
 414    global blob_marks
 415
 416    parser.next()
 417    mark = parser.get_mark()
 418    parser.next()
 419    data = parser.get_data()
 420    blob_marks[mark] = data
 421    parser.next()
 422    return
 423
 424def parse_commit(parser):
 425    global marks, blob_marks, bmarks, parsed_refs
 426
 427    from_mark = merge_mark = None
 428
 429    ref = parser[1]
 430    parser.next()
 431
 432    commit_mark = parser.get_mark()
 433    parser.next()
 434    author = parser.get_author()
 435    parser.next()
 436    committer = parser.get_author()
 437    parser.next()
 438    data = parser.get_data()
 439    parser.next()
 440    if parser.check('from'):
 441        from_mark = parser.get_mark()
 442        parser.next()
 443    if parser.check('merge'):
 444        merge_mark = parser.get_mark()
 445        parser.next()
 446        if parser.check('merge'):
 447            die('octopus merges are not supported yet')
 448
 449    files = {}
 450
 451    for line in parser:
 452        if parser.check('M'):
 453            t, m, mark_ref, path = line.split(' ')
 454            mark = int(mark_ref[1:])
 455            f = { 'mode' : hgmode(m), 'data' : blob_marks[mark] }
 456        elif parser.check('D'):
 457            t, path = line.split(' ')
 458            f = { 'deleted' : True }
 459        else:
 460            die('Unknown file command: %s' % line)
 461        files[path] = f
 462
 463    def getfilectx(repo, memctx, f):
 464        of = files[f]
 465        if 'deleted' in of:
 466            raise IOError
 467        is_exec = of['mode'] == 'x'
 468        is_link = of['mode'] == 'l'
 469        return context.memfilectx(f, of['data'], is_link, is_exec, None)
 470
 471    repo = parser.repo
 472
 473    user, date, tz = author
 474    extra = {}
 475
 476    if committer != author:
 477        extra['committer'] = "%s %u %u" % committer
 478
 479    if from_mark:
 480        p1 = repo.changelog.node(mark_to_rev(from_mark))
 481    else:
 482        p1 = '\0' * 20
 483
 484    if merge_mark:
 485        p2 = repo.changelog.node(mark_to_rev(merge_mark))
 486    else:
 487        p2 = '\0' * 20
 488
 489    ctx = context.memctx(repo, (p1, p2), data,
 490            files.keys(), getfilectx,
 491            user, (date, tz), extra)
 492
 493    node = repo.commitctx(ctx)
 494
 495    rev = repo[node].rev()
 496
 497    parsed_refs[ref] = node
 498
 499    marks.new_mark(rev, commit_mark)
 500
 501def parse_reset(parser):
 502    ref = parser[1]
 503    parser.next()
 504    # ugh
 505    if parser.check('commit'):
 506        parse_commit(parser)
 507        return
 508    if not parser.check('from'):
 509        return
 510    from_mark = parser.get_mark()
 511    parser.next()
 512
 513    node = parser.repo.changelog.node(mark_to_rev(from_mark))
 514    parsed_refs[ref] = node
 515
 516def parse_tag(parser):
 517    name = parser[1]
 518    parser.next()
 519    from_mark = parser.get_mark()
 520    parser.next()
 521    tagger = parser.get_author()
 522    parser.next()
 523    data = parser.get_data()
 524    parser.next()
 525
 526    # nothing to do
 527
 528def do_export(parser):
 529    global parsed_refs, bmarks, peer
 530
 531    parser.next()
 532
 533    for line in parser.each_block('done'):
 534        if parser.check('blob'):
 535            parse_blob(parser)
 536        elif parser.check('commit'):
 537            parse_commit(parser)
 538        elif parser.check('reset'):
 539            parse_reset(parser)
 540        elif parser.check('tag'):
 541            parse_tag(parser)
 542        elif parser.check('feature'):
 543            pass
 544        else:
 545            die('unhandled export command: %s' % line)
 546
 547    for ref, node in parsed_refs.iteritems():
 548        if ref.startswith('refs/heads/branches'):
 549            pass
 550        elif ref.startswith('refs/heads/'):
 551            bmark = ref[len('refs/heads/'):]
 552            if bmark in bmarks:
 553                old = bmarks[bmark].hex()
 554            else:
 555                old = ''
 556            if not bookmarks.pushbookmark(parser.repo, bmark, old, node):
 557                continue
 558        elif ref.startswith('refs/tags/'):
 559            tag = ref[len('refs/tags/'):]
 560            parser.repo.tag([tag], node, None, True, None, {})
 561        print "ok %s" % ref
 562
 563    print
 564
 565    if peer:
 566        parser.repo.push(peer, force=False)
 567
 568def main(args):
 569    global prefix, dirname, branches, bmarks
 570    global marks, blob_marks, parsed_refs
 571    global peer
 572
 573    alias = args[1]
 574    url = args[2]
 575    peer = None
 576
 577    gitdir = os.environ['GIT_DIR']
 578    dirname = os.path.join(gitdir, 'hg', alias)
 579    branches = {}
 580    bmarks = {}
 581    blob_marks = {}
 582    parsed_refs = {}
 583
 584    repo = get_repo(url, alias)
 585    prefix = 'refs/hg/%s' % alias
 586
 587    if not os.path.exists(dirname):
 588        os.makedirs(dirname)
 589
 590    marks_path = os.path.join(dirname, 'marks-hg')
 591    marks = Marks(marks_path)
 592
 593    parser = Parser(repo)
 594    for line in parser:
 595        if parser.check('capabilities'):
 596            do_capabilities(parser)
 597        elif parser.check('list'):
 598            do_list(parser)
 599        elif parser.check('import'):
 600            do_import(parser)
 601        elif parser.check('export'):
 602            do_export(parser)
 603        else:
 604            die('unhandled command: %s' % line)
 605        sys.stdout.flush()
 606
 607    marks.store()
 608
 609sys.exit(main(sys.argv))