Andrew's git - gitweb.git/blob - contrib/remote-helpers/git-remote-bzr

   1#!/usr/bin/env python
   2#
   3# Copyright (c) 2012 Felipe Contreras
   4#
   5
   6#
   7# Just copy to your ~/bin, or anywhere in your $PATH.
   8# Then you can clone with:
   9# % git clone bzr::/path/to/bzr/repo/or/url
  10#
  11# For example:
  12# % git clone bzr::$HOME/myrepo
  13# or
  14# % git clone bzr::lp:myrepo
  15#
  16
  17import sys
  18
  19import bzrlib
  20if hasattr(bzrlib, "initialize"):
  21    bzrlib.initialize()
  22
  23import bzrlib.plugin
  24bzrlib.plugin.load_plugins()
  25
  26import bzrlib.generate_ids
  27import bzrlib.transport
  28
  29import sys
  30import os
  31import json
  32import re
  33import StringIO
  34
  35NAME_RE = re.compile('^([^<>]+)')
  36AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  37RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  38
  39def die(msg, *args):
  40    sys.stderr.write('ERROR: %s\n' % (msg % args))
  41    sys.exit(1)
  42
  43def warn(msg, *args):
  44    sys.stderr.write('WARNING: %s\n' % (msg % args))
  45
  46def gittz(tz):
  47    return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  48
  49class Marks:
  50
  51    def __init__(self, path):
  52        self.path = path
  53        self.tips = {}
  54        self.marks = {}
  55        self.rev_marks = {}
  56        self.last_mark = 0
  57        self.load()
  58
  59    def load(self):
  60        if not os.path.exists(self.path):
  61            return
  62
  63        tmp = json.load(open(self.path))
  64        self.tips = tmp['tips']
  65        self.marks = tmp['marks']
  66        self.last_mark = tmp['last-mark']
  67
  68        for rev, mark in self.marks.iteritems():
  69            self.rev_marks[mark] = rev
  70
  71    def dict(self):
  72        return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  73
  74    def store(self):
  75        json.dump(self.dict(), open(self.path, 'w'))
  76
  77    def __str__(self):
  78        return str(self.dict())
  79
  80    def from_rev(self, rev):
  81        return self.marks[rev]
  82
  83    def to_rev(self, mark):
  84        return self.rev_marks[mark]
  85
  86    def next_mark(self):
  87        self.last_mark += 1
  88        return self.last_mark
  89
  90    def get_mark(self, rev):
  91        self.last_mark += 1
  92        self.marks[rev] = self.last_mark
  93        return self.last_mark
  94
  95    def is_marked(self, rev):
  96        return self.marks.has_key(rev)
  97
  98    def new_mark(self, rev, mark):
  99        self.marks[rev] = mark
 100        self.rev_marks[mark] = rev
 101        self.last_mark = mark
 102
 103    def get_tip(self, branch):
 104        return self.tips.get(branch, None)
 105
 106    def set_tip(self, branch, tip):
 107        self.tips[branch] = tip
 108
 109class Parser:
 110
 111    def __init__(self, repo):
 112        self.repo = repo
 113        self.line = self.get_line()
 114
 115    def get_line(self):
 116        return sys.stdin.readline().strip()
 117
 118    def __getitem__(self, i):
 119        return self.line.split()[i]
 120
 121    def check(self, word):
 122        return self.line.startswith(word)
 123
 124    def each_block(self, separator):
 125        while self.line != separator:
 126            yield self.line
 127            self.line = self.get_line()
 128
 129    def __iter__(self):
 130        return self.each_block('')
 131
 132    def next(self):
 133        self.line = self.get_line()
 134        if self.line == 'done':
 135            self.line = None
 136
 137    def get_mark(self):
 138        i = self.line.index(':') + 1
 139        return int(self.line[i:])
 140
 141    def get_data(self):
 142        if not self.check('data'):
 143            return None
 144        i = self.line.index(' ') + 1
 145        size = int(self.line[i:])
 146        return sys.stdin.read(size)
 147
 148    def get_author(self):
 149        m = RAW_AUTHOR_RE.match(self.line)
 150        if not m:
 151            return None
 152        _, name, email, date, tz = m.groups()
 153        committer = '%s <%s>' % (name, email)
 154        tz = int(tz)
 155        tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 156        return (committer, int(date), tz)
 157
 158def rev_to_mark(rev):
 159    global marks
 160    return marks.from_rev(rev)
 161
 162def mark_to_rev(mark):
 163    global marks
 164    return marks.to_rev(mark)
 165
 166def fixup_user(user):
 167    name = mail = None
 168    user = user.replace('"', '')
 169    m = AUTHOR_RE.match(user)
 170    if m:
 171        name = m.group(1)
 172        mail = m.group(2).strip()
 173    else:
 174        m = NAME_RE.match(user)
 175        if m:
 176            name = m.group(1).strip()
 177
 178    return '%s <%s>' % (name, mail)
 179
 180def get_filechanges(cur, prev):
 181    modified = {}
 182    removed = {}
 183
 184    changes = cur.changes_from(prev)
 185
 186    for path, fid, kind in changes.added:
 187        modified[path] = fid
 188    for path, fid, kind in changes.removed:
 189        removed[path] = None
 190    for path, fid, kind, mod, _ in changes.modified:
 191        modified[path] = fid
 192    for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 193        removed[oldpath] = None
 194        if kind == 'directory':
 195            lst = cur.list_files(from_dir=newpath, recursive=True)
 196            for path, file_class, kind, fid, entry in lst:
 197                if kind != 'directory':
 198                    modified[newpath + '/' + path] = fid
 199        else:
 200            modified[newpath] = fid
 201
 202    return modified, removed
 203
 204def export_files(tree, files):
 205    global marks, filenodes
 206
 207    final = []
 208    for path, fid in files.iteritems():
 209        kind = tree.kind(fid)
 210
 211        h = tree.get_file_sha1(fid)
 212
 213        if kind == 'symlink':
 214            d = tree.get_symlink_target(fid)
 215            mode = '120000'
 216        elif kind == 'file':
 217
 218            if tree.is_executable(fid):
 219                mode = '100755'
 220            else:
 221                mode = '100644'
 222
 223            # is the blog already exported?
 224            if h in filenodes:
 225                mark = filenodes[h]
 226                final.append((mode, mark, path.encode('utf-8')))
 227                continue
 228
 229            d = tree.get_file_text(fid)
 230        elif kind == 'directory':
 231            continue
 232        else:
 233            die("Unhandled kind '%s' for path '%s'" % (kind, path))
 234
 235        mark = marks.next_mark()
 236        filenodes[h] = mark
 237
 238        print "blob"
 239        print "mark :%u" % mark
 240        print "data %d" % len(d)
 241        print d
 242
 243        final.append((mode, mark, path.encode('utf-8')))
 244
 245    return final
 246
 247def export_branch(branch, name):
 248    global prefix, dirname
 249
 250    ref = '%s/heads/%s' % (prefix, name)
 251    tip = marks.get_tip(name)
 252
 253    repo = branch.repository
 254    repo.lock_read()
 255    revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 256    count = 0
 257
 258    revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 259
 260    for revid in revs:
 261
 262        rev = repo.get_revision(revid)
 263
 264        parents = rev.parent_ids
 265        time = rev.timestamp
 266        tz = rev.timezone
 267        committer = rev.committer.encode('utf-8')
 268        committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 269        authors = rev.get_apparent_authors()
 270        if authors:
 271            author = authors[0].encode('utf-8')
 272            author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 273        else:
 274            author = committer
 275        msg = rev.message.encode('utf-8')
 276
 277        msg += '\n'
 278
 279        if len(parents) == 0:
 280            parent = bzrlib.revision.NULL_REVISION
 281        else:
 282            parent = parents[0]
 283
 284        cur_tree = repo.revision_tree(revid)
 285        prev = repo.revision_tree(parent)
 286        modified, removed = get_filechanges(cur_tree, prev)
 287
 288        modified_final = export_files(cur_tree, modified)
 289
 290        if len(parents) == 0:
 291            print 'reset %s' % ref
 292
 293        print "commit %s" % ref
 294        print "mark :%d" % (marks.get_mark(revid))
 295        print "author %s" % (author)
 296        print "committer %s" % (committer)
 297        print "data %d" % (len(msg))
 298        print msg
 299
 300        for i, p in enumerate(parents):
 301            try:
 302                m = rev_to_mark(p)
 303            except KeyError:
 304                # ghost?
 305                continue
 306            if i == 0:
 307                print "from :%s" % m
 308            else:
 309                print "merge :%s" % m
 310
 311        for f in removed:
 312            print "D %s" % (f,)
 313        for f in modified_final:
 314            print "M %s :%u %s" % f
 315        print
 316
 317        count += 1
 318        if (count % 100 == 0):
 319            print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 320            print "#############################################################"
 321
 322    repo.unlock()
 323
 324    revid = branch.last_revision()
 325
 326    # make sure the ref is updated
 327    print "reset %s" % ref
 328    print "from :%u" % rev_to_mark(revid)
 329    print
 330
 331    marks.set_tip(name, revid)
 332
 333def export_tag(repo, name):
 334    global tags
 335    try:
 336        print "reset refs/tags/%s" % name
 337        print "from :%u" % rev_to_mark(tags[name])
 338        print
 339    except KeyError:
 340        warn("TODO: fetch tag '%s'" % name)
 341
 342def do_import(parser):
 343    global dirname
 344
 345    branch = parser.repo
 346    path = os.path.join(dirname, 'marks-git')
 347
 348    print "feature done"
 349    if os.path.exists(path):
 350        print "feature import-marks=%s" % path
 351    print "feature export-marks=%s" % path
 352    sys.stdout.flush()
 353
 354    while parser.check('import'):
 355        ref = parser[1]
 356        if ref.startswith('refs/heads/'):
 357            name = ref[len('refs/heads/'):]
 358            export_branch(branch, name)
 359        if ref.startswith('refs/tags/'):
 360            name = ref[len('refs/tags/'):]
 361            export_tag(branch, name)
 362        parser.next()
 363
 364    print 'done'
 365
 366    sys.stdout.flush()
 367
 368def parse_blob(parser):
 369    global blob_marks
 370
 371    parser.next()
 372    mark = parser.get_mark()
 373    parser.next()
 374    data = parser.get_data()
 375    blob_marks[mark] = data
 376    parser.next()
 377
 378class CustomTree():
 379
 380    def __init__(self, repo, revid, parents, files):
 381        global files_cache
 382
 383        self.repo = repo
 384        self.revid = revid
 385        self.parents = parents
 386        self.updates = {}
 387
 388        def copy_tree(revid):
 389            files = files_cache[revid] = {}
 390            tree = repo.repository.revision_tree(revid)
 391            repo.lock_read()
 392            try:
 393                for path, entry in tree.iter_entries_by_dir():
 394                    files[path] = entry.file_id
 395            finally:
 396                repo.unlock()
 397            return files
 398
 399        if len(parents) == 0:
 400            self.base_id = bzrlib.revision.NULL_REVISION
 401            self.base_files = {}
 402        else:
 403            self.base_id = parents[0]
 404            self.base_files = files_cache.get(self.base_id, None)
 405            if not self.base_files:
 406                self.base_files = copy_tree(self.base_id)
 407
 408        self.files = files_cache[revid] = self.base_files.copy()
 409
 410        for path, f in files.iteritems():
 411            fid = self.files.get(path, None)
 412            if not fid:
 413                fid = bzrlib.generate_ids.gen_file_id(path)
 414            f['path'] = path
 415            self.updates[fid] = f
 416
 417    def last_revision(self):
 418        return self.base_id
 419
 420    def iter_changes(self):
 421        changes = []
 422
 423        def get_parent(dirname, basename):
 424            parent_fid = self.base_files.get(dirname, None)
 425            if parent_fid:
 426                return parent_fid
 427            parent_fid = self.files.get(dirname, None)
 428            if parent_fid:
 429                return parent_fid
 430            if basename == '':
 431                return None
 432            fid = bzrlib.generate_ids.gen_file_id(path)
 433            d = add_entry(fid, dirname, 'directory')
 434            return fid
 435
 436        def add_entry(fid, path, kind, mode = None):
 437            dirname, basename = os.path.split(path)
 438            parent_fid = get_parent(dirname, basename)
 439
 440            executable = False
 441            if mode == '100755':
 442                executable = True
 443            elif mode == '120000':
 444                kind = 'symlink'
 445
 446            change = (fid,
 447                    (None, path),
 448                    True,
 449                    (False, True),
 450                    (None, parent_fid),
 451                    (None, basename),
 452                    (None, kind),
 453                    (None, executable))
 454            self.files[path] = change[0]
 455            changes.append(change)
 456            return change
 457
 458        def update_entry(fid, path, kind, mode = None):
 459            dirname, basename = os.path.split(path)
 460            parent_fid = get_parent(dirname, basename)
 461
 462            executable = False
 463            if mode == '100755':
 464                executable = True
 465            elif mode == '120000':
 466                kind = 'symlink'
 467
 468            change = (fid,
 469                    (path, path),
 470                    True,
 471                    (True, True),
 472                    (None, parent_fid),
 473                    (None, basename),
 474                    (None, kind),
 475                    (None, executable))
 476            self.files[path] = change[0]
 477            changes.append(change)
 478            return change
 479
 480        def remove_entry(fid, path, kind):
 481            dirname, basename = os.path.split(path)
 482            parent_fid = get_parent(dirname, basename)
 483            change = (fid,
 484                    (path, None),
 485                    True,
 486                    (True, False),
 487                    (parent_fid, None),
 488                    (None, None),
 489                    (None, None),
 490                    (None, None))
 491            del self.files[path]
 492            changes.append(change)
 493            return change
 494
 495        for fid, f in self.updates.iteritems():
 496            path = f['path']
 497
 498            if 'deleted' in f:
 499                remove_entry(fid, path, 'file')
 500                continue
 501
 502            if path in self.base_files:
 503                update_entry(fid, path, 'file', f['mode'])
 504            else:
 505                add_entry(fid, path, 'file', f['mode'])
 506
 507        return changes
 508
 509    def get_file_with_stat(self, file_id, path=None):
 510        return (StringIO.StringIO(self.updates[file_id]['data']), None)
 511
 512    def get_symlink_target(self, file_id):
 513        return self.updates[file_id]['data']
 514
 515def c_style_unescape(string):
 516    if string[0] == string[-1] == '"':
 517        return string.decode('string-escape')[1:-1]
 518    return string
 519
 520def parse_commit(parser):
 521    global marks, blob_marks, bmarks, parsed_refs
 522    global mode
 523
 524    parents = []
 525
 526    ref = parser[1]
 527    parser.next()
 528
 529    if ref != 'refs/heads/master':
 530        die("bzr doesn't support multiple branches; use 'master'")
 531
 532    commit_mark = parser.get_mark()
 533    parser.next()
 534    author = parser.get_author()
 535    parser.next()
 536    committer = parser.get_author()
 537    parser.next()
 538    data = parser.get_data()
 539    parser.next()
 540    if parser.check('from'):
 541        parents.append(parser.get_mark())
 542        parser.next()
 543    while parser.check('merge'):
 544        parents.append(parser.get_mark())
 545        parser.next()
 546
 547    files = {}
 548
 549    for line in parser:
 550        if parser.check('M'):
 551            t, m, mark_ref, path = line.split(' ', 3)
 552            mark = int(mark_ref[1:])
 553            f = { 'mode' : m, 'data' : blob_marks[mark] }
 554        elif parser.check('D'):
 555            t, path = line.split(' ')
 556            f = { 'deleted' : True }
 557        else:
 558            die('Unknown file command: %s' % line)
 559        path = c_style_unescape(path).decode('utf-8')
 560        files[path] = f
 561
 562    repo = parser.repo
 563
 564    committer, date, tz = committer
 565    parents = [str(mark_to_rev(p)) for p in parents]
 566    revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 567    props = {}
 568    props['branch-nick'] = repo.nick
 569
 570    mtree = CustomTree(repo, revid, parents, files)
 571    changes = mtree.iter_changes()
 572
 573    repo.lock_write()
 574    try:
 575        builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 576        try:
 577            list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 578            builder.finish_inventory()
 579            builder.commit(data.decode('utf-8', 'replace'))
 580        except Exception, e:
 581            builder.abort()
 582            raise
 583    finally:
 584        repo.unlock()
 585
 586    parsed_refs[ref] = revid
 587    marks.new_mark(revid, commit_mark)
 588
 589def parse_reset(parser):
 590    global parsed_refs
 591
 592    ref = parser[1]
 593    parser.next()
 594
 595    if ref != 'refs/heads/master':
 596        die("bzr doesn't support multiple branches; use 'master'")
 597
 598    # ugh
 599    if parser.check('commit'):
 600        parse_commit(parser)
 601        return
 602    if not parser.check('from'):
 603        return
 604    from_mark = parser.get_mark()
 605    parser.next()
 606
 607    parsed_refs[ref] = mark_to_rev(from_mark)
 608
 609def do_export(parser):
 610    global parsed_refs, dirname, peer
 611
 612    parser.next()
 613
 614    for line in parser.each_block('done'):
 615        if parser.check('blob'):
 616            parse_blob(parser)
 617        elif parser.check('commit'):
 618            parse_commit(parser)
 619        elif parser.check('reset'):
 620            parse_reset(parser)
 621        elif parser.check('tag'):
 622            pass
 623        elif parser.check('feature'):
 624            pass
 625        else:
 626            die('unhandled export command: %s' % line)
 627
 628    repo = parser.repo
 629
 630    for ref, revid in parsed_refs.iteritems():
 631        if ref == 'refs/heads/master':
 632            repo.generate_revision_history(revid, marks.get_tip('master'))
 633            if peer:
 634                try:
 635                    repo.push(peer, stop_revision=revid)
 636                except bzrlib.errors.DivergedBranches:
 637                    print "error %s non-fast forward" % ref
 638                    continue
 639            else:
 640                wt = repo.bzrdir.open_workingtree()
 641                wt.update()
 642        print "ok %s" % ref
 643    print
 644
 645def do_capabilities(parser):
 646    global dirname
 647
 648    print "import"
 649    print "export"
 650    print "refspec refs/heads/*:%s/heads/*" % prefix
 651
 652    path = os.path.join(dirname, 'marks-git')
 653
 654    if os.path.exists(path):
 655        print "*import-marks %s" % path
 656    print "*export-marks %s" % path
 657
 658    print
 659
 660def do_list(parser):
 661    global tags
 662    print "? refs/heads/%s" % 'master'
 663
 664    history = parser.repo.revision_history()
 665    for tag, revid in parser.repo.tags.get_tag_dict().items():
 666        if revid not in history:
 667            continue
 668        print "? refs/tags/%s" % tag
 669        tags[tag] = revid
 670    print "@refs/heads/%s HEAD" % 'master'
 671    print
 672
 673def get_repo(url, alias):
 674    global dirname, peer
 675
 676    origin = bzrlib.bzrdir.BzrDir.open(url)
 677    branch = origin.open_branch()
 678
 679    if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 680        clone_path = os.path.join(dirname, 'clone')
 681        remote_branch = branch
 682        if os.path.exists(clone_path):
 683            # pull
 684            d = bzrlib.bzrdir.BzrDir.open(clone_path)
 685            branch = d.open_branch()
 686            result = branch.pull(remote_branch, [], None, False)
 687        else:
 688            # clone
 689            d = origin.sprout(clone_path, None,
 690                    hardlink=True, create_tree_if_local=False,
 691                    source_branch=remote_branch)
 692            branch = d.open_branch()
 693            branch.bind(remote_branch)
 694
 695        peer = remote_branch
 696    else:
 697        peer = None
 698
 699    return branch
 700
 701def main(args):
 702    global marks, prefix, dirname
 703    global tags, filenodes
 704    global blob_marks
 705    global parsed_refs
 706    global files_cache
 707
 708    alias = args[1]
 709    url = args[2]
 710
 711    prefix = 'refs/bzr/%s' % alias
 712    tags = {}
 713    filenodes = {}
 714    blob_marks = {}
 715    parsed_refs = {}
 716    files_cache = {}
 717
 718    gitdir = os.environ['GIT_DIR']
 719    dirname = os.path.join(gitdir, 'bzr', alias)
 720
 721    if not os.path.exists(dirname):
 722        os.makedirs(dirname)
 723
 724    repo = get_repo(url, alias)
 725
 726    marks_path = os.path.join(dirname, 'marks-int')
 727    marks = Marks(marks_path)
 728
 729    parser = Parser(repo)
 730    for line in parser:
 731        if parser.check('capabilities'):
 732            do_capabilities(parser)
 733        elif parser.check('list'):
 734            do_list(parser)
 735        elif parser.check('import'):
 736            do_import(parser)
 737        elif parser.check('export'):
 738            do_export(parser)
 739        else:
 740            die('unhandled command: %s' % line)
 741        sys.stdout.flush()
 742
 743    marks.store()
 744
 745sys.exit(main(sys.argv))