Andrew's git - gitweb.git/blob - contrib/remote-helpers/git-remote-bzr

   1#!/usr/bin/env python
   2#
   3# Copyright (c) 2012 Felipe Contreras
   4#
   5
   6#
   7# Just copy to your ~/bin, or anywhere in your $PATH.
   8# Then you can clone with:
   9# % git clone bzr::/path/to/bzr/repo/or/url
  10#
  11# For example:
  12# % git clone bzr::$HOME/myrepo
  13# or
  14# % git clone bzr::lp:myrepo
  15#
  16
  17import sys
  18
  19import bzrlib
  20if hasattr(bzrlib, "initialize"):
  21    bzrlib.initialize()
  22
  23import bzrlib.plugin
  24bzrlib.plugin.load_plugins()
  25
  26import bzrlib.generate_ids
  27import bzrlib.transport
  28import bzrlib.errors
  29
  30import sys
  31import os
  32import json
  33import re
  34import StringIO
  35
  36NAME_RE = re.compile('^([^<>]+)')
  37AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  38RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  39
  40def die(msg, *args):
  41    sys.stderr.write('ERROR: %s\n' % (msg % args))
  42    sys.exit(1)
  43
  44def warn(msg, *args):
  45    sys.stderr.write('WARNING: %s\n' % (msg % args))
  46
  47def gittz(tz):
  48    return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  49
  50class Marks:
  51
  52    def __init__(self, path):
  53        self.path = path
  54        self.tips = {}
  55        self.marks = {}
  56        self.rev_marks = {}
  57        self.last_mark = 0
  58        self.load()
  59
  60    def load(self):
  61        if not os.path.exists(self.path):
  62            return
  63
  64        tmp = json.load(open(self.path))
  65        self.tips = tmp['tips']
  66        self.marks = tmp['marks']
  67        self.last_mark = tmp['last-mark']
  68
  69        for rev, mark in self.marks.iteritems():
  70            self.rev_marks[mark] = rev
  71
  72    def dict(self):
  73        return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  74
  75    def store(self):
  76        json.dump(self.dict(), open(self.path, 'w'))
  77
  78    def __str__(self):
  79        return str(self.dict())
  80
  81    def from_rev(self, rev):
  82        return self.marks[rev]
  83
  84    def to_rev(self, mark):
  85        return self.rev_marks[mark]
  86
  87    def next_mark(self):
  88        self.last_mark += 1
  89        return self.last_mark
  90
  91    def get_mark(self, rev):
  92        self.last_mark += 1
  93        self.marks[rev] = self.last_mark
  94        return self.last_mark
  95
  96    def is_marked(self, rev):
  97        return self.marks.has_key(rev)
  98
  99    def new_mark(self, rev, mark):
 100        self.marks[rev] = mark
 101        self.rev_marks[mark] = rev
 102        self.last_mark = mark
 103
 104    def get_tip(self, branch):
 105        return self.tips.get(branch, None)
 106
 107    def set_tip(self, branch, tip):
 108        self.tips[branch] = tip
 109
 110class Parser:
 111
 112    def __init__(self, repo):
 113        self.repo = repo
 114        self.line = self.get_line()
 115
 116    def get_line(self):
 117        return sys.stdin.readline().strip()
 118
 119    def __getitem__(self, i):
 120        return self.line.split()[i]
 121
 122    def check(self, word):
 123        return self.line.startswith(word)
 124
 125    def each_block(self, separator):
 126        while self.line != separator:
 127            yield self.line
 128            self.line = self.get_line()
 129
 130    def __iter__(self):
 131        return self.each_block('')
 132
 133    def next(self):
 134        self.line = self.get_line()
 135        if self.line == 'done':
 136            self.line = None
 137
 138    def get_mark(self):
 139        i = self.line.index(':') + 1
 140        return int(self.line[i:])
 141
 142    def get_data(self):
 143        if not self.check('data'):
 144            return None
 145        i = self.line.index(' ') + 1
 146        size = int(self.line[i:])
 147        return sys.stdin.read(size)
 148
 149    def get_author(self):
 150        m = RAW_AUTHOR_RE.match(self.line)
 151        if not m:
 152            return None
 153        _, name, email, date, tz = m.groups()
 154        committer = '%s <%s>' % (name, email)
 155        tz = int(tz)
 156        tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 157        return (committer, int(date), tz)
 158
 159def rev_to_mark(rev):
 160    global marks
 161    return marks.from_rev(rev)
 162
 163def mark_to_rev(mark):
 164    global marks
 165    return marks.to_rev(mark)
 166
 167def fixup_user(user):
 168    name = mail = None
 169    user = user.replace('"', '')
 170    m = AUTHOR_RE.match(user)
 171    if m:
 172        name = m.group(1)
 173        mail = m.group(2).strip()
 174    else:
 175        m = NAME_RE.match(user)
 176        if m:
 177            name = m.group(1).strip()
 178
 179    return '%s <%s>' % (name, mail)
 180
 181def get_filechanges(cur, prev):
 182    modified = {}
 183    removed = {}
 184
 185    changes = cur.changes_from(prev)
 186
 187    def u(s):
 188        return s.encode('utf-8')
 189
 190    for path, fid, kind in changes.added:
 191        modified[u(path)] = fid
 192    for path, fid, kind in changes.removed:
 193        removed[u(path)] = None
 194    for path, fid, kind, mod, _ in changes.modified:
 195        modified[u(path)] = fid
 196    for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 197        removed[u(oldpath)] = None
 198        if kind == 'directory':
 199            lst = cur.list_files(from_dir=newpath, recursive=True)
 200            for path, file_class, kind, fid, entry in lst:
 201                if kind != 'directory':
 202                    modified[u(newpath + '/' + path)] = fid
 203        else:
 204            modified[u(newpath)] = fid
 205
 206    return modified, removed
 207
 208def export_files(tree, files):
 209    global marks, filenodes
 210
 211    final = []
 212    for path, fid in files.iteritems():
 213        kind = tree.kind(fid)
 214
 215        h = tree.get_file_sha1(fid)
 216
 217        if kind == 'symlink':
 218            d = tree.get_symlink_target(fid)
 219            mode = '120000'
 220        elif kind == 'file':
 221
 222            if tree.is_executable(fid):
 223                mode = '100755'
 224            else:
 225                mode = '100644'
 226
 227            # is the blog already exported?
 228            if h in filenodes:
 229                mark = filenodes[h]
 230                final.append((mode, mark, path))
 231                continue
 232
 233            d = tree.get_file_text(fid)
 234        elif kind == 'directory':
 235            continue
 236        else:
 237            die("Unhandled kind '%s' for path '%s'" % (kind, path))
 238
 239        mark = marks.next_mark()
 240        filenodes[h] = mark
 241
 242        print "blob"
 243        print "mark :%u" % mark
 244        print "data %d" % len(d)
 245        print d
 246
 247        final.append((mode, mark, path))
 248
 249    return final
 250
 251def export_branch(branch, name):
 252    global prefix, dirname
 253
 254    ref = '%s/heads/%s' % (prefix, name)
 255    tip = marks.get_tip(name)
 256
 257    repo = branch.repository
 258    repo.lock_read()
 259    revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 260    count = 0
 261
 262    revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 263
 264    for revid in revs:
 265
 266        rev = repo.get_revision(revid)
 267
 268        parents = rev.parent_ids
 269        time = rev.timestamp
 270        tz = rev.timezone
 271        committer = rev.committer.encode('utf-8')
 272        committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 273        authors = rev.get_apparent_authors()
 274        if authors:
 275            author = authors[0].encode('utf-8')
 276            author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 277        else:
 278            author = committer
 279        msg = rev.message.encode('utf-8')
 280
 281        msg += '\n'
 282
 283        if len(parents) == 0:
 284            parent = bzrlib.revision.NULL_REVISION
 285        else:
 286            parent = parents[0]
 287
 288        cur_tree = repo.revision_tree(revid)
 289        prev = repo.revision_tree(parent)
 290        modified, removed = get_filechanges(cur_tree, prev)
 291
 292        modified_final = export_files(cur_tree, modified)
 293
 294        if len(parents) == 0:
 295            print 'reset %s' % ref
 296
 297        print "commit %s" % ref
 298        print "mark :%d" % (marks.get_mark(revid))
 299        print "author %s" % (author)
 300        print "committer %s" % (committer)
 301        print "data %d" % (len(msg))
 302        print msg
 303
 304        for i, p in enumerate(parents):
 305            try:
 306                m = rev_to_mark(p)
 307            except KeyError:
 308                # ghost?
 309                continue
 310            if i == 0:
 311                print "from :%s" % m
 312            else:
 313                print "merge :%s" % m
 314
 315        for f in removed:
 316            print "D %s" % (f,)
 317        for f in modified_final:
 318            print "M %s :%u %s" % f
 319        print
 320
 321        count += 1
 322        if (count % 100 == 0):
 323            print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 324            print "#############################################################"
 325
 326    repo.unlock()
 327
 328    revid = branch.last_revision()
 329
 330    # make sure the ref is updated
 331    print "reset %s" % ref
 332    print "from :%u" % rev_to_mark(revid)
 333    print
 334
 335    marks.set_tip(name, revid)
 336
 337def export_tag(repo, name):
 338    global tags
 339    print "reset refs/tags/%s" % name
 340    print "from :%u" % rev_to_mark(tags[name])
 341    print
 342
 343def do_import(parser):
 344    global dirname
 345
 346    branch = parser.repo
 347    path = os.path.join(dirname, 'marks-git')
 348
 349    print "feature done"
 350    if os.path.exists(path):
 351        print "feature import-marks=%s" % path
 352    print "feature export-marks=%s" % path
 353    sys.stdout.flush()
 354
 355    while parser.check('import'):
 356        ref = parser[1]
 357        if ref.startswith('refs/heads/'):
 358            name = ref[len('refs/heads/'):]
 359            export_branch(branch, name)
 360        if ref.startswith('refs/tags/'):
 361            name = ref[len('refs/tags/'):]
 362            export_tag(branch, name)
 363        parser.next()
 364
 365    print 'done'
 366
 367    sys.stdout.flush()
 368
 369def parse_blob(parser):
 370    global blob_marks
 371
 372    parser.next()
 373    mark = parser.get_mark()
 374    parser.next()
 375    data = parser.get_data()
 376    blob_marks[mark] = data
 377    parser.next()
 378
 379class CustomTree():
 380
 381    def __init__(self, repo, revid, parents, files):
 382        global files_cache
 383
 384        self.repo = repo
 385        self.revid = revid
 386        self.parents = parents
 387        self.updates = {}
 388
 389        def copy_tree(revid):
 390            files = files_cache[revid] = {}
 391            tree = repo.repository.revision_tree(revid)
 392            repo.lock_read()
 393            try:
 394                for path, entry in tree.iter_entries_by_dir():
 395                    files[path] = entry.file_id
 396            finally:
 397                repo.unlock()
 398            return files
 399
 400        if len(parents) == 0:
 401            self.base_id = bzrlib.revision.NULL_REVISION
 402            self.base_files = {}
 403        else:
 404            self.base_id = parents[0]
 405            self.base_files = files_cache.get(self.base_id, None)
 406            if not self.base_files:
 407                self.base_files = copy_tree(self.base_id)
 408
 409        self.files = files_cache[revid] = self.base_files.copy()
 410
 411        for path, f in files.iteritems():
 412            fid = self.files.get(path, None)
 413            if not fid:
 414                fid = bzrlib.generate_ids.gen_file_id(path)
 415            f['path'] = path
 416            self.updates[fid] = f
 417
 418    def last_revision(self):
 419        return self.base_id
 420
 421    def iter_changes(self):
 422        changes = []
 423
 424        def get_parent(dirname, basename):
 425            parent_fid = self.base_files.get(dirname, None)
 426            if parent_fid:
 427                return parent_fid
 428            parent_fid = self.files.get(dirname, None)
 429            if parent_fid:
 430                return parent_fid
 431            if basename == '':
 432                return None
 433            fid = bzrlib.generate_ids.gen_file_id(path)
 434            d = add_entry(fid, dirname, 'directory')
 435            return fid
 436
 437        def add_entry(fid, path, kind, mode = None):
 438            dirname, basename = os.path.split(path)
 439            parent_fid = get_parent(dirname, basename)
 440
 441            executable = False
 442            if mode == '100755':
 443                executable = True
 444            elif mode == '120000':
 445                kind = 'symlink'
 446
 447            change = (fid,
 448                    (None, path),
 449                    True,
 450                    (False, True),
 451                    (None, parent_fid),
 452                    (None, basename),
 453                    (None, kind),
 454                    (None, executable))
 455            self.files[path] = change[0]
 456            changes.append(change)
 457            return change
 458
 459        def update_entry(fid, path, kind, mode = None):
 460            dirname, basename = os.path.split(path)
 461            parent_fid = get_parent(dirname, basename)
 462
 463            executable = False
 464            if mode == '100755':
 465                executable = True
 466            elif mode == '120000':
 467                kind = 'symlink'
 468
 469            change = (fid,
 470                    (path, path),
 471                    True,
 472                    (True, True),
 473                    (None, parent_fid),
 474                    (None, basename),
 475                    (None, kind),
 476                    (None, executable))
 477            self.files[path] = change[0]
 478            changes.append(change)
 479            return change
 480
 481        def remove_entry(fid, path, kind):
 482            dirname, basename = os.path.split(path)
 483            parent_fid = get_parent(dirname, basename)
 484            change = (fid,
 485                    (path, None),
 486                    True,
 487                    (True, False),
 488                    (parent_fid, None),
 489                    (None, None),
 490                    (None, None),
 491                    (None, None))
 492            del self.files[path]
 493            changes.append(change)
 494            return change
 495
 496        for fid, f in self.updates.iteritems():
 497            path = f['path']
 498
 499            if 'deleted' in f:
 500                remove_entry(fid, path, 'file')
 501                continue
 502
 503            if path in self.base_files:
 504                update_entry(fid, path, 'file', f['mode'])
 505            else:
 506                add_entry(fid, path, 'file', f['mode'])
 507
 508        return changes
 509
 510    def get_file_with_stat(self, file_id, path=None):
 511        return (StringIO.StringIO(self.updates[file_id]['data']), None)
 512
 513    def get_symlink_target(self, file_id):
 514        return self.updates[file_id]['data']
 515
 516def c_style_unescape(string):
 517    if string[0] == string[-1] == '"':
 518        return string.decode('string-escape')[1:-1]
 519    return string
 520
 521def parse_commit(parser):
 522    global marks, blob_marks, bmarks, parsed_refs
 523    global mode
 524
 525    parents = []
 526
 527    ref = parser[1]
 528    parser.next()
 529
 530    if ref != 'refs/heads/master':
 531        die("bzr doesn't support multiple branches; use 'master'")
 532
 533    commit_mark = parser.get_mark()
 534    parser.next()
 535    author = parser.get_author()
 536    parser.next()
 537    committer = parser.get_author()
 538    parser.next()
 539    data = parser.get_data()
 540    parser.next()
 541    if parser.check('from'):
 542        parents.append(parser.get_mark())
 543        parser.next()
 544    while parser.check('merge'):
 545        parents.append(parser.get_mark())
 546        parser.next()
 547
 548    files = {}
 549
 550    for line in parser:
 551        if parser.check('M'):
 552            t, m, mark_ref, path = line.split(' ', 3)
 553            mark = int(mark_ref[1:])
 554            f = { 'mode' : m, 'data' : blob_marks[mark] }
 555        elif parser.check('D'):
 556            t, path = line.split(' ')
 557            f = { 'deleted' : True }
 558        else:
 559            die('Unknown file command: %s' % line)
 560        path = c_style_unescape(path).decode('utf-8')
 561        files[path] = f
 562
 563    repo = parser.repo
 564
 565    committer, date, tz = committer
 566    parents = [str(mark_to_rev(p)) for p in parents]
 567    revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 568    props = {}
 569    props['branch-nick'] = repo.nick
 570
 571    mtree = CustomTree(repo, revid, parents, files)
 572    changes = mtree.iter_changes()
 573
 574    repo.lock_write()
 575    try:
 576        builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 577        try:
 578            list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 579            builder.finish_inventory()
 580            builder.commit(data.decode('utf-8', 'replace'))
 581        except Exception, e:
 582            builder.abort()
 583            raise
 584    finally:
 585        repo.unlock()
 586
 587    parsed_refs[ref] = revid
 588    marks.new_mark(revid, commit_mark)
 589
 590def parse_reset(parser):
 591    global parsed_refs
 592
 593    ref = parser[1]
 594    parser.next()
 595
 596    if ref != 'refs/heads/master':
 597        die("bzr doesn't support multiple branches; use 'master'")
 598
 599    # ugh
 600    if parser.check('commit'):
 601        parse_commit(parser)
 602        return
 603    if not parser.check('from'):
 604        return
 605    from_mark = parser.get_mark()
 606    parser.next()
 607
 608    parsed_refs[ref] = mark_to_rev(from_mark)
 609
 610def do_export(parser):
 611    global parsed_refs, dirname, peer
 612
 613    parser.next()
 614
 615    for line in parser.each_block('done'):
 616        if parser.check('blob'):
 617            parse_blob(parser)
 618        elif parser.check('commit'):
 619            parse_commit(parser)
 620        elif parser.check('reset'):
 621            parse_reset(parser)
 622        elif parser.check('tag'):
 623            pass
 624        elif parser.check('feature'):
 625            pass
 626        else:
 627            die('unhandled export command: %s' % line)
 628
 629    repo = parser.repo
 630
 631    for ref, revid in parsed_refs.iteritems():
 632        if ref == 'refs/heads/master':
 633            repo.generate_revision_history(revid, marks.get_tip('master'))
 634            revno, revid = repo.last_revision_info()
 635            if peer:
 636                if hasattr(peer, "import_last_revision_info_and_tags"):
 637                    peer.import_last_revision_info_and_tags(repo, revno, revid)
 638                else:
 639                    peer.import_last_revision_info(repo.repository, revno, revid)
 640            else:
 641                wt = repo.bzrdir.open_workingtree()
 642                wt.update()
 643        print "ok %s" % ref
 644    print
 645
 646def do_capabilities(parser):
 647    global dirname
 648
 649    print "import"
 650    print "export"
 651    print "refspec refs/heads/*:%s/heads/*" % prefix
 652
 653    path = os.path.join(dirname, 'marks-git')
 654
 655    if os.path.exists(path):
 656        print "*import-marks %s" % path
 657    print "*export-marks %s" % path
 658
 659    print
 660
 661def ref_is_valid(name):
 662    return not True in [c in name for c in '~^: \\']
 663
 664def do_list(parser):
 665    global tags
 666    print "? refs/heads/%s" % 'master'
 667
 668    branch = parser.repo
 669    branch.lock_read()
 670    for tag, revid in branch.tags.get_tag_dict().items():
 671        try:
 672            branch.revision_id_to_dotted_revno(revid)
 673        except bzrlib.errors.NoSuchRevision:
 674            continue
 675        if not ref_is_valid(tag):
 676            continue
 677        print "? refs/tags/%s" % tag
 678        tags[tag] = revid
 679    branch.unlock()
 680    print "@refs/heads/%s HEAD" % 'master'
 681    print
 682
 683def get_repo(url, alias):
 684    global dirname, peer
 685
 686    origin = bzrlib.bzrdir.BzrDir.open(url)
 687    branch = origin.open_branch()
 688
 689    if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 690        clone_path = os.path.join(dirname, 'clone')
 691        remote_branch = branch
 692        if os.path.exists(clone_path):
 693            # pull
 694            d = bzrlib.bzrdir.BzrDir.open(clone_path)
 695            branch = d.open_branch()
 696            result = branch.pull(remote_branch, [], None, False)
 697        else:
 698            # clone
 699            d = origin.sprout(clone_path, None,
 700                    hardlink=True, create_tree_if_local=False,
 701                    source_branch=remote_branch)
 702            branch = d.open_branch()
 703            branch.bind(remote_branch)
 704
 705        peer = remote_branch
 706    else:
 707        peer = None
 708
 709    return branch
 710
 711def main(args):
 712    global marks, prefix, dirname
 713    global tags, filenodes
 714    global blob_marks
 715    global parsed_refs
 716    global files_cache
 717
 718    alias = args[1]
 719    url = args[2]
 720
 721    prefix = 'refs/bzr/%s' % alias
 722    tags = {}
 723    filenodes = {}
 724    blob_marks = {}
 725    parsed_refs = {}
 726    files_cache = {}
 727
 728    gitdir = os.environ['GIT_DIR']
 729    dirname = os.path.join(gitdir, 'bzr', alias)
 730
 731    if not os.path.exists(dirname):
 732        os.makedirs(dirname)
 733
 734    repo = get_repo(url, alias)
 735
 736    marks_path = os.path.join(dirname, 'marks-int')
 737    marks = Marks(marks_path)
 738
 739    parser = Parser(repo)
 740    for line in parser:
 741        if parser.check('capabilities'):
 742            do_capabilities(parser)
 743        elif parser.check('list'):
 744            do_list(parser)
 745        elif parser.check('import'):
 746            do_import(parser)
 747        elif parser.check('export'):
 748            do_export(parser)
 749        else:
 750            die('unhandled command: %s' % line)
 751        sys.stdout.flush()
 752
 753    marks.store()
 754
 755sys.exit(main(sys.argv))