Andrew's git - gitweb.git/blob - contrib/remote-helpers/git-remote-bzr

   1#!/usr/bin/env python
   2#
   3# Copyright (c) 2012 Felipe Contreras
   4#
   5
   6#
   7# Just copy to your ~/bin, or anywhere in your $PATH.
   8# Then you can clone with:
   9# % git clone bzr::/path/to/bzr/repo/or/url
  10#
  11# For example:
  12# % git clone bzr::$HOME/myrepo
  13# or
  14# % git clone bzr::lp:myrepo
  15#
  16
  17import sys
  18
  19import bzrlib
  20if hasattr(bzrlib, "initialize"):
  21    bzrlib.initialize()
  22
  23import bzrlib.plugin
  24bzrlib.plugin.load_plugins()
  25
  26import bzrlib.generate_ids
  27import bzrlib.transport
  28import bzrlib.errors
  29
  30import sys
  31import os
  32import json
  33import re
  34import StringIO
  35import atexit, shutil, hashlib, urlparse, subprocess
  36
  37NAME_RE = re.compile('^([^<>]+)')
  38AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  39RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  40
  41def die(msg, *args):
  42    sys.stderr.write('ERROR: %s\n' % (msg % args))
  43    sys.exit(1)
  44
  45def warn(msg, *args):
  46    sys.stderr.write('WARNING: %s\n' % (msg % args))
  47
  48def gittz(tz):
  49    return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  50
  51class Marks:
  52
  53    def __init__(self, path):
  54        self.path = path
  55        self.tips = {}
  56        self.marks = {}
  57        self.rev_marks = {}
  58        self.last_mark = 0
  59        self.load()
  60
  61    def load(self):
  62        if not os.path.exists(self.path):
  63            return
  64
  65        tmp = json.load(open(self.path))
  66        self.tips = tmp['tips']
  67        self.marks = tmp['marks']
  68        self.last_mark = tmp['last-mark']
  69
  70        for rev, mark in self.marks.iteritems():
  71            self.rev_marks[mark] = rev
  72
  73    def dict(self):
  74        return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  75
  76    def store(self):
  77        json.dump(self.dict(), open(self.path, 'w'))
  78
  79    def __str__(self):
  80        return str(self.dict())
  81
  82    def from_rev(self, rev):
  83        return self.marks[rev]
  84
  85    def to_rev(self, mark):
  86        return self.rev_marks[mark]
  87
  88    def next_mark(self):
  89        self.last_mark += 1
  90        return self.last_mark
  91
  92    def get_mark(self, rev):
  93        self.last_mark += 1
  94        self.marks[rev] = self.last_mark
  95        return self.last_mark
  96
  97    def is_marked(self, rev):
  98        return rev in self.marks
  99
 100    def new_mark(self, rev, mark):
 101        self.marks[rev] = mark
 102        self.rev_marks[mark] = rev
 103        self.last_mark = mark
 104
 105    def get_tip(self, branch):
 106        return self.tips.get(branch, None)
 107
 108    def set_tip(self, branch, tip):
 109        self.tips[branch] = tip
 110
 111class Parser:
 112
 113    def __init__(self, repo):
 114        self.repo = repo
 115        self.line = self.get_line()
 116
 117    def get_line(self):
 118        return sys.stdin.readline().strip()
 119
 120    def __getitem__(self, i):
 121        return self.line.split()[i]
 122
 123    def check(self, word):
 124        return self.line.startswith(word)
 125
 126    def each_block(self, separator):
 127        while self.line != separator:
 128            yield self.line
 129            self.line = self.get_line()
 130
 131    def __iter__(self):
 132        return self.each_block('')
 133
 134    def next(self):
 135        self.line = self.get_line()
 136        if self.line == 'done':
 137            self.line = None
 138
 139    def get_mark(self):
 140        i = self.line.index(':') + 1
 141        return int(self.line[i:])
 142
 143    def get_data(self):
 144        if not self.check('data'):
 145            return None
 146        i = self.line.index(' ') + 1
 147        size = int(self.line[i:])
 148        return sys.stdin.read(size)
 149
 150    def get_author(self):
 151        m = RAW_AUTHOR_RE.match(self.line)
 152        if not m:
 153            return None
 154        _, name, email, date, tz = m.groups()
 155        committer = '%s <%s>' % (name, email)
 156        tz = int(tz)
 157        tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 158        return (committer, int(date), tz)
 159
 160def rev_to_mark(rev):
 161    global marks
 162    return marks.from_rev(rev)
 163
 164def mark_to_rev(mark):
 165    global marks
 166    return marks.to_rev(mark)
 167
 168def fixup_user(user):
 169    name = mail = None
 170    user = user.replace('"', '')
 171    m = AUTHOR_RE.match(user)
 172    if m:
 173        name = m.group(1)
 174        mail = m.group(2).strip()
 175    else:
 176        m = NAME_RE.match(user)
 177        if m:
 178            name = m.group(1).strip()
 179
 180    return '%s <%s>' % (name, mail)
 181
 182def get_filechanges(cur, prev):
 183    modified = {}
 184    removed = {}
 185
 186    changes = cur.changes_from(prev)
 187
 188    def u(s):
 189        return s.encode('utf-8')
 190
 191    for path, fid, kind in changes.added:
 192        modified[u(path)] = fid
 193    for path, fid, kind in changes.removed:
 194        removed[u(path)] = None
 195    for path, fid, kind, mod, _ in changes.modified:
 196        modified[u(path)] = fid
 197    for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 198        removed[u(oldpath)] = None
 199        if kind == 'directory':
 200            lst = cur.list_files(from_dir=newpath, recursive=True)
 201            for path, file_class, kind, fid, entry in lst:
 202                if kind != 'directory':
 203                    modified[u(newpath + '/' + path)] = fid
 204        else:
 205            modified[u(newpath)] = fid
 206
 207    return modified, removed
 208
 209def export_files(tree, files):
 210    global marks, filenodes
 211
 212    final = []
 213    for path, fid in files.iteritems():
 214        kind = tree.kind(fid)
 215
 216        h = tree.get_file_sha1(fid)
 217
 218        if kind == 'symlink':
 219            d = tree.get_symlink_target(fid)
 220            mode = '120000'
 221        elif kind == 'file':
 222
 223            if tree.is_executable(fid):
 224                mode = '100755'
 225            else:
 226                mode = '100644'
 227
 228            # is the blob already exported?
 229            if h in filenodes:
 230                mark = filenodes[h]
 231                final.append((mode, mark, path))
 232                continue
 233
 234            d = tree.get_file_text(fid)
 235        elif kind == 'directory':
 236            continue
 237        else:
 238            die("Unhandled kind '%s' for path '%s'" % (kind, path))
 239
 240        mark = marks.next_mark()
 241        filenodes[h] = mark
 242
 243        print "blob"
 244        print "mark :%u" % mark
 245        print "data %d" % len(d)
 246        print d
 247
 248        final.append((mode, mark, path))
 249
 250    return final
 251
 252def export_branch(branch, name):
 253    global prefix
 254
 255    ref = '%s/heads/%s' % (prefix, name)
 256    tip = marks.get_tip(name)
 257
 258    repo = branch.repository
 259    repo.lock_read()
 260    revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 261    count = 0
 262
 263    revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 264
 265    for revid in revs:
 266
 267        rev = repo.get_revision(revid)
 268
 269        parents = rev.parent_ids
 270        time = rev.timestamp
 271        tz = rev.timezone
 272        committer = rev.committer.encode('utf-8')
 273        committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 274        authors = rev.get_apparent_authors()
 275        if authors:
 276            author = authors[0].encode('utf-8')
 277            author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 278        else:
 279            author = committer
 280        msg = rev.message.encode('utf-8')
 281
 282        msg += '\n'
 283
 284        if len(parents) == 0:
 285            parent = bzrlib.revision.NULL_REVISION
 286        else:
 287            parent = parents[0]
 288
 289        cur_tree = repo.revision_tree(revid)
 290        prev = repo.revision_tree(parent)
 291        modified, removed = get_filechanges(cur_tree, prev)
 292
 293        modified_final = export_files(cur_tree, modified)
 294
 295        if len(parents) == 0:
 296            print 'reset %s' % ref
 297
 298        print "commit %s" % ref
 299        print "mark :%d" % (marks.get_mark(revid))
 300        print "author %s" % (author)
 301        print "committer %s" % (committer)
 302        print "data %d" % (len(msg))
 303        print msg
 304
 305        for i, p in enumerate(parents):
 306            try:
 307                m = rev_to_mark(p)
 308            except KeyError:
 309                # ghost?
 310                continue
 311            if i == 0:
 312                print "from :%s" % m
 313            else:
 314                print "merge :%s" % m
 315
 316        for f in removed:
 317            print "D %s" % (f,)
 318        for f in modified_final:
 319            print "M %s :%u %s" % f
 320        print
 321
 322        count += 1
 323        if (count % 100 == 0):
 324            print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 325            print "#############################################################"
 326
 327    repo.unlock()
 328
 329    revid = branch.last_revision()
 330
 331    # make sure the ref is updated
 332    print "reset %s" % ref
 333    print "from :%u" % rev_to_mark(revid)
 334    print
 335
 336    marks.set_tip(name, revid)
 337
 338def export_tag(repo, name):
 339    global tags, prefix
 340
 341    ref = '%s/tags/%s' % (prefix, name)
 342    print "reset %s" % ref
 343    print "from :%u" % rev_to_mark(tags[name])
 344    print
 345
 346def do_import(parser):
 347    global dirname
 348
 349    branch = parser.repo
 350    path = os.path.join(dirname, 'marks-git')
 351
 352    print "feature done"
 353    if os.path.exists(path):
 354        print "feature import-marks=%s" % path
 355    print "feature export-marks=%s" % path
 356    sys.stdout.flush()
 357
 358    while parser.check('import'):
 359        ref = parser[1]
 360        if ref.startswith('refs/heads/'):
 361            name = ref[len('refs/heads/'):]
 362            export_branch(branch, name)
 363        if ref.startswith('refs/tags/'):
 364            name = ref[len('refs/tags/'):]
 365            export_tag(branch, name)
 366        parser.next()
 367
 368    print 'done'
 369
 370    sys.stdout.flush()
 371
 372def parse_blob(parser):
 373    global blob_marks
 374
 375    parser.next()
 376    mark = parser.get_mark()
 377    parser.next()
 378    data = parser.get_data()
 379    blob_marks[mark] = data
 380    parser.next()
 381
 382class CustomTree():
 383
 384    def __init__(self, repo, revid, parents, files):
 385        global files_cache
 386
 387        self.repo = repo
 388        self.revid = revid
 389        self.parents = parents
 390        self.updates = {}
 391
 392        def copy_tree(revid):
 393            files = files_cache[revid] = {}
 394            tree = repo.repository.revision_tree(revid)
 395            repo.lock_read()
 396            try:
 397                for path, entry in tree.iter_entries_by_dir():
 398                    files[path] = entry.file_id
 399            finally:
 400                repo.unlock()
 401            return files
 402
 403        if len(parents) == 0:
 404            self.base_id = bzrlib.revision.NULL_REVISION
 405            self.base_files = {}
 406        else:
 407            self.base_id = parents[0]
 408            self.base_files = files_cache.get(self.base_id, None)
 409            if not self.base_files:
 410                self.base_files = copy_tree(self.base_id)
 411
 412        self.files = files_cache[revid] = self.base_files.copy()
 413
 414        for path, f in files.iteritems():
 415            fid = self.files.get(path, None)
 416            if not fid:
 417                fid = bzrlib.generate_ids.gen_file_id(path)
 418            f['path'] = path
 419            self.updates[fid] = f
 420
 421    def last_revision(self):
 422        return self.base_id
 423
 424    def iter_changes(self):
 425        changes = []
 426
 427        def get_parent(dirname, basename):
 428            parent_fid = self.base_files.get(dirname, None)
 429            if parent_fid:
 430                return parent_fid
 431            parent_fid = self.files.get(dirname, None)
 432            if parent_fid:
 433                return parent_fid
 434            if basename == '':
 435                return None
 436            fid = bzrlib.generate_ids.gen_file_id(path)
 437            d = add_entry(fid, dirname, 'directory')
 438            return fid
 439
 440        def add_entry(fid, path, kind, mode = None):
 441            dirname, basename = os.path.split(path)
 442            parent_fid = get_parent(dirname, basename)
 443
 444            executable = False
 445            if mode == '100755':
 446                executable = True
 447            elif mode == '120000':
 448                kind = 'symlink'
 449
 450            change = (fid,
 451                    (None, path),
 452                    True,
 453                    (False, True),
 454                    (None, parent_fid),
 455                    (None, basename),
 456                    (None, kind),
 457                    (None, executable))
 458            self.files[path] = change[0]
 459            changes.append(change)
 460            return change
 461
 462        def update_entry(fid, path, kind, mode = None):
 463            dirname, basename = os.path.split(path)
 464            parent_fid = get_parent(dirname, basename)
 465
 466            executable = False
 467            if mode == '100755':
 468                executable = True
 469            elif mode == '120000':
 470                kind = 'symlink'
 471
 472            change = (fid,
 473                    (path, path),
 474                    True,
 475                    (True, True),
 476                    (None, parent_fid),
 477                    (None, basename),
 478                    (None, kind),
 479                    (None, executable))
 480            self.files[path] = change[0]
 481            changes.append(change)
 482            return change
 483
 484        def remove_entry(fid, path, kind):
 485            dirname, basename = os.path.split(path)
 486            parent_fid = get_parent(dirname, basename)
 487            change = (fid,
 488                    (path, None),
 489                    True,
 490                    (True, False),
 491                    (parent_fid, None),
 492                    (None, None),
 493                    (None, None),
 494                    (None, None))
 495            del self.files[path]
 496            changes.append(change)
 497            return change
 498
 499        for fid, f in self.updates.iteritems():
 500            path = f['path']
 501
 502            if 'deleted' in f:
 503                remove_entry(fid, path, 'file')
 504                continue
 505
 506            if path in self.base_files:
 507                update_entry(fid, path, 'file', f['mode'])
 508            else:
 509                add_entry(fid, path, 'file', f['mode'])
 510
 511        return changes
 512
 513    def get_file_with_stat(self, file_id, path=None):
 514        return (StringIO.StringIO(self.updates[file_id]['data']), None)
 515
 516    def get_symlink_target(self, file_id):
 517        return self.updates[file_id]['data']
 518
 519def c_style_unescape(string):
 520    if string[0] == string[-1] == '"':
 521        return string.decode('string-escape')[1:-1]
 522    return string
 523
 524def parse_commit(parser):
 525    global marks, blob_marks, parsed_refs
 526    global mode
 527
 528    parents = []
 529
 530    ref = parser[1]
 531    parser.next()
 532
 533    if ref != 'refs/heads/master':
 534        die("bzr doesn't support multiple branches; use 'master'")
 535
 536    commit_mark = parser.get_mark()
 537    parser.next()
 538    author = parser.get_author()
 539    parser.next()
 540    committer = parser.get_author()
 541    parser.next()
 542    data = parser.get_data()
 543    parser.next()
 544    if parser.check('from'):
 545        parents.append(parser.get_mark())
 546        parser.next()
 547    while parser.check('merge'):
 548        parents.append(parser.get_mark())
 549        parser.next()
 550
 551    files = {}
 552
 553    for line in parser:
 554        if parser.check('M'):
 555            t, m, mark_ref, path = line.split(' ', 3)
 556            mark = int(mark_ref[1:])
 557            f = { 'mode' : m, 'data' : blob_marks[mark] }
 558        elif parser.check('D'):
 559            t, path = line.split(' ')
 560            f = { 'deleted' : True }
 561        else:
 562            die('Unknown file command: %s' % line)
 563        path = c_style_unescape(path).decode('utf-8')
 564        files[path] = f
 565
 566    repo = parser.repo
 567
 568    committer, date, tz = committer
 569    parents = [str(mark_to_rev(p)) for p in parents]
 570    revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 571    props = {}
 572    props['branch-nick'] = repo.nick
 573
 574    mtree = CustomTree(repo, revid, parents, files)
 575    changes = mtree.iter_changes()
 576
 577    repo.lock_write()
 578    try:
 579        builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 580        try:
 581            list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 582            builder.finish_inventory()
 583            builder.commit(data.decode('utf-8', 'replace'))
 584        except Exception, e:
 585            builder.abort()
 586            raise
 587    finally:
 588        repo.unlock()
 589
 590    parsed_refs[ref] = revid
 591    marks.new_mark(revid, commit_mark)
 592
 593def parse_reset(parser):
 594    global parsed_refs
 595
 596    ref = parser[1]
 597    parser.next()
 598
 599    if ref != 'refs/heads/master':
 600        die("bzr doesn't support multiple branches; use 'master'")
 601
 602    # ugh
 603    if parser.check('commit'):
 604        parse_commit(parser)
 605        return
 606    if not parser.check('from'):
 607        return
 608    from_mark = parser.get_mark()
 609    parser.next()
 610
 611    parsed_refs[ref] = mark_to_rev(from_mark)
 612
 613def do_export(parser):
 614    global parsed_refs, dirname, peer
 615
 616    parser.next()
 617
 618    for line in parser.each_block('done'):
 619        if parser.check('blob'):
 620            parse_blob(parser)
 621        elif parser.check('commit'):
 622            parse_commit(parser)
 623        elif parser.check('reset'):
 624            parse_reset(parser)
 625        elif parser.check('tag'):
 626            pass
 627        elif parser.check('feature'):
 628            pass
 629        else:
 630            die('unhandled export command: %s' % line)
 631
 632    repo = parser.repo
 633
 634    for ref, revid in parsed_refs.iteritems():
 635        if ref == 'refs/heads/master':
 636            repo.generate_revision_history(revid, marks.get_tip('master'))
 637            if peer:
 638                try:
 639                    repo.push(peer, stop_revision=revid)
 640                except bzrlib.errors.DivergedBranches:
 641                    print "error %s non-fast forward" % ref
 642                    continue
 643            else:
 644                wt = repo.bzrdir.open_workingtree()
 645                wt.update()
 646        print "ok %s" % ref
 647
 648    print
 649
 650def do_capabilities(parser):
 651    global dirname
 652
 653    print "import"
 654    print "export"
 655    print "refspec refs/heads/*:%s/heads/*" % prefix
 656    print "refspec refs/tags/*:%s/tags/*" % prefix
 657
 658    path = os.path.join(dirname, 'marks-git')
 659
 660    if os.path.exists(path):
 661        print "*import-marks %s" % path
 662    print "*export-marks %s" % path
 663
 664    print
 665
 666def ref_is_valid(name):
 667    return not True in [c in name for c in '~^: \\']
 668
 669def do_list(parser):
 670    global tags
 671    print "? refs/heads/%s" % 'master'
 672
 673    branch = parser.repo
 674    branch.lock_read()
 675    for tag, revid in branch.tags.get_tag_dict().items():
 676        try:
 677            branch.revision_id_to_dotted_revno(revid)
 678        except bzrlib.errors.NoSuchRevision:
 679            continue
 680        if not ref_is_valid(tag):
 681            continue
 682        print "? refs/tags/%s" % tag
 683        tags[tag] = revid
 684    branch.unlock()
 685    print "@refs/heads/%s HEAD" % 'master'
 686    print
 687
 688def get_repo(url, alias):
 689    global dirname, peer
 690
 691    origin = bzrlib.bzrdir.BzrDir.open(url)
 692    branch = origin.open_branch()
 693
 694    if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 695        clone_path = os.path.join(dirname, 'clone')
 696        remote_branch = branch
 697        if os.path.exists(clone_path):
 698            # pull
 699            d = bzrlib.bzrdir.BzrDir.open(clone_path)
 700            branch = d.open_branch()
 701            result = branch.pull(remote_branch, [], None, False)
 702        else:
 703            # clone
 704            d = origin.sprout(clone_path, None,
 705                    hardlink=True, create_tree_if_local=False,
 706                    source_branch=remote_branch)
 707            branch = d.open_branch()
 708            branch.bind(remote_branch)
 709
 710        peer = remote_branch
 711    else:
 712        peer = None
 713
 714    return branch
 715
 716def fix_path(alias, orig_url):
 717    url = urlparse.urlparse(orig_url, 'file')
 718    if url.scheme != 'file' or os.path.isabs(url.path):
 719        return
 720    abs_url = urlparse.urljoin("%s/" % os.getcwd(), orig_url)
 721    cmd = ['git', 'config', 'remote.%s.url' % alias, "bzr::%s" % abs_url]
 722    subprocess.call(cmd)
 723
 724def main(args):
 725    global marks, prefix, dirname
 726    global tags, filenodes
 727    global blob_marks
 728    global parsed_refs
 729    global files_cache
 730    global is_tmp
 731
 732    alias = args[1]
 733    url = args[2]
 734
 735    tags = {}
 736    filenodes = {}
 737    blob_marks = {}
 738    parsed_refs = {}
 739    files_cache = {}
 740    marks = None
 741
 742    if alias[5:] == url:
 743        is_tmp = True
 744        alias = hashlib.sha1(alias).hexdigest()
 745    else:
 746        is_tmp = False
 747
 748    prefix = 'refs/bzr/%s' % alias
 749    gitdir = os.environ['GIT_DIR']
 750    dirname = os.path.join(gitdir, 'bzr', alias)
 751
 752    if not is_tmp:
 753        fix_path(alias, url)
 754
 755    if not os.path.exists(dirname):
 756        os.makedirs(dirname)
 757
 758    repo = get_repo(url, alias)
 759
 760    marks_path = os.path.join(dirname, 'marks-int')
 761    marks = Marks(marks_path)
 762
 763    parser = Parser(repo)
 764    for line in parser:
 765        if parser.check('capabilities'):
 766            do_capabilities(parser)
 767        elif parser.check('list'):
 768            do_list(parser)
 769        elif parser.check('import'):
 770            do_import(parser)
 771        elif parser.check('export'):
 772            do_export(parser)
 773        else:
 774            die('unhandled command: %s' % line)
 775        sys.stdout.flush()
 776
 777def bye():
 778    if not marks:
 779        return
 780    if not is_tmp:
 781        marks.store()
 782    else:
 783        shutil.rmtree(dirname)
 784
 785atexit.register(bye)
 786sys.exit(main(sys.argv))