3eb07dc93c1a30a0c7f8d4c5b07273bced3be5a0
1#!/usr/bin/env python
2#
3# Copyright (c) 2012 Felipe Contreras
4#
5
6# Inspired by Rocco Rutte's hg-fast-export
7
8# Just copy to your ~/bin, or anywhere in your $PATH.
9# Then you can clone with:
10# git clone hg::/path/to/mercurial/repo/
11
12from mercurial import hg, ui, bookmarks, context, util, encoding, node, error
13
14import re
15import sys
16import os
17import json
18import shutil
19import subprocess
20import urllib
21
22#
23# If you want to switch to hg-git compatibility mode:
24# git config --global remote-hg.hg-git-compat true
25#
26# If you are not in hg-git-compat mode and want to disable the tracking of
27# named branches:
28# git config --global remote-hg.track-branches false
29#
30# If you don't want to force pushes (and thus risk creating new remote heads):
31# git config --global remote-hg.force-push false
32#
33# If you want the equivalent of hg's clone/pull--insecure option:
34# git config remote-hg.insecure true
35#
36# git:
37# Sensible defaults for git.
38# hg bookmarks are exported as git branches, hg branches are prefixed
39# with 'branches/', HEAD is a special case.
40#
41# hg:
42# Emulate hg-git.
43# Only hg bookmarks are exported as git branches.
44# Commits are modified to preserve hg information and allow bidirectionality.
45#
46
47NAME_RE = re.compile('^([^<>]+)')
48AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
49AUTHOR_HG_RE = re.compile('^(.*?) ?<(.*?)(?:>(.+)?)?$')
50RAW_AUTHOR_RE = re.compile('^(\w+) (?:(.+)? )?<(.*)> (\d+) ([+-]\d+)')
51
52def die(msg, *args):
53 sys.stderr.write('ERROR: %s\n' % (msg % args))
54 sys.exit(1)
55
56def warn(msg, *args):
57 sys.stderr.write('WARNING: %s\n' % (msg % args))
58
59def gitmode(flags):
60 return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
61
62def gittz(tz):
63 return '%+03d%02d' % (-tz / 3600, -tz % 3600 / 60)
64
65def hgmode(mode):
66 m = { '100755': 'x', '120000': 'l' }
67 return m.get(mode, '')
68
69def hghex(node):
70 return hg.node.hex(node)
71
72def get_config(config):
73 cmd = ['git', 'config', '--get', config]
74 process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
75 output, _ = process.communicate()
76 return output
77
78class Marks:
79
80 def __init__(self, path):
81 self.path = path
82 self.tips = {}
83 self.marks = {}
84 self.rev_marks = {}
85 self.last_mark = 0
86
87 self.load()
88
89 def load(self):
90 if not os.path.exists(self.path):
91 return
92
93 tmp = json.load(open(self.path))
94
95 self.tips = tmp['tips']
96 self.marks = tmp['marks']
97 self.last_mark = tmp['last-mark']
98
99 for rev, mark in self.marks.iteritems():
100 self.rev_marks[mark] = int(rev)
101
102 def dict(self):
103 return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
104
105 def store(self):
106 json.dump(self.dict(), open(self.path, 'w'))
107
108 def __str__(self):
109 return str(self.dict())
110
111 def from_rev(self, rev):
112 return self.marks[str(rev)]
113
114 def to_rev(self, mark):
115 return self.rev_marks[mark]
116
117 def get_mark(self, rev):
118 self.last_mark += 1
119 self.marks[str(rev)] = self.last_mark
120 return self.last_mark
121
122 def new_mark(self, rev, mark):
123 self.marks[str(rev)] = mark
124 self.rev_marks[mark] = rev
125 self.last_mark = mark
126
127 def is_marked(self, rev):
128 return self.marks.has_key(str(rev))
129
130 def get_tip(self, branch):
131 return self.tips.get(branch, 0)
132
133 def set_tip(self, branch, tip):
134 self.tips[branch] = tip
135
136class Parser:
137
138 def __init__(self, repo):
139 self.repo = repo
140 self.line = self.get_line()
141
142 def get_line(self):
143 return sys.stdin.readline().strip()
144
145 def __getitem__(self, i):
146 return self.line.split()[i]
147
148 def check(self, word):
149 return self.line.startswith(word)
150
151 def each_block(self, separator):
152 while self.line != separator:
153 yield self.line
154 self.line = self.get_line()
155
156 def __iter__(self):
157 return self.each_block('')
158
159 def next(self):
160 self.line = self.get_line()
161 if self.line == 'done':
162 self.line = None
163
164 def get_mark(self):
165 i = self.line.index(':') + 1
166 return int(self.line[i:])
167
168 def get_data(self):
169 if not self.check('data'):
170 return None
171 i = self.line.index(' ') + 1
172 size = int(self.line[i:])
173 return sys.stdin.read(size)
174
175 def get_author(self):
176 global bad_mail
177
178 ex = None
179 m = RAW_AUTHOR_RE.match(self.line)
180 if not m:
181 return None
182 _, name, email, date, tz = m.groups()
183 if name and 'ext:' in name:
184 m = re.match('^(.+?) ext:\((.+)\)$', name)
185 if m:
186 name = m.group(1)
187 ex = urllib.unquote(m.group(2))
188
189 if email != bad_mail:
190 if name:
191 user = '%s <%s>' % (name, email)
192 else:
193 user = '<%s>' % (email)
194 else:
195 user = name
196
197 if ex:
198 user += ex
199
200 tz = int(tz)
201 tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
202 return (user, int(date), -tz)
203
204def export_file(fc):
205 d = fc.data()
206 print "M %s inline %s" % (gitmode(fc.flags()), fc.path())
207 print "data %d" % len(d)
208 print d
209
210def get_filechanges(repo, ctx, parent):
211 modified = set()
212 added = set()
213 removed = set()
214
215 cur = ctx.manifest()
216 prev = repo[parent].manifest().copy()
217
218 for fn in cur:
219 if fn in prev:
220 if (cur.flags(fn) != prev.flags(fn) or cur[fn] != prev[fn]):
221 modified.add(fn)
222 del prev[fn]
223 else:
224 added.add(fn)
225 removed |= set(prev.keys())
226
227 return added | modified, removed
228
229def fixup_user_git(user):
230 name = mail = None
231 user = user.replace('"', '')
232 m = AUTHOR_RE.match(user)
233 if m:
234 name = m.group(1)
235 mail = m.group(2).strip()
236 else:
237 m = NAME_RE.match(user)
238 if m:
239 name = m.group(1).strip()
240 return (name, mail)
241
242def fixup_user_hg(user):
243 def sanitize(name):
244 # stole this from hg-git
245 return re.sub('[<>\n]', '?', name.lstrip('< ').rstrip('> '))
246
247 m = AUTHOR_HG_RE.match(user)
248 if m:
249 name = sanitize(m.group(1))
250 mail = sanitize(m.group(2))
251 ex = m.group(3)
252 if ex:
253 name += ' ext:(' + urllib.quote(ex) + ')'
254 else:
255 name = sanitize(user)
256 if '@' in user:
257 mail = name
258 else:
259 mail = None
260
261 return (name, mail)
262
263def fixup_user(user):
264 global mode, bad_mail
265
266 if mode == 'git':
267 name, mail = fixup_user_git(user)
268 else:
269 name, mail = fixup_user_hg(user)
270
271 if not name:
272 name = bad_name
273 if not mail:
274 mail = bad_mail
275
276 return '%s <%s>' % (name, mail)
277
278def get_repo(url, alias):
279 global dirname, peer
280
281 myui = ui.ui()
282 myui.setconfig('ui', 'interactive', 'off')
283 myui.fout = sys.stderr
284
285 try:
286 if get_config('remote-hg.insecure') == 'true\n':
287 myui.setconfig('web', 'cacerts', '')
288 except subprocess.CalledProcessError:
289 pass
290
291 if hg.islocal(url):
292 repo = hg.repository(myui, url)
293 else:
294 local_path = os.path.join(dirname, 'clone')
295 if not os.path.exists(local_path):
296 try:
297 peer, dstpeer = hg.clone(myui, {}, url, local_path, update=True, pull=True)
298 except:
299 die('Repository error')
300 repo = dstpeer.local()
301 else:
302 repo = hg.repository(myui, local_path)
303 try:
304 peer = hg.peer(myui, {}, url)
305 except:
306 die('Repository error')
307 repo.pull(peer, heads=None, force=True)
308
309 return repo
310
311def rev_to_mark(rev):
312 global marks
313 return marks.from_rev(rev)
314
315def mark_to_rev(mark):
316 global marks
317 return marks.to_rev(mark)
318
319def export_ref(repo, name, kind, head):
320 global prefix, marks, mode
321
322 ename = '%s/%s' % (kind, name)
323 tip = marks.get_tip(ename)
324
325 # mercurial takes too much time checking this
326 if tip and tip == head.rev():
327 # nothing to do
328 return
329 revs = xrange(tip, head.rev() + 1)
330 count = 0
331
332 revs = [rev for rev in revs if not marks.is_marked(rev)]
333
334 for rev in revs:
335
336 c = repo[rev]
337 (manifest, user, (time, tz), files, desc, extra) = repo.changelog.read(c.node())
338 rev_branch = extra['branch']
339
340 author = "%s %d %s" % (fixup_user(user), time, gittz(tz))
341 if 'committer' in extra:
342 user, time, tz = extra['committer'].rsplit(' ', 2)
343 committer = "%s %s %s" % (user, time, gittz(int(tz)))
344 else:
345 committer = author
346
347 parents = [p for p in repo.changelog.parentrevs(rev) if p >= 0]
348
349 if len(parents) == 0:
350 modified = c.manifest().keys()
351 removed = []
352 else:
353 modified, removed = get_filechanges(repo, c, parents[0])
354
355 if mode == 'hg':
356 extra_msg = ''
357
358 if rev_branch != 'default':
359 extra_msg += 'branch : %s\n' % rev_branch
360
361 renames = []
362 for f in c.files():
363 if f not in c.manifest():
364 continue
365 rename = c.filectx(f).renamed()
366 if rename:
367 renames.append((rename[0], f))
368
369 for e in renames:
370 extra_msg += "rename : %s => %s\n" % e
371
372 for key, value in extra.iteritems():
373 if key in ('author', 'committer', 'encoding', 'message', 'branch', 'hg-git'):
374 continue
375 else:
376 extra_msg += "extra : %s : %s\n" % (key, urllib.quote(value))
377
378 desc += '\n'
379 if extra_msg:
380 desc += '\n--HG--\n' + extra_msg
381
382 if len(parents) == 0 and rev:
383 print 'reset %s/%s' % (prefix, ename)
384
385 print "commit %s/%s" % (prefix, ename)
386 print "mark :%d" % (marks.get_mark(rev))
387 print "author %s" % (author)
388 print "committer %s" % (committer)
389 print "data %d" % (len(desc))
390 print desc
391
392 if len(parents) > 0:
393 print "from :%s" % (rev_to_mark(parents[0]))
394 if len(parents) > 1:
395 print "merge :%s" % (rev_to_mark(parents[1]))
396
397 for f in modified:
398 export_file(c.filectx(f))
399 for f in removed:
400 print "D %s" % (f)
401 print
402
403 count += 1
404 if (count % 100 == 0):
405 print "progress revision %d '%s' (%d/%d)" % (rev, name, count, len(revs))
406 print "#############################################################"
407
408 # make sure the ref is updated
409 print "reset %s/%s" % (prefix, ename)
410 print "from :%u" % rev_to_mark(rev)
411 print
412
413 marks.set_tip(ename, rev)
414
415def export_tag(repo, tag):
416 export_ref(repo, tag, 'tags', repo[tag])
417
418def export_bookmark(repo, bmark):
419 head = bmarks[bmark]
420 export_ref(repo, bmark, 'bookmarks', head)
421
422def export_branch(repo, branch):
423 tip = get_branch_tip(repo, branch)
424 head = repo[tip]
425 export_ref(repo, branch, 'branches', head)
426
427def export_head(repo):
428 global g_head
429 export_ref(repo, g_head[0], 'bookmarks', g_head[1])
430
431def do_capabilities(parser):
432 global prefix, dirname
433
434 print "import"
435 print "export"
436 print "refspec refs/heads/branches/*:%s/branches/*" % prefix
437 print "refspec refs/heads/*:%s/bookmarks/*" % prefix
438 print "refspec refs/tags/*:%s/tags/*" % prefix
439
440 path = os.path.join(dirname, 'marks-git')
441
442 if os.path.exists(path):
443 print "*import-marks %s" % path
444 print "*export-marks %s" % path
445
446 print
447
448def get_branch_tip(repo, branch):
449 global branches
450
451 heads = branches.get(branch, None)
452 if not heads:
453 return None
454
455 # verify there's only one head
456 if (len(heads) > 1):
457 warn("Branch '%s' has more than one head, consider merging" % branch)
458 # older versions of mercurial don't have this
459 if hasattr(repo, "branchtip"):
460 return repo.branchtip(branch)
461
462 return heads[0]
463
464def list_head(repo, cur):
465 global g_head, bmarks
466
467 head = bookmarks.readcurrent(repo)
468 if head:
469 node = repo[head]
470 else:
471 # fake bookmark from current branch
472 head = cur
473 node = repo['.']
474 if not node:
475 node = repo['tip']
476 if not node:
477 return
478 if head == 'default':
479 head = 'master'
480 bmarks[head] = node
481
482 print "@refs/heads/%s HEAD" % head
483 g_head = (head, node)
484
485def do_list(parser):
486 global branches, bmarks, mode, track_branches
487
488 repo = parser.repo
489 for bmark, node in bookmarks.listbookmarks(repo).iteritems():
490 bmarks[bmark] = repo[node]
491
492 cur = repo.dirstate.branch()
493
494 list_head(repo, cur)
495
496 if track_branches:
497 for branch in repo.branchmap():
498 heads = repo.branchheads(branch)
499 if len(heads):
500 branches[branch] = heads
501
502 for branch in branches:
503 print "? refs/heads/branches/%s" % branch
504
505 for bmark in bmarks:
506 print "? refs/heads/%s" % bmark
507
508 for tag, node in repo.tagslist():
509 if tag == 'tip':
510 continue
511 print "? refs/tags/%s" % tag
512
513 print
514
515def do_import(parser):
516 repo = parser.repo
517
518 path = os.path.join(dirname, 'marks-git')
519
520 print "feature done"
521 if os.path.exists(path):
522 print "feature import-marks=%s" % path
523 print "feature export-marks=%s" % path
524 sys.stdout.flush()
525
526 tmp = encoding.encoding
527 encoding.encoding = 'utf-8'
528
529 # lets get all the import lines
530 while parser.check('import'):
531 ref = parser[1]
532
533 if (ref == 'HEAD'):
534 export_head(repo)
535 elif ref.startswith('refs/heads/branches/'):
536 branch = ref[len('refs/heads/branches/'):]
537 export_branch(repo, branch)
538 elif ref.startswith('refs/heads/'):
539 bmark = ref[len('refs/heads/'):]
540 export_bookmark(repo, bmark)
541 elif ref.startswith('refs/tags/'):
542 tag = ref[len('refs/tags/'):]
543 export_tag(repo, tag)
544
545 parser.next()
546
547 encoding.encoding = tmp
548
549 print 'done'
550
551def parse_blob(parser):
552 global blob_marks
553
554 parser.next()
555 mark = parser.get_mark()
556 parser.next()
557 data = parser.get_data()
558 blob_marks[mark] = data
559 parser.next()
560
561def get_merge_files(repo, p1, p2, files):
562 for e in repo[p1].files():
563 if e not in files:
564 if e not in repo[p1].manifest():
565 continue
566 f = { 'ctx' : repo[p1][e] }
567 files[e] = f
568
569def parse_commit(parser):
570 global marks, blob_marks, parsed_refs
571 global mode
572
573 from_mark = merge_mark = None
574
575 ref = parser[1]
576 parser.next()
577
578 commit_mark = parser.get_mark()
579 parser.next()
580 author = parser.get_author()
581 parser.next()
582 committer = parser.get_author()
583 parser.next()
584 data = parser.get_data()
585 parser.next()
586 if parser.check('from'):
587 from_mark = parser.get_mark()
588 parser.next()
589 if parser.check('merge'):
590 merge_mark = parser.get_mark()
591 parser.next()
592 if parser.check('merge'):
593 die('octopus merges are not supported yet')
594
595 files = {}
596
597 for line in parser:
598 if parser.check('M'):
599 t, m, mark_ref, path = line.split(' ', 3)
600 mark = int(mark_ref[1:])
601 f = { 'mode' : hgmode(m), 'data' : blob_marks[mark] }
602 elif parser.check('D'):
603 t, path = line.split(' ', 1)
604 f = { 'deleted' : True }
605 else:
606 die('Unknown file command: %s' % line)
607 files[path] = f
608
609 def getfilectx(repo, memctx, f):
610 of = files[f]
611 if 'deleted' in of:
612 raise IOError
613 if 'ctx' in of:
614 return of['ctx']
615 is_exec = of['mode'] == 'x'
616 is_link = of['mode'] == 'l'
617 rename = of.get('rename', None)
618 return context.memfilectx(f, of['data'],
619 is_link, is_exec, rename)
620
621 repo = parser.repo
622
623 user, date, tz = author
624 extra = {}
625
626 if committer != author:
627 extra['committer'] = "%s %u %u" % committer
628
629 if from_mark:
630 p1 = repo.changelog.node(mark_to_rev(from_mark))
631 else:
632 p1 = '\0' * 20
633
634 if merge_mark:
635 p2 = repo.changelog.node(mark_to_rev(merge_mark))
636 else:
637 p2 = '\0' * 20
638
639 #
640 # If files changed from any of the parents, hg wants to know, but in git if
641 # nothing changed from the first parent, nothing changed.
642 #
643 if merge_mark:
644 get_merge_files(repo, p1, p2, files)
645
646 # Check if the ref is supposed to be a named branch
647 if ref.startswith('refs/heads/branches/'):
648 extra['branch'] = ref[len('refs/heads/branches/'):]
649
650 if mode == 'hg':
651 i = data.find('\n--HG--\n')
652 if i >= 0:
653 tmp = data[i + len('\n--HG--\n'):].strip()
654 for k, v in [e.split(' : ', 1) for e in tmp.split('\n')]:
655 if k == 'rename':
656 old, new = v.split(' => ', 1)
657 files[new]['rename'] = old
658 elif k == 'branch':
659 extra[k] = v
660 elif k == 'extra':
661 ek, ev = v.split(' : ', 1)
662 extra[ek] = urllib.unquote(ev)
663 data = data[:i]
664
665 ctx = context.memctx(repo, (p1, p2), data,
666 files.keys(), getfilectx,
667 user, (date, tz), extra)
668
669 tmp = encoding.encoding
670 encoding.encoding = 'utf-8'
671
672 node = repo.commitctx(ctx)
673
674 encoding.encoding = tmp
675
676 rev = repo[node].rev()
677
678 parsed_refs[ref] = node
679 marks.new_mark(rev, commit_mark)
680
681def parse_reset(parser):
682 global parsed_refs
683
684 ref = parser[1]
685 parser.next()
686 # ugh
687 if parser.check('commit'):
688 parse_commit(parser)
689 return
690 if not parser.check('from'):
691 return
692 from_mark = parser.get_mark()
693 parser.next()
694
695 node = parser.repo.changelog.node(mark_to_rev(from_mark))
696 parsed_refs[ref] = node
697
698def parse_tag(parser):
699 name = parser[1]
700 parser.next()
701 from_mark = parser.get_mark()
702 parser.next()
703 tagger = parser.get_author()
704 parser.next()
705 data = parser.get_data()
706 parser.next()
707
708 # nothing to do
709
710def do_export(parser):
711 global parsed_refs, bmarks, peer
712
713 p_bmarks = []
714
715 parser.next()
716
717 for line in parser.each_block('done'):
718 if parser.check('blob'):
719 parse_blob(parser)
720 elif parser.check('commit'):
721 parse_commit(parser)
722 elif parser.check('reset'):
723 parse_reset(parser)
724 elif parser.check('tag'):
725 parse_tag(parser)
726 elif parser.check('feature'):
727 pass
728 else:
729 die('unhandled export command: %s' % line)
730
731 for ref, node in parsed_refs.iteritems():
732 if ref.startswith('refs/heads/branches'):
733 print "ok %s" % ref
734 elif ref.startswith('refs/heads/'):
735 bmark = ref[len('refs/heads/'):]
736 p_bmarks.append((bmark, node))
737 continue
738 elif ref.startswith('refs/tags/'):
739 tag = ref[len('refs/tags/'):]
740 if mode == 'git':
741 msg = 'Added tag %s for changeset %s' % (tag, hghex(node[:6]));
742 parser.repo.tag([tag], node, msg, False, None, {})
743 else:
744 parser.repo.tag([tag], node, None, True, None, {})
745 print "ok %s" % ref
746 else:
747 # transport-helper/fast-export bugs
748 continue
749
750 if peer:
751 parser.repo.push(peer, force=force_push)
752
753 # handle bookmarks
754 for bmark, node in p_bmarks:
755 ref = 'refs/heads/' + bmark
756 new = hghex(node)
757
758 if bmark in bmarks:
759 old = bmarks[bmark].hex()
760 else:
761 old = ''
762
763 if bmark == 'master' and 'master' not in parser.repo._bookmarks:
764 # fake bookmark
765 pass
766 elif bookmarks.pushbookmark(parser.repo, bmark, old, new):
767 # updated locally
768 pass
769 else:
770 print "error %s" % ref
771 continue
772
773 if peer:
774 if not peer.pushkey('bookmarks', bmark, old, new):
775 print "error %s" % ref
776 continue
777
778 print "ok %s" % ref
779
780 print
781
782def fix_path(alias, repo, orig_url):
783 repo_url = util.url(repo.url())
784 url = util.url(orig_url)
785 if str(url) == str(repo_url):
786 return
787 cmd = ['git', 'config', 'remote.%s.url' % alias, "hg::%s" % repo_url]
788 subprocess.call(cmd)
789
790def main(args):
791 global prefix, dirname, branches, bmarks
792 global marks, blob_marks, parsed_refs
793 global peer, mode, bad_mail, bad_name
794 global track_branches, force_push
795
796 alias = args[1]
797 url = args[2]
798 peer = None
799
800 hg_git_compat = False
801 track_branches = True
802 force_push = True
803
804 try:
805 if get_config('remote-hg.hg-git-compat') == 'true\n':
806 hg_git_compat = True
807 track_branches = False
808 if get_config('remote-hg.track-branches') == 'false\n':
809 track_branches = False
810 if get_config('remote-hg.force-push') == 'false\n':
811 force_push = False
812 except subprocess.CalledProcessError:
813 pass
814
815 if hg_git_compat:
816 mode = 'hg'
817 bad_mail = 'none@none'
818 bad_name = ''
819 else:
820 mode = 'git'
821 bad_mail = 'unknown'
822 bad_name = 'Unknown'
823
824 if alias[4:] == url:
825 is_tmp = True
826 alias = util.sha1(alias).hexdigest()
827 else:
828 is_tmp = False
829
830 gitdir = os.environ['GIT_DIR']
831 dirname = os.path.join(gitdir, 'hg', alias)
832 branches = {}
833 bmarks = {}
834 blob_marks = {}
835 parsed_refs = {}
836
837 repo = get_repo(url, alias)
838 prefix = 'refs/hg/%s' % alias
839
840 if not is_tmp:
841 fix_path(alias, peer or repo, url)
842
843 if not os.path.exists(dirname):
844 os.makedirs(dirname)
845
846 marks_path = os.path.join(dirname, 'marks-hg')
847 marks = Marks(marks_path)
848
849 parser = Parser(repo)
850 for line in parser:
851 if parser.check('capabilities'):
852 do_capabilities(parser)
853 elif parser.check('list'):
854 do_list(parser)
855 elif parser.check('import'):
856 do_import(parser)
857 elif parser.check('export'):
858 do_export(parser)
859 else:
860 die('unhandled command: %s' % line)
861 sys.stdout.flush()
862
863 if not is_tmp:
864 marks.store()
865 else:
866 shutil.rmtree(dirname)
867
868sys.exit(main(sys.argv))