ff8972579ecbd60eb42307c0ae093c1ba52d905b
1#!/usr/bin/env python
2#
3# Copyright (c) 2012 Felipe Contreras
4#
5
6# Inspired by Rocco Rutte's hg-fast-export
7
8# Just copy to your ~/bin, or anywhere in your $PATH.
9# Then you can clone with:
10# git clone hg::/path/to/mercurial/repo/
11
12from mercurial import hg, ui, bookmarks, context, util, encoding, node
13
14import re
15import sys
16import os
17import json
18import shutil
19import subprocess
20import urllib
21
22#
23# If you want to switch to hg-git compatibility mode:
24# git config --global remote-hg.hg-git-compat true
25#
26# If you are not in hg-git-compat mode and want to disable the tracking of
27# named branches:
28# git config --global remote-hg.track-branches false
29#
30# If you don't want to force pushes (and thus risk creating new remote heads):
31# git config --global remote-hg.force-push false
32#
33# git:
34# Sensible defaults for git.
35# hg bookmarks are exported as git branches, hg branches are prefixed
36# with 'branches/', HEAD is a special case.
37#
38# hg:
39# Emulate hg-git.
40# Only hg bookmarks are exported as git branches.
41# Commits are modified to preserve hg information and allow bidirectionality.
42#
43
44NAME_RE = re.compile('^([^<>]+)')
45AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
46AUTHOR_HG_RE = re.compile('^(.*?) ?<(.*?)(?:>(.+)?)?$')
47RAW_AUTHOR_RE = re.compile('^(\w+) (?:(.+)? )?<(.*)> (\d+) ([+-]\d+)')
48
49def die(msg, *args):
50 sys.stderr.write('ERROR: %s\n' % (msg % args))
51 sys.exit(1)
52
53def warn(msg, *args):
54 sys.stderr.write('WARNING: %s\n' % (msg % args))
55
56def gitmode(flags):
57 return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
58
59def gittz(tz):
60 return '%+03d%02d' % (-tz / 3600, -tz % 3600 / 60)
61
62def hgmode(mode):
63 m = { '100755': 'x', '120000': 'l' }
64 return m.get(mode, '')
65
66def hghex(node):
67 return hg.node.hex(node)
68
69def get_config(config):
70 cmd = ['git', 'config', '--get', config]
71 process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
72 output, _ = process.communicate()
73 return output
74
75class Marks:
76
77 def __init__(self, path):
78 self.path = path
79 self.tips = {}
80 self.marks = {}
81 self.rev_marks = {}
82 self.last_mark = 0
83
84 self.load()
85
86 def load(self):
87 if not os.path.exists(self.path):
88 return
89
90 tmp = json.load(open(self.path))
91
92 self.tips = tmp['tips']
93 self.marks = tmp['marks']
94 self.last_mark = tmp['last-mark']
95
96 for rev, mark in self.marks.iteritems():
97 self.rev_marks[mark] = int(rev)
98
99 def dict(self):
100 return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
101
102 def store(self):
103 json.dump(self.dict(), open(self.path, 'w'))
104
105 def __str__(self):
106 return str(self.dict())
107
108 def from_rev(self, rev):
109 return self.marks[str(rev)]
110
111 def to_rev(self, mark):
112 return self.rev_marks[mark]
113
114 def get_mark(self, rev):
115 self.last_mark += 1
116 self.marks[str(rev)] = self.last_mark
117 return self.last_mark
118
119 def new_mark(self, rev, mark):
120 self.marks[str(rev)] = mark
121 self.rev_marks[mark] = rev
122 self.last_mark = mark
123
124 def is_marked(self, rev):
125 return self.marks.has_key(str(rev))
126
127 def get_tip(self, branch):
128 return self.tips.get(branch, 0)
129
130 def set_tip(self, branch, tip):
131 self.tips[branch] = tip
132
133class Parser:
134
135 def __init__(self, repo):
136 self.repo = repo
137 self.line = self.get_line()
138
139 def get_line(self):
140 return sys.stdin.readline().strip()
141
142 def __getitem__(self, i):
143 return self.line.split()[i]
144
145 def check(self, word):
146 return self.line.startswith(word)
147
148 def each_block(self, separator):
149 while self.line != separator:
150 yield self.line
151 self.line = self.get_line()
152
153 def __iter__(self):
154 return self.each_block('')
155
156 def next(self):
157 self.line = self.get_line()
158 if self.line == 'done':
159 self.line = None
160
161 def get_mark(self):
162 i = self.line.index(':') + 1
163 return int(self.line[i:])
164
165 def get_data(self):
166 if not self.check('data'):
167 return None
168 i = self.line.index(' ') + 1
169 size = int(self.line[i:])
170 return sys.stdin.read(size)
171
172 def get_author(self):
173 global bad_mail
174
175 ex = None
176 m = RAW_AUTHOR_RE.match(self.line)
177 if not m:
178 return None
179 _, name, email, date, tz = m.groups()
180 if name and 'ext:' in name:
181 m = re.match('^(.+?) ext:\((.+)\)$', name)
182 if m:
183 name = m.group(1)
184 ex = urllib.unquote(m.group(2))
185
186 if email != bad_mail:
187 if name:
188 user = '%s <%s>' % (name, email)
189 else:
190 user = '<%s>' % (email)
191 else:
192 user = name
193
194 if ex:
195 user += ex
196
197 tz = int(tz)
198 tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
199 return (user, int(date), -tz)
200
201def export_file(fc):
202 d = fc.data()
203 print "M %s inline %s" % (gitmode(fc.flags()), fc.path())
204 print "data %d" % len(d)
205 print d
206
207def get_filechanges(repo, ctx, parent):
208 modified = set()
209 added = set()
210 removed = set()
211
212 cur = ctx.manifest()
213 prev = repo[parent].manifest().copy()
214
215 for fn in cur:
216 if fn in prev:
217 if (cur.flags(fn) != prev.flags(fn) or cur[fn] != prev[fn]):
218 modified.add(fn)
219 del prev[fn]
220 else:
221 added.add(fn)
222 removed |= set(prev.keys())
223
224 return added | modified, removed
225
226def fixup_user_git(user):
227 name = mail = None
228 user = user.replace('"', '')
229 m = AUTHOR_RE.match(user)
230 if m:
231 name = m.group(1)
232 mail = m.group(2).strip()
233 else:
234 m = NAME_RE.match(user)
235 if m:
236 name = m.group(1).strip()
237 return (name, mail)
238
239def fixup_user_hg(user):
240 def sanitize(name):
241 # stole this from hg-git
242 return re.sub('[<>\n]', '?', name.lstrip('< ').rstrip('> '))
243
244 m = AUTHOR_HG_RE.match(user)
245 if m:
246 name = sanitize(m.group(1))
247 mail = sanitize(m.group(2))
248 ex = m.group(3)
249 if ex:
250 name += ' ext:(' + urllib.quote(ex) + ')'
251 else:
252 name = sanitize(user)
253 if '@' in user:
254 mail = name
255 else:
256 mail = None
257
258 return (name, mail)
259
260def fixup_user(user):
261 global mode, bad_mail
262
263 if mode == 'git':
264 name, mail = fixup_user_git(user)
265 else:
266 name, mail = fixup_user_hg(user)
267
268 if not name:
269 name = bad_name
270 if not mail:
271 mail = bad_mail
272
273 return '%s <%s>' % (name, mail)
274
275def get_repo(url, alias):
276 global dirname, peer
277
278 myui = ui.ui()
279 myui.setconfig('ui', 'interactive', 'off')
280 myui.fout = sys.stderr
281
282 if hg.islocal(url):
283 repo = hg.repository(myui, url)
284 else:
285 local_path = os.path.join(dirname, 'clone')
286 if not os.path.exists(local_path):
287 peer, dstpeer = hg.clone(myui, {}, url, local_path, update=False, pull=True)
288 repo = dstpeer.local()
289 else:
290 repo = hg.repository(myui, local_path)
291 peer = hg.peer(myui, {}, url)
292 repo.pull(peer, heads=None, force=True)
293
294 return repo
295
296def rev_to_mark(rev):
297 global marks
298 return marks.from_rev(rev)
299
300def mark_to_rev(mark):
301 global marks
302 return marks.to_rev(mark)
303
304def export_ref(repo, name, kind, head):
305 global prefix, marks, mode
306
307 ename = '%s/%s' % (kind, name)
308 tip = marks.get_tip(ename)
309
310 # mercurial takes too much time checking this
311 if tip and tip == head.rev():
312 # nothing to do
313 return
314 revs = xrange(tip, head.rev() + 1)
315 count = 0
316
317 revs = [rev for rev in revs if not marks.is_marked(rev)]
318
319 for rev in revs:
320
321 c = repo[rev]
322 (manifest, user, (time, tz), files, desc, extra) = repo.changelog.read(c.node())
323 rev_branch = extra['branch']
324
325 author = "%s %d %s" % (fixup_user(user), time, gittz(tz))
326 if 'committer' in extra:
327 user, time, tz = extra['committer'].rsplit(' ', 2)
328 committer = "%s %s %s" % (user, time, gittz(int(tz)))
329 else:
330 committer = author
331
332 parents = [p for p in repo.changelog.parentrevs(rev) if p >= 0]
333
334 if len(parents) == 0:
335 modified = c.manifest().keys()
336 removed = []
337 else:
338 modified, removed = get_filechanges(repo, c, parents[0])
339
340 if mode == 'hg':
341 extra_msg = ''
342
343 if rev_branch != 'default':
344 extra_msg += 'branch : %s\n' % rev_branch
345
346 renames = []
347 for f in c.files():
348 if f not in c.manifest():
349 continue
350 rename = c.filectx(f).renamed()
351 if rename:
352 renames.append((rename[0], f))
353
354 for e in renames:
355 extra_msg += "rename : %s => %s\n" % e
356
357 for key, value in extra.iteritems():
358 if key in ('author', 'committer', 'encoding', 'message', 'branch', 'hg-git'):
359 continue
360 else:
361 extra_msg += "extra : %s : %s\n" % (key, urllib.quote(value))
362
363 desc += '\n'
364 if extra_msg:
365 desc += '\n--HG--\n' + extra_msg
366
367 if len(parents) == 0 and rev:
368 print 'reset %s/%s' % (prefix, ename)
369
370 print "commit %s/%s" % (prefix, ename)
371 print "mark :%d" % (marks.get_mark(rev))
372 print "author %s" % (author)
373 print "committer %s" % (committer)
374 print "data %d" % (len(desc))
375 print desc
376
377 if len(parents) > 0:
378 print "from :%s" % (rev_to_mark(parents[0]))
379 if len(parents) > 1:
380 print "merge :%s" % (rev_to_mark(parents[1]))
381
382 for f in modified:
383 export_file(c.filectx(f))
384 for f in removed:
385 print "D %s" % (f)
386 print
387
388 count += 1
389 if (count % 100 == 0):
390 print "progress revision %d '%s' (%d/%d)" % (rev, name, count, len(revs))
391 print "#############################################################"
392
393 # make sure the ref is updated
394 print "reset %s/%s" % (prefix, ename)
395 print "from :%u" % rev_to_mark(rev)
396 print
397
398 marks.set_tip(ename, rev)
399
400def export_tag(repo, tag):
401 export_ref(repo, tag, 'tags', repo[tag])
402
403def export_bookmark(repo, bmark):
404 head = bmarks[bmark]
405 export_ref(repo, bmark, 'bookmarks', head)
406
407def export_branch(repo, branch):
408 tip = get_branch_tip(repo, branch)
409 head = repo[tip]
410 export_ref(repo, branch, 'branches', head)
411
412def export_head(repo):
413 global g_head
414 export_ref(repo, g_head[0], 'bookmarks', g_head[1])
415
416def do_capabilities(parser):
417 global prefix, dirname
418
419 print "import"
420 print "export"
421 print "refspec refs/heads/branches/*:%s/branches/*" % prefix
422 print "refspec refs/heads/*:%s/bookmarks/*" % prefix
423 print "refspec refs/tags/*:%s/tags/*" % prefix
424
425 path = os.path.join(dirname, 'marks-git')
426
427 if os.path.exists(path):
428 print "*import-marks %s" % path
429 print "*export-marks %s" % path
430
431 print
432
433def get_branch_tip(repo, branch):
434 global branches
435
436 heads = branches.get(branch, None)
437 if not heads:
438 return None
439
440 # verify there's only one head
441 if (len(heads) > 1):
442 warn("Branch '%s' has more than one head, consider merging" % branch)
443 # older versions of mercurial don't have this
444 if hasattr(repo, "branchtip"):
445 return repo.branchtip(branch)
446
447 return heads[0]
448
449def list_head(repo, cur):
450 global g_head, bmarks
451
452 head = bookmarks.readcurrent(repo)
453 if head:
454 node = repo[head]
455 else:
456 # fake bookmark from current branch
457 head = cur
458 node = repo['.']
459 if not node:
460 node = repo['tip']
461 if not node:
462 return
463 if head == 'default':
464 head = 'master'
465 bmarks[head] = node
466
467 print "@refs/heads/%s HEAD" % head
468 g_head = (head, node)
469
470def do_list(parser):
471 global branches, bmarks, mode, track_branches
472
473 repo = parser.repo
474 for bmark, node in bookmarks.listbookmarks(repo).iteritems():
475 bmarks[bmark] = repo[node]
476
477 cur = repo.dirstate.branch()
478
479 list_head(repo, cur)
480
481 if track_branches:
482 for branch in repo.branchmap():
483 heads = repo.branchheads(branch)
484 if len(heads):
485 branches[branch] = heads
486
487 for branch in branches:
488 print "? refs/heads/branches/%s" % branch
489
490 for bmark in bmarks:
491 print "? refs/heads/%s" % bmark
492
493 for tag, node in repo.tagslist():
494 if tag == 'tip':
495 continue
496 print "? refs/tags/%s" % tag
497
498 print
499
500def do_import(parser):
501 repo = parser.repo
502
503 path = os.path.join(dirname, 'marks-git')
504
505 print "feature done"
506 if os.path.exists(path):
507 print "feature import-marks=%s" % path
508 print "feature export-marks=%s" % path
509 sys.stdout.flush()
510
511 tmp = encoding.encoding
512 encoding.encoding = 'utf-8'
513
514 # lets get all the import lines
515 while parser.check('import'):
516 ref = parser[1]
517
518 if (ref == 'HEAD'):
519 export_head(repo)
520 elif ref.startswith('refs/heads/branches/'):
521 branch = ref[len('refs/heads/branches/'):]
522 export_branch(repo, branch)
523 elif ref.startswith('refs/heads/'):
524 bmark = ref[len('refs/heads/'):]
525 export_bookmark(repo, bmark)
526 elif ref.startswith('refs/tags/'):
527 tag = ref[len('refs/tags/'):]
528 export_tag(repo, tag)
529
530 parser.next()
531
532 encoding.encoding = tmp
533
534 print 'done'
535
536def parse_blob(parser):
537 global blob_marks
538
539 parser.next()
540 mark = parser.get_mark()
541 parser.next()
542 data = parser.get_data()
543 blob_marks[mark] = data
544 parser.next()
545
546def get_merge_files(repo, p1, p2, files):
547 for e in repo[p1].files():
548 if e not in files:
549 if e not in repo[p1].manifest():
550 continue
551 f = { 'ctx' : repo[p1][e] }
552 files[e] = f
553
554def parse_commit(parser):
555 global marks, blob_marks, parsed_refs
556 global mode
557
558 from_mark = merge_mark = None
559
560 ref = parser[1]
561 parser.next()
562
563 commit_mark = parser.get_mark()
564 parser.next()
565 author = parser.get_author()
566 parser.next()
567 committer = parser.get_author()
568 parser.next()
569 data = parser.get_data()
570 parser.next()
571 if parser.check('from'):
572 from_mark = parser.get_mark()
573 parser.next()
574 if parser.check('merge'):
575 merge_mark = parser.get_mark()
576 parser.next()
577 if parser.check('merge'):
578 die('octopus merges are not supported yet')
579
580 files = {}
581
582 for line in parser:
583 if parser.check('M'):
584 t, m, mark_ref, path = line.split(' ', 3)
585 mark = int(mark_ref[1:])
586 f = { 'mode' : hgmode(m), 'data' : blob_marks[mark] }
587 elif parser.check('D'):
588 t, path = line.split(' ', 1)
589 f = { 'deleted' : True }
590 else:
591 die('Unknown file command: %s' % line)
592 files[path] = f
593
594 def getfilectx(repo, memctx, f):
595 of = files[f]
596 if 'deleted' in of:
597 raise IOError
598 if 'ctx' in of:
599 return of['ctx']
600 is_exec = of['mode'] == 'x'
601 is_link = of['mode'] == 'l'
602 rename = of.get('rename', None)
603 return context.memfilectx(f, of['data'],
604 is_link, is_exec, rename)
605
606 repo = parser.repo
607
608 user, date, tz = author
609 extra = {}
610
611 if committer != author:
612 extra['committer'] = "%s %u %u" % committer
613
614 if from_mark:
615 p1 = repo.changelog.node(mark_to_rev(from_mark))
616 else:
617 p1 = '\0' * 20
618
619 if merge_mark:
620 p2 = repo.changelog.node(mark_to_rev(merge_mark))
621 else:
622 p2 = '\0' * 20
623
624 #
625 # If files changed from any of the parents, hg wants to know, but in git if
626 # nothing changed from the first parent, nothing changed.
627 #
628 if merge_mark:
629 get_merge_files(repo, p1, p2, files)
630
631 # Check if the ref is supposed to be a named branch
632 if ref.startswith('refs/heads/branches/'):
633 extra['branch'] = ref[len('refs/heads/branches/'):]
634
635 if mode == 'hg':
636 i = data.find('\n--HG--\n')
637 if i >= 0:
638 tmp = data[i + len('\n--HG--\n'):].strip()
639 for k, v in [e.split(' : ', 1) for e in tmp.split('\n')]:
640 if k == 'rename':
641 old, new = v.split(' => ', 1)
642 files[new]['rename'] = old
643 elif k == 'branch':
644 extra[k] = v
645 elif k == 'extra':
646 ek, ev = v.split(' : ', 1)
647 extra[ek] = urllib.unquote(ev)
648 data = data[:i]
649
650 ctx = context.memctx(repo, (p1, p2), data,
651 files.keys(), getfilectx,
652 user, (date, tz), extra)
653
654 tmp = encoding.encoding
655 encoding.encoding = 'utf-8'
656
657 node = repo.commitctx(ctx)
658
659 encoding.encoding = tmp
660
661 rev = repo[node].rev()
662
663 parsed_refs[ref] = node
664 marks.new_mark(rev, commit_mark)
665
666def parse_reset(parser):
667 global parsed_refs
668
669 ref = parser[1]
670 parser.next()
671 # ugh
672 if parser.check('commit'):
673 parse_commit(parser)
674 return
675 if not parser.check('from'):
676 return
677 from_mark = parser.get_mark()
678 parser.next()
679
680 node = parser.repo.changelog.node(mark_to_rev(from_mark))
681 parsed_refs[ref] = node
682
683def parse_tag(parser):
684 name = parser[1]
685 parser.next()
686 from_mark = parser.get_mark()
687 parser.next()
688 tagger = parser.get_author()
689 parser.next()
690 data = parser.get_data()
691 parser.next()
692
693 # nothing to do
694
695def do_export(parser):
696 global parsed_refs, bmarks, peer
697
698 p_bmarks = []
699
700 parser.next()
701
702 for line in parser.each_block('done'):
703 if parser.check('blob'):
704 parse_blob(parser)
705 elif parser.check('commit'):
706 parse_commit(parser)
707 elif parser.check('reset'):
708 parse_reset(parser)
709 elif parser.check('tag'):
710 parse_tag(parser)
711 elif parser.check('feature'):
712 pass
713 else:
714 die('unhandled export command: %s' % line)
715
716 for ref, node in parsed_refs.iteritems():
717 if ref.startswith('refs/heads/branches'):
718 print "ok %s" % ref
719 elif ref.startswith('refs/heads/'):
720 bmark = ref[len('refs/heads/'):]
721 p_bmarks.append((bmark, node))
722 continue
723 elif ref.startswith('refs/tags/'):
724 tag = ref[len('refs/tags/'):]
725 if mode == 'git':
726 msg = 'Added tag %s for changeset %s' % (tag, hghex(node[:6]));
727 parser.repo.tag([tag], node, msg, False, None, {})
728 else:
729 parser.repo.tag([tag], node, None, True, None, {})
730 print "ok %s" % ref
731 else:
732 # transport-helper/fast-export bugs
733 continue
734
735 if peer:
736 parser.repo.push(peer, force=force_push)
737
738 # handle bookmarks
739 for bmark, node in p_bmarks:
740 ref = 'refs/heads/' + bmark
741 new = hghex(node)
742
743 if bmark in bmarks:
744 old = bmarks[bmark].hex()
745 else:
746 old = ''
747
748 if bmark == 'master' and 'master' not in parser.repo._bookmarks:
749 # fake bookmark
750 pass
751 elif bookmarks.pushbookmark(parser.repo, bmark, old, new):
752 # updated locally
753 pass
754 else:
755 print "error %s" % ref
756 continue
757
758 if peer:
759 if not peer.pushkey('bookmarks', bmark, old, new):
760 print "error %s" % ref
761 continue
762
763 print "ok %s" % ref
764
765 print
766
767def fix_path(alias, repo, orig_url):
768 repo_url = util.url(repo.url())
769 url = util.url(orig_url)
770 if str(url) == str(repo_url):
771 return
772 cmd = ['git', 'config', 'remote.%s.url' % alias, "hg::%s" % repo_url]
773 subprocess.call(cmd)
774
775def main(args):
776 global prefix, dirname, branches, bmarks
777 global marks, blob_marks, parsed_refs
778 global peer, mode, bad_mail, bad_name
779 global track_branches, force_push
780
781 alias = args[1]
782 url = args[2]
783 peer = None
784
785 hg_git_compat = False
786 track_branches = True
787 force_push = True
788
789 try:
790 if get_config('remote-hg.hg-git-compat') == 'true\n':
791 hg_git_compat = True
792 track_branches = False
793 if get_config('remote-hg.track-branches') == 'false\n':
794 track_branches = False
795 if get_config('remote-hg.force-push') == 'false\n':
796 force_push = False
797 except subprocess.CalledProcessError:
798 pass
799
800 if hg_git_compat:
801 mode = 'hg'
802 bad_mail = 'none@none'
803 bad_name = ''
804 else:
805 mode = 'git'
806 bad_mail = 'unknown'
807 bad_name = 'Unknown'
808
809 if alias[4:] == url:
810 is_tmp = True
811 alias = util.sha1(alias).hexdigest()
812 else:
813 is_tmp = False
814
815 gitdir = os.environ['GIT_DIR']
816 dirname = os.path.join(gitdir, 'hg', alias)
817 branches = {}
818 bmarks = {}
819 blob_marks = {}
820 parsed_refs = {}
821
822 repo = get_repo(url, alias)
823 prefix = 'refs/hg/%s' % alias
824
825 if not is_tmp:
826 fix_path(alias, peer or repo, url)
827
828 if not os.path.exists(dirname):
829 os.makedirs(dirname)
830
831 marks_path = os.path.join(dirname, 'marks-hg')
832 marks = Marks(marks_path)
833
834 parser = Parser(repo)
835 for line in parser:
836 if parser.check('capabilities'):
837 do_capabilities(parser)
838 elif parser.check('list'):
839 do_list(parser)
840 elif parser.check('import'):
841 do_import(parser)
842 elif parser.check('export'):
843 do_export(parser)
844 else:
845 die('unhandled command: %s' % line)
846 sys.stdout.flush()
847
848 if not is_tmp:
849 marks.store()
850 else:
851 shutil.rmtree(dirname)
852
853sys.exit(main(sys.argv))