=head1 Invocation
- git-archimport [ -h ] [ -v ] [ -T ] [ -t tempdir ] <archive>/<branch> [ <archive>/<branch> ]
+ git-archimport [ -h ] [ -v ] [ -o ] [ -a ] [ -f ] [ -T ]
+ [ -D depth] [ -t tempdir ] <archive>/<branch> [ <archive>/<branch> ]
Imports a project from one or more Arch repositories. It will follow branches
and repositories within the namespaces defined by the <archive/branch>
-parameters suppplied. If it cannot find the remote branch a merge comes from
+parameters supplied. If it cannot find the remote branch a merge comes from
it will just import it as a regular commit. If it can find it, it will mark it
as a merge whenever possible.
- audit shell-escaping of filenames
- hide our private tags somewhere smarter
- find a way to make "cat *patches | patch" safe even when patchfiles are missing newlines
+ - sort and apply patches by graphing ancestry relations instead of just
+ relying in dates supplied in the changeset itself.
+ tla ancestry-graph -m could be helpful here...
=head1 Devel tricks
$ENV{"GIT_DIR"} = $git_dir;
my $ptag_dir = "$git_dir/archimport/tags";
-our($opt_h,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
+our($opt_h,$opt_f,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
sub usage() {
print STDERR <<END;
Usage: ${\basename $0} # fetch/update GIT from Arch
- [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
+ [ -h ] [ -v ] [ -o ] [ -a ] [ -f ] [ -T ] [ -D depth ] [ -t tempdir ]
repository/arch-branch [ repository/arch-branch] ...
END
exit(1);
}
-getopts("Thvat:D:") or usage();
+getopts("fThvat:D:") or usage();
usage if $opt_h;
@ARGV >= 1 or usage();
# $arch_branches:
# values associated with keys:
# =1 - Arch version / git 'branch' detected via abrowse on a limit
-# >1 - Arch version / git 'branch' of an auxilliary branch we've merged
+# >1 - Arch version / git 'branch' of an auxiliary branch we've merged
my %arch_branches = map { $_ => 1 } @ARGV;
$ENV{'TMPDIR'} = $opt_t if $opt_t; # $ENV{TMPDIR} will affect tempdir() calls:
my %unreachable = (); # Arch repositories we can't access :<
my @psets = (); # the collection
my %psets = (); # the collection, by name
+my %stats = ( # Track which strategy we used to import:
+ get_tag => 0, replay => 0, get_new => 0, get_delta => 0,
+ simple_changeset => 0, import_or_tag => 0
+);
my %rptags = (); # my reverse private tags
# to map a SHA1 to a commitid
*git_branchname = $opt_o ? *old_style_branchname : *tree_dirname;
-# process patchsets
-foreach my $ps (@psets) {
- $ps->{branch} = git_branchname($ps->{id});
+sub process_patchset_accurate {
+ my $ps = shift;
+
+ # switch to that branch if we're not already in that branch:
+ if (-e "$git_dir/refs/heads/$ps->{branch}") {
+ system('git-checkout','-f',$ps->{branch}) == 0 or die "$! $?\n";
- #
- # ensure we have a clean state
- #
- if (`git-diff-files`) {
- die "Unclean tree when about to process $ps->{id} " .
- " - did we fail to commit cleanly before?";
- }
- die $! if $?;
-
- #
- # skip commits already in repo
- #
- if (ptag($ps->{id})) {
- $opt_v && print " * Skipping already imported: $ps->{id}\n";
- next;
+ # remove any old stuff that got leftover:
+ my $rm = safe_pipe_capture('git-ls-files','--others','-z');
+ rmtree(split(/\0/,$rm)) if $rm;
}
+
+ # Apply the import/changeset/merge into the working tree
+ my $dir = sync_to_ps($ps);
+ # read the new log entry:
+ my @commitlog = safe_pipe_capture($TLA,'cat-log','-d',$dir,$ps->{id});
+ die "Error in cat-log: $!" if $?;
+ chomp @commitlog;
+
+ # grab variables we want from the log, new fields get added to $ps:
+ # (author, date, email, summary, message body ...)
+ parselog($ps, \@commitlog);
+
+ if ($ps->{id} =~ /--base-0$/ && $ps->{id} ne $psets[0]{id}) {
+ # this should work when importing continuations
+ if ($ps->{tag} && (my $branchpoint = eval { ptag($ps->{tag}) })) {
+
+ # find where we are supposed to branch from
+ system('git-checkout','-f','-b',$ps->{branch},
+ $branchpoint) == 0 or die "$! $?\n";
+
+ # remove any old stuff that got leftover:
+ my $rm = safe_pipe_capture('git-ls-files','--others','-z');
+ rmtree(split(/\0/,$rm)) if $rm;
- print " * Starting to work on $ps->{id}\n";
+ # If we trust Arch with the fact that this is just
+ # a tag, and it does not affect the state of the tree
+ # then we just tag and move on
+ tag($ps->{id}, $branchpoint);
+ ptag($ps->{id}, $branchpoint);
+ print " * Tagged $ps->{id} at $branchpoint\n";
+ return 0;
+ } else {
+ warn "Tagging from unknown id unsupported\n" if $ps->{tag};
+ }
+ # allow multiple bases/imports here since Arch supports cherry-picks
+ # from unrelated trees
+ }
+
+ # update the index with all the changes we got
+ system('git-diff-files --name-only -z | '.
+ 'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
+ system('git-ls-files --others -z | '.
+ 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
+ return 1;
+}
+# the native changeset processing strategy. This is very fast, but
+# does not handle permissions or any renames involving directories
+sub process_patchset_fast {
+ my $ps = shift;
#
# create the branch if needed
#
tag($ps->{id}, $branchpoint);
ptag($ps->{id}, $branchpoint);
print " * Tagged $ps->{id} at $branchpoint\n";
- next;
+ return 0;
}
die $! if $?;
}
#
if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
apply_import($ps) or die $!;
+ $stats{import_or_tag}++;
$import=0;
} elsif ($ps->{type} eq 's') {
apply_cset($ps);
+ $stats{simple_changeset}++;
}
#
# prepare update git's index, based on what arch knows
# about the pset, resolve parents, etc
#
- my $tree;
my @commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
die "Error in cat-archive-log: $!" if $?;
# imports don't give us good info
# on added files. Shame on them
if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
- system('git-ls-files --others -z | '.
- 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
system('git-ls-files --deleted -z | '.
'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
+ system('git-ls-files --others -z | '.
+ 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
}
# TODO: handle removed_directories and renamed_directories:
-
- if (my $add = $ps->{new_files}) {
- while (@$add) {
- my @slice = splice(@$add, 0, 100);
- system('git-update-index','--add','--',@slice) == 0 or
- die "Error in git-update-index --add: $! $?\n";
- }
- }
-
+
if (my $del = $ps->{removed_files}) {
unlink @$del;
while (@$del) {
unless (-d dirname($to)) {
mkpath(dirname($to)); # will die on err
}
- print "moving $from $to";
+ # print "moving $from $to";
rename($from, $to) or die "Error renaming '$from' '$to': $!\n";
system('git-update-index','--remove','--',$from) == 0 or
die "Error in git-update-index --remove: $! $?\n";
system('git-update-index','--add','--',$to) == 0 or
die "Error in git-update-index --add: $! $?\n";
}
+ }
+ if (my $add = $ps->{new_files}) {
+ while (@$add) {
+ my @slice = splice(@$add, 0, 100);
+ system('git-update-index','--add','--',@slice) == 0 or
+ die "Error in git-update-index --add: $! $?\n";
+ }
}
if (my $mod = $ps->{modified_files}) {
die "Error in git-update-index: $! $?\n";
}
}
+ return 1; # we successfully applied the changeset
+}
+
+if ($opt_f) {
+ print "Will import patchsets using the fast strategy\n",
+ "Renamed directories and permission changes will be missed\n";
+ *process_patchset = *process_patchset_fast;
+} else {
+ print "Using the default (accurate) import strategy.\n",
+ "Things may be a bit slow\n";
+ *process_patchset = *process_patchset_accurate;
+}
+
+foreach my $ps (@psets) {
+ # process patchsets
+ $ps->{branch} = git_branchname($ps->{id});
+
+ #
+ # ensure we have a clean state
+ #
+ if (my $dirty = `git-diff-files`) {
+ die "Unclean tree when about to process $ps->{id} " .
+ " - did we fail to commit cleanly before?\n$dirty";
+ }
+ die $! if $?;
+ #
+ # skip commits already in repo
+ #
+ if (ptag($ps->{id})) {
+ $opt_v && print " * Skipping already imported: $ps->{id}\n";
+ next;
+ }
+
+ print " * Starting to work on $ps->{id}\n";
+
+ process_patchset($ps) or next;
+
# warn "errors when running git-update-index! $!";
- $tree = `git-write-tree`;
+ my $tree = `git-write-tree`;
die "cannot write tree $!" if $?;
chomp $tree;
$opt_v && print " + parents: ",join(' ',@par),"\n";
}
+if ($opt_v) {
+ foreach (sort keys %stats) {
+ print" $_: $stats{$_}\n";
+ }
+}
+exit 0;
+
+# used by the accurate strategy:
+sub sync_to_ps {
+ my $ps = shift;
+ my $tree_dir = $tmp.'/'.tree_dirname($ps->{id});
+
+ $opt_v && print "sync_to_ps($ps->{id}) method: ";
+
+ if (-d $tree_dir) {
+ if ($ps->{type} eq 't') {
+ $opt_v && print "get (tag)\n";
+ # looks like a tag-only or (worse,) a mixed tags/changeset branch,
+ # can't rely on replay to work correctly on these
+ rmtree($tree_dir);
+ safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
+ $stats{get_tag}++;
+ } else {
+ my $tree_id = arch_tree_id($tree_dir);
+ if ($ps->{parent_id} && ($ps->{parent_id} eq $tree_id)) {
+ # the common case (hopefully)
+ $opt_v && print "replay\n";
+ safe_pipe_capture($TLA,'replay','-d',$tree_dir,$ps->{id});
+ $stats{replay}++;
+ } else {
+ # getting one tree is usually faster than getting two trees
+ # and applying the delta ...
+ rmtree($tree_dir);
+ $opt_v && print "apply-delta\n";
+ safe_pipe_capture($TLA,'get','--no-pristine',
+ $ps->{id},$tree_dir);
+ $stats{get_delta}++;
+ }
+ }
+ } else {
+ # new branch work
+ $opt_v && print "get (new tree)\n";
+ safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
+ $stats{get_new}++;
+ }
+
+ # added -I flag to rsync since we're going to fast! AIEEEEE!!!!
+ system('rsync','-aI','--delete','--exclude',$git_dir,
+# '--exclude','.arch-inventory',
+ '--exclude','.arch-ids','--exclude','{arch}',
+ '--exclude','+*','--exclude',',*',
+ "$tree_dir/",'./') == 0 or die "Cannot rsync $tree_dir: $! $?";
+ return $tree_dir;
+}
+
sub apply_import {
my $ps = shift;
my $bname = git_branchname($ps->{id});
if (`find $tmp/changeset/patches -type f -name '*.patch'`) {
# this can be sped up considerably by doing
# (find | xargs cat) | patch
- # but that cna get mucked up by patches
+ # but that can get mucked up by patches
# with missing trailing newlines or the standard
# 'missing newline' flag in the patch - possibly
# produced with an old/buggy diff.
# skip Arch control files, unescape pika-escaped files
foreach my $k (keys %want_headers) {
next unless (defined $ps->{$k});
- my @tmp;
+ my @tmp = ();
foreach my $t (@{$ps->{$k}}) {
next unless length ($t);
next if $t =~ m!\{arch\}/!;
}
push @tmp, $t;
}
- $ps->{$k} = \@tmp if scalar @tmp;
+ $ps->{$k} = \@tmp;
}
}
# now walk up to the mergepoint collecting what patches we have
my $branchtip = git_rev_parse($ps->{branch});
- my @ancestors = `git-rev-list --merge-order $branchtip ^$mergebase`;
+ my @ancestors = `git-rev-list --topo-order $branchtip ^$mergebase`;
my %have; # collected merges this branch has
foreach my $merge (@{$ps->{merges}}) {
$have{$merge} = 1;
# see what the remote branch has - these are the merges we
# will want to have in a consecutive series from the mergebase
my $otherbranchtip = git_rev_parse($branch);
- my @needraw = `git-rev-list --merge-order $otherbranchtip ^$mergebase`;
+ my @needraw = `git-rev-list --topo-order $otherbranchtip ^$mergebase`;
my @need;
foreach my $needps (@needraw) { # get the psets
$needps = commitid2pset($needps);
}
-# an alterative to `command` that allows input to be passed as an array
+# an alternative to `command` that allows input to be passed as an array
# to work around shell problems with weird characters in arguments
sub safe_pipe_capture {
my @output;
@output = (<$child>);
close $child or die join(' ',@_).": $! $?";
} else {
- exec(@_) or die $?; # exec() can fail the executable can't be found
+ exec(@_) or die "$! $?"; # exec() can fail the executable can't be found
}
return wantarray ? @output : join('',@output);
}