#
# Known limitations:
#
-# - Poor performance in the best case: it takes forever to check
-# whether we're up-to-date (on fetch or push) or to fetch a few
-# revisions from a large wiki, because we use exclusively a
-# page-based synchronization. We could switch to a wiki-wide
-# synchronization when the synchronization involves few revisions
-# but the wiki is large.
+# - Several strategies are provided to fetch modifications from the
+# wiki, but no automatic heuristics is provided, the user has
+# to understand and chose which strategy is appropriate for him.
#
# - Git renames could be turned into MediaWiki renames (see TODO
# below)
#
-# - login/password support requires the user to write the password
-# cleartext in a file (see TODO below).
-#
# - No way to import "one page, and all pages included in it"
#
# - Multiple remote MediaWikis have not been very well tested.
# used to reflect file creation or deletion in diff.
use constant NULL_SHA1 => "0000000000000000000000000000000000000000";
+# Used on Git's side to reflect empty edit messages on the wiki
+use constant EMPTY_MESSAGE => '*Empty MediaWiki Message*';
+
my $remotename = $ARGV[0];
my $url = $ARGV[1];
my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories"));
chomp(@tracked_categories);
-# Import media files too.
+# Import media files on pull
my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport");
chomp($import_media);
$import_media = ($import_media eq "true");
+# Export media files on push
+my $export_media = run_git("config --get --bool remote.". $remotename .".mediaexport");
+chomp($export_media);
+$export_media = !($export_media eq "false");
+
my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin");
-# TODO: ideally, this should be able to read from keyboard, but we're
-# inside a remote helper, so our stdin is connect to git, not to a
-# terminal.
+# Note: mwPassword is discourraged. Use the credential system instead.
my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword");
my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain");
chomp($wiki_login);
chomp($shallow_import);
$shallow_import = ($shallow_import eq "true");
+# Fetch (clone and pull) by revisions instead of by pages. This behavior
+# is more efficient when we have a wiki with lots of pages and we fetch
+# the revisions quite often so that they concern only few pages.
+# Possible values:
+# - by_rev: perform one query per new revision on the remote wiki
+# - by_page: query each tracked page for new revision
+my $fetch_strategy = run_git("config --get remote.$remotename.fetchStrategy");
+unless ($fetch_strategy) {
+ $fetch_strategy = run_git("config --get mediawiki.fetchStrategy");
+}
+chomp($fetch_strategy);
+unless ($fetch_strategy) {
+ $fetch_strategy = "by_page";
+}
+
# Dumb push: don't update notes and mediawiki ref to reflect the last push.
#
# Configurable with mediawiki.dumbPush, or per-remote with
## credential API management (generic functions)
-sub credential_from_url {
- my $url = shift;
- my $parsed = URI->new($url);
- my %credential;
-
- if ($parsed->scheme) {
- $credential{protocol} = $parsed->scheme;
- }
- if ($parsed->host) {
- $credential{host} = $parsed->host;
- }
- if ($parsed->path) {
- $credential{path} = $parsed->path;
- }
- if ($parsed->userinfo) {
- if ($parsed->userinfo =~ /([^:]*):(.*)/) {
- $credential{username} = $1;
- $credential{password} = $2;
- } else {
- $credential{username} = $parsed->userinfo;
- }
- }
-
- return %credential;
-}
-
sub credential_read {
my %credential;
my $reader = shift;
sub credential_write {
my $credential = shift;
my $writer = shift;
+ # url overwrites other fields, so it must come first
+ print $writer "url=$credential->{url}\n" if exists $credential->{url};
while (my ($key, $value) = each(%$credential) ) {
- if ($value) {
+ if (length $value && $key ne 'url') {
print $writer "$key=$value\n";
}
}
$mediawiki = MediaWiki::API->new;
$mediawiki->{config}->{api_url} = "$url/api.php";
if ($wiki_login) {
- my %credential = credential_from_url($url);
+ my %credential = (url => $url);
$credential{username} = $wiki_login;
$credential{password} = $wiki_passwd;
credential_run("fill", \%credential);
sub get_mw_pages {
mw_connect_maybe();
+ print STDERR "Listing pages on remote wiki...\n";
+
my %pages; # hash on page titles to avoid duplicates
my $user_defined;
if (@tracked_pages) {
get_all_mediafiles(\%pages);
}
}
+ print STDERR (scalar keys %pages) . " pages found.\n";
return %pages;
}
# Remember the timestamp corresponding to a revision id.
my %basetimestamps;
+# Get the last remote revision without taking in account which pages are
+# tracked or not. This function makes a single request to the wiki thus
+# avoid a loop onto all tracked pages. This is useful for the fetch-by-rev
+# option.
+sub get_last_global_remote_rev {
+ mw_connect_maybe();
+
+ my $query = {
+ action => 'query',
+ list => 'recentchanges',
+ prop => 'revisions',
+ rclimit => '1',
+ rcdir => 'older',
+ };
+ my $result = $mediawiki->api($query);
+ return $result->{query}->{recentchanges}[0]->{revid};
+}
+
+# Get the last remote revision concerning the tracked pages and the tracked
+# categories.
sub get_last_remote_revision {
mw_connect_maybe();
my $max_rev_num = 0;
+ print STDERR "Getting last revision id on tracked pages...\n";
+
foreach my $page (@pages) {
my $id = $page->{pageid};
mw_connect_maybe();
- my %pages_hash = get_mw_pages();
- my @pages = values(%pages_hash);
-
print STDERR "Searching revisions...\n";
my $last_local = get_last_local_revision();
my $fetch_from = $last_local + 1;
} else {
print STDERR ", fetching from here.\n";
}
- my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
- # Creation of the fast-import stream
- print STDERR "Fetching & writing export data...\n";
+ my $n = 0;
+ if ($fetch_strategy eq "by_rev") {
+ print STDERR "Fetching & writing export data by revs...\n";
+ $n = mw_import_ref_by_revs($fetch_from);
+ } elsif ($fetch_strategy eq "by_page") {
+ print STDERR "Fetching & writing export data by pages...\n";
+ $n = mw_import_ref_by_pages($fetch_from);
+ } else {
+ print STDERR "fatal: invalid fetch strategy \"$fetch_strategy\".\n";
+ print STDERR "Check your configuration variables remote.$remotename.fetchStrategy and mediawiki.fetchStrategy\n";
+ exit 1;
+ }
+
+ if ($fetch_from == 1 && $n == 0) {
+ print STDERR "You appear to have cloned an empty MediaWiki.\n";
+ # Something has to be done remote-helper side. If nothing is done, an error is
+ # thrown saying that HEAD is refering to unknown object 0000000000000000000
+ # and the clone fails.
+ }
+}
+
+sub mw_import_ref_by_pages {
+
+ my $fetch_from = shift;
+ my %pages_hash = get_mw_pages();
+ my @pages = values(%pages_hash);
+
+ my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
@revisions = sort {$a->{revid} <=> $b->{revid}} @revisions;
my @revision_ids = map $_->{revid}, @revisions;
- $n = 0;
+ return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
+}
+
+sub mw_import_ref_by_revs {
+
+ my $fetch_from = shift;
+ my %pages_hash = get_mw_pages();
+
+ my $last_remote = get_last_global_remote_rev();
+ my @revision_ids = $fetch_from..$last_remote;
+ return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
+}
+
+# Import revisions given in second argument (array of integers).
+# Only pages appearing in the third argument (hash indexed by page titles)
+# will be imported.
+sub mw_import_revids {
+ my $fetch_from = shift;
+ my $revision_ids = shift;
+ my $pages = shift;
+
+ my $n = 0;
+ my $n_actual = 0;
my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined
- foreach my $pagerevid (@revision_ids) {
+ foreach my $pagerevid (@$revision_ids) {
+ # Count page even if we skip it, since we display
+ # $n/$total and $total includes skipped pages.
+ $n++;
+
# fetch the content of the pages
my $query = {
action => 'query',
my $result = $mediawiki->api($query);
+ if (!$result) {
+ die "Failed to retrieve modified page for revision $pagerevid";
+ }
+
+ if (defined($result->{query}->{badrevids}->{$pagerevid})) {
+ # The revision id does not exist on the remote wiki.
+ next;
+ }
+
+ if (!defined($result->{query}->{pages})) {
+ die "Invalid revision $pagerevid.";
+ }
+
my @result_pages = values(%{$result->{query}->{pages}});
my $result_page = $result_pages[0];
my $rev = $result_pages[0]->{revisions}->[0];
- $n++;
-
my $page_title = $result_page->{title};
+
+ if (!exists($pages->{$page_title})) {
+ print STDERR "$n/", scalar(@$revision_ids),
+ ": Skipping revision #$rev->{revid} of $page_title\n";
+ next;
+ }
+
+ $n_actual++;
+
my %commit;
$commit{author} = $rev->{user} || 'Anonymous';
- $commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*';
+ $commit{comment} = $rev->{comment} || EMPTY_MESSAGE;
$commit{title} = mediawiki_smudge_filename($page_title);
$commit{mw_revision} = $rev->{revid};
$commit{content} = mediawiki_smudge($rev->{'*'});
# Differentiates classic pages and media files.
my ($namespace, $filename) = $page_title =~ /^([^:]*):(.*)$/;
my %mediafile;
- if ($namespace && get_mw_namespace_id($namespace) == get_mw_namespace_id("File")) {
- %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
+ if ($namespace) {
+ my $id = get_mw_namespace_id($namespace);
+ if ($id && $id == get_mw_namespace_id("File")) {
+ %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
+ }
}
# If this is a revision of the media page for new version
# of a file do one common commit for both file and media page.
# Else do commit only for that page.
- print STDERR "$n/", scalar(@revision_ids), ": Revision #$rev->{revid} of $commit{title}\n";
- import_file_revision(\%commit, ($fetch_from == 1), $n, \%mediafile);
+ print STDERR "$n/", scalar(@$revision_ids), ": Revision #$rev->{revid} of $commit{title}\n";
+ import_file_revision(\%commit, ($fetch_from == 1), $n_actual, \%mediafile);
}
- if ($fetch_from == 1 && $n == 0) {
- print STDERR "You appear to have cloned an empty MediaWiki.\n";
- # Something has to be done remote-helper side. If nothing is done, an error is
- # thrown saying that HEAD is refering to unknown object 0000000000000000000
- # and the clone fails.
- }
+ return $n_actual;
}
sub error_non_fast_forward {
my $oldrevid = shift;
my $newrevid;
+ if ($summary eq EMPTY_MESSAGE) {
+ $summary = '';
+ }
+
my $new_sha1 = $diff_info_split[3];
my $old_sha1 = $diff_info_split[2];
my $page_created = ($old_sha1 eq NULL_SHA1);
$extension = "";
}
if ($extension eq "mw") {
+ my $ns = get_mw_namespace_id_for_page($complete_file_name);
+ if ($ns && $ns == get_mw_namespace_id("File") && (!$export_media)) {
+ print STDERR "Ignoring media file related page: $complete_file_name\n";
+ return ($oldrevid, "ok");
+ }
my $file_content;
if ($page_deleted) {
# Deleting a page usually requires
}
$newrevid = $result->{edit}->{newrevid};
print STDERR "Pushed file: $new_sha1 - $title\n";
- } else {
+ } elsif ($export_media) {
$newrevid = mw_upload_file($complete_file_name, $new_sha1,
$extension, $page_deleted,
$summary);
+ } else {
+ print STDERR "Ignoring media file $title\n";
}
$newrevid = ($newrevid or $oldrevid);
return ($newrevid, "ok");
if ($last_local_revid > 0) {
my $parsed_sha1 = $remoteorigin_sha1;
# Find a path from last MediaWiki commit to pushed commit
+ print STDERR "Computing path from local to remote ...\n";
+ my @local_ancestry = split(/\n/, run_git("rev-list --boundary --parents $local ^$parsed_sha1"));
+ my %local_ancestry;
+ foreach my $line (@local_ancestry) {
+ if (my ($child, $parents) = $line =~ m/^-?([a-f0-9]+) ([a-f0-9 ]+)/) {
+ foreach my $parent (split(' ', $parents)) {
+ $local_ancestry{$parent} = $child;
+ }
+ } elsif (!$line =~ m/^([a-f0-9]+)/) {
+ die "Unexpected output from git rev-list: $line";
+ }
+ }
while ($parsed_sha1 ne $HEAD_sha1) {
- my @commit_info = grep(/^$parsed_sha1/, split(/\n/, run_git("rev-list --children $local")));
- if (!@commit_info) {
+ my $child = $local_ancestry{$parsed_sha1};
+ if (!$child) {
+ printf STDERR "Cannot find a path in history from remote commit to last commit\n";
return error_non_fast_forward($remote);
}
- my @commit_info_split = split(/ |\n/, $commit_info[0]);
- # $commit_info_split[1] is the sha1 of the commit to export
- # $commit_info_split[0] is the sha1 of its direct child
- push(@commit_pairs, \@commit_info_split);
- $parsed_sha1 = $commit_info_split[1];
+ push(@commit_pairs, [$parsed_sha1, $child]);
+ $parsed_sha1 = $child;
}
} else {
# No remote mediawiki revision. Export the whole
}
}
unless ($dumb_push) {
- run_git("notes --ref=$remotename/mediawiki add -m \"mediawiki_revision: $mw_revision\" $sha1_commit");
+ run_git("notes --ref=$remotename/mediawiki add -f -m \"mediawiki_revision: $mw_revision\" $sha1_commit");
run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child");
}
}
# Look at configuration file, if the record for that namespace is
# already cached. Namespaces are stored in form:
# "Name_of_namespace:Id_namespace", ex.: "File:6".
- my @temp = split(/[ \n]/, run_git("config --get-all remote."
+ my @temp = split(/[\n]/, run_git("config --get-all remote."
. $remotename .".namespaceCache"));
chomp(@temp);
foreach my $ns (@temp) {
my ($n, $id) = split(/:/, $ns);
- $namespace_id{$n} = $id;
+ if ($id eq 'notANameSpace') {
+ $namespace_id{$n} = {is_namespace => 0};
+ } else {
+ $namespace_id{$n} = {is_namespace => 1, id => $id};
+ }
$cached_mw_namespace_id{$n} = 1;
}
}
while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) {
if (defined($ns->{id}) && defined($ns->{canonical})) {
- $namespace_id{$ns->{canonical}} = $ns->{id};
+ $namespace_id{$ns->{canonical}} = {is_namespace => 1, id => $ns->{id}};
if ($ns->{'*'}) {
# alias (e.g. french Fichier: as alias for canonical File:)
- $namespace_id{$ns->{'*'}} = $ns->{id};
+ $namespace_id{$ns->{'*'}} = {is_namespace => 1, id => $ns->{id}};
}
}
}
}
- my $id = $namespace_id{$name};
+ my $ns = $namespace_id{$name};
+ my $id;
- if (defined $id) {
- # Store explicitely requested namespaces on disk
- if (!exists $cached_mw_namespace_id{$name}) {
- run_git("config --add remote.". $remotename
- .".namespaceCache \"". $name .":". $id ."\"");
- $cached_mw_namespace_id{$name} = 1;
- }
- return $id;
+ unless (defined $ns) {
+ print STDERR "No such namespace $name on MediaWiki.\n";
+ $ns = {is_namespace => 0};
+ $namespace_id{$name} = $ns;
+ }
+
+ if ($ns->{is_namespace}) {
+ $id = $ns->{id};
+ }
+
+ # Store "notANameSpace" as special value for inexisting namespaces
+ my $store_id = ($id || 'notANameSpace');
+
+ # Store explicitely requested namespaces on disk
+ if (!exists $cached_mw_namespace_id{$name}) {
+ run_git("config --add remote.". $remotename
+ .".namespaceCache \"". $name .":". $store_id ."\"");
+ $cached_mw_namespace_id{$name} = 1;
+ }
+ return $id;
+}
+
+sub get_mw_namespace_id_for_page {
+ if (my ($namespace) = $_[0] =~ /^([^:]*):/) {
+ return get_mw_namespace_id($namespace);
} else {
- die "No such namespace $name on MediaWiki.";
+ return;
}
}