Andrew's git - gitweb.git/blob

4bf990bd58040f2644cf79a0301fb9e7ac41b27d
   1#! /usr/bin/perl
   2
   3# Copyright (C) 2011
   4#     Jérémie Nikaes <jeremie.nikaes@ensimag.imag.fr>
   5#     Arnaud Lacurie <arnaud.lacurie@ensimag.imag.fr>
   6#     Claire Fousse <claire.fousse@ensimag.imag.fr>
   7#     David Amouyal <david.amouyal@ensimag.imag.fr>
   8#     Matthieu Moy <matthieu.moy@grenoble-inp.fr>
   9# License: GPL v2 or later
  10
  11# Gateway between Git and MediaWiki.
  12#   https://github.com/Bibzball/Git-Mediawiki/wiki
  13#
  14# Known limitations:
  15#
  16# - Poor performance in the best case: it takes forever to check
  17#   whether we're up-to-date (on fetch or push) or to fetch a few
  18#   revisions from a large wiki, because we use exclusively a
  19#   page-based synchronization. We could switch to a wiki-wide
  20#   synchronization when the synchronization involves few revisions
  21#   but the wiki is large.
  22#
  23# - Git renames could be turned into MediaWiki renames (see TODO
  24#   below)
  25#
  26# - login/password support requires the user to write the password
  27#   cleartext in a file (see TODO below).
  28#
  29# - No way to import "one page, and all pages included in it"
  30#
  31# - Multiple remote MediaWikis have not been very well tested.
  32
  33use strict;
  34use MediaWiki::API;
  35use DateTime::Format::ISO8601;
  36
  37# By default, use UTF-8 to communicate with Git and the user
  38binmode STDERR, ":utf8";
  39binmode STDOUT, ":utf8";
  40
  41use URI::Escape;
  42use IPC::Open2;
  43
  44use warnings;
  45
  46# Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced
  47use constant SLASH_REPLACEMENT => "%2F";
  48
  49# It's not always possible to delete pages (may require some
  50# priviledges). Deleted pages are replaced with this content.
  51use constant DELETED_CONTENT => "[[Category:Deleted]]\n";
  52
  53# It's not possible to create empty pages. New empty files in Git are
  54# sent with this content instead.
  55use constant EMPTY_CONTENT => "<!-- empty page -->\n";
  56
  57# used to reflect file creation or deletion in diff.
  58use constant NULL_SHA1 => "0000000000000000000000000000000000000000";
  59
  60my $remotename = $ARGV[0];
  61my $url = $ARGV[1];
  62
  63# Accept both space-separated and multiple keys in config file.
  64# Spaces should be written as _ anyway because we'll use chomp.
  65my @tracked_pages = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".pages"));
  66chomp(@tracked_pages);
  67
  68# Just like @tracked_pages, but for MediaWiki categories.
  69my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories"));
  70chomp(@tracked_categories);
  71
  72# Import media files too.
  73my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport");
  74chomp($import_media);
  75$import_media = ($import_media eq "true");
  76
  77my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin");
  78# TODO: ideally, this should be able to read from keyboard, but we're
  79# inside a remote helper, so our stdin is connect to git, not to a
  80# terminal.
  81my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword");
  82my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain");
  83chomp($wiki_login);
  84chomp($wiki_passwd);
  85chomp($wiki_domain);
  86
  87# Import only last revisions (both for clone and fetch)
  88my $shallow_import = run_git("config --get --bool remote.". $remotename .".shallow");
  89chomp($shallow_import);
  90$shallow_import = ($shallow_import eq "true");
  91
  92# Dumb push: don't update notes and mediawiki ref to reflect the last push.
  93#
  94# Configurable with mediawiki.dumbPush, or per-remote with
  95# remote.<remotename>.dumbPush.
  96#
  97# This means the user will have to re-import the just-pushed
  98# revisions. On the other hand, this means that the Git revisions
  99# corresponding to MediaWiki revisions are all imported from the wiki,
 100# regardless of whether they were initially created in Git or from the
 101# web interface, hence all users will get the same history (i.e. if
 102# the push from Git to MediaWiki loses some information, everybody
 103# will get the history with information lost). If the import is
 104# deterministic, this means everybody gets the same sha1 for each
 105# MediaWiki revision.
 106my $dumb_push = run_git("config --get --bool remote.$remotename.dumbPush");
 107unless ($dumb_push) {
 108        $dumb_push = run_git("config --get --bool mediawiki.dumbPush");
 109}
 110chomp($dumb_push);
 111$dumb_push = ($dumb_push eq "true");
 112
 113my $wiki_name = $url;
 114$wiki_name =~ s/[^\/]*:\/\///;
 115# If URL is like http://user:password@example.com/, we clearly don't
 116# want the password in $wiki_name. While we're there, also remove user
 117# and '@' sign, to avoid author like MWUser@HTTPUser@host.com
 118$wiki_name =~ s/^.*@//;
 119
 120# Commands parser
 121my $entry;
 122my @cmd;
 123while (<STDIN>) {
 124        chomp;
 125        @cmd = split(/ /);
 126        if (defined($cmd[0])) {
 127                # Line not blank
 128                if ($cmd[0] eq "capabilities") {
 129                        die("Too many arguments for capabilities") unless (!defined($cmd[1]));
 130                        mw_capabilities();
 131                } elsif ($cmd[0] eq "list") {
 132                        die("Too many arguments for list") unless (!defined($cmd[2]));
 133                        mw_list($cmd[1]);
 134                } elsif ($cmd[0] eq "import") {
 135                        die("Invalid arguments for import") unless ($cmd[1] ne "" && !defined($cmd[2]));
 136                        mw_import($cmd[1]);
 137                } elsif ($cmd[0] eq "option") {
 138                        die("Too many arguments for option") unless ($cmd[1] ne "" && $cmd[2] ne "" && !defined($cmd[3]));
 139                        mw_option($cmd[1],$cmd[2]);
 140                } elsif ($cmd[0] eq "push") {
 141                        mw_push($cmd[1]);
 142                } else {
 143                        print STDERR "Unknown command. Aborting...\n";
 144                        last;
 145                }
 146        } else {
 147                # blank line: we should terminate
 148                last;
 149        }
 150
 151        BEGIN { $| = 1 } # flush STDOUT, to make sure the previous
 152                         # command is fully processed.
 153}
 154
 155########################## Functions ##############################
 156
 157## credential API management (generic functions)
 158
 159sub credential_from_url {
 160        my $url = shift;
 161        my $parsed = URI->new($url);
 162        my %credential;
 163
 164        if ($parsed->scheme) {
 165                $credential{protocol} = $parsed->scheme;
 166        }
 167        if ($parsed->host) {
 168                $credential{host} = $parsed->host;
 169        }
 170        if ($parsed->path) {
 171                $credential{path} = $parsed->path;
 172        }
 173        if ($parsed->userinfo) {
 174                if ($parsed->userinfo =~ /([^:]*):(.*)/) {
 175                        $credential{username} = $1;
 176                        $credential{password} = $2;
 177                } else {
 178                        $credential{username} = $parsed->userinfo;
 179                }
 180        }
 181
 182        return %credential;
 183}
 184
 185sub credential_read {
 186        my %credential;
 187        my $reader = shift;
 188        my $op = shift;
 189        while (<$reader>) {
 190                my ($key, $value) = /([^=]*)=(.*)/;
 191                if (not defined $key) {
 192                        die "ERROR receiving response from git credential $op:\n$_\n";
 193                }
 194                $credential{$key} = $value;
 195        }
 196        return %credential;
 197}
 198
 199sub credential_write {
 200        my $credential = shift;
 201        my $writer = shift;
 202        while (my ($key, $value) = each(%$credential) ) {
 203                if ($value) {
 204                        print $writer "$key=$value\n";
 205                }
 206        }
 207}
 208
 209sub credential_run {
 210        my $op = shift;
 211        my $credential = shift;
 212        my $pid = open2(my $reader, my $writer, "git credential $op");
 213        credential_write($credential, $writer);
 214        print $writer "\n";
 215        close($writer);
 216
 217        if ($op eq "fill") {
 218                %$credential = credential_read($reader, $op);
 219        } else {
 220                if (<$reader>) {
 221                        die "ERROR while running git credential $op:\n$_";
 222                }
 223        }
 224        close($reader);
 225        waitpid($pid, 0);
 226        my $child_exit_status = $? >> 8;
 227        if ($child_exit_status != 0) {
 228                die "'git credential $op' failed with code $child_exit_status.";
 229        }
 230}
 231
 232# MediaWiki API instance, created lazily.
 233my $mediawiki;
 234
 235sub mw_connect_maybe {
 236        if ($mediawiki) {
 237                return;
 238        }
 239        $mediawiki = MediaWiki::API->new;
 240        $mediawiki->{config}->{api_url} = "$url/api.php";
 241        if ($wiki_login) {
 242                my %credential = credential_from_url($url);
 243                $credential{username} = $wiki_login;
 244                $credential{password} = $wiki_passwd;
 245                credential_run("fill", \%credential);
 246                my $request = {lgname => $credential{username},
 247                               lgpassword => $credential{password},
 248                               lgdomain => $wiki_domain};
 249                if ($mediawiki->login($request)) {
 250                        credential_run("approve", \%credential);
 251                        print STDERR "Logged in mediawiki user \"$credential{username}\".\n";
 252                } else {
 253                        print STDERR "Failed to log in mediawiki user \"$credential{username}\" on $url\n";
 254                        print STDERR "  (error " .
 255                                $mediawiki->{error}->{code} . ': ' .
 256                                $mediawiki->{error}->{details} . ")\n";
 257                        credential_run("reject", \%credential);
 258                        exit 1;
 259                }
 260        }
 261}
 262
 263## Functions for listing pages on the remote wiki
 264sub get_mw_tracked_pages {
 265        my $pages = shift;
 266        get_mw_page_list(\@tracked_pages, $pages);
 267}
 268
 269sub get_mw_page_list {
 270        my $page_list = shift;
 271        my $pages = shift;
 272        my @some_pages = @$page_list;
 273        while (@some_pages) {
 274                my $last = 50;
 275                if ($#some_pages < $last) {
 276                        $last = $#some_pages;
 277                }
 278                my @slice = @some_pages[0..$last];
 279                get_mw_first_pages(\@slice, $pages);
 280                @some_pages = @some_pages[51..$#some_pages];
 281        }
 282}
 283
 284sub get_mw_tracked_categories {
 285        my $pages = shift;
 286        foreach my $category (@tracked_categories) {
 287                if (index($category, ':') < 0) {
 288                        # Mediawiki requires the Category
 289                        # prefix, but let's not force the user
 290                        # to specify it.
 291                        $category = "Category:" . $category;
 292                }
 293                my $mw_pages = $mediawiki->list( {
 294                        action => 'query',
 295                        list => 'categorymembers',
 296                        cmtitle => $category,
 297                        cmlimit => 'max' } )
 298                        || die $mediawiki->{error}->{code} . ': '
 299                                . $mediawiki->{error}->{details};
 300                foreach my $page (@{$mw_pages}) {
 301                        $pages->{$page->{title}} = $page;
 302                }
 303        }
 304}
 305
 306sub get_mw_all_pages {
 307        my $pages = shift;
 308        # No user-provided list, get the list of pages from the API.
 309        my $mw_pages = $mediawiki->list({
 310                action => 'query',
 311                list => 'allpages',
 312                aplimit => 'max'
 313        });
 314        if (!defined($mw_pages)) {
 315                print STDERR "fatal: could not get the list of wiki pages.\n";
 316                print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 317                print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 318                exit 1;
 319        }
 320        foreach my $page (@{$mw_pages}) {
 321                $pages->{$page->{title}} = $page;
 322        }
 323}
 324
 325# queries the wiki for a set of pages. Meant to be used within a loop
 326# querying the wiki for slices of page list.
 327sub get_mw_first_pages {
 328        my $some_pages = shift;
 329        my @some_pages = @{$some_pages};
 330
 331        my $pages = shift;
 332
 333        # pattern 'page1|page2|...' required by the API
 334        my $titles = join('|', @some_pages);
 335
 336        my $mw_pages = $mediawiki->api({
 337                action => 'query',
 338                titles => $titles,
 339        });
 340        if (!defined($mw_pages)) {
 341                print STDERR "fatal: could not query the list of wiki pages.\n";
 342                print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 343                print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 344                exit 1;
 345        }
 346        while (my ($id, $page) = each(%{$mw_pages->{query}->{pages}})) {
 347                if ($id < 0) {
 348                        print STDERR "Warning: page $page->{title} not found on wiki\n";
 349                } else {
 350                        $pages->{$page->{title}} = $page;
 351                }
 352        }
 353}
 354
 355# Get the list of pages to be fetched according to configuration.
 356sub get_mw_pages {
 357        mw_connect_maybe();
 358
 359        my %pages; # hash on page titles to avoid duplicates
 360        my $user_defined;
 361        if (@tracked_pages) {
 362                $user_defined = 1;
 363                # The user provided a list of pages titles, but we
 364                # still need to query the API to get the page IDs.
 365                get_mw_tracked_pages(\%pages);
 366        }
 367        if (@tracked_categories) {
 368                $user_defined = 1;
 369                get_mw_tracked_categories(\%pages);
 370        }
 371        if (!$user_defined) {
 372                get_mw_all_pages(\%pages);
 373        }
 374        if ($import_media) {
 375                print STDERR "Getting media files for selected pages...\n";
 376                if ($user_defined) {
 377                        get_linked_mediafiles(\%pages);
 378                } else {
 379                        get_all_mediafiles(\%pages);
 380                }
 381        }
 382        return %pages;
 383}
 384
 385# usage: $out = run_git("command args");
 386#        $out = run_git("command args", "raw"); # don't interpret output as UTF-8.
 387sub run_git {
 388        my $args = shift;
 389        my $encoding = (shift || "encoding(UTF-8)");
 390        open(my $git, "-|:$encoding", "git " . $args);
 391        my $res = do { local $/; <$git> };
 392        close($git);
 393
 394        return $res;
 395}
 396
 397
 398sub get_all_mediafiles {
 399        my $pages = shift;
 400        # Attach list of all pages for media files from the API,
 401        # they are in a different namespace, only one namespace
 402        # can be queried at the same moment
 403        my $mw_pages = $mediawiki->list({
 404                action => 'query',
 405                list => 'allpages',
 406                apnamespace => get_mw_namespace_id("File"),
 407                aplimit => 'max'
 408        });
 409        if (!defined($mw_pages)) {
 410                print STDERR "fatal: could not get the list of pages for media files.\n";
 411                print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
 412                print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
 413                exit 1;
 414        }
 415        foreach my $page (@{$mw_pages}) {
 416                $pages->{$page->{title}} = $page;
 417        }
 418}
 419
 420sub get_linked_mediafiles {
 421        my $pages = shift;
 422        my @titles = map $_->{title}, values(%{$pages});
 423
 424        # The query is split in small batches because of the MW API limit of
 425        # the number of links to be returned (500 links max).
 426        my $batch = 10;
 427        while (@titles) {
 428                if ($#titles < $batch) {
 429                        $batch = $#titles;
 430                }
 431                my @slice = @titles[0..$batch];
 432
 433                # pattern 'page1|page2|...' required by the API
 434                my $mw_titles = join('|', @slice);
 435
 436                # Media files could be included or linked from
 437                # a page, get all related
 438                my $query = {
 439                        action => 'query',
 440                        prop => 'links|images',
 441                        titles => $mw_titles,
 442                        plnamespace => get_mw_namespace_id("File"),
 443                        pllimit => 'max'
 444                };
 445                my $result = $mediawiki->api($query);
 446
 447                while (my ($id, $page) = each(%{$result->{query}->{pages}})) {
 448                        my @media_titles;
 449                        if (defined($page->{links})) {
 450                                my @link_titles = map $_->{title}, @{$page->{links}};
 451                                push(@media_titles, @link_titles);
 452                        }
 453                        if (defined($page->{images})) {
 454                                my @image_titles = map $_->{title}, @{$page->{images}};
 455                                push(@media_titles, @image_titles);
 456                        }
 457                        if (@media_titles) {
 458                                get_mw_page_list(\@media_titles, $pages);
 459                        }
 460                }
 461
 462                @titles = @titles[($batch+1)..$#titles];
 463        }
 464}
 465
 466sub get_mw_mediafile_for_page_revision {
 467        # Name of the file on Wiki, with the prefix.
 468        my $filename = shift;
 469        my $timestamp = shift;
 470        my %mediafile;
 471
 472        # Search if on a media file with given timestamp exists on
 473        # MediaWiki. In that case download the file.
 474        my $query = {
 475                action => 'query',
 476                prop => 'imageinfo',
 477                titles => "File:" . $filename,
 478                iistart => $timestamp,
 479                iiend => $timestamp,
 480                iiprop => 'timestamp|archivename|url',
 481                iilimit => 1
 482        };
 483        my $result = $mediawiki->api($query);
 484
 485        my ($fileid, $file) = each( %{$result->{query}->{pages}} );
 486        # If not defined it means there is no revision of the file for
 487        # given timestamp.
 488        if (defined($file->{imageinfo})) {
 489                $mediafile{title} = $filename;
 490
 491                my $fileinfo = pop(@{$file->{imageinfo}});
 492                $mediafile{timestamp} = $fileinfo->{timestamp};
 493                # Mediawiki::API's download function doesn't support https URLs
 494                # and can't download old versions of files.
 495                print STDERR "\tDownloading file $mediafile{title}, version $mediafile{timestamp}\n";
 496                $mediafile{content} = download_mw_mediafile($fileinfo->{url});
 497        }
 498        return %mediafile;
 499}
 500
 501sub download_mw_mediafile {
 502        my $url = shift;
 503
 504        my $response = $mediawiki->{ua}->get($url);
 505        if ($response->code == 200) {
 506                return $response->decoded_content;
 507        } else {
 508                print STDERR "Error downloading mediafile from :\n";
 509                print STDERR "URL: $url\n";
 510                print STDERR "Server response: " . $response->code . " " . $response->message . "\n";
 511                exit 1;
 512        }
 513}
 514
 515sub get_last_local_revision {
 516        # Get note regarding last mediawiki revision
 517        my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null");
 518        my @note_info = split(/ /, $note);
 519
 520        my $lastrevision_number;
 521        if (!(defined($note_info[0]) && $note_info[0] eq "mediawiki_revision:")) {
 522                print STDERR "No previous mediawiki revision found";
 523                $lastrevision_number = 0;
 524        } else {
 525                # Notes are formatted : mediawiki_revision: #number
 526                $lastrevision_number = $note_info[1];
 527                chomp($lastrevision_number);
 528                print STDERR "Last local mediawiki revision found is $lastrevision_number";
 529        }
 530        return $lastrevision_number;
 531}
 532
 533# Remember the timestamp corresponding to a revision id.
 534my %basetimestamps;
 535
 536sub get_last_remote_revision {
 537        mw_connect_maybe();
 538
 539        my %pages_hash = get_mw_pages();
 540        my @pages = values(%pages_hash);
 541
 542        my $max_rev_num = 0;
 543
 544        foreach my $page (@pages) {
 545                my $id = $page->{pageid};
 546
 547                my $query = {
 548                        action => 'query',
 549                        prop => 'revisions',
 550                        rvprop => 'ids|timestamp',
 551                        pageids => $id,
 552                };
 553
 554                my $result = $mediawiki->api($query);
 555
 556                my $lastrev = pop(@{$result->{query}->{pages}->{$id}->{revisions}});
 557
 558                $basetimestamps{$lastrev->{revid}} = $lastrev->{timestamp};
 559
 560                $max_rev_num = ($lastrev->{revid} > $max_rev_num ? $lastrev->{revid} : $max_rev_num);
 561        }
 562
 563        print STDERR "Last remote revision found is $max_rev_num.\n";
 564        return $max_rev_num;
 565}
 566
 567# Clean content before sending it to MediaWiki
 568sub mediawiki_clean {
 569        my $string = shift;
 570        my $page_created = shift;
 571        # Mediawiki does not allow blank space at the end of a page and ends with a single \n.
 572        # This function right trims a string and adds a \n at the end to follow this rule
 573        $string =~ s/\s+$//;
 574        if ($string eq "" && $page_created) {
 575                # Creating empty pages is forbidden.
 576                $string = EMPTY_CONTENT;
 577        }
 578        return $string."\n";
 579}
 580
 581# Filter applied on MediaWiki data before adding them to Git
 582sub mediawiki_smudge {
 583        my $string = shift;
 584        if ($string eq EMPTY_CONTENT) {
 585                $string = "";
 586        }
 587        # This \n is important. This is due to mediawiki's way to handle end of files.
 588        return $string."\n";
 589}
 590
 591sub mediawiki_clean_filename {
 592        my $filename = shift;
 593        $filename =~ s/@{[SLASH_REPLACEMENT]}/\//g;
 594        # [, ], |, {, and } are forbidden by MediaWiki, even URL-encoded.
 595        # Do a variant of URL-encoding, i.e. looks like URL-encoding,
 596        # but with _ added to prevent MediaWiki from thinking this is
 597        # an actual special character.
 598        $filename =~ s/[\[\]\{\}\|]/sprintf("_%%_%x", ord($&))/ge;
 599        # If we use the uri escape before
 600        # we should unescape here, before anything
 601
 602        return $filename;
 603}
 604
 605sub mediawiki_smudge_filename {
 606        my $filename = shift;
 607        $filename =~ s/\//@{[SLASH_REPLACEMENT]}/g;
 608        $filename =~ s/ /_/g;
 609        # Decode forbidden characters encoded in mediawiki_clean_filename
 610        $filename =~ s/_%_([0-9a-fA-F][0-9a-fA-F])/sprintf("%c", hex($1))/ge;
 611        return $filename;
 612}
 613
 614sub literal_data {
 615        my ($content) = @_;
 616        print STDOUT "data ", bytes::length($content), "\n", $content;
 617}
 618
 619sub literal_data_raw {
 620        # Output possibly binary content.
 621        my ($content) = @_;
 622        # Avoid confusion between size in bytes and in characters
 623        utf8::downgrade($content);
 624        binmode STDOUT, ":raw";
 625        print STDOUT "data ", bytes::length($content), "\n", $content;
 626        binmode STDOUT, ":utf8";
 627}
 628
 629sub mw_capabilities {
 630        # Revisions are imported to the private namespace
 631        # refs/mediawiki/$remotename/ by the helper and fetched into
 632        # refs/remotes/$remotename later by fetch.
 633        print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n";
 634        print STDOUT "import\n";
 635        print STDOUT "list\n";
 636        print STDOUT "push\n";
 637        print STDOUT "\n";
 638}
 639
 640sub mw_list {
 641        # MediaWiki do not have branches, we consider one branch arbitrarily
 642        # called master, and HEAD pointing to it.
 643        print STDOUT "? refs/heads/master\n";
 644        print STDOUT "\@refs/heads/master HEAD\n";
 645        print STDOUT "\n";
 646}
 647
 648sub mw_option {
 649        print STDERR "remote-helper command 'option $_[0]' not yet implemented\n";
 650        print STDOUT "unsupported\n";
 651}
 652
 653sub fetch_mw_revisions_for_page {
 654        my $page = shift;
 655        my $id = shift;
 656        my $fetch_from = shift;
 657        my @page_revs = ();
 658        my $query = {
 659                action => 'query',
 660                prop => 'revisions',
 661                rvprop => 'ids',
 662                rvdir => 'newer',
 663                rvstartid => $fetch_from,
 664                rvlimit => 500,
 665                pageids => $id,
 666        };
 667
 668        my $revnum = 0;
 669        # Get 500 revisions at a time due to the mediawiki api limit
 670        while (1) {
 671                my $result = $mediawiki->api($query);
 672
 673                # Parse each of those 500 revisions
 674                foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) {
 675                        my $page_rev_ids;
 676                        $page_rev_ids->{pageid} = $page->{pageid};
 677                        $page_rev_ids->{revid} = $revision->{revid};
 678                        push(@page_revs, $page_rev_ids);
 679                        $revnum++;
 680                }
 681                last unless $result->{'query-continue'};
 682                $query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid};
 683        }
 684        if ($shallow_import && @page_revs) {
 685                print STDERR "  Found 1 revision (shallow import).\n";
 686                @page_revs = sort {$b->{revid} <=> $a->{revid}} (@page_revs);
 687                return $page_revs[0];
 688        }
 689        print STDERR "  Found ", $revnum, " revision(s).\n";
 690        return @page_revs;
 691}
 692
 693sub fetch_mw_revisions {
 694        my $pages = shift; my @pages = @{$pages};
 695        my $fetch_from = shift;
 696
 697        my @revisions = ();
 698        my $n = 1;
 699        foreach my $page (@pages) {
 700                my $id = $page->{pageid};
 701
 702                print STDERR "page $n/", scalar(@pages), ": ". $page->{title} ."\n";
 703                $n++;
 704                my @page_revs = fetch_mw_revisions_for_page($page, $id, $fetch_from);
 705                @revisions = (@page_revs, @revisions);
 706        }
 707
 708        return ($n, @revisions);
 709}
 710
 711sub import_file_revision {
 712        my $commit = shift;
 713        my %commit = %{$commit};
 714        my $full_import = shift;
 715        my $n = shift;
 716        my $mediafile = shift;
 717        my %mediafile;
 718        if ($mediafile) {
 719                %mediafile = %{$mediafile};
 720        }
 721
 722        my $title = $commit{title};
 723        my $comment = $commit{comment};
 724        my $content = $commit{content};
 725        my $author = $commit{author};
 726        my $date = $commit{date};
 727
 728        print STDOUT "commit refs/mediawiki/$remotename/master\n";
 729        print STDOUT "mark :$n\n";
 730        print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
 731        literal_data($comment);
 732
 733        # If it's not a clone, we need to know where to start from
 734        if (!$full_import && $n == 1) {
 735                print STDOUT "from refs/mediawiki/$remotename/master^0\n";
 736        }
 737        if ($content ne DELETED_CONTENT) {
 738                print STDOUT "M 644 inline $title.mw\n";
 739                literal_data($content);
 740                if (%mediafile) {
 741                        print STDOUT "M 644 inline $mediafile{title}\n";
 742                        literal_data_raw($mediafile{content});
 743                }
 744                print STDOUT "\n\n";
 745        } else {
 746                print STDOUT "D $title.mw\n";
 747        }
 748
 749        # mediawiki revision number in the git note
 750        if ($full_import && $n == 1) {
 751                print STDOUT "reset refs/notes/$remotename/mediawiki\n";
 752        }
 753        print STDOUT "commit refs/notes/$remotename/mediawiki\n";
 754        print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
 755        literal_data("Note added by git-mediawiki during import");
 756        if (!$full_import && $n == 1) {
 757                print STDOUT "from refs/notes/$remotename/mediawiki^0\n";
 758        }
 759        print STDOUT "N inline :$n\n";
 760        literal_data("mediawiki_revision: " . $commit{mw_revision});
 761        print STDOUT "\n\n";
 762}
 763
 764# parse a sequence of
 765# <cmd> <arg1>
 766# <cmd> <arg2>
 767# \n
 768# (like batch sequence of import and sequence of push statements)
 769sub get_more_refs {
 770        my $cmd = shift;
 771        my @refs;
 772        while (1) {
 773                my $line = <STDIN>;
 774                if ($line =~ m/^$cmd (.*)$/) {
 775                        push(@refs, $1);
 776                } elsif ($line eq "\n") {
 777                        return @refs;
 778                } else {
 779                        die("Invalid command in a '$cmd' batch: ". $_);
 780                }
 781        }
 782}
 783
 784sub mw_import {
 785        # multiple import commands can follow each other.
 786        my @refs = (shift, get_more_refs("import"));
 787        foreach my $ref (@refs) {
 788                mw_import_ref($ref);
 789        }
 790        print STDOUT "done\n";
 791}
 792
 793sub mw_import_ref {
 794        my $ref = shift;
 795        # The remote helper will call "import HEAD" and
 796        # "import refs/heads/master".
 797        # Since HEAD is a symbolic ref to master (by convention,
 798        # followed by the output of the command "list" that we gave),
 799        # we don't need to do anything in this case.
 800        if ($ref eq "HEAD") {
 801                return;
 802        }
 803
 804        mw_connect_maybe();
 805
 806        my %pages_hash = get_mw_pages();
 807        my @pages = values(%pages_hash);
 808
 809        print STDERR "Searching revisions...\n";
 810        my $last_local = get_last_local_revision();
 811        my $fetch_from = $last_local + 1;
 812        if ($fetch_from == 1) {
 813                print STDERR ", fetching from beginning.\n";
 814        } else {
 815                print STDERR ", fetching from here.\n";
 816        }
 817        my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
 818
 819        # Creation of the fast-import stream
 820        print STDERR "Fetching & writing export data...\n";
 821
 822        $n = 0;
 823        my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined
 824
 825        foreach my $pagerevid (sort {$a->{revid} <=> $b->{revid}} @revisions) {
 826                # fetch the content of the pages
 827                my $query = {
 828                        action => 'query',
 829                        prop => 'revisions',
 830                        rvprop => 'content|timestamp|comment|user|ids',
 831                        revids => $pagerevid->{revid},
 832                };
 833
 834                my $result = $mediawiki->api($query);
 835
 836                my $rev = pop(@{$result->{query}->{pages}->{$pagerevid->{pageid}}->{revisions}});
 837
 838                $n++;
 839
 840                my $page_title = $result->{query}->{pages}->{$pagerevid->{pageid}}->{title};
 841                my %commit;
 842                $commit{author} = $rev->{user} || 'Anonymous';
 843                $commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*';
 844                $commit{title} = mediawiki_smudge_filename($page_title);
 845                $commit{mw_revision} = $pagerevid->{revid};
 846                $commit{content} = mediawiki_smudge($rev->{'*'});
 847
 848                if (!defined($rev->{timestamp})) {
 849                        $last_timestamp++;
 850                } else {
 851                        $last_timestamp = $rev->{timestamp};
 852                }
 853                $commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp);
 854
 855                # Differentiates classic pages and media files.
 856                my ($namespace, $filename) = $page_title =~ /^([^:]*):(.*)$/;
 857                my %mediafile;
 858                if ($namespace && get_mw_namespace_id($namespace) == get_mw_namespace_id("File")) {
 859                        %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
 860                }
 861                # If this is a revision of the media page for new version
 862                # of a file do one common commit for both file and media page.
 863                # Else do commit only for that page.
 864                print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n";
 865                import_file_revision(\%commit, ($fetch_from == 1), $n, \%mediafile);
 866        }
 867
 868        if ($fetch_from == 1 && $n == 0) {
 869                print STDERR "You appear to have cloned an empty MediaWiki.\n";
 870                # Something has to be done remote-helper side. If nothing is done, an error is
 871                # thrown saying that HEAD is refering to unknown object 0000000000000000000
 872                # and the clone fails.
 873        }
 874}
 875
 876sub error_non_fast_forward {
 877        my $advice = run_git("config --bool advice.pushNonFastForward");
 878        chomp($advice);
 879        if ($advice ne "false") {
 880                # Native git-push would show this after the summary.
 881                # We can't ask it to display it cleanly, so print it
 882                # ourselves before.
 883                print STDERR "To prevent you from losing history, non-fast-forward updates were rejected\n";
 884                print STDERR "Merge the remote changes (e.g. 'git pull') before pushing again. See the\n";
 885                print STDERR "'Note about fast-forwards' section of 'git push --help' for details.\n";
 886        }
 887        print STDOUT "error $_[0] \"non-fast-forward\"\n";
 888        return 0;
 889}
 890
 891sub mw_upload_file {
 892        my $complete_file_name = shift;
 893        my $new_sha1 = shift;
 894        my $extension = shift;
 895        my $file_deleted = shift;
 896        my $summary = shift;
 897        my $newrevid;
 898        my $path = "File:" . $complete_file_name;
 899        my %hashFiles = get_allowed_file_extensions();
 900        if (!exists($hashFiles{$extension})) {
 901                print STDERR "$complete_file_name is not a permitted file on this wiki.\n";
 902                print STDERR "Check the configuration of file uploads in your mediawiki.\n";
 903                return $newrevid;
 904        }
 905        # Deleting and uploading a file requires a priviledged user
 906        if ($file_deleted) {
 907                mw_connect_maybe();
 908                my $query = {
 909                        action => 'delete',
 910                        title => $path,
 911                        reason => $summary
 912                };
 913                if (!$mediawiki->edit($query)) {
 914                        print STDERR "Failed to delete file on remote wiki\n";
 915                        print STDERR "Check your permissions on the remote site. Error code:\n";
 916                        print STDERR $mediawiki->{error}->{code} . ':' . $mediawiki->{error}->{details};
 917                        exit 1;
 918                }
 919        } else {
 920                # Don't let perl try to interpret file content as UTF-8 => use "raw"
 921                my $content = run_git("cat-file blob $new_sha1", "raw");
 922                if ($content ne "") {
 923                        mw_connect_maybe();
 924                        $mediawiki->{config}->{upload_url} =
 925                                "$url/index.php/Special:Upload";
 926                        $mediawiki->edit({
 927                                action => 'upload',
 928                                filename => $complete_file_name,
 929                                comment => $summary,
 930                                file => [undef,
 931                                         $complete_file_name,
 932                                         Content => $content],
 933                                ignorewarnings => 1,
 934                        }, {
 935                                skip_encoding => 1
 936                        } ) || die $mediawiki->{error}->{code} . ':'
 937                                 . $mediawiki->{error}->{details};
 938                        my $last_file_page = $mediawiki->get_page({title => $path});
 939                        $newrevid = $last_file_page->{revid};
 940                        print STDERR "Pushed file: $new_sha1 - $complete_file_name.\n";
 941                } else {
 942                        print STDERR "Empty file $complete_file_name not pushed.\n";
 943                }
 944        }
 945        return $newrevid;
 946}
 947
 948sub mw_push_file {
 949        my $diff_info = shift;
 950        # $diff_info contains a string in this format:
 951        # 100644 100644 <sha1_of_blob_before_commit> <sha1_of_blob_now> <status>
 952        my @diff_info_split = split(/[ \t]/, $diff_info);
 953
 954        # Filename, including .mw extension
 955        my $complete_file_name = shift;
 956        # Commit message
 957        my $summary = shift;
 958        # MediaWiki revision number. Keep the previous one by default,
 959        # in case there's no edit to perform.
 960        my $oldrevid = shift;
 961        my $newrevid;
 962
 963        my $new_sha1 = $diff_info_split[3];
 964        my $old_sha1 = $diff_info_split[2];
 965        my $page_created = ($old_sha1 eq NULL_SHA1);
 966        my $page_deleted = ($new_sha1 eq NULL_SHA1);
 967        $complete_file_name = mediawiki_clean_filename($complete_file_name);
 968
 969        my ($title, $extension) = $complete_file_name =~ /^(.*)\.([^\.]*)$/;
 970        if (!defined($extension)) {
 971                $extension = "";
 972        }
 973        if ($extension eq "mw") {
 974                my $file_content;
 975                if ($page_deleted) {
 976                        # Deleting a page usually requires
 977                        # special priviledges. A common
 978                        # convention is to replace the page
 979                        # with this content instead:
 980                        $file_content = DELETED_CONTENT;
 981                } else {
 982                        $file_content = run_git("cat-file blob $new_sha1");
 983                }
 984
 985                mw_connect_maybe();
 986
 987                my $result = $mediawiki->edit( {
 988                        action => 'edit',
 989                        summary => $summary,
 990                        title => $title,
 991                        basetimestamp => $basetimestamps{$oldrevid},
 992                        text => mediawiki_clean($file_content, $page_created),
 993                                  }, {
 994                                          skip_encoding => 1 # Helps with names with accentuated characters
 995                                  });
 996                if (!$result) {
 997                        if ($mediawiki->{error}->{code} == 3) {
 998                                # edit conflicts, considered as non-fast-forward
 999                                print STDERR 'Warning: Error ' .
1000                                    $mediawiki->{error}->{code} .
1001                                    ' from mediwiki: ' . $mediawiki->{error}->{details} .
1002                                    ".\n";
1003                                return ($oldrevid, "non-fast-forward");
1004                        } else {
1005                                # Other errors. Shouldn't happen => just die()
1006                                die 'Fatal: Error ' .
1007                                    $mediawiki->{error}->{code} .
1008                                    ' from mediwiki: ' . $mediawiki->{error}->{details};
1009                        }
1010                }
1011                $newrevid = $result->{edit}->{newrevid};
1012                print STDERR "Pushed file: $new_sha1 - $title\n";
1013        } else {
1014                $newrevid = mw_upload_file($complete_file_name, $new_sha1,
1015                                           $extension, $page_deleted,
1016                                           $summary);
1017        }
1018        $newrevid = ($newrevid or $oldrevid);
1019        return ($newrevid, "ok");
1020}
1021
1022sub mw_push {
1023        # multiple push statements can follow each other
1024        my @refsspecs = (shift, get_more_refs("push"));
1025        my $pushed;
1026        for my $refspec (@refsspecs) {
1027                my ($force, $local, $remote) = $refspec =~ /^(\+)?([^:]*):([^:]*)$/
1028                    or die("Invalid refspec for push. Expected <src>:<dst> or +<src>:<dst>");
1029                if ($force) {
1030                        print STDERR "Warning: forced push not allowed on a MediaWiki.\n";
1031                }
1032                if ($local eq "") {
1033                        print STDERR "Cannot delete remote branch on a MediaWiki\n";
1034                        print STDOUT "error $remote cannot delete\n";
1035                        next;
1036                }
1037                if ($remote ne "refs/heads/master") {
1038                        print STDERR "Only push to the branch 'master' is supported on a MediaWiki\n";
1039                        print STDOUT "error $remote only master allowed\n";
1040                        next;
1041                }
1042                if (mw_push_revision($local, $remote)) {
1043                        $pushed = 1;
1044                }
1045        }
1046
1047        # Notify Git that the push is done
1048        print STDOUT "\n";
1049
1050        if ($pushed && $dumb_push) {
1051                print STDERR "Just pushed some revisions to MediaWiki.\n";
1052                print STDERR "The pushed revisions now have to be re-imported, and your current branch\n";
1053                print STDERR "needs to be updated with these re-imported commits. You can do this with\n";
1054                print STDERR "\n";
1055                print STDERR "  git pull --rebase\n";
1056                print STDERR "\n";
1057        }
1058}
1059
1060sub mw_push_revision {
1061        my $local = shift;
1062        my $remote = shift; # actually, this has to be "refs/heads/master" at this point.
1063        my $last_local_revid = get_last_local_revision();
1064        print STDERR ".\n"; # Finish sentence started by get_last_local_revision()
1065        my $last_remote_revid = get_last_remote_revision();
1066        my $mw_revision = $last_remote_revid;
1067
1068        # Get sha1 of commit pointed by local HEAD
1069        my $HEAD_sha1 = run_git("rev-parse $local 2>/dev/null"); chomp($HEAD_sha1);
1070        # Get sha1 of commit pointed by remotes/$remotename/master
1071        my $remoteorigin_sha1 = run_git("rev-parse refs/remotes/$remotename/master 2>/dev/null");
1072        chomp($remoteorigin_sha1);
1073
1074        if ($last_local_revid > 0 &&
1075            $last_local_revid < $last_remote_revid) {
1076                return error_non_fast_forward($remote);
1077        }
1078
1079        if ($HEAD_sha1 eq $remoteorigin_sha1) {
1080                # nothing to push
1081                return 0;
1082        }
1083
1084        # Get every commit in between HEAD and refs/remotes/origin/master,
1085        # including HEAD and refs/remotes/origin/master
1086        my @commit_pairs = ();
1087        if ($last_local_revid > 0) {
1088                my $parsed_sha1 = $remoteorigin_sha1;
1089                # Find a path from last MediaWiki commit to pushed commit
1090                while ($parsed_sha1 ne $HEAD_sha1) {
1091                        my @commit_info =  grep(/^$parsed_sha1/, split(/\n/, run_git("rev-list --children $local")));
1092                        if (!@commit_info) {
1093                                return error_non_fast_forward($remote);
1094                        }
1095                        my @commit_info_split = split(/ |\n/, $commit_info[0]);
1096                        # $commit_info_split[1] is the sha1 of the commit to export
1097                        # $commit_info_split[0] is the sha1 of its direct child
1098                        push(@commit_pairs, \@commit_info_split);
1099                        $parsed_sha1 = $commit_info_split[1];
1100                }
1101        } else {
1102                # No remote mediawiki revision. Export the whole
1103                # history (linearized with --first-parent)
1104                print STDERR "Warning: no common ancestor, pushing complete history\n";
1105                my $history = run_git("rev-list --first-parent --children $local");
1106                my @history = split('\n', $history);
1107                @history = @history[1..$#history];
1108                foreach my $line (reverse @history) {
1109                        my @commit_info_split = split(/ |\n/, $line);
1110                        push(@commit_pairs, \@commit_info_split);
1111                }
1112        }
1113
1114        foreach my $commit_info_split (@commit_pairs) {
1115                my $sha1_child = @{$commit_info_split}[0];
1116                my $sha1_commit = @{$commit_info_split}[1];
1117                my $diff_infos = run_git("diff-tree -r --raw -z $sha1_child $sha1_commit");
1118                # TODO: we could detect rename, and encode them with a #redirect on the wiki.
1119                # TODO: for now, it's just a delete+add
1120                my @diff_info_list = split(/\0/, $diff_infos);
1121                # Keep the subject line of the commit message as mediawiki comment for the revision
1122                my $commit_msg = run_git("log --no-walk --format=\"%s\" $sha1_commit");
1123                chomp($commit_msg);
1124                # Push every blob
1125                while (@diff_info_list) {
1126                        my $status;
1127                        # git diff-tree -z gives an output like
1128                        # <metadata>\0<filename1>\0
1129                        # <metadata>\0<filename2>\0
1130                        # and we've split on \0.
1131                        my $info = shift(@diff_info_list);
1132                        my $file = shift(@diff_info_list);
1133                        ($mw_revision, $status) = mw_push_file($info, $file, $commit_msg, $mw_revision);
1134                        if ($status eq "non-fast-forward") {
1135                                # we may already have sent part of the
1136                                # commit to MediaWiki, but it's too
1137                                # late to cancel it. Stop the push in
1138                                # the middle, but still give an
1139                                # accurate error message.
1140                                return error_non_fast_forward($remote);
1141                        }
1142                        if ($status ne "ok") {
1143                                die("Unknown error from mw_push_file()");
1144                        }
1145                }
1146                unless ($dumb_push) {
1147                        run_git("notes --ref=$remotename/mediawiki add -m \"mediawiki_revision: $mw_revision\" $sha1_commit");
1148                        run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child");
1149                }
1150        }
1151
1152        print STDOUT "ok $remote\n";
1153        return 1;
1154}
1155
1156sub get_allowed_file_extensions {
1157        mw_connect_maybe();
1158
1159        my $query = {
1160                action => 'query',
1161                meta => 'siteinfo',
1162                siprop => 'fileextensions'
1163                };
1164        my $result = $mediawiki->api($query);
1165        my @file_extensions= map $_->{ext},@{$result->{query}->{fileextensions}};
1166        my %hashFile = map {$_ => 1}@file_extensions;
1167
1168        return %hashFile;
1169}
1170
1171# In memory cache for MediaWiki namespace ids.
1172my %namespace_id;
1173
1174# Namespaces whose id is cached in the configuration file
1175# (to avoid duplicates)
1176my %cached_mw_namespace_id;
1177
1178# Return MediaWiki id for a canonical namespace name.
1179# Ex.: "File", "Project".
1180sub get_mw_namespace_id {
1181        mw_connect_maybe();
1182        my $name = shift;
1183
1184        if (!exists $namespace_id{$name}) {
1185                # Look at configuration file, if the record for that namespace is
1186                # already cached. Namespaces are stored in form:
1187                # "Name_of_namespace:Id_namespace", ex.: "File:6".
1188                my @temp = split(/[ \n]/, run_git("config --get-all remote."
1189                                                . $remotename .".namespaceCache"));
1190                chomp(@temp);
1191                foreach my $ns (@temp) {
1192                        my ($n, $id) = split(/:/, $ns);
1193                        $namespace_id{$n} = $id;
1194                        $cached_mw_namespace_id{$n} = 1;
1195                }
1196        }
1197
1198        if (!exists $namespace_id{$name}) {
1199                print STDERR "Namespace $name not found in cache, querying the wiki ...\n";
1200                # NS not found => get namespace id from MW and store it in
1201                # configuration file.
1202                my $query = {
1203                        action => 'query',
1204                        meta => 'siteinfo',
1205                        siprop => 'namespaces'
1206                };
1207                my $result = $mediawiki->api($query);
1208
1209                while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) {
1210                        if (defined($ns->{id}) && defined($ns->{canonical})) {
1211                                $namespace_id{$ns->{canonical}} = $ns->{id};
1212                                if ($ns->{'*'}) {
1213                                        # alias (e.g. french Fichier: as alias for canonical File:)
1214                                        $namespace_id{$ns->{'*'}} = $ns->{id};
1215                                }
1216                        }
1217                }
1218        }
1219
1220        my $id = $namespace_id{$name};
1221
1222        if (defined $id) {
1223                # Store explicitely requested namespaces on disk
1224                if (!exists $cached_mw_namespace_id{$name}) {
1225                        run_git("config --add remote.". $remotename
1226                                .".namespaceCache \"". $name .":". $id ."\"");
1227                        $cached_mw_namespace_id{$name} = 1;
1228                }
1229                return $id;
1230        } else {
1231                die "No such namespace $name on MediaWiki.";
1232        }
1233}