Merge branch 'tr/diff-words-test'

* tr/diff-words-test:
t4034 (diff --word-diff): add a minimum Perl drier test vector
t4034 (diff --word-diff): style suggestions
userdiff: simplify word-diff safeguard
t4034: bulk verify builtin word regex sanity

diff --git a/t/t4034-diff-words.sh b/t/t4034-diff-words.sh
index 8096d8a337867b4afdc4b061fdc314fdd8eac185..37aeab0d5c7415a54384c90708626f6a5807acee 100755 (executable)
--- a/t/t4034-diff-words.sh
+++ b/t/t4034-diff-words.sh
@@ -4,331 +4,307 @@ test_description='word diff colors'
 
 . ./test-lib.sh
 
-test_expect_success setup '
-
-       git config diff.color.old red &&
-       git config diff.color.new green &&
-       git config diff.color.func magenta
+cat >pre.simple <<-\EOF
+       h(4)
 
-'
+       a = b + c
+EOF
+cat >post.simple <<-\EOF
+       h(4),hh[44]
 
-word_diff () {
-       test_must_fail git diff --no-index "$@" pre post > output &&
-       test_decode_color <output >output.decrypted &&
-       test_cmp expect output.decrypted
-}
+       a = b + c
 
-cat > pre <<\EOF
-h(4)
+       aa = a
 
-a = b + c
+       aeff = aeff * ( aaa )
 EOF
+cat >expect.letter-runs-are-words <<-\EOF
+       <BOLD>diff --git a/pre b/post<RESET>
+       <BOLD>index 330b04f..5ed8eff 100644<RESET>
+       <BOLD>--- a/pre<RESET>
+       <BOLD>+++ b/post<RESET>
+       <CYAN>@@ -1,3 +1,7 @@<RESET>
+       h(4),<GREEN>hh<RESET>[44]
 
-cat > post <<\EOF
-h(4),hh[44]
-
-a = b + c
+       a = b + c<RESET>
 
-aa = a
+       <GREEN>aa = a<RESET>
 
-aeff = aeff * ( aaa )
+       <GREEN>aeff = aeff * ( aaa<RESET> )
 EOF
+cat >expect.non-whitespace-is-word <<-\EOF
+       <BOLD>diff --git a/pre b/post<RESET>
+       <BOLD>index 330b04f..5ed8eff 100644<RESET>
+       <BOLD>--- a/pre<RESET>
+       <BOLD>+++ b/post<RESET>
+       <CYAN>@@ -1,3 +1,7 @@<RESET>
+       h(4)<GREEN>,hh[44]<RESET>
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-<RED>h(4)<RESET><GREEN>h(4),hh[44]<RESET>
+       a = b + c<RESET>
 
-a = b + c<RESET>
+       <GREEN>aa = a<RESET>
 
-<GREEN>aa = a<RESET>
-
-<GREEN>aeff = aeff * ( aaa )<RESET>
+       <GREEN>aeff = aeff * ( aaa )<RESET>
 EOF
 
-test_expect_success 'word diff with runs of whitespace' '
+word_diff () {
+       test_must_fail git diff --no-index "$@" pre post >output &&
+       test_decode_color <output >output.decrypted &&
+       test_cmp expect output.decrypted
+}
 
-       word_diff --color-words
+test_language_driver () {
+       lang=$1
+       test_expect_success "diff driver '$lang'" '
+               cp "$TEST_DIRECTORY/t4034/'"$lang"'/pre" \
+                       "$TEST_DIRECTORY/t4034/'"$lang"'/post" \
+                       "$TEST_DIRECTORY/t4034/'"$lang"'/expect" . &&
+               echo "* diff='"$lang"'" >.gitattributes &&
+               word_diff --color-words
+       '
+}
 
+test_expect_success setup '
+       git config diff.color.old red &&
+       git config diff.color.new green &&
+       git config diff.color.func magenta
 '
 
-test_expect_success '--word-diff=color' '
-
-       word_diff --word-diff=color
-
+test_expect_success 'set up pre and post with runs of whitespace' '
+       cp pre.simple pre &&
+       cp post.simple post
 '
 
-test_expect_success '--color --word-diff=color' '
-
+test_expect_success 'word diff with runs of whitespace' '
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 330b04f..5ed8eff 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1,3 +1,7 @@<RESET>
+               <RED>h(4)<RESET><GREEN>h(4),hh[44]<RESET>
+
+               a = b + c<RESET>
+
+               <GREEN>aa = a<RESET>
+
+               <GREEN>aeff = aeff * ( aaa )<RESET>
+       EOF
+       word_diff --color-words &&
+       word_diff --word-diff=color &&
        word_diff --color --word-diff=color
-
 '
 
-sed 's/#.*$//' > expect <<EOF
-diff --git a/pre b/post
-index 330b04f..5ed8eff 100644
---- a/pre
-+++ b/post
-@@ -1,3 +1,7 @@
--h(4)
-+h(4),hh[44]
-~
- # significant space
-~
- a = b + c
-~
-~
-+aa = a
-~
-~
-+aeff = aeff * ( aaa )
-~
-EOF
-
 test_expect_success '--word-diff=porcelain' '
-
+       sed 's/#.*$//' >expect <<-\EOF &&
+               diff --git a/pre b/post
+               index 330b04f..5ed8eff 100644
+               --- a/pre
+               +++ b/post
+               @@ -1,3 +1,7 @@
+               -h(4)
+               +h(4),hh[44]
+               ~
+                # significant space
+               ~
+                a = b + c
+               ~
+               ~
+               +aa = a
+               ~
+               ~
+               +aeff = aeff * ( aaa )
+               ~
+       EOF
        word_diff --word-diff=porcelain
-
 '
 
-cat > expect <<EOF
-diff --git a/pre b/post
-index 330b04f..5ed8eff 100644
---- a/pre
-+++ b/post
-@@ -1,3 +1,7 @@
-[-h(4)-]{+h(4),hh[44]+}
-
-a = b + c
-
-{+aa = a+}
-
-{+aeff = aeff * ( aaa )+}
-EOF
-
 test_expect_success '--word-diff=plain' '
+       cat >expect <<-\EOF &&
+               diff --git a/pre b/post
+               index 330b04f..5ed8eff 100644
+               --- a/pre
+               +++ b/post
+               @@ -1,3 +1,7 @@
+               [-h(4)-]{+h(4),hh[44]+}
 
-       word_diff --word-diff=plain
-
-'
+               a = b + c
 
-test_expect_success '--word-diff=plain --no-color' '
+               {+aa = a+}
 
+               {+aeff = aeff * ( aaa )+}
+       EOF
+       word_diff --word-diff=plain &&
        word_diff --word-diff=plain --no-color
-
 '
 
-cat > expect <<EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-<RED>[-h(4)-]<RESET><GREEN>{+h(4),hh[44]+}<RESET>
-
-a = b + c<RESET>
-
-<GREEN>{+aa = a+}<RESET>
+test_expect_success '--word-diff=plain --color' '
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 330b04f..5ed8eff 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1,3 +1,7 @@<RESET>
+               <RED>[-h(4)-]<RESET><GREEN>{+h(4),hh[44]+}<RESET>
 
-<GREEN>{+aeff = aeff * ( aaa )+}<RESET>
-EOF
+               a = b + c<RESET>
 
-test_expect_success '--word-diff=plain --color' '
+               <GREEN>{+aa = a+}<RESET>
 
+               <GREEN>{+aeff = aeff * ( aaa )+}<RESET>
+       EOF
        word_diff --word-diff=plain --color
-
 '
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1 +1 @@<RESET>
-<RED>h(4)<RESET><GREEN>h(4),hh[44]<RESET>
-<CYAN>@@ -3,0 +4,4 @@<RESET> <RESET><MAGENTA>a = b + c<RESET>
-
-<GREEN>aa = a<RESET>
-
-<GREEN>aeff = aeff * ( aaa )<RESET>
-EOF
-
 test_expect_success 'word diff without context' '
-
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 330b04f..5ed8eff 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1 +1 @@<RESET>
+               <RED>h(4)<RESET><GREEN>h(4),hh[44]<RESET>
+               <CYAN>@@ -3,0 +4,4 @@<RESET> <RESET><MAGENTA>a = b + c<RESET>
+
+               <GREEN>aa = a<RESET>
+
+               <GREEN>aeff = aeff * ( aaa )<RESET>
+       EOF
        word_diff --color-words --unified=0
-
 '
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-h(4),<GREEN>hh<RESET>[44]
-
-a = b + c<RESET>
-
-<GREEN>aa = a<RESET>
-
-<GREEN>aeff = aeff * ( aaa<RESET> )
-EOF
-cp expect expect.letter-runs-are-words
-
 test_expect_success 'word diff with a regular expression' '
-
+       cp expect.letter-runs-are-words expect &&
        word_diff --color-words="[a-z]+"
-
 '
 
-test_expect_success 'set a diff driver' '
+test_expect_success 'set up a diff driver' '
        git config diff.testdriver.wordRegex "[^[:space:]]" &&
-       cat <<EOF > .gitattributes
-pre diff=testdriver
-post diff=testdriver
-EOF
+       cat <<-\EOF >.gitattributes
+               pre diff=testdriver
+               post diff=testdriver
+       EOF
 '
 
 test_expect_success 'option overrides .gitattributes' '
-
+       cp expect.letter-runs-are-words expect &&
        word_diff --color-words="[a-z]+"
-
 '
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-h(4)<GREEN>,hh[44]<RESET>
-
-a = b + c<RESET>
-
-<GREEN>aa = a<RESET>
-
-<GREEN>aeff = aeff * ( aaa )<RESET>
-EOF
-cp expect expect.non-whitespace-is-word
-
 test_expect_success 'use regex supplied by driver' '
-
+       cp expect.non-whitespace-is-word expect &&
        word_diff --color-words
-
 '
 
-test_expect_success 'set diff.wordRegex option' '
+test_expect_success 'set up diff.wordRegex option' '
        git config diff.wordRegex "[[:alnum:]]+"
 '
 
-cp expect.letter-runs-are-words expect
-
 test_expect_success 'command-line overrides config' '
+       cp expect.letter-runs-are-words expect &&
        word_diff --color-words="[a-z]+"
 '
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-h(4),<GREEN>{+hh+}<RESET>[44]
-
-a = b + c<RESET>
+test_expect_success 'command-line overrides config: --word-diff-regex' '
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 330b04f..5ed8eff 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1,3 +1,7 @@<RESET>
+               h(4),<GREEN>{+hh+}<RESET>[44]
 
-<GREEN>{+aa = a+}<RESET>
+               a = b + c<RESET>
 
-<GREEN>{+aeff = aeff * ( aaa+}<RESET> )
-EOF
+               <GREEN>{+aa = a+}<RESET>
 
-test_expect_success 'command-line overrides config: --word-diff-regex' '
+               <GREEN>{+aeff = aeff * ( aaa+}<RESET> )
+       EOF
        word_diff --color --word-diff-regex="[a-z]+"
 '
 
-cp expect.non-whitespace-is-word expect
-
 test_expect_success '.gitattributes override config' '
+       cp expect.non-whitespace-is-word expect &&
        word_diff --color-words
 '
 
-test_expect_success 'remove diff driver regex' '
-       git config --unset diff.testdriver.wordRegex
+test_expect_success 'setup: remove diff driver regex' '
+       test_might_fail git config --unset diff.testdriver.wordRegex
 '
 
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 330b04f..5ed8eff 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1,3 +1,7 @@<RESET>
-h(4),<GREEN>hh[44<RESET>]
-
-a = b + c<RESET>
+test_expect_success 'use configured regex' '
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 330b04f..5ed8eff 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1,3 +1,7 @@<RESET>
+               h(4),<GREEN>hh[44<RESET>]
 
-<GREEN>aa = a<RESET>
+               a = b + c<RESET>
 
-<GREEN>aeff = aeff * ( aaa<RESET> )
-EOF
+               <GREEN>aa = a<RESET>
 
-test_expect_success 'use configured regex' '
+               <GREEN>aeff = aeff * ( aaa<RESET> )
+       EOF
        word_diff --color-words
 '
 
-echo 'aaa (aaa)' > pre
-echo 'aaa (aaa) aaa' > post
-
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index c29453b..be22f37 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1 +1 @@<RESET>
-aaa (aaa) <GREEN>aaa<RESET>
-EOF
-
 test_expect_success 'test parsing words for newline' '
-
+       echo "aaa (aaa)" >pre &&
+       echo "aaa (aaa) aaa" >post &&
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index c29453b..be22f37 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1 +1 @@<RESET>
+               aaa (aaa) <GREEN>aaa<RESET>
+       EOF
        word_diff --color-words="a+"
-
-
 '
 
-echo '(:' > pre
-echo '(' > post
-
-cat > expect <<\EOF
-<BOLD>diff --git a/pre b/post<RESET>
-<BOLD>index 289cb9d..2d06f37 100644<RESET>
-<BOLD>--- a/pre<RESET>
-<BOLD>+++ b/post<RESET>
-<CYAN>@@ -1 +1 @@<RESET>
-(<RED>:<RESET>
-EOF
-
 test_expect_success 'test when words are only removed at the end' '
-
+       echo "(:" >pre &&
+       echo "(" >post &&
+       cat >expect <<-\EOF &&
+               <BOLD>diff --git a/pre b/post<RESET>
+               <BOLD>index 289cb9d..2d06f37 100644<RESET>
+               <BOLD>--- a/pre<RESET>
+               <BOLD>+++ b/post<RESET>
+               <CYAN>@@ -1 +1 @@<RESET>
+               (<RED>:<RESET>
+       EOF
        word_diff --color-words=.
-
 '
 
-cat > expect <<\EOF
-diff --git a/pre b/post
-index 289cb9d..2d06f37 100644
---- a/pre
-+++ b/post
-@@ -1 +1 @@
--(:
-+(
-EOF
-
 test_expect_success '--word-diff=none' '
-
+       echo "(:" >pre &&
+       echo "(" >post &&
+       cat >expect <<-\EOF &&
+               diff --git a/pre b/post
+               index 289cb9d..2d06f37 100644
+               --- a/pre
+               +++ b/post
+               @@ -1 +1 @@
+               -(:
+               +(
+       EOF
        word_diff --word-diff=plain --word-diff=none
-
 '
 
+test_language_driver bibtex
+test_language_driver cpp
+test_language_driver csharp
+test_language_driver fortran
+test_language_driver html
+test_language_driver java
+test_language_driver objc
+test_language_driver pascal
+test_language_driver perl
+test_language_driver php
+test_language_driver python
+test_language_driver ruby
+test_language_driver tex
+
 test_done

diff --git a/t/t4034/bibtex/expect b/t/t4034/bibtex/expect
new file mode 100644 (file)
index 0000000..a157774
--- /dev/null
+++ b/t/t4034/bibtex/expect
@@ -0,0 +1,15 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 95cd55b..ddcba9b 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,9 +1,10 @@<RESET>
+@article{aldous1987uie,<RESET>
+  title={{Ultimate instability of exponential back-off protocol for acknowledgment-based transmission control of random access communication channels}},<RESET>
+  author={Aldous, <RED>D.<RESET><GREEN>David<RESET>},
+  journal={Information Theory, IEEE Transactions on},<RESET>
+  volume={<RED>33<RESET><GREEN>Bogus.<RESET>},
+  number={<RED>2<RESET><GREEN>4<RESET>},
+  pages={219--223},<RESET>
+  year=<GREEN>1987,<RESET>
+<GREEN>  note={This is in fact a rather funny read since ethernet works well in practice. The<RESET> {<RED>1987<RESET><GREEN>\em pre} reference is the right one, however.<RESET>}<RED>,<RESET>
+}<RESET>

diff --git a/t/t4034/bibtex/post b/t/t4034/bibtex/post
new file mode 100644 (file)
index 0000000..ddcba9b
--- /dev/null
+++ b/t/t4034/bibtex/post
@@ -0,0 +1,10 @@
+@article{aldous1987uie,
+  title={{Ultimate instability of exponential back-off protocol for acknowledgment-based transmission control of random access communication channels}},
+  author={Aldous, David},
+  journal={Information Theory, IEEE Transactions on},
+  volume={Bogus.},
+  number={4},
+  pages={219--223},
+  year=1987,
+  note={This is in fact a rather funny read since ethernet works well in practice. The {\em pre} reference is the right one, however.}
+}

diff --git a/t/t4034/bibtex/pre b/t/t4034/bibtex/pre
new file mode 100644 (file)
index 0000000..95cd55b
--- /dev/null
+++ b/t/t4034/bibtex/pre
@@ -0,0 +1,9 @@
+@article{aldous1987uie,
+  title={{Ultimate instability of exponential back-off protocol for acknowledgment-based transmission control of random access communication channels}},
+  author={Aldous, D.},
+  journal={Information Theory, IEEE Transactions on},
+  volume={33},
+  number={2},
+  pages={219--223},
+  year={1987},
+}

diff --git a/t/t4034/cpp/expect b/t/t4034/cpp/expect
new file mode 100644 (file)
index 0000000..37d1ea2
--- /dev/null
+++ b/t/t4034/cpp/expect
@@ -0,0 +1,36 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 23d5c8a..7e8c026 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,19 +1,19 @@<RESET>
+Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <GREEN>bar(x);<RESET> }
+cout<<"Hello World<RED>!<RESET><GREEN>?<RESET>\n"<<endl;
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y
+<RED>a<RESET><GREEN>x<RESET>::<RED>b<RESET><GREEN>y<RESET>

diff --git a/t/t4034/cpp/post b/t/t4034/cpp/post
new file mode 100644 (file)
index 0000000..7e8c026
--- /dev/null
+++ b/t/t4034/cpp/post
@@ -0,0 +1,19 @@
+Foo() : x(0&42) { bar(x); }
+cout<<"Hello World?\n"<<endl;
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y
+x::y

diff --git a/t/t4034/cpp/pre b/t/t4034/cpp/pre
new file mode 100644 (file)
index 0000000..23d5c8a
--- /dev/null
+++ b/t/t4034/cpp/pre
@@ -0,0 +1,19 @@
+Foo():x(0&&1){}
+cout<<"Hello World!\n"<<endl;
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y
+a::b

diff --git a/t/t4034/csharp/expect b/t/t4034/csharp/expect
new file mode 100644 (file)
index 0000000..e5d1dd2
--- /dev/null
+++ b/t/t4034/csharp/expect
@@ -0,0 +1,35 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 9106d63..dd5f421 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,18 +1,18 @@<RESET>
+Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <GREEN>bar(x);<RESET> }
+cout<<"Hello World<RED>!<RESET><GREEN>?<RESET>\n"<<endl;
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y

diff --git a/t/t4034/csharp/post b/t/t4034/csharp/post
new file mode 100644 (file)
index 0000000..dd5f421
--- /dev/null
+++ b/t/t4034/csharp/post
@@ -0,0 +1,18 @@
+Foo() : x(0&42) { bar(x); }
+cout<<"Hello World?\n"<<endl;
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y

diff --git a/t/t4034/csharp/pre b/t/t4034/csharp/pre
new file mode 100644 (file)
index 0000000..9106d63
--- /dev/null
+++ b/t/t4034/csharp/pre
@@ -0,0 +1,18 @@
+Foo():x(0&&1){}
+cout<<"Hello World!\n"<<endl;
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y

diff --git a/t/t4034/fortran/expect b/t/t4034/fortran/expect
new file mode 100644 (file)
index 0000000..b233dbd
--- /dev/null
+++ b/t/t4034/fortran/expect
@@ -0,0 +1,10 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 87f0d0b..d308da2 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,5 +1,5 @@<RESET>
+print *, "Hello World<RED>!<RESET><GREEN>?<RESET>"
+
+DO10I = 1,10<RESET>
+<RED>DO10I<RESET><GREEN>DO 10 I<RESET> = 1,10
+<RED>DO10I<RESET><GREEN>DO 1 0 I<RESET> = 1,10

diff --git a/t/t4034/fortran/post b/t/t4034/fortran/post
new file mode 100644 (file)
index 0000000..d308da2
--- /dev/null
+++ b/t/t4034/fortran/post
@@ -0,0 +1,5 @@
+print *, "Hello World?"
+
+DO10I = 1,10
+DO 10 I = 1,10
+DO 1 0 I = 1,10

diff --git a/t/t4034/fortran/pre b/t/t4034/fortran/pre
new file mode 100644 (file)
index 0000000..87f0d0b
--- /dev/null
+++ b/t/t4034/fortran/pre
@@ -0,0 +1,5 @@
+print *, "Hello World!"
+
+DO10I = 1,10
+DO10I = 1,10
+DO10I = 1,10

diff --git a/t/t4034/html/expect b/t/t4034/html/expect
new file mode 100644 (file)
index 0000000..447b49a
--- /dev/null
+++ b/t/t4034/html/expect
@@ -0,0 +1,8 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 8ca4aea..46921e5 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,3 +1,3 @@<RESET>
+<tag <GREEN>newattr="newvalue"<RESET>><GREEN>added<RESET> content</tag>
+<tag attr=<RED>"value"<RESET><GREEN>"newvalue"<RESET>><RED>content<RESET><GREEN>changed<RESET></tag>
+<<RED>tag<RESET><GREEN>newtag<RESET>>content <RED>&entity;<RESET><GREEN>&newentity;<RESET><<RED>/tag<RESET><GREEN>/newtag<RESET>>

diff --git a/t/t4034/html/post b/t/t4034/html/post
new file mode 100644 (file)
index 0000000..46921e5
--- /dev/null
+++ b/t/t4034/html/post
@@ -0,0 +1,3 @@
+<tag newattr="newvalue">added content</tag>
+<tag attr="newvalue">changed</tag>
+<newtag>content &newentity;</newtag>

diff --git a/t/t4034/html/pre b/t/t4034/html/pre
new file mode 100644 (file)
index 0000000..8ca4aea
--- /dev/null
+++ b/t/t4034/html/pre
@@ -0,0 +1,3 @@
+<tag>content</tag>
+<tag attr="value">content</tag>
+<tag>content &entity;</tag>

diff --git a/t/t4034/java/expect b/t/t4034/java/expect
new file mode 100644 (file)
index 0000000..37d1ea2
--- /dev/null
+++ b/t/t4034/java/expect
@@ -0,0 +1,36 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 23d5c8a..7e8c026 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,19 +1,19 @@<RESET>
+Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <GREEN>bar(x);<RESET> }
+cout<<"Hello World<RED>!<RESET><GREEN>?<RESET>\n"<<endl;
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y
+<RED>a<RESET><GREEN>x<RESET>::<RED>b<RESET><GREEN>y<RESET>

diff --git a/t/t4034/java/post b/t/t4034/java/post
new file mode 100644 (file)
index 0000000..7e8c026
--- /dev/null
+++ b/t/t4034/java/post
@@ -0,0 +1,19 @@
+Foo() : x(0&42) { bar(x); }
+cout<<"Hello World?\n"<<endl;
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y
+x::y

diff --git a/t/t4034/java/pre b/t/t4034/java/pre
new file mode 100644 (file)
index 0000000..23d5c8a
--- /dev/null
+++ b/t/t4034/java/pre
@@ -0,0 +1,19 @@
+Foo():x(0&&1){}
+cout<<"Hello World!\n"<<endl;
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y
+a::b

diff --git a/t/t4034/objc/expect b/t/t4034/objc/expect
new file mode 100644 (file)
index 0000000..e5d1dd2
--- /dev/null
+++ b/t/t4034/objc/expect
@@ -0,0 +1,35 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 9106d63..dd5f421 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,18 +1,18 @@<RESET>
+Foo() : x(0<RED>&&1<RESET><GREEN>&42<RESET>) { <GREEN>bar(x);<RESET> }
+cout<<"Hello World<RED>!<RESET><GREEN>?<RESET>\n"<<endl;
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y

diff --git a/t/t4034/objc/post b/t/t4034/objc/post
new file mode 100644 (file)
index 0000000..dd5f421
--- /dev/null
+++ b/t/t4034/objc/post
@@ -0,0 +1,18 @@
+Foo() : x(0&42) { bar(x); }
+cout<<"Hello World?\n"<<endl;
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y

diff --git a/t/t4034/objc/pre b/t/t4034/objc/pre
new file mode 100644 (file)
index 0000000..9106d63
--- /dev/null
+++ b/t/t4034/objc/pre
@@ -0,0 +1,18 @@
+Foo():x(0&&1){}
+cout<<"Hello World!\n"<<endl;
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y

diff --git a/t/t4034/pascal/expect b/t/t4034/pascal/expect
new file mode 100644 (file)
index 0000000..2ce4230
--- /dev/null
+++ b/t/t4034/pascal/expect
@@ -0,0 +1,35 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 077046c..8865e6b 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,18 +1,18 @@<RESET>
+writeln("Hello World<RED>!<RESET><GREEN>?<RESET>");
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y
+<RED>a<RESET><GREEN>x<RESET>::<RED>b<RESET><GREEN>y<RESET>

diff --git a/t/t4034/pascal/post b/t/t4034/pascal/post
new file mode 100644 (file)
index 0000000..8865e6b
--- /dev/null
+++ b/t/t4034/pascal/post
@@ -0,0 +1,18 @@
+writeln("Hello World?");
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y
+x::y

diff --git a/t/t4034/pascal/pre b/t/t4034/pascal/pre
new file mode 100644 (file)
index 0000000..077046c
--- /dev/null
+++ b/t/t4034/pascal/pre
@@ -0,0 +1,18 @@
+writeln("Hello World!");
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y
+a::b

diff --git a/t/t4034/perl/expect b/t/t4034/perl/expect
new file mode 100644 (file)
index 0000000..a1deb6b
--- /dev/null
+++ b/t/t4034/perl/expect
@@ -0,0 +1,13 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index f6610d3..e8b72ef 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -4,8 +4,8 @@<RESET>
+
+package Frotz;<RESET>
+sub new {<RESET>
+       my <GREEN>(<RESET>$class<GREEN>, %opts)<RESET> = <RED>shift<RESET><GREEN>@_<RESET>;
+       return bless { <GREEN>xyzzy => "nitfol", %opts<RESET> }, $class;
+}<RESET>
+
+__END__<RESET>

diff --git a/t/t4034/perl/post b/t/t4034/perl/post
new file mode 100644 (file)
index 0000000..e8b72ef
--- /dev/null
+++ b/t/t4034/perl/post
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+use strict;
+
+package Frotz;
+sub new {
+       my ($class, %opts) = @_;
+       return bless { xyzzy => "nitfol", %opts }, $class;
+}
+
+__END__
+=head1 NAME
+
+frotz - Frotz
+
+=head1 SYNOPSIS
+
+  use frotz;
+
+  $nitfol = new Frotz();
+
+=cut

diff --git a/t/t4034/perl/pre b/t/t4034/perl/pre
new file mode 100644 (file)
index 0000000..f6610d3
--- /dev/null
+++ b/t/t4034/perl/pre
@@ -0,0 +1,22 @@
+#!/usr/bin/perl
+
+use strict;
+
+package Frotz;
+sub new {
+       my $class = shift;
+       return bless {}, $class;
+}
+
+__END__
+=head1 NAME
+
+frotz - Frotz
+
+=head1 SYNOPSIS
+
+  use frotz;
+
+  $nitfol = new Frotz();
+
+=cut

diff --git a/t/t4034/php/expect b/t/t4034/php/expect
new file mode 100644 (file)
index 0000000..0404408
--- /dev/null
+++ b/t/t4034/php/expect
@@ -0,0 +1,35 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index cf6e06b..4420a49 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,18 +1,18 @@<RESET>
+<GREEN>(<RESET>$var<GREEN>)<RESET> $ var
+<?="Hello World<RED>!<RESET><GREEN>?<RESET>"?>
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y

diff --git a/t/t4034/php/post b/t/t4034/php/post
new file mode 100644 (file)
index 0000000..4420a49
--- /dev/null
+++ b/t/t4034/php/post
@@ -0,0 +1,18 @@
+($var) $ var
+<?="Hello World?"?>
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y

diff --git a/t/t4034/php/pre b/t/t4034/php/pre
new file mode 100644 (file)
index 0000000..cf6e06b
--- /dev/null
+++ b/t/t4034/php/pre
@@ -0,0 +1,18 @@
+$var $var
+<?= "Hello World!" ?>
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y

diff --git a/t/t4034/python/expect b/t/t4034/python/expect
new file mode 100644 (file)
index 0000000..8abb8a4
--- /dev/null
+++ b/t/t4034/python/expect
@@ -0,0 +1,34 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 438f776..68baf34 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,17 +1,17 @@<RESET>
+print<RED>u<RESET> "Hello World<RED>!<RESET><GREEN>?<RESET>\n"<GREEN>; print<RESET>
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>) u<RESET>'<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>?<RED>b<RESET><GREEN>y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y

diff --git a/t/t4034/python/post b/t/t4034/python/post
new file mode 100644 (file)
index 0000000..68baf34
--- /dev/null
+++ b/t/t4034/python/post
@@ -0,0 +1,17 @@
+print "Hello World?\n"; print
+(1) (-1e10) (0xabcdef) u'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y

diff --git a/t/t4034/python/pre b/t/t4034/python/pre
new file mode 100644 (file)
index 0000000..438f776
--- /dev/null
+++ b/t/t4034/python/pre
@@ -0,0 +1,17 @@
+print u"Hello World!\n"
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y

diff --git a/t/t4034/ruby/expect b/t/t4034/ruby/expect
new file mode 100644 (file)
index 0000000..16e1dd5
--- /dev/null
+++ b/t/t4034/ruby/expect
@@ -0,0 +1,34 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 30ed9a1..7678f14 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,17 +1,17 @@<RESET>
+10.downto(1) {|<RED>x<RESET><GREEN>y<RESET>| puts <RED>x<RESET><GREEN>y<RESET>}
+<GREEN>(<RESET>1<GREEN>) (<RESET>-1e10<GREEN>) (<RESET>0xabcdef<GREEN>)<RESET> '<RED>x<RESET><GREEN>y<RESET>'
+[<RED>a<RESET><GREEN>x<RESET>] <RED>a<RESET><GREEN>x<RESET>-><RED>b a<RESET><GREEN>y x<RESET>.<RED>b<RESET><GREEN>y<RESET>
+!<RED>a<RESET><GREEN>x<RESET> ~<RED>a a<RESET><GREEN>x x<RESET>++ <RED>a<RESET><GREEN>x<RESET>-- <RED>a<RESET><GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>*<RED>b a<RESET><GREEN>y x<RESET>/<RED>b a<RESET><GREEN>y x<RESET>%<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>+<RED>b a<RESET><GREEN>y x<RESET>-<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<<RED>b a<RESET><GREEN>y x<RESET>>><RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET><<RED>b a<RESET><GREEN>y x<RESET><=<RED>b a<RESET><GREEN>y x<RESET>><RED>b a<RESET><GREEN>y x<RESET>>=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>==<RED>b a<RESET><GREEN>y x<RESET>!=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>^<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>|<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>&&<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>||<RED>b<RESET>
+<RED>a?b<RESET><GREEN>y<RESET>
+<GREEN>x?y<RESET>:z
+<RED>a<RESET><GREEN>x<RESET>=<RED>b a<RESET><GREEN>y x<RESET>+=<RED>b a<RESET><GREEN>y x<RESET>-=<RED>b a<RESET><GREEN>y x<RESET>*=<RED>b a<RESET><GREEN>y x<RESET>/=<RED>b a<RESET><GREEN>y x<RESET>%=<RED>b a<RESET><GREEN>y x<RESET><<=<RED>b a<RESET><GREEN>y x<RESET>>>=<RED>b a<RESET><GREEN>y x<RESET>&=<RED>b a<RESET><GREEN>y x<RESET>^=<RED>b a<RESET><GREEN>y x<RESET>|=<RED>b<RESET>
+<RED>a<RESET><GREEN>y<RESET>
+<GREEN>x<RESET>,y

diff --git a/t/t4034/ruby/post b/t/t4034/ruby/post
new file mode 100644 (file)
index 0000000..7678f14
--- /dev/null
+++ b/t/t4034/ruby/post
@@ -0,0 +1,17 @@
+10.downto(1) {|y| puts y}
+(1) (-1e10) (0xabcdef) 'y'
+[x] x->y x.y
+!x ~x x++ x-- x*y x&y
+x*y x/y x%y
+x+y x-y
+x<<y x>>y
+x<y x<=y x>y x>=y
+x==y x!=y
+x&y
+x^y
+x|y
+x&&y
+x||y
+x?y:z
+x=y x+=y x-=y x*=y x/=y x%=y x<<=y x>>=y x&=y x^=y x|=y
+x,y

diff --git a/t/t4034/ruby/pre b/t/t4034/ruby/pre
new file mode 100644 (file)
index 0000000..30ed9a1
--- /dev/null
+++ b/t/t4034/ruby/pre
@@ -0,0 +1,17 @@
+10.downto(1) {|x| puts x}
+1 -1e10 0xabcdef 'x'
+[a] a->b a.b
+!a ~a a++ a-- a*b a&b
+a*b a/b a%b
+a+b a-b
+a<<b a>>b
+a<b a<=b a>b a>=b
+a==b a!=b
+a&b
+a^b
+a|b
+a&&b
+a||b
+a?b:z
+a=b a+=b a-=b a*=b a/=b a%=b a<<=b a>>=b a&=b a^=b a|=b
+a,y

diff --git a/t/t4034/tex/expect b/t/t4034/tex/expect
new file mode 100644 (file)
index 0000000..604969b
--- /dev/null
+++ b/t/t4034/tex/expect
@@ -0,0 +1,9 @@
+<BOLD>diff --git a/pre b/post<RESET>
+<BOLD>index 2b2dfcb..65cab61 100644<RESET>
+<BOLD>--- a/pre<RESET>
+<BOLD>+++ b/post<RESET>
+<CYAN>@@ -1,4 +1,4 @@<RESET>
+\section{Something <GREEN>new<RESET>}
+<RED>\emph<RESET><GREEN>\textbf<RESET>{Macro style}
+{<RED>\em<RESET><GREEN>\bfseries<RESET> State toggle style}
+\\[<RED>1em<RESET><GREEN>1cm<RESET>]

diff --git a/t/t4034/tex/post b/t/t4034/tex/post
new file mode 100644 (file)
index 0000000..65cab61
--- /dev/null
+++ b/t/t4034/tex/post
@@ -0,0 +1,4 @@
+\section{Something new}
+\textbf{Macro style}
+{\bfseries State toggle style}
+\\[1cm]

diff --git a/t/t4034/tex/pre b/t/t4034/tex/pre
new file mode 100644 (file)
index 0000000..2b2dfcb
--- /dev/null
+++ b/t/t4034/tex/pre
@@ -0,0 +1,4 @@
+\section{Something}
+\emph{Macro style}
+{\em State toggle style}
+\\[1em]

diff --git a/userdiff.c b/userdiff.c
index 9ebf231ea5ee2e7fd1f71c9557a6537d29428d8d..1ff47977d549992ada8c1236187d4516b648ee7d 100644 (file)
--- a/userdiff.c
+++ b/userdiff.c
@@ -8,9 +8,11 @@ static int ndrivers;
 static int drivers_alloc;
 
 #define PATTERNS(name, pattern, word_regex)                    \
-       { name, NULL, -1, { pattern, REG_EXTENDED }, word_regex }
+       { name, NULL, -1, { pattern, REG_EXTENDED },            \
+         word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" }
 #define IPATTERN(name, pattern, word_regex)                    \
-       { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, word_regex }
+       { name, NULL, -1, { pattern, REG_EXTENDED | REG_ICASE }, \
+         word_regex "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" }
 static struct userdiff_driver builtin_drivers[] = {
 IPATTERN("fortran",
         "!^([C*]|[ \t]*!)\n"
@@ -24,10 +26,9 @@ IPATTERN("fortran",
          * Don't worry about format statements without leading digits since
          * they would have been matched above as a variable anyway. */
         "|[-+]?[0-9.]+([AaIiDdEeFfLlTtXx][Ss]?[-+]?[0-9.]*)?(_[a-zA-Z0-9][a-zA-Z0-9_]*)?"
-        "|//|\\*\\*|::|[/<>=]="
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|//|\\*\\*|::|[/<>=]="),
 PATTERNS("html", "^[ \t]*(<[Hh][1-6][ \t].*>.*)$",
-        "[^<>= \t]+|[^[:space:]]|[\x80-\xff]+"),
+        "[^<>= \t]+"),
 PATTERNS("java",
         "!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
         "^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$",
@@ -35,8 +36,7 @@ PATTERNS("java",
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
         "|[-+*/<>%&^|=!]="
-        "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"),
 PATTERNS("objc",
         /* Negate C statements that can look like functions */
         "!^[ \t]*(do|for|if|else|return|switch|while)\n"
@@ -49,8 +49,7 @@ PATTERNS("objc",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
-        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
 PATTERNS("pascal",
         "^(((class[ \t]+)?(procedure|function)|constructor|destructor|interface|"
                "implementation|initialization|finalization)[ \t]*.*)$"
@@ -59,8 +58,7 @@ PATTERNS("pascal",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
-        "|<>|<=|>=|:=|\\.\\."
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|<>|<=|>=|:=|\\.\\."),
 PATTERNS("perl",
         "^[ \t]*package .*;\n"
         "^[ \t]*sub .* \\{\n"
@@ -76,33 +74,29 @@ PATTERNS("perl",
         "|&&|\\|\\||//|\\+\\+|--|\\*\\*|\\.\\.\\.?"
         "|[-+*/%.^&<>=!|]="
         "|=~|!~"
-        "|<<|<>|<=>|>>"
-        "|[^[:space:]]"),
+        "|<<|<>|<=>|>>"),
 PATTERNS("php",
         "^[\t ]*(((public|protected|private|static)[\t ]+)*function.*)$\n"
         "^[\t ]*(class.*)$",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
-        "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"),
 PATTERNS("python", "^[ \t]*((class|def)[ \t].*)$",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
-        "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"),
         /* -- */
 PATTERNS("ruby", "^[ \t]*((class|module|def)[ \t].*)$",
         /* -- */
         "(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
-        "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"),
 PATTERNS("bibtex", "(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
         "[={}\"]|[^={}\" \t]+"),
 PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
-        "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+|[^[:space:]]"),
+        "\\\\[a-zA-Z@]+|\\\\.|[a-zA-Z0-9\x80-\xff]+"),
 PATTERNS("cpp",
         /* Jump targets or access declarations */
         "!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:.*$\n"
@@ -113,8 +107,7 @@ PATTERNS("cpp",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
-        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
 PATTERNS("csharp",
         /* Keywords */
         "!^[ \t]*(do|while|for|if|else|instanceof|new|return|switch|case|throw|catch|using)\n"
@@ -129,8 +122,7 @@ PATTERNS("csharp",
         /* -- */
         "[a-zA-Z_][a-zA-Z0-9_]*"
         "|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
-        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"
-        "|[^[:space:]]|[\x80-\xff]+"),
+        "|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
 { "default", NULL, -1, { NULL, 0 } },
 };
 #undef PATTERNS