nameget removes successful entries from queue
authorAndrew Lorimer <andrew@lorimer.id.au>
Wed, 4 Jul 2018 11:44:28 +0000 (21:44 +1000)
committerAndrew Lorimer <andrew@lorimer.id.au>
Wed, 4 Jul 2018 11:44:28 +0000 (21:44 +1000)
nameget.sh
index e8fbbfa367c6a1c3cfef6cf68ca7cf580867a78d..c0088ab2be47925263153b92f9b33667d6f24464 100755 (executable)
@@ -25,5 +25,7 @@ fi
 otherargs=$@                                                                                                   # set aside remaining arguments for passing to wget
 
 while read -r url filename tail; do
 otherargs=$@                                                                                                   # set aside remaining arguments for passing to wget
 
 while read -r url filename tail; do
-  wget --cut-dirs=100 -r -np -nc --accept -e robots=off -O "$outdir$filename" "$url" $otherargs
+  if wget --cut-dirs=100 -r -np -nc --accept -e robots=off -O "$outdir$filename" "$url" $otherargs
+               sed '/$filename/d' $list
+       fi
 done < $list
 done < $list