Andrew's git - scripts.git/blob

   1#! /bin/bash
   2
   3# get a list of urls and corresponding output file names, then parse them to wget
   4
   5LOGTAG="nameget.sh"
   6args=("$@")
   7list=${args[0]}
   8
   9if [[ $list == "" ]]
  10        then
  11                printf  "\n\x1b[31mWrong number of arguments\x1b[0m\n\n"
  12                printf "Usage: nameget LIST [DESTINATION] [ARGS]\n       where LIST is a text file of format [url] [name] (one pair per line)\n             [DESTINATION] is a path appended to the start of each [name] in LIST\n             [ARGS] is passed directly to wget"
  13                exit 1
  14fi
  15
  16if [[ -d ${args[1]} ]]; then                                    # check if args[2] is [DESTINATION] or [ARGS]
  17        outdir=${args[1]}
  18        if [[ ! $outdir =~ /$ ]]; then                  # check if [DESTINATION] has a trailing /
  19            outdir=$outdir/
  20        fi
  21        shift 2
  22else
  23        shift 1
  24fi
  25
  26otherargs=$@                                                                                                    # set aside remaining arguments for passing to wget
  27
  28logger -s -t $LOGTAG "starting downloading files in $list"
  29
  30while read -r url filename; do
  31  destarg = ''
  32  if [ -n "$filename" ]; then
  33    $destarg = " -O $outdir$filename"
  34
  35  if output=$(wget --cut-dirs=100 -np -e robots=off -P $outdir$destarg  $otherargs $url); then
  36                sed -i "/$filename/d" $list
  37                echo $filename
  38                logger -s -t $LOGTAG "downloaded $filename"
  39        else
  40                logger -s -t $LOGTAG "downloading of $filename from $url failed: $output"
  41        fi
  42done < $list