Andrew's git - scripts.git/blob

bb3d2a1ce27ff99f7c6742683ea02efbc49986a8
   1#! /bin/bash
   2#
   3# Get a list of urls and optional corresponding output paths, then send them to wget
   4# For more info, see `nameget.sh --help`
   5#   Andrew Lorimer   https://lorimer.id.au
   6#
   7
   8VERSION="0.9"
   9LOGTAG="$(basename "$0")" 
  10LOGFILE=/var/log/nameget
  11BADWORDS="error\|failure\|unable\|denied\|directory"
  12outdir="./"
  13
  14function output() {
  15  if [ $test ] || [ "$EUID" -ne 0 ]; then
  16    printf "$2%s\e[0m\n" "$1"
  17  else
  18    logger -s -t $LOGTAG "$1" >> $LOGFILE
  19  fi
  20}
  21
  22usagelong="\e[34m$(basename "$0")\e[0m $VERSION
  23Get a list of urls and optional corresponding output paths, then send them to wget
  24
  25\e[1mUSAGE:\e[0m
  26  $(basename "$0") [OPTIONS]
  27
  28\e[1mOPTIONS:\e[0m
  29  -l, --list
  30    Text file where each line contains a url and filename separated by a space. Filename is optional. Lines starting with # are ignored.
  31    Default: ./queue.txt
  32
  33  -h, --help
  34    Show this help message
  35  
  36  -t, --test
  37    Dry run - parses LIST and prints wget commands but does not execute them
  38    
  39  -d, --destination \e[4m<destination>\e[0m
  40    Directory in which to save each file in LIST
  41    Default: ./
  42    
  43  -a, --args \e[4m<args>\e[0m
  44    String of shell arguments which are passed verbatim to wget\n"
  45
  46usageshort="\e[1mUSAGE:\e[0m
  47  $(basename "$0") [OPTIONS] -l LIST
  48
  49For more information, see \e[34m$(basename "$0") --help\e[0m\n"
  50
  51if [ "$EUID" -ne 0 ]; then
  52  LOGFILE=/dev/null   # redirect log because we're probably not running on a cron job
  53fi
  54
  55#
  56# Validate arguments
  57#
  58
  59while [ $# -gt 0 ]; do
  60  case "$1" in
  61    -l|--list)
  62      list=(${2-})
  63      shift
  64      ;;
  65    -h|--help)
  66      printf "$usagelong" 
  67      exit
  68      ;;
  69    -t|--test)
  70      test=1
  71      shift
  72      ;;
  73    -d|--destination)
  74      outdir=(${2-})
  75      if [ -z "$outdir" ] || [ "$outdir" = " " ]; then
  76        printf "\x1b[31mOption $1 requires an argument\x1b[0m\n\n"
  77        printf "$usageshort"
  78        exit 1
  79      fi
  80      shift
  81      ;;
  82    -a|--args)
  83      otherargs=" ${2-}"
  84      if [ -z "$otherargs" ] || [ "$otherargs" = " " ]; then
  85        printf "\x1b[31mOption $1 requires an argument\x1b[0m\n\n"
  86        printf "$usageshort"
  87        exit 1
  88      fi
  89      shift
  90      ;;
  91    --)
  92      break
  93      ;;
  94    *)
  95      printf "\x1b[31mInvalid argument $1\x1b[0m\n\n"
  96      printf "$usageshort"
  97      exit 1
  98      ;;
  99  esac
 100  shift
 101done
 102
 103if [ -z "$list" ] || [ "$list" = " " ]; then
 104  printf "\x1b[31mList location not specified\x1b[0m\n\n" >&2
 105  printf "$usageshort" >&2
 106  exit 1
 107fi
 108
 109output "Welcome to $(basename "$0") $VERSION" "\e[34m"
 110
 111if [ "$test" ]; then
 112  output "Running in test mode"
 113  output "Finished parsing options"
 114fi
 115
 116
 117if [[ ! $outdir =~ /$ ]]; then  # check if [DESTINATION] has a trailing /
 118    outdir=$outdir/
 119 fi
 120
 121#
 122# Recurse through list
 123#
 124
 125
 126output "Starting downloading files in $(readlink -f $list) to $outdir" "\n\e[1m"
 127
 128while read -r url filename; do
 129
 130  # Ignore if line is commented
 131  [[ "$url" =~ ^\s*#.*  ]] && continue
 132
 133  # parse output filename
 134  destarg=''    # placeholder for -O argument (output filename)
 135  if [ -z "$filename" ] || [ "filename" = "" ]; then
 136    filename="$(echo $url | sed 's/\/\$//; s/.*\///; s/[_ \.]/-/g; s/\(.*\)/\L\1/; s/^the-//; s/-the\(-movie\)\?\(-film\)\?//g; s/^a-//; s/-\(20\|19\)[0-9]\{2\}.*-\(.*\)/.\2/g; s/-[0-9]\{3,4\}p.*-\(.*\)/.\1/g; s/-ii-/-2-/g; s/-iii-/-3-/g; s/-iv-/-4-/g; s/-v-/-5-/g; s/-vi-/-6-/g; s/-vii-/-7-/g; s/-viii-/-8-/g; s/-viiii-/-9-/g; s/-x-/-10-/g;')"
 137  fi
 138  destarg=" -O $outdir$filename"
 139  prettyname="$filename from $url"
 140  if [ "$EUID" -eq 0 ]; then
 141    nvarg=" -nv"
 142  fi
 143
 144  # compile wget command (for debugging purposes)
 145  command="wget$nvarg -R "*index.html*" -c -nd -nH --cut-dirs=100 -np -e robots=off -P $outdir$destarg$otherargs $url"
 146
 147  if [ "$test" ]; then
 148    output "$command" "    "
 149  else
 150    if [ "$EUID" -ne 0 ]; then # get stdout of command
 151      wgetout=$($command | tee /dev/tty)  # get stdout of command
 152    else
 153      wgetout=$($command 2>&1 > /dev/null)  # get stderr of command
 154    fi
 155    exitcode=$?
 156    failures=$(echo "$wgetout" | grep -i "$badwords")
 157    if [ $exitcode -eq 0 ] && [ -z "$failures" ]; then
 158      sed -i '/\"$url\"/d' $list
 159      output "Downloaded $prettyname" "\e[32m"
 160    else
 161      output "Downloading of $prettyname failed with code $exitcode: $(echo $failures | paste -sd \; -)" "\e[31m"
 162    fi
 163  fi
 164
 165done < $list