Andrew's git / scripts.git /
3999cba57619dbd2b0a1ce6c45b59b288eccd2e3
   1#! /bin/bash
   2
   3# get urls from duckduckgo, then parse them to wget to download
   4
   5args=("$@")
   6site=${args[0]}
   7extension=${args[1]}
   8outdir=${args[2]}
   9filetype=""
  10
  11if (($# == 0 ))
  12        then
  13                printf  "\n\x1b[31mWrong number of arguments\x1b[0m\n\n"
  14                printf "Usage: duckduckget SITE [FILETYPE] [DESTINATION] \n       where SITE is the domain to search for files\n             [FILETYPE] is extension without preceeding dot\n             [DESTINATION] is the output directory relative to working directory"
  15                exit 1
  16fi
  17
  18if [[ $outdir == "" ]]
  19        then
  20                outdir="`pwd`/"
  21fi
  22
  23if ! [[ $outdir =~ /$ ]]
  24        then
  25                outdir=$outdir/
  26fi
  27
  28if [[ $extension != "" ]]
  29        then
  30                filetype="filetype:$extension"
  31fi
  32
  33urls=`curl -silent https://duckduckgo.com/html/\?q\=site:$site%20$filetype | grep "$extension$" | tr -d ' \t\r' | grep -v '^[0-9]' | awk '{print "http://" $0}'`
  34
  35for url in $urls; do
  36        wget --cut-dirs=100 -P $outdir $url
  37done