Shell Programming and Scripting

# This passes the request from the inputted searchphrase by the user, to go and lookup the google search engine and post the result in the output file mylist.html
wget  -t1  -E -e robots=off -awGet.log -T 200 -H -Priserless -O mylist1.html -U "Mozilla" "http://www.google.co.uk/search?q=${searchphrase}&btnG=Search"
# put a newline at the start of a link
cat mylist1.html |sed -e 's#<a href#\n<a href#g' >mylist2.html
# put a newline at the end of link 
cat mylist2.html |sed -e 's#</a>#</a>\n#g' >mylist3.html
# now we need to extract all lines that begin with <a href="http:
cat mylist3.html |sed -n -e '/^<a href="http:/p' >mylist4.html
# now remove urls that contain google
cat mylist4.html |sed -n -e '/\.google/!p' >mylist5.html
# now remove urls that are numeric
cat mylist5.html |sed -n -e '/[0-9]*[0-9]*[0-9]*\.[0-9]*[0-9]*[0-9]*\.[0-9]*[0-9]*[0-9]*\.[0-9]*[0-9]*[0-9]*/!p' >mylist6.html
# Remove all HTML tags 
cat mylist6.html |sed -e :a -e 's/<[^>]*>//g;'  >mylist7.html
# Delete <a href=" before every text line
cat mylist6.html |sed -e 's#<a href="#\n#' >mylist8.html
# Delete everything after '"', and get rid of blankspace
cat mylist8.html |sed -e 's#"#\n"#' |sed -n -e '/"/!p' |sed -e '/./!d' >mylist9.html
# Remove 'http://' before every text, and get rid of blankspace
cat mylist9.html |sed -e 's#http://#\n#' |sed -e '/./!d' >mylist10.html
# Remove textline after '/' from the file
cat mylist10.html |sed -e 's#/#\n/#' |sed -n -e '/^\//!p' |sed -e '/./!d' >mylist11.html
# Join texts from file13 with file10
sdiff mylist11.html mylist9.html >mylist12.html
# clickable hyperlink
awk '{ printf "<a href=\"%s\">%s</a>\n", $0, $0 }' mylist9.html >mylist13.html