#!/bin/bash
#
#
#
# For this script to work, you will first need to create a DotMedListings dir in your /home/XXXX/ directory.
#
#
#

declare link=""                #Will store the link for each iteration
declare linkStart=""            #Defines the type of equipment to crawl. To be found in Find Listings For Sale or Wanted On DOTmed.com
declare brand=""            #output
declare price=""            #output
declare currency=""            #output
declare condition=""            #output
declare dateListing=""            #output
declare country=""            #output
declare title=""            #output
declare description=""            #output
declare equipment=""            #output
declare yom=""                #output
declare -i totalCrawl=1            #Variable to define the scope of the crawl (total number of listing to crawl)
declare fileNameBase=""            #Used for the name of the output file via curl: Corresponds to the name of the equipment
declare fileName=""            #Definitve name of the Output file: dateCrawl + fileBaseName
declare dateCrawl=$(date +"%d-%m-%y")    #Date of the crawl used for the name
declare -i offset=0            #Base iteration of the offset. Gets +1 after each iteration
declare -i firstIndex=1            #index for the while - Gets +1 after each iteration but starts on 1 instead of 0 (for the offset).
declare nameToHome=$(cd ~ ; pwd | sed -n 's/.home.\([^\/]*\).*/\1/p')    #name for the path of the file to search if already created

echo
echo 
echo "************* Give the link to the equipment type, from https:// to /all included (last '/' excluded): *************"
read linkStart
echo
echo "************* Now, the total number of listings for the equipment: *************"
read totalCrawl
echo 
echo

#
# Naming the output file
#

fileNameBase=$(curl -s "$linkStart/offset/$offset/all?key=&limit=1&price_sort=descending&cond=all&continent_filter=0&zip=&distance=5&att_1=0&att_row_num=1&additionalkeywords=&country=ES" | sed -n '/^[[:blank:]]*<li itemscope itemtype="http:..data-vocabulary.org.Breadcrumb"><span itemprop="title">.*$/ s/^.*itemprop="title">\([^<]*\).*$/\1/p')

fileName=$dateCrawl"-"$fileNameBase".csv"

#
# Looking if it already exists
#

if test -f "/home/$nameToHome/DotMedListings/$fileName"
then
    echo
    echo "************* WARNING ************* WARNING *************"
    echo "************* You already crawled that today! *************"
    echo "************* Delete the file or try another *************"
    echo "************* WARNING ************* WARNING *************"
    echo
#
# If not, starting the script
#

else

    echo
    echo
    echo
    echo "************* You will find your result in ~/DotMedListings/$fileName *************"
    echo
    echo
    echo

    echo "brand;equipment;title;description;price;currency;condition;dateListing;country;YoM" >> ~/DotMedListings/"$fileName"    #defining each category for the crawl.


    while [ $firstIndex -le $totalCrawl ]        #Starting the crawling loop
    do

        awk -v t1="$firstIndex" -v t2="$totalCrawl" 'BEGIN{print (t1/t2) * 100}'        # Prints the percentage advancement instead of having the curl info.

        link=$(curl -s "$linkStart/offset/$offset/all?key=&limit=1&price_sort=descending&cond=all&continent_filter=0&zip=&distance=5&att_1=0&att_row_num=1&additionalkeywords=&country=ES" | sed -n '/href.*view more/ s/.*href="\([^"]*\).*/\1/p')    # Get the corredponding listing. If it's the first iteration then it will get the first listing for the equipment.

        curl -s "https://www.dotmed.com$link" -o ~/curl"$totalCrawl".xml        #Saves one curl for the first listing to avoid various curls for the same listing

#
# Getting the info out of the curl
#

        brand=$(sed -n "/^[[:blank:]]*<h1 itemprop='name'.*/ s/^.*itemprop='brand'>\([^<]*\).*/\1/p" ~/curl"$totalCrawl".xml)
        equipment=$(sed -n '/^[[:blank:]]*<meta property="og:url".*$/ s/.*"https:\/\/www.dotmed.com\/listing\/\([^\/]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        price=$(sed -n "/^[[:blank:]]*<ul><li class=.left.>Price.*$/ s/^.*amount=\([^&]*\).*/\1/p" ~/curl"$totalCrawl".xml)
        currency=$(sed -n '/^[[:blank:]]*<ul><li class=.left.>Price.*$/ s/^.*currency_from=\([^"]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        condition=$(sed -n '/^[[:blank:]]*<ul><li class="left">Condition:.*$/ s/^.*content=.used.>\([^<]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        dateListing=$(sed -n '/^[[:blank:]]*<ul><li class="left">Date updated.*$/ s/^.*id="date_updated">\([^<]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        country=$(sed -n "/^[[:blank:]]*<p class=.nation.>.*$/ s/^.*'This listing comes from \([^']*\).*/\1/p" ~/curl"$totalCrawl".xml)
        title=$(sed -n '/^[[:blank:]]*<meta property="og:title".*$/ s/.*content="\([^-]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        description=$(sed -n '/^[[:blank:]]*<meta property="og:description".*$/ s/.*content="\([^-]*\).*/\1/p' ~/curl"$totalCrawl".xml)
        yom=$(sed -n '/^.*Specifications: Year of Manufacture.*$/ s/^.*Specifications: Year of Manufacture,\([^,]*\).*/\1/p' ~/curl"$totalCrawl".xml)

# 
# Sending the info to the output file
#


        echo $brand";"$equipment";"$title";"$description";"$price";"$currency";"$condition";"$dateListing";"$country";"$yom >> ~/DotMedListings/"$fileName"

        rm ~/curl"$totalCrawl".xml    # Deleting the curl file to leave space for the next iteration. FYI, I nammed the curl file with the number of crawls to be done to be able to launch simutaniously the script and be able to crawl various equipments at a time.

#
# Resetting for the next iteration.
#

        link=""    
        brand=""
        price=""
        currency=""
        condition=""
        dateListing=""
        country=""
        title=""    
        description=""
        equipment=""
        yom=""

        (( firstIndex++ ))
        (( offset++ ))
    done

    echo
    echo
    echo 
    echo "************* Done! Again, you will find the result in ~/DotMedListings/$fileName *************"
    echo
    echo
    echo
fi