Shell Programming and Scripting

#!/bin/bash
sftp user@domain.com <<EOF
cd somedir
mget *.csv
quit
EOF

#!/bin/bash

# pipe ls of remote folder to file
# to compare to figure out which files
# are newer
b=`sftp user@ftp.domain.com <<EOF
cd folder_with_hourly_dataset
ls
quit
EOF`
echo $b > remotelist.txt

# sanitize that file, so that all we
# see are the .csv entries
list=$(cat remotelist.txt)
for i in $list
do
    echo $i | grep \.csv$ >> sortedremotelist.txt
done

# create local list of files
ls *.csv >> locallist.txt

# compare local/remote list and then
# sanitize the output file into a file
# with one .csv per line so we can 
# use a for loop to sftp get it
diff locallist.txt sortedremotelist.txt > get
cut -c3- get > get2
grep \.csv get2 > get3

# now go get that list of files
c=$(cat get3)
for i in $c
do
    d=`user@domain.com <<EOF
    cd directory_with_hourly_data
    mget $i
    quit
    EOF`
    echo "-------------"
    echo "getting file: "$i
done

# clean up files for next time
cat /dev/null > remotelist.txt
cat /dev/null > sortedremotelist.txt
cat /dev/null > get
cat /dev/null > get2
cat /dev/null > get3