Sponsored Content
Top Forums Shell Programming and Scripting Recursive FTP -- here at last. Post 32388 by Perderabo on Saturday 23rd of November 2002 02:49:57 PM
Old 11-23-2002
Debian HardFeed Part 2

Code:
#
#  Function to decode an "ls -l" line.

lsdcode() {

	typeset -Z2 nmonth day
	typeset -i8 octal

	#
	#  get the line, get the first character, split line into words

	line="$1"
	char1=${line%%${line#?}}
	IFS=" "
	set -A  things -- $line
	IFS=""

	#
	#  We may have a "total" line which needs to be ignored

	if [[ ${things[0]} = total ]] ; then
		set -A  lsdc --  skip 000 000000000000 x x
		return 0
	fi

	#
	#
	parser=1
	month=${things[5]}
	xmonth=$(conv_month $month)
	if  conv_month $month > /dev/null ; then
		parser=1
	else
		parser=0
	fi

	if ((parser)); then
		#
		# Strict Left to Right Parse Routine 
		#
		# Break out the fields that we want.  This technique requires
		# that the user, group, and size fields never run together and
		# so they must have at least one space between them.  But it 
		# allows some limited support of filenames with embedded spaces.

		echo "$line" | IFS=" " read permstring junk junk junk junk \
						month day swing rawname
		if [[ $char1 = l ]] ; then
			link=${rawname#*-\> }
			name=${rawname% -\>*}
		else
			name="$rawname"
			link=""
		fi
	else
		#
		# Outside to Inside Parse Routine 
		#
		# Break out the fields that we want.  This technique requires 
		# that no white space exist in the filename.  But the user, 
		# group, and size  fields may sometimes run together without 
		# causing a problem.

		echo "WARNING:" badly formatted line in directory listing for:    >&2
		echo "        " "${line}"                                         >&2
		echo "        " attempting outside-to-inside scan                 >&2
		echo                                                              >&2

		((pname=${#things[*]}-1))
		if [[ $char1 = l ]] ; then
			link=${things[pname]}
			((pname=pname-2))
		else
			link=
		fi
		permstring=${things[0]}
		name=${things[pname]}
		month=${things[pname-3]}
		day=${things[pname-2]}
		swing=${things[pname-1]}
		if  conv_month $month > /dev/null ; then
			:
		else
			echo "ERROR: " outside-to-inside scan has also failed >&2
			echo "       " giving up on:                          >&2
			echo "       " "$line"                                >&2
			echo                                                  >&2
			set -A  lsdc --  skip 000 000000000000 x x
			return 0
		fi
	fi


	#
	#  Ignore . and ..

	if [[ $name = . || $name = .. ]] ; then
			set -A  lsdc --  skip 000 000000000000 x x
			return 0
	fi

	#
	#  decode permissions  (the permission string is first word

	set -A perms -- $(print -- ${permstring#?} | sed 's/./& /g')
	extras=0
	[[ ${perms[2]} = S ]] && { ((extras=extras+4000)); perms[2]=- ; }
	[[ ${perms[2]} = s ]] && { ((extras=extras+4000)); perms[2]=x ; }
	[[ ${perms[5]} = S ]] && { ((extras=extras+2000)); perms[5]=- ; }
	[[ ${perms[5]} = s ]] && { ((extras=extras+2000)); perms[5]=x ; }
	[[ ${perms[8]} = T ]] && { ((extras=extras+1000)); perms[8]=- ; }
	[[ ${perms[8]} = t ]] && { ((extras=extras+1000)); perms[8]=x ; }

	binary=2#$(print -- ${perms[@]} | sed 's/ //g;s/-/0/g;s/[^0]/1/g')
	((octal=binary))
	result=$(echo $octal)
	result=${result#??}
	((result=result+extras))

	#
	# Decode date and time and convert it to yyyymmddhhmm

	nmonth=$(conv_month $month)
	if [[ $swing = *:* ]] ; then
		if [[ $nmonth > $THISMONTH ]] ; then
			((year=LASTYEAR))
		else
			((year=THISYEAR))
		time1=${swing%???}
		time2=${swing#???}
		time="${time1}${time2}"
		fi
        else
                year=$swing
		time="0000"
        fi

	#
	#  Output the final record

	set -A lsdc -- ${char1} ${result} ${year}${nmonth}${day}${time} ${name} ${link}
	return
}


#
#  Function to process a remote file
#  We will not overwrite and existing file unless we in "freshen" mode.
#  And unless we are in "freshen" mode, it is an error for a file to
#  pre-exist.

process_remote_file() {
	VMESS="${VMESS} is a remote file that"
	do_get=0
	if [[ -f $name ]] ; then
		VMESS="${VMESS} already exists"
		if ((OPT_FRESHEN)) ; then
			line2=$(ls -ld "$name")
			lsdcode "$line2" 
			char12=${lsdc[0]}
			mode2=${lsdc[1]}
			datestamp2=${lsdc[2]}
			name2=${lsdc[3]}
			link2=${lsdc[4]}
			if [[ $datestamp > $datestamp2 ]] ; then
				VMESS="${VMESS} but is out-of-date and"
				do_get=1
			else
				VMESS="${VMESS} and is current"
			fi
		else
			VMESS="${VMESS} and cannot be retrieved"
			echo WARNING: no get since $name exists in ${localpath} >&2
		fi
	else
		do_get=1
	fi
	if ((do_get)) ; then
		print -p get \""$name"\"
		waitfor $name
		VMESS="${VMESS} has been retrieved"
		if ((OPT_MODE)) ; then
			chmod $mode "$name"
		fi
	fi
	if (($OPT_VERBOSE)) ; then
		echo "$VMESS"
	fi
	return 0
}


#  Function to process a remote directory
#  To this function, a remote directory is just an object that
#  may need to be duplicated in the current directory

process_remote_directory() {

	VMESS="${VMESS} is a remote directory that"
	if ((OPT_DIRECTORIES)) ; then
		if exists $name  ; then
			if [[ ! -d $name ]] ; then
				VMESS="${VMESS} cannot be created due to pre-existing object"
				echo WARNING: no mkdir since $name exists in ${localpath} >&2
			else
				VMESS="${VMESS} already exists"
			fi
		else
			mkdir "$name"
			VMESS="${VMESS} has been created locally"
			if ((OPT_MODE)) ; then
				chmod $mode "$name"
			fi
		fi
	else
		VMESS="${VMESS} has been ignored"
	fi
	if (($OPT_VERBOSE)) ; then
		echo "$VMESS"
	fi
	if ((OPT_RECURS)) ; then
		if [[ -d "$name" ]] ; then
			cd "$name"
			print -p lcd \""$name"\"
			exec 4<&-
			obtain_and_process_remote_ls "$name"
			print -p cd ..
			print -p lcd ..
			cd ..
			exec 4< ${DIR_FILE_NAME[LEV]}
			lineno=0
			while (( lineno != ${DIR_LINE_NUM[LEV]})) ; do
				read -u4 junk
				((lineno=lineno+1))
			done
		fi
	fi
	return 0
}


#
#  Function to process a remote symlink
#  Note that we deal with th symlink only --  not
#  the object (if any) that the link points to.

process_remote_symlink() {
	VMESS="${VMESS} is a remote symlink that"
	if ((OPT_SYMLINKS)) ; then
		if exists "$name" ; then
			if [[ ! -L $name ]] ; then
				VMESS="${VMESS} cannot be created due to pre-existing object"
				echo WARNING: no symlink since $name exists in ${localpath} >&2
			else
				VMESS="${VMESS} already exists"
			fi
		else
			ln -s "$link" "$name"
			VMESS="${VMESS} has been duplicated locally"
		fi
	else
		VMESS="${VMESS} has been ignored"
	fi
	if (($OPT_VERBOSE)) ; then
		echo "$VMESS"
	fi
}


#
#  If a remote object is not a file, directory, or
#  symlink, we come here.  

process_remote_weirdo() {
	VMESS="${VMESS} is a remote unknown object that has been ignored"
	return 0
	}

#
#  This function obtains an "ls" listing from the remote ftp system.  Then it 
#  scans the listing line by line to figure out what to do.  It will completely 
#  process the current directory.

obtain_and_process_remote_ls() {

	typeset rdir tmpfile okfile   ## local scope variables ##
	rdir=$1

	#
	#  Set up variables or modify them if we have recursed

	((LEV=LEV+1))
	tmpfile=/tmp/HardFeed.tp.$$.${LEV}
	okfile=/tmp/HardFeed.ok.$$.${LEV}
	if ((LEV == 1)) ; then
		localpath=$STARTPATH
		remotepath=$rdir
	else
		localpath=${localpath}/$rdir
		remotepath=${remotepath}/$rdir

	fi

	#
	#  Get a copy of the remote dir output in a local file
	#  called $tmpfile 

	print -p cd \""$rdir"\"
	print -p $OPT_DIRCMD $tmpfile
	print -p $OPT_DIRCMD $okfile
	waitfor $okfile
	DIR_FILE_NAME[LEV]=$tmpfile
	DIR_LINE_NUM[LEV]=0
	exec 4< $tmpfile

	#
	#  process each line
	#

	while read -u4 line ; do
		((DIR_LINE_NUM[LEV]=${DIR_LINE_NUM[LEV]}+1))
		lsdcode "$line" 
		char1=${lsdc[0]}
		mode=${lsdc[1]}
		datestamp=${lsdc[2]}
		name=${lsdc[3]}
		link=${lsdc[4]}
		VMESS="${remotepath}/${name}"
		case $char1 in
		skip)   ;;
		-)      process_remote_file
			;;
		d)      process_remote_directory
			;;
		l)      process_remote_symlink
			;;
		*)      process_remote_weirdo
			;;
		esac
	done 

#
#  We may have recursed...so we must put everything back the way
#  we found it

	localpath=${localpath%$rdir}
	localpath=${localpath%/}
	remotepath=${remotepath%$rdir}
	remotepath=${remotepath%/}
	rm $tmpfile
	rm $okfile
	((LEV=LEV-1))

	return 0
}


#
#  Main Program
#


ftp -inv >&3 2>&1 |&
print -p open $SYSTEM
print -p user $USER $PASSWORD
print -p binary

i=0
while ((OPT_CMDS>i)) ; do
	print -p ${OPT_CMDS_LIST[i]}
	((i=i+1))
done

obtain_and_process_remote_ls $DIRECTORY

print -p bye
wait
exit 0


Last edited by Perderabo; 03-16-2004 at 04:30 PM..
These 2 Users Gave Thanks to Perderabo For This Post:
 

10 More Discussions You Might Find Interesting

1. UNIX for Dummies Questions & Answers

Recursive FTP

I am trying to write a recursive FTP script and have come to a point where I need to test if the file is either a normal ascii file or a directory. My question is how do I test if the file is either ascii or directory. (1 Reply)
Discussion started by: aslamg
1 Replies

2. UNIX for Dummies Questions & Answers

recursive effect!!

I run the following command in some of my folders... and ended up with a huge mess!! find . -type f -exec perl -e 's/blabla/zzzxxxx/gi' -p -i.bak {} \; I had to kill the process and later when I checked with one of my folders.. ls vaditerm.dt.bak vaditerm.dt.bak.bak... (2 Replies)
Discussion started by: sskb
2 Replies

3. Shell Programming and Scripting

perl + Net::FTP::Recursive

Problem: It will not advance to the next user in the list. It always dies right after it sends the 2/2 files from the first users dir. $USERLIST="/export/home/mxdooley/perl_ftp/userlist"; $USER_DIR="/export/home/mxdooley/perl_ftp/homes";... (2 Replies)
Discussion started by: Optimus_P
2 Replies

4. Shell Programming and Scripting

recursive rcp

I wrote a shell script (AIX) to extract the file "/rep1/toto" from all the hosts referred in a list and send them to one local directory named ~/$host-$file with the hostname as prefix rcp -p user@host:/rep1/$file ~/$host-$file where file = toto ==> it works ! I would do the same thing... (6 Replies)
Discussion started by: Nicol
6 Replies

5. UNIX for Advanced & Expert Users

recursive sorting

In the ls command, -t option and -R option dont work simultaneously. ls -t ---> lists the files with sorting based on file date ls -R ---> lists the files recursively. How to make utilize both in the same command.? I want to sort the recursive files listing.. (1 Reply)
Discussion started by: fermisoft
1 Replies

6. Cybersecurity

Recursive SFTP

Hello, I need to transfer files from Serve1 to Server2. Previously I was using scp command. Now I have to use sftp (due to audit issues). The problem with sftp is (atleast to my level of knowledge) we cannot transfer dirs (and files within that dir). Is there a way to solve this? Looks like... (1 Reply)
Discussion started by: MohanTJ
1 Replies

7. UNIX for Dummies Questions & Answers

recursive wc on a directory?

Hi all, I need to count the number of lines in all the files under a directory (several levels deep). I am feeling extremely dumb, but I don't know how to do that. Needless to say, I am not a shell script wiz... Any advice? thanks in advance! (13 Replies)
Discussion started by: bimba17
13 Replies

8. UNIX for Dummies Questions & Answers

Recursive Permissions???

Is there anyway that I can change permissions on a directory and all its sub-directories and files using one single "chmod" command?? (5 Replies)
Discussion started by: the_red_dove
5 Replies

9. UNIX for Dummies Questions & Answers

recursive search and ftp

Could someone help me in recursive search and ftp'ing the files to remote server? The host machine will have /dir1/dira/list_of_files1 /dir1/dirb/list_of_files2 /dir1/dirc/list_of_files3 . . . so., I need to search from dir1 recursively (only one level down) and find all the files that... (1 Reply)
Discussion started by: brahmi
1 Replies

10. OS X (Apple)

Search recursive

before posting, I have tried to find my answer elsewhere. no luck. I need to find a file buried in a folder somewhere. Master folder has 10 sub folders. each sub folder has folders too. I found this but it does nothing I am on Mac and use Applescript. do shell script "find... (2 Replies)
Discussion started by: sbrady
2 Replies
All times are GMT -4. The time now is 05:05 AM.
Unix & Linux Forums Content Copyright 1993-2022. All Rights Reserved.
Privacy Policy