Filesystems, Disks and Memory

vxfs )
DUMP=/sbin/vdump ; RDUMP=/sbin/vdump
RESTORE=/sbin/vrestore ; RRESTORE=/sbin/vrestore
;;
* )
DUMP=cpio
;;
esac
;; #end of sysv4
*osf* )
case $FSTYPE in
ufs )
DUMP=/usr/sbin/dump ; RDUMP=/usr/sbin/rdump
RESTORE=/usr/sbin/restore ; RRESTORE=/usr/sbin/rrestore
D_OPTS="-$LEVEL~-b~$B_FACTOR~-d~$DENSITY~-s~$SIZE~-c~-u~-f"
R_OPTS="-t~-b~$B_FACTOR~-y~-f"
;;
advfs )
DUMP=/sbin/vdump ; RDUMP=/sbin/vdump
RESTORE=/sbin/vrestore ; RRESTORE=/sbin/vrestore
D_OPTS="-$LEVEL~-b~$B_FACTOR~-N~-x~8~-u~-f"
R_OPTS="-t~-f"
;;
* )
DUMP=cpio
;;
esac
;; #end of osf
*sco* )
case $FSTYPE in
XENIX )
;;
* )
DUMP=cpio
;;
esac
;; #end of sco
#The lfs is the 'local file system' found on Auspex servers, which
#report as sunos4 systems when you do a uname -sr
*sunos* )
case $FSTYPE in
4.2|lfs )
;;
* )
DUMP=/usr/bin/cpio
;;
esac
;; #end of sunos
*solaris* )
case $FSTYPE in
ufs )
DUMP=/usr/sbin/ufsdump ; RDUMP=/usr/sbin/ufsdump
RESTORE=/usr/sbin/ufsrestore ; RRESTORE=/usr/sbin/ufsrestore
;;
vxfs )
DUMP=/usr/sbin/vxdump ; RDUMP=/usr/sbin/vxdump
RESTORE=/usr/sbin/vxrestore ; RRESTORE=/usr/sbin/vxrestore
D_OPTS="-nu~-$LEVEL~-b~$B_FACTOR~-B~1000000000~-f"
R_OPTS="-ty~-b~$B_FACTOR~-f"
;;

* )
DUMP=/usr/bin/cpio
;;
esac
;; #end of solaris
*ultrix* )
case $FSTYPE in
ufs )
;;
* )
DUMP=cpio
;;
esac
;; #end of ultrix
esac

#If the filesystem type was not found, use cpio
if [ "`basename $DUMP`" = cpio ] ; then
RDUMP=$DUMP ; RESTORE=$DUMP ; RRESTORE=$DUMP
D_OPTS='-oacB' ; R_OPTS='-ictB'
fi

echo "$LINE:$DUMP:$RDUMP:$D_OPTS:$RESTORE:$RRESTORE:$R_OPTS:$LEVEL:$R_RSH:$B_FACTOR" >>$INCLIST

done #done determining the dump commands and options to use with each file sys

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "Tape would not rewind with command: $L_MT $L_F $DEVICE $L_REWIND" exit
else
DEVICE="${DEVICE_BASE}/label.level.$LEVEL"
touch $DEVICE
Log_err $? "Could not create backup file: $DEVICE" exit
fi

#sleep 20

LABEL="BACKUP.LABEL"
NUMFILES=`cat $INCLIST|wc -l|sed 's/ //g'`
NUMFILES=`expr $NUMFILES + 1`
cd $TMP

echo |tee -a $LOG
echo "---------------------------------------------------------" |tee -a $LOG
echo "Placing label of $LABEL as first file on $DEVICE" |tee -a $LOG
echo "(and verifying that $DEVICE is set to NO-REW...)" |tee -a $LOG
echo |tee -a $LOG
echo "Displaying contents of the label..." |tee -a $LOG
echo |tee -a $LOG

#Start making $LABEL that will go on the tape

cat >$LABEL <<EOF_HEAD

----------------------------------------------------------------------
This tape is a level $LEVEL backup made `date`
The following is a table of contents of this tape.
It is in the following format:
host:fs name:fs type:OS Version:dump cmmd:rdump cmmd:dump options \\
:restore cmmd: rrestore cmmd:restore options:LEVEL \\
:Client rsh command:Blocking factor
----------------------------------------------------------------------

EOF_HEAD

cat $INCLIST >>$LABEL

#Create one variable with list of hosts
for FSNAME in `awk -F: '{print $1}' $INCLIST|sort -u`
do
echo $N "${FSNAME}_&_$C"
done >$TMP/$X.LIST.OF.SYSTEMS.BEING.BACKED.UP

echo >> $TMP/$X.LIST.OF.SYSTEMS.BEING.BACKED.UP

SAVESET="`cat $TMP/$X.LIST.OF.SYSTEMS.BEING.BACKED.UP|sed 's/_&_$//'`"
SAVESETLOGDIR=${LOGDIR}/$SAVESET
[ -d $SAVESETLOGDIR ] || mkdir $SAVESETLOGDIR

cat >>$LABEL <<EOF_LABEL

----------------------------------------------------------------------
Also, the last file on this tape is a tar file containing a complete
flat file index of the tape. To read it, issue the following commands:
cd $TMP
$L_MT $L_F $DEVICE fsf $NUMFILES
tar xvf $DEVICE

EOF_LABEL

#Add label to the log
cat $LABEL|tee -a $LOG

tar cf $DEVICE $LABEL >/dev/null
Log_err $? "Could not tar to tape $DEVICE!" exit

rm -f $LABEL

if [ "$TAPE" = Y ] ; then
TAPE_DID_REWIND=`tar tvf $DEVICE $LABEL 2>/dev/null|grep "$LABEL"`
[ -z "$TAPE_DID_REWIND" ] && STAT=0 || STAT=1
Log_err $STAT "$DEVICE is NOT a NO-REWIND device! (Try ${DEVICE}n)" exit

$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "Tape would not rewind with command: $L_MT $L_F $DEVICE $L_REWIND" exit
fi

sleep 2

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE fsf 1
Log_err $? "Tape would not fsf with command: $L_MT $L_F $DEVICE fsf 1" exit
fi

sleep 2

#Initialize the file system counter
COUNT=1

echo |tee -a $LOG
echo "============================================================"|tee -a $LOG
echo "Now beginning the backups of all the systems listed above..."|tee -a $LOG
echo "============================================================"|tee -a $LOG
echo |tee -a $LOG

for LINE in `cat $INCLIST`
do

#Parse out values from the line
CLIENT=`echo $LINE|awk -F: '{print $1}'`
FILE_SYS=`echo $LINE|awk -F: '{print $2}'`
FSTYPE=`echo $LINE|awk -F: '{print $3}'`
REMOTE_OSnR=`echo $LINE|awk -F: '{print $4}'`
export REMOTE_OSnR
DUMP=`echo $LINE|awk -F: '{print $5}'`
RDUMP=`echo $LINE|awk -F: '{print $6}'`
D_OPTS=`echo $LINE|awk -F: '{print $7}'|sed 's/~/ /g'`
DOPTS=`echo $LINE|awk -F: '{print $7}'` #Used to create a unique log file
RESTORE=`echo $LINE|awk -F: '{print $8}'`
RRESTORE=`echo $LINE|awk -F: '{print $9}'`
R_OPTS=`echo $LINE|awk -F: '{print $10}'|sed 's/~/ /g'`
LEVEL=`echo $LINE|awk -F: '{print $11}'`
R_RSH=`echo $LINE|awk -F: '{print $12}'`
B_FACTOR=`echo $LINE|awk -F: '{print $13}'`
FSNAME=`echo ${FILE_SYS}|sed 's-/-_-g'`

[ "$FSNAME" = "_" ] && FSNAME='_root'
DUMPBASE=`basename $DUMP`

if [ "$TAPE" != Y ] ; then
DEVICE="$DEVICE_BASE/${CLIENT}$FSNAME.$DUMPBASE.level.$LEVEL"
touch $DEVICE
fi

. ${BINDIR}/rempath.sh
Check_if_client_is_remote

PRE_n_POST_SH="$X.PRE_n_POST_BACKUP.sh"

cat >$TMP/$PRE_n_POST_SH <<EOF_PRE_POST
#!/bin/sh
[ -f /usr/local/bin/\$1 ] && $R_LS /usr/local/bin/\$1
EOF_PRE_POST

chmod 755 $TMP/$PRE_n_POST_SH
[ -n "$RSH_IF_REMOTE" ] && rcp $TMP/$PRE_n_POST_SH $CLIENT:$TMP

PREBACKUP=`$RSH_IF_REMOTE $CLIENT_IF_REMOTE $TMP/$PRE_n_POST_SH \
prebackup.sh`
POSTBACKUP=`$RSH_IF_REMOTE $CLIENT_IF_REMOTE $TMP/$PRE_n_POST_SH \
postbackup.sh`

rm -f $TMP/$PRE_n_POST_SH
[ -n "$RSH_IF_REMOTE" ] && $RSH_IF_REMOTE $CLIENT_IF_REMOTE \
$R_RM -f $TMP/$PRE_n_POST_SH

CURRENT_HOST_LOG=${LOGDIR}/${CLIENT}.current.log
RUNNING_HOST_LOG=${LOGDIR}/${CLIENT}.running.log

FSLOG=${SAVESETLOGDIR}/${CLIENT}${FSNAME}
cp /dev/null $FSLOG
Log_err $? "Could not create $FSLOG"

#Check to see if this is the first file system being backed up
#For this client. If so, run the prebackup script if there is one.

FIRST_FILE_SYS=`grep $CLIENT $INCLIST \
| awk -F: '{print $2}' | head -1 | grep -c "^${FILE_SYS}$"`

if [ $FIRST_FILE_SYS -eq 1 ] ; then

cp /dev/null $CURRENT_HOST_LOG
Log_err $? "Could not create $CURRENT_HOST_LOG"

echo "\n================================" |tee -a $CURRENT_HOST_LOG
echo "Beginning $DUMP of $CLIENT: `date`" |tee -a $CURRENT_HOST_LOG

if [ -n "$PREBACKUP" ] ; then
$RSH_IF_REMOTE $CLIENT_IF_REMOTE /usr/local/bin/prebackup.sh |tee -a $CURRENT_HOST_LOG
fi
fi

echo "\n-------------------------------------------------" |tee -a $FSLOG

if [ "$CLIENT" = "$SERVER" ] ; then

if [ $DUMPBASE != cpio ] ; then

echo $N "Backing up ${CLIENT}:${FILE_SYS} level $LEVEL $C" |tee -a $FSLOG
echo "(File system ${COUNT})" |tee -a $FSLOG

#sleep 5
echo "Using Command: $DUMP $D_OPTS $DEVICE $FILE_SYS" |tee -a $FSLOG
$DUMP $D_OPTS $DEVICE $FILE_SYS 2>&1 |tee -a $FSLOG
else

if [ $LEVEL -eq 0 ] ; then
NEWER=''
else

#This strange section of code is to ensure that if a user does an
#incremental backup before they do a full backup, it will cause a
#full backup, and record that a full was done

cd $FILE_SYS

#Start the record that we are doing a level $LEVEL backup
touch DO_NOT_REMOVE.level.$LEVEL.cpio.touchfile

#Get the "number" of this level's log file. If the number is one,
#then that means that there are no DO_NOT_REMOVE file from any backups
#that had a lower level than this one. That means that it needs to
#be a full backup

FILENUM=`ls DO_NOT_REMOVE.level*|grep -n "DO_NOT_REMOVE.level\.$LEVEL\." \
|awk -F: '{print $1}'`

#This shouldn't ever happen, since I just made one, but you never know...
[ -n "$FILENUM" ] || FILENUM=1

#Get the number of the DO_NOT_REMOVE file that is one level less
#than this one
FILENUM=`expr $FILENUM - 1`

if [ $FILENUM -eq 0 ] ; then

#If the number is zero, then that means this is the first backup on
#this machine, so we have to do a full. Record that we did a full.
NEWER=''
touch DO_NOT_REMOVE.level.0.cpio.touchfile

else
#If there is a file that is one level lower, use it for the -newer value
NEWER="-newer `ls DO_NOT_REMOVE.level.*|head -$FILENUM|tail -1`"
fi
fi

[ -n "`echo $LOCAL_OSnR|grep irix`" ] && XDEV=mount || XDEV=xdev

echo "Using Command: find . $NEWER -$XDEV -print|$DUMP $D_OPTS > $DEVICE"\
|tee -a $FSLOG

cd $FILE_SYS
find . $NEWER -$XDEV -print | $DUMP $D_OPTS > $DEVICE 2>>$FSLOG

fi

if [ $? -eq 0 ] ; then

DATE=`date +"%a %D %T"`

echo $N "$DUMP $LEVEL Done $DATE from $CLIENT $C" |tee -a $FSLOG
echo "$FILE_SYS on ${SERVER}:${DEVICE} $COUNT " |tee -a $FSLOG

else

DATE=`date +"%a %D %T"`

echo "$DUMP $LEVEL NOT Done $DATE from $CLIENT $FILE_SYS on " \
|tee -a $FSLOG
echo "${SERVER}:${DEVICE} $COUNT" |tee -a $FSLOG

fi

#This next section is for the xfsdump command, which makes multiple
#tape files per dump!

FILECOUNT=`grep ' media file [0-9]' $FSLOG |tail -1 \
|sed 's/.* media file //' |sed 's/ .*//g'`
[ -n "$FILECOUNT" ] || FILECOUNT=0
FILECOUNT=`expr $FILECOUNT + 1`
sed "s+$LINE+$LINE:$FILECOUNT+" $INCLIST >$TMP/$$.inclist.tmp
mv $TMP/$$.inclist.tmp $INCLIST

COUNT=`expr $COUNT + $FILECOUNT`

cat $FSLOG >> $CURRENT_HOST_LOG

else #Else, it's a remote dump

echo $N "Backing up ${CLIENT}:${FILE_SYS} level $LEVEL $C" |tee -a $FSLOG
echo "(File system ${COUNT})" |tee -a $FSLOG

TMP_DUMP_SH="$X.tmp.dump.script.sh"

if [ $DUMPBASE != cpio ] ; then
#sleep 5
echo $N "Using Command: $L_RSH $CLIENT $RDUMP $D_OPTS $SERVER:$DEVICE $FILE_SYS" |tee -a $FSLOG

#First, we make a little dump shell script that will run on the client

cat >$TMP/$TMP_DUMP_SH <<EOF_TMP_DUMP
#!/bin/sh

$R_CD $FILE_SYS
$R_RM -f $TMP/$X.$FSNAME.$DOPTS.stat 2>/dev/null
$RDUMP $D_OPTS $SERVER:$DEVICE $FILE_SYS 2>&1
$R_ECHO \$? >$TMP/$X.$FSNAME.$DOPTS.stat
EOF_TMP_DUMP

else

echo $N "Using Command: $L_RSH $CLIENT \" $R_CD $FILE_SYS ; $R_TOUCH DO_NOT_REMOVE.level.$LEVEL.cpio.touchfile ; find . -newer DO_NOT_REMOVE.level.`expr $LEVEL - 1`.cpio.touchfile -xdev -print | $DUMP $D_OPTS | ($R_RSH $SERVER $L_DD of=$DEVICE bs=5120 \" )" |tee -a $FSLOG

#First, we make a little cpio shell script that will run on the client

cat >$TMP/$TMP_DUMP_SH <<EOF_R_CPIO.SH
#!/bin/sh

$R_RM -f $TMP/$X.$FSNAME.$DOPTS.stat 2>/dev/null

$R_CD $FILE_SYS
$R_TOUCH DO_NOT_REMOVE.level.$LEVEL.cpio.touchfile
$R_ECHO \$? >> $TMP/$X.$FSNAME.$DOPTS.stat

#To understand how this section works, see the comments in the
#local backup section for the cpio section

if [ $LEVEL -eq 0 ] ; then
NEWER=''
else
$R_TOUCH DO_NOT_REMOVE.level.$LEVEL.cpio.touchfile
FILENUM=\`$R_LS DO_NOT_REMOVE.level* \
|$R_GREP -n "DO_NOT_REMOVE.level\.$LEVEL\." \
|$R_AWK -F: '{print \$1}'\`

[ -n "\$FILENUM" ] || FILENUM=1

FILENUM=\`$R_EXPR \$FILENUM - 1\`

if [ \$FILENUM -eq 0 ] ; then
NEWER=''
$R_TOUCH DO_NOT_REMOVE.level.0.cpio.touchfile
else
NEWER="-newer \`ls DO_NOT_REMOVE.level.*|head -\$FILENUM|tail -1\`"
fi
fi

$R_ECHO \$? >> $TMP/$X.$FSNAME.$DOPTS.stat

[ -n "\`$TMP/config.guess|grep irix\`" ] && XDEV=mount || XDEV=xdev

$R_FIND . \$NEWER -\$XDEV -print | $DUMP $D_OPTS \
|($R_RSH $SERVER $L_DD of=$DEVICE bs=5120 )
$R_ECHO \$? >> $TMP/$X.$FSNAME.$DOPTS.stat

EOF_R_CPIO.SH

Log_err $? "Could not create $TMP/$TMP_DUMP_SH"

fi

#Then we rcp it over and run it
chmod 755 $TMP/$TMP_DUMP_SH
Log_err $? "Could not chmod $TMP/$TMP_DUMP_SH" exit

rcp $TMP/$TMP_DUMP_SH $CLIENT:$TMP/$TMP_DUMP_SH
Log_err $? "Could not rcp $TMP/$TMP_DUMP_SH to $CLIENT" exit

$L_RSH $CLIENT $TMP/$TMP_DUMP_SH |tee -a $FSLOG

RESULT=`$L_RSH $CLIENT $R_SORT -u $TMP/$X.$FSNAME.$DOPTS.stat|tail -1`
#$L_RSH $CLIENT $R_RM -f $TMP/$TMP_DUMP_SH

if [ "$RESULT" = 0 ] ; then

DATE=`date +"%a %D %T"`

echo $N "$DUMP $LEVEL Done $DATE from $CLIENT: $C" |tee -a $FSLOG
echo "$FILE_SYS on ${SERVER}:${DEVICE} $COUNT" |tee -a $FSLOG

else

DATE=`date +"%a %D %T"`

echo $N "$DUMP $LEVEL NOT Done $DATE from $CLIENT $C" |tee -a $FSLOG
echo "$FILE_SYS on ${SERVER}:${DEVICE} $COUNT" |tee -a $FSLOG fi

fi

#This next section is for the xfsdump command, which makes multiple
#tape files per dump!

FILECOUNT=`grep ' media file [0-9]' $FSLOG |tail -1 \
|sed 's/.* media file //' |sed 's/ .*//g'`
[ -n "$FILECOUNT" ] || FILECOUNT=0
FILECOUNT=`expr $FILECOUNT + 1`
sed "s+$LINE+$LINE:$FILECOUNT+" $INCLIST >$TMP/$$.inclist.tmp
mv $TMP/$$.inclist.tmp $INCLIST

COUNT=`expr $COUNT + $FILECOUNT`

$L_RSH $CLIENT $R_RM -f $TMP/$X.$FSNAME.$DOPTS.stat

cat $FSLOG >>$CURRENT_HOST_LOG
Log_err $? "Could not append $FSLOG to $CURRENT_HOST_LOG"

fi

LAST_FILE_SYS=`grep $CLIENT $INCLIST \
| awk -F: '{print $2}' | tail -1 | grep -c "^${FILE_SYS}$"`

if [ $LAST_FILE_SYS -eq 1 ] ; then

if [ -n "$POSTBACKUP" ] ; then
$RSH_IF_REMOTE $CLIENT_IF_REMOTE /usr/local/bin/postbackup.sh |tee -a $CURRENT_HOST_LOG
fi

#Append this file sys's backup log to the current log of this backup
cat $CURRENT_HOST_LOG >> $LOG
Log_err $? "Could not append $CURRENT_HOST_LOG to $LOG"

#Append this backup's log to the running master backup log for this host
cat $CURRENT_HOST_LOG >> $RUNNING_HOST_LOG
Log_err $? "Could not append $CURRENT_HOST_LOG to $RUNNING_HOST_LOG"

#Keep only 10000 lines in the running log
tail -10000 $RUNNING_HOST_LOG >$TMP/$X.running.tmp.log
Log_err $? "Could not create $TMP/$X.running.tmp.log"

cp $TMP/$X.running.tmp.log $RUNNING_HOST_LOG
Log_err $? "Could not cp $TMP/$X.master.tmp.log to $LOG"

fi

done


if [ $COUNT -gt 1 ] ; then
echo "Backups complete... Beginning reading of tape...\n" |tee -a $LOG
#sleep 60

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "Tape would not rewind with command: $L_MT $L_F $DEVICE $L_REWIND" exit
fi

echo "Rewind Complete" |tee -a $LOG

#sleep 20

#This must start at '2' because the "label" that is put on the tape is
#Actually the first partition on the tape.

READCOUNT=2

for LINE in `cat $INCLIST`
do

sleep 2

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "$L_MT $L_F: error ${L_REWIND}'ing tape." exit
fi

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE fsf `expr $READCOUNT - 1`
Log_err $? "$L_MT $L_F: error fsf'ing tape." exit
fi

sleep 2

#Parse out values from the line
CLIENT=`echo $LINE|awk -F: '{print $1}'`
FILE_SYS=`echo $LINE|awk -F: '{print $2}'`
FSTYPE=`echo $LINE|awk -F: '{print $3}'`
REMOTE_OSnR=`echo $LINE|awk -F: '{print $4}'`
export REMOTE_OSnR
DUMP=`echo $LINE|awk -F: '{print $5}'`
RDUMP=`echo $LINE|awk -F: '{print $6}'`
D_OPTS=`echo $LINE|awk -F: '{print $7}'|sed 's/~/ /g'`
DOPTS=`echo $LINE|awk -F: '{print $7}'` #Used to create a unique log file
RESTORE=`echo $LINE|awk -F: '{print $8}'`
RRESTORE=`echo $LINE|awk -F: '{print $9}'`
R_OPTS=`echo $LINE|awk -F: '{print $10}'|sed 's/~/ /g'`
LEVEL=`echo $LINE|awk -F: '{print $11}'`
R_RSH=`echo $LINE|awk -F: '{print $12}'`
B_FACTOR=`echo $LINE|awk -F: '{print $13}'`
FILECOUNT=`echo $LINE|awk -F: '{print $14}'`
FSNAME=`echo ${FILE_SYS}|sed 's-/-_-g'`

[ "$FSNAME" = "_" ] && FSNAME='_root'

if [ "$TAPE" != Y ] ; then
DEVICE="$DEVICE_BASE/${CLIENT}$FSNAME.$DUMPBASE.level.$LEVEL"
touch $DEVICE
fi

DUMPBASE=`basename $DUMP`

echo |tee -a $LOG
echo "=======================================================" |tee -a $LOG
echo $N " Positioning to Pt# $READCOUNT w/cmd: $C" |tee -a $LOG
echo "$L_MT $L_F $DEVICE fsf `expr $READCOUNT - 1`" |tee -a $LOG

echo " Reading File system $CLIENT:$FILE_SYS" |tee -a $LOG
echo " This was a $FSTYPE file system on a $REMOTE_OSnR system."|tee -a $LOG
echo " (It was backed up with $DUMPBASE.)" |tee -a $LOG

if [ $REMOTE_OSnR = $LOCAL_OSnR -a $DUMPBASE != cpio ] ; then

echo " Using Command: $RESTORE $R_OPTS $DEVICE" |tee -a $LOG

$RESTORE $R_OPTS $DEVICE \
|sed "s/^/$DUMPBASE Pt# $READCOUNT: > /" |tee -a $LOG

else

if [ $DUMPBASE = cpio ] ; then
#Read the file w/ cpio

echo $N " Using Command: $L_DD if=$DEVICE bs=5120 $C" |tee -a $LOG
echo "|($L_RSH $CLIENT $RESTORE -ictv )" |tee -a $LOG
echo |tee -a $LOG

$L_DD if=$DEVICE bs=5120|($L_RSH $CLIENT $RESTORE -ictv )\
|sed "s/^/$DUMPBASE Pt# $READCOUNT: > /" |tee -a $LOG

else

echo " Using Command: $L_RSH $CLIENT $RRESTORE $R_OPTS $SERVER:$DEVICE "
echo |tee -a $LOG

$L_RSH $CLIENT $RRESTORE $R_OPTS $SERVER:$DEVICE \
|sed "s/^/$DUMPBASE Pt# $READCOUNT: > /"|tee -a $LOG
fi

fi

Log_err $? "Could not read filesystem $READCOUNT!"

READCOUNT=`expr $READCOUNT + $FILECOUNT`

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "$L_MT $L_F: error ${L_REWIND}'ing tape." exit
fi
done

#Append the output log to the tape for disaster purposes
if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_REWIND
Log_err $? "$L_MT $L_F: error ${L_REWIND}'ing tape." exit

$L_MT $L_F $DEVICE fsf `expr $READCOUNT - 1`
Log_err $? "$L_MT $L_F: error fsf'ing tape." exit
else
DEVICE="$DEVICE_BASE/index.level.$LEVEL"
touch $DEVICE
fi

tar cvf $DEVICE $LOG

echo "Tape read successful..." |tee -a $LOG

if [ -f /usr/local/bin/afterread.sh ] ; then
echo "Running the commands found in /usr/local/bin/afterread.sh" | tee -a $LOG
/usr/local/bin/afterread.sh
fi

if [ "$TAPE" = Y ] ; then
$L_MT $L_F $DEVICE $L_OFFLINE
Log_err $? "$L_MT $L_F: error ${L_OFFLINE}'ing tape." exit
fi

echo "Tape ejected" |tee -a $LOG

fi

Mail_logs

#EOF

Wow. Heck of a script. Not much fun trying to get it to work on my test system.
Could not find your problem but by the way you describe it, I still think there is something wrong with the way it is counting the backups.

Options:
1. add one last backup to make it 'work'. SURE, this doesn't
fix the problem but may make the script work again
- and the most important thing is getting the backups.
Get a test system to do the rest of the options so you
can work on it while letting the 'fix' allow backups.
2. turn on the DEBUG on it and run it - find out where it's breaking
give it a small number of backups to do and see if it's still
giving an error. Check the logs and all output. Make sure
you KNOW the correct number of backups you are doing.
Check your fsf options in the script and by checking the tape
during testing.
3. Look again - something had to have changed
- a new patch on the system
- a new backup added
- check those log files throughly - insure ALL the backups you
think are working ARE working

As far as that script goes, tell me about it. I have actually narrowed the problem down to a specific host in the backups. With trial and error, I found the culprit. It was a more recent addition to the backup schedule, but one that had worked without errors for a while. I can't say for sure, because I still don't know the cause, but what you say about counting the file marks sounds right. Thanks for all the help. When I do discover the root of this problem, I'll post back.

After banging my head on the keyboard for a while, I built up enough courage to post this. The problem was an I/O error with one of the machines in the backup list. Apparently, 3 slices of one physical disk weren't functioning properly. It appeared to be a SCSI issue. I'm still not sure why nobody reported this to me in 2 weeks (I don't use the filesystems), but that's beside the point. I never understood why and it was careless to never figure out, but that one machine never logged the backup process in detail, like the other machines. I ended up altering my jobs to backup individual filesystems and found that some had I/O errors and caused the job to count the tape files wrong. So the confusion can be blamed on my laziness. I've learned alot more about work ethics than I have about backups, in this case. Thanks for all the help.