Shell Programming and Scripting

#!/bin/sh

filepathlist="filepathlist.txt"
filepathlistcomp="filepathlistcomp.txt"
duplicatefilenamelist="duplicatefilenamelist.txt"

echo "" > "$filenamelist"
echo "" > "$filepathlistcomp"
find > "$filepathlist"

while read path ;do
 filename=`basename "$path"`
 dupes=0
 while read pathcomp ;do
  filenamecomp=`basename "$pathcomp"`
  if [ "$filename" = "$filecomp" ];then
   if [ $dupes -gt 0 ];then
    echo "$filename" >> "$duplicatefilenamelist"
   fi
   dupes=1
  else
   echo "$path" >> "$filepathlistcomp" 
  fi
 done < "$filepathlist"
 
 echo "" >> "$duplicatefilenamelist"
 "$filepathlist" < "$filepathlistcomp"
done < "$filepathlist"

#!/bin/bash
FILES=/dev/shm/filelist
find -type f | awk -F'/' '{print $NF}' | sort | uniq -d > $FILES
while read F
do
	find -type f -name "$F"
	echo
done < $FILES

#!/bin/bash
# Usage: find-dup [Path [Name]]
if [ -z "$1" ]
then
    read -p "Path to scan: " DIR    # Ask for a base path if not given as argument
else
    DIR="$1"
    if [ -z "$2" ]
    then
        read -p "File Names: " NAME    # Ask for a file pattern if not given as argument
        [ -n "$NAME" ] && NAME="-name $NAME"
    fi
fi
cd $DIR || exit 1
LIST=/dev/shm/filelist    # to store the temp filelists (ramdisk)
find -type f $NAME | awk -F'/' '{print $NF}' | sort | uniq -d > $LIST-1
while read F
do
    find -type f -name "$F" > $LIST-2
    i=0
    unset FILE
    while read L    # Creates an array with duplicate files
    do    ((i++)); FILE[$i]="$L"
    done < $LIST-2
    FILE[0]="Do not delete"
    OPT=""
    for ((i=0; i<${#FILE[@]}; i++))    # Displays the files with numbers for deletion
    do    OPT+=$i; echo -e "$i. ${FILE[$i]}"
    done
    K1=""
    until [[ $K1 = [$OPT] ]]
    do    read -s -n1 K1 <&1
    done
    if (($K1))
    then
        read -s -n1 -p "Confirm deletion of ${FILE[$K1]} (Y/N): " K2 <&1
        [[ $K2 = [yY] ]] && { echo; rm -v "${FILE[$K1]}"; } || echo "No deletion"
    else
        echo "No deletion"
    fi
    echo
done < $LIST-1

perl -MFile::Find::Duplicates -e'
    @dupes = find_duplicate_files("dir1", "dir2");
    printf "Files %s (of size %d) hash to %s\n", 
      (join "," , @{$_->files}), $_->size, $_->md5
        for @dupes'

#!/bin/sh

# Filenames in shared memory directory
filepathlist=/dev/shm/filelist
filepathlistcomp=/dev/shm/filelistcomp

# Usage help printed
usage="$0 [-f list_file] [Directory]"

# Option processing
while test $# -gt 0 ; do
 case "$1" in
  -f) usefilelist=true; filelist="$2" shift 2 ;;          
  --help) echo $usage; exit 1 ;;
  --*) break ;;
  -*) echo $usage; exit 1 ;;
  *)  break ;;
 esac
done

# store search directory given as command line argument
if [ ! -z "$2" ]; then
 finddir="$2"
else
 finddir="."
fi


if $usefilelist ;then
 cp "$filelist" "$filepathlist"
else
 find "$finddir" -type f > "$filepathlist"
fi

echo -n "" > "$filepathlistcomp"

while true ;do
 read path < "$filepathlist"
 filename=`basename "$path"`
 printfirst=true
 if [ "$path" = "" ];then
  exit
 fi
 while read pathcomp ;do
  if [ "$path" != "$pathcomp" ];then
   filenamecomp=`basename "$pathcomp"`
   if [ "$filename" = "$filenamecomp" ];then
     if [ $printfirst = true ];then
       echo "" #new line for new set
       echo "$path"
       printfirst=false
     fi
     echo "$pathcomp"    
   else
     echo "$pathcomp" >> "$filepathlistcomp"
   fi
  fi
 done < "$filepathlist"
 cp "$filepathlistcomp" "$filepathlist"

 echo -n "" > "$filepathlistcomp"
done

#!/bin/sh

# Filenames in shared memory directory
filepathlist=/dev/shm/filelist
filepathlistcomp=/dev/shm/filelistcomp
filedupeslist=/dev/shm/filedupeslist

# Usage help printed
usage="$0 [-f list_file] [Directory]"

# Option processing
while test $# -gt 0 ; do
 case "$1" in
  -f) usefilelist=true; filelist="$2" shift 2 ;;          
  --help) echo $usage; exit 1 ;;
  --*) break ;;
  -*) echo $usage; exit 1 ;;
  *)  break ;;
 esac
done

# store search directory given as command line argument
if [ ! -z "$2" ]; then
 finddir="$2"
else
 finddir="."
fi

if $usefilelist ;then
 cp "$filelist" "$filepathlist"
else
 find "$finddir" -type f > "$filepathlist"
fi

cat "$filepathlist" | awk -F'/' '{print $NF}' | sort | uniq -d > "$filedupeslist"
grep -f "$filedupeslist" "$filepathlist" > "$filepathlistcomp"

while read filedupe ;do
 
 echo -n "" > "$filepathlist"

 while read path ;do
  if [ "$path" = "" ];then
   break
  fi
  filename=`basename "$path"`
  if [ "$filename" = "$filedupe" ];then
   echo "$path"
  else
   echo "$path" >> "$filepathlist"
  fi
 done < "$filepathlistcomp"
 
 cp "$filepathlist" "$filepathlistcomp"
 echo ""

done < "$filedupeslist"

#!/bin/sh

# Filenames in shared memory directory
filepathlist=/dev/shm/filelist4
filepathlistcomp=/dev/shm/filelistcomp4
filedupeslist=/dev/shm/filedupeslist4

# Usage help printed
usage="$0 [-f list_file] [Directory]"

# Option processing
while test $# -gt 0 ; do
 case "$1" in
  -f) usefilelist=true; filelist="$2" shift 2 ;;          
  --help) echo $usage; exit 1 ;;
  --*) break ;;
  -*) echo $usage; exit 1 ;;
  *)  break ;;
 esac
done

# store search directory given as command line argument
if [ ! -z "$2" ]; then
 finddir="$2"
else
 finddir="."
fi

if $usefilelist ;then
 cp "$filelist" "$filepathlist"
else
 find "$finddir" -type f > "$filepathlist"
fi

cat "$filepathlist" | awk -F'/' '{print $NF}' | sort | uniq -d > "$filedupeslist"

while read filedupe ;do
 grep "$filedupe" "$filepathlist" > "$filepathlistcomp"
 
 while read path ;do
  if [ "$path" = "" ];then
   break
  fi
  filename=`basename "$path"`
  if [ "$filename" = "$filedupe" ];then
   echo "$path"
  fi
 done < "$filepathlistcomp"

 echo ""

done < "$filedupeslist"

perl -MFile::Find -e'
  $d = shift || die "$0 dir\n";
  find { 
    wanted => sub {
      -f and push @{$u{$_}}, $File::Find::name;
      }
    }, $d;
  @{$u{$_}} > 1 and printf "found %s in: \n\n%s\n\n", 
    $_, join $/, @{$u{$_}} for keys %u;    
  ' <dirname>