|
|
Command to remove duplicate lines with perl,sed,awk
10-19-2010
# cat file hello hello hello hello monkey donkey hello hello drink vay dance drink
# ./fsed.sedv1.uniq file hello hello monkey donkey drink vay dance
# ## fsed-Sedv1-Uniq ##
#!/bin/bash
xsed="";sedarr=""
while read -r l
do
x=( $( echo $(sed '=' 1 | sed -n 'N;s/\n/ /;p' | sed -n "s/^\(.\).*$l/\1/p") | sed 's/ .*//') );
xsed=("$xsed $x" )
done <"$1"
fsed=( $(echo ${xsed[@]}|sed 's/ /\n/g' | sed -n '/^1/p'|sed -n '1p') )
sedarr=("$fsed" )
for i in ${xsed[@]}
do
sedarr=( "$sedarr $( echo ${xsed[@]}|sed 's/ /\n/g' | sed -ne "/^$i/p"| sed -n '1p' | sed -e "/[${sedarr[@]}]/d" )" )
done
for i in ${sedarr[@]}
do
sed -n "$i p" "$1"
done# cat newfile hello hello hello hello monkey donkey hello hello4 drink dance2 dance drink4 hello hello1 donkey2 hello hello1 hello hello2 hello hello5 donkey3 donkey2 hello hello3 hello hello3 hello hello5 monkey3 dance3 dance3 monkey3 dance3
# ./fsed.sedv2.uniq newfile hello hello monkey donkey hello hello4 drink dance2 dance drink4 hello hello1 donkey2 hello hello2 hello hello5 donkey3 hello hello3 monkey3 dance3
# ## fsed-Sedv2-Uniq ##
#!/bin/bash
xsed="" ;uniq="" ;sedarr="" ;fsed=""
while read -r l
do
x=( $( echo $(sed '=' 1 | sed -n 'N;s/\n/ /;p' | sed -n "s/\(.*\) \b$l\b/\1/p") ) );
xsed=("$xsed ${x}\b\|" )
done <"$1"
fsed=( $(echo ${xsed[@]}|sed 's/ /\n/g' | sed -n '/^1/p'|sed -n '1p') )
sedar=("\b$fsed" )
for i in ${xsed[@]}
do
newi=$(echo $i | sed 's/..$//')
sedar=( $(echo $sedar|sed 's/..$//') )
sedax=$(echo "${xsed[@]}"|sed 's/ /\n/g' | sed -ne "/^${newi}/p"| sed -n '1p'|sed -e "/${sedar[@]}/d" )
x=("$(echo ${sedar[@]}|sed 's/\\|/\\b&\\b/g')" )
sedar=("${x}\|${sedax}" )
done
for i in $(echo ${sedar[@]} | sed 's/[^0-9]/ /g')
do
sed -n "$i p" "$1"
donePS:there are maybe some bugs!!..I dont guaranteed works wery well(like slow results)
|
|
|
|