# this prints only one line for each distinct record in oldfile
awk '!x[$0]++' oldfile > newfile

# this completely removes all occurrances of all duplicated lines
awk 'x[$0]++ == 2' oldfile > t.sed
grep -v -f t.sed  oldfile > newfile