ABC1 012345 header
ABC2 7890-000
ABC3 012345 Content Header
ABC5 593.0000 587.4800
ABC5 593.5000 587.6580
ABC5 593.5000 587.6580
ABC1 67890 header
ABC2 1234-0001
ABC3 67890 Content Header
ABC5 594.5000 588.5361
ABC5 602.0000 595.0642
ABC5 602.0000 595.0642
ABC1 234567 header            #   This section removed from 
ABC2 2345-0009                #   the script because there
ABC3 23456 Content Header     #   are no duplicates found 
ABC5 590.5000 588.5361        #   In this section
ABC5 600.5000 594.1603        #
ABC1 019283 header
ABC2 5748-0101
ABC3 019283 Content Header
ABC5 590.5000 588.5361
ABC5 590.5000 588.5361
ABC5 600.5000 594.1603

 
ABC1 012345 header
ABC3 012345 Content Header
ABC5 593.5000 587.6580
 
ABC1 67890 header
ABC3 67890 Content Header
ABC5 602.0000 595.0642
 
ABC1 019283 header
ABC3 019283 Content Header
ABC5 590.5000 588.5361
ABC5 600.5000 594.1603

 
# This will remove the duplicate lines and place the unique lines in file.out
awk '{ if ($1=="ABC1") {delete arr}
      if ( !arr[$0]++ ) { print $0 } }' file.dat > file.out
 
# This will create the duplicate report keeping ABC1 as the section
# header and keeping the Content header as well
awk '/ABC[13]/&&h=$2;_[h,$2,$3]++==1' file.dat > file.tmp
 
#   Seperates the sections by adding an empty line
awk '/ABC1/{print ""}1' file.tmp > file.tmp2
 
# Removes any ABC1 section without any duplicates.  This is done by 
# searching for ABC3 and a blank line after it.  If found it will remove
# only that line, then it will go back and search for the ABC1 header with
# a blank line after it and remove that line as well as the blank line.
sed '/ABC3/{N;/\n *$/D;}' file.tmp2 > file.tmp3
sed '/ABC1/{N;/\n *$/d;}' file.tmp3 > file.dup