Shell Programming and Scripting

my $in_file     =  '/temp/tmp/t'; # file contains your example data
my $out_file    =  '/temp/tmp/new_file.txt'; # New file with dups removed
my $line;
my @inla;
my @outla;

open ( my $in_file_fh,  '<', $in_file  ) or die "Can't open $in_file $!\n";
open ( my $out_file_fh, '>', $out_file ) or die "Can't open $out_file $!\n";

DATA: while ( $line = <$in_file_fh> ) {  # Read input file
        # If line starts with '#DATA' write to out file
        # also write to out file the next line which is a '#VALID x' line
        if ( $line =~ /^\#DATA/ ) {
          print $out_file_fh $line;
          foreach (1..1) {
            $line = <$in_file_fh>;
            print $out_file_fh $line;
          }
          # Read lines until '#END' line is read
          while ( $line = <$in_file_fh> ) {
            if ( $line =~ /^\#END/ ){
              # Create an anoymous hash from input lines in arrary(@inla) which removes duplicates
              # and place results in array: @outla
              @outla = keys %{{ map{$_=>1}@inla}};
              # write sorted out array to out file
              print $out_file_fh ( sort @outla );
              # write '#END' line to out file
              print $out_file_fh $line;
              # exit inner loop back to main loop and start over
              last DATA;
            }
            # Load lines between the '#VALID x' line and the '#END' line into array
            push @inla, $line;
          }
        }
      }

$ cat new_file.txt
#DATA
#VALID 1
a
all
an
and
are
as
awk
below
case
could
data
does
dupes
duplicates
ends
english
examples
file
find
footer
from
given
happens
have
header
however
i
identify
in
input
is
issue
it
language
large
need
not
of
or
output
perl
perso-arabic
real
removing
repeated
result
sample
script
section
since
so
sort
that
the
them
time
up
what
which
with
within
words
#END