Shell Programming and Scripting

[ There_EX could_MD be_VB more_RBR  casualties_NNS in_IN the_DT mishap_NN ,_, ''_null]

[ There_EX could_MD be_VB more_RBR  casualties_NNS]
[ in_IN the_DT mishap_NN ,_, ''_null]

NN
NNS
NNP
followed by space

to_IN
in_IN
towards_IN
across_IN
for_IN
into_IN
up to _IN

$
$
$ cat -n data.txt
     1  [ There_EX could_MD be_VB more_RBR  casualties_NNS in_IN the_DT mishap_NN ,_, ''_null]
     2  [ Some other text here_NNP towards_IN  mary had_NN upto_IN a_NNS across_IN little lamb_NNP nope_IN a false alarm.]

$
$ cat -n preposition_splitter.pl
     1  #!/usr/bin/perl
     2  use strict;
     3
     4  # We set up a hash to store the precedence words. Hashes will
     5  # allow fast lookup as we iterate through the words. We look for
     6  # words that end with any key in the "precedence" hash.
     7  my %precedence = ( 'NN' => 1,
     8                     'NNS' => 1,
     9                     'NNP' => 1
    10                   );
    11
    12  # And another hash for the preposition markers. Same idea as above.
    13  my %marker = (  'to_IN' => 1,
    14                  'in_IN' => 1,
    15                  'towards_IN' => 1,
    16                  'across_IN' => 1,
    17                  'for_IN' => 1,
    18                  'into_IN' => 1,
    19                  'upto_IN' => 1
    20               );
    21
    22  # Set up the data file
    23  my $file = "data.txt";
    24
    25  # String variable to hold the tokens until we reach a "potential" newline
    26  my $str;
    27
    28  # Marker to be set if we reach a word that should potentially be followed
    29  # by a newline
    30  my $potential_nl = 0;
    31
    32  # Open the file, loop through each line and, within each line,
    33  # loop through each word.
    34  open (FH, "<", $file) or die "Can't open $file: $!";
    35  while (<FH>) {
    36      # Remove the EOL character
    37      chomp;
    38      # A note about what we mean by a "word". Perl's "\w" metacharacter includes the
    39      # following: a-zA-Z_ i.e. upper and lower case letters and the underscore. If the
    40      # data has other characters besides these, we add them to our character class.
    41      # Hence the following characters have been added to our definition of
    42      # a word: "]","[", ",", "'", "."
    43      while (/\s*([\w\]\[,'\.]+)\s*/g) {
    44          my $word = $1;
    45          my $suffix;
    46          ($suffix = $word) =~ s/^.*_//;
    47          #printf("Line: %d %s\n", $., $word);
    48
    49          # If the suffix was found in our precedence hash, then
    50          # we set the potential newline marker
    51          if (defined $precedence{$suffix}) { $potential_nl = 1 }
    52
    53          # If the newline marker was set in the earlier iteration and
    54          # the current word is present in the "marker" hash, then
    55          # (a)  we print the string adding brackets at beginning/end if needed
    56          # (b)  flush the string $str and reset the newline marker
    57          if ($potential_nl and defined $marker{$word}) {
    58              if ( $str !~ m/^\s*\[/ ) {$str = "[".$str }
    59              if ( $str !~ m/\s*\]$/ ) {$str .= "]" }
    60              print $str, "\n";
    61              $str = "";
    62              $potential_nl = 0;
    63          } elsif ($word =~ m/\]$/) {  # do the same thing if we reach the end of line
    64              $str .= $word;
    65              if ( $str !~ m/^\s*\[/ ) {$str = "[".$str }
    66              if ( $str !~ m/\s*\]$/ ) {$str .= "]" }
    67              print $str, "\n";
    68              $str = "";
    69              $potential_nl = 0;
    70              next;
    71          }
    72          $str .= $word." ";
    73      }
    74  }
    75  close (FH) or die "Can't close $file: $!";
    76

$
$
$ perl preposition_splitter.pl
[ There_EX could_MD be_VB more_RBR casualties_NNS ]
[in_IN the_DT mishap_NN ,_, ''_null]
[ Some other text here_NNP ]
[towards_IN mary had_NN ]
[upto_IN a_NNS ]
[across_IN little lamb_NNP nope_IN a false alarm.]

$
$