Shell Programming and Scripting

#!/usr/bin/env ksh
# 4 colums of output:
#   1 - interval or bin number
#   2 - total observations during the interval
#   3 - new observations during the interval
#   4 - new observations during long interval
#
#   if lbin_size is 0, then the 4th column contains
#   the number of ip addresses that were observed
#   for the very first time. lbin_size defaults to 10; change
#   to 0 to enable running script in 'single interval' mode. 
awk  -v lbin_size=${2:-10} -v bin_size=${1:-1} '
    function dump( )
    {
        if( NR == 1 )
            return;

        new_count = 0;
        for( u in unique )              # compute total in this bin that were not in last bin
            if( last_bin[u] == 0 )
                new_count++;

        if( lbin_size> 1 )                  # long interval is turned on
        {
            if( ++lidx >= lbin_size )           # spot for next list; roll if needed
            {
                lidx = 0;
                lwrap = 1;
            }


            if( !lwrap )
                printf( "%5d %5d %5d %5s\n", bin+1, total, new_count, "  -" );  # no value until we have wrapped round
            else
            {                                   # compute the unique addresses in the long interval
                for( l in llist )               # go through each long interval list to weed out duplicates
                {
                    split( llist[l], a, " " );
                    for( i = 1; i <= length( a ); i++ )
                        lunique[a[i]] = 1;                  # get unique set across the long interval
                }

                ltotal = 0;
                for( u in lunique )
                    ltotal++;                   # finally total the unique addresses seen in long interval

                printf( "%5d %5d %5d %5d\n", bin+1, total, new_count, ltotal );
            }

            llist[lidx] = "";               # clear the next list
        }
        else                                # long interval off, show stats and since beginning of time value in col 4
            printf( "%5d %5d %5d %6d\n", bin+1, total, new_count, brand_new );

        brand_new = 0;                      # reset
        bin++;
    }

    BEGIN {
        # comment next line out if no header is needed
        if( lbin_size > 0 )
            printf( "%5s %5s %5s %5s\n", "INT", "TOT", "NEW-S", "NEW-L" );
        else
            printf( "%5s %5s %5s %6s\n", "INT", "TOT", "NEW-S", "1ST-TM" );
        lwrap = 0;
        lidx = 0;
    }

    {
        if( $1+0 >= next_bin )              # short interval expires
        {
            dump( );                        # write a line of data
            next_bin = $1 + bin_size;       # set new expiry time

            delete last_bin;
            for( u in unique )              # copy hits from this bin
                last_bin[u] = 1;
            delete unique;
            total = 0;
        }

        if( lbin_size> 1 ) 
                llist[lidx] = llist[lidx] $2 " ";   # add this to list of addresses for the long interval

        if( !seen[$2]++ )                   # never seen at all
            brand_new++;                    # never seen before count

        unique[$2] = 1;                     # track unique addrs in the short interval
        total++;
    }
    END {
        if( total )
            dump( );
    }
'

899787600 78.169.38.0      1 address not in prev group (all as it's the first group)

899787601 52.72.7.0          2 addresses not in previous group
899787601 52.72.7.0         ( duplicate in this group))
899787601 154.225.0.0

899787602 118.82.0.0     2 addresses not in previous group
899787602 252.184.37.0

899787603 211.20.38.0        1 address not in previous group

899787604 211.20.38.0       1 address not in previous group
899787604 64.184.37.0

899787605 116.96.9.0     1 address not in previous group

899787606 118.82.0.0     2 address not in previous group
899787606 202.184.37.0

899787607 202.184.37.0      0 address not in previous group