Shell Programming and Scripting

# 4 colums of output:
#   1 - interval or bin number
#   2 - total observations during the interval
#   3 - unique observations during the interval
#   4 - new observations during long interval or new first time observations
#
#   if lbin_size is 0, then the 4th column contains
#   the number of ip addresses that were observed
#   for the very first time.
awk  -v lbin_size=${2:-10} -v bin_size=${1:-1} '
    function dump( )
    {
        if( lbin_size > 1 )                 # long interval turned on
        {
            if( ++lidx >= lbin_size )           # spot for next list; roll if needed
            {
                lidx = 0;
                lwrap = 1;
            }


            if( !lwrap )
                printf( "%5d %5d %5d %5s\n", bin+1, total, new_count, "  -" );  # no value until we have wrapped round
            else
            {                                   # compute the unique addresses in the long interval
                for( l in llist )               # go through each long interval list to weed out duplicates
                {
                    split( llist[l], a, " " );
                    for( i = 1; i <= length( a ); i++ )
                        lunique[a[i]] = 1;                  # get unique set across the long interval
                }

                ltotal = 0;
                for( u in lunique )
                    ltotal++;                   # finally total the unique addresses seen in long interval

                printf( "%5d %5d %5d %5d\n", bin+1, total, new_count, ltotal );
            }

            llist[lidx] = "";               # clear the next list
        }
        else                                # long interval off, show stats and since beginning of time value in col 4
            printf( "%5d %5d %5d %6d\n", bin+1, total, length( unique ), brand_new );

        brand_new = 0;                      # reset
        bin++;
    }

    BEGIN {
        # comment next line out if no header is needed
        if( lbin_size > 0 )
            printf( "%5s %5s %5s %5s\n", "INT", "TOT", "NEW-S", "NEW-L" );
        else
            printf( "%5s %5s %5s %6s\n", "INT", "TOT", "NEW-S", "1ST-TM" );
        lwrap = 0;
        lidx = 0;
    }

    {
        if( $1+0 >= next_bin )              # short interval expires
        {
            if( NR > 1 )
            {
                dump( );                        # write a line of data
                delete unique;
            }

            next_bin = $1 + bin_size;       # set new expiry time
            total = 0;
        }

        if( lbin_size > 0 )
            llist[lidx] = llist[lidx] $2 " ";   # add this to list of addresses for the long interval

        if( !seen[$2]++ )                   # never seen at all
            brand_new++;                    # never seen before count

        unique[$2] = 1;                     # track unique addrs in the short interval
        total++;
    }
    END {
        if( total )
            dump( );
    }
'