Shell Programming and Scripting

# This sed expression removes IP-like sequences if they are surrounded by a / at each end.
# e.g. src="http://web.com/libs/1.6.1.0/file.js" would have the "/1.6.1.0/" bit removed.
# A basic regex is used to remain Posix compatible and because the Free BSD / Apple OS X
# sed use -E to specify extended regex, while GNU sed uses -r (both may accept both?).
local sedRemoveExpr="s:\/[0-9]*\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]*\/::g"

# This grep extended regex will match all remaining IP-like sequences. -E to specify
# extended regex is specified by Posix and is used in all versions of grep.
local grepExtractExpr="[0-9]+\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]+"

# grep options: -E = extended regex, -o = only matching. sort -u removes duplicates.
local ipLikeAddressMatches=$(sed "$sedRemoveExpr" < "$tempFileName" | \
                             grep -Eo "$grepExtractExpr" | sort -u)

local ipLikeAddressMatches=$(awk '
BEGIN { IP_RE="[0-9]+\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]+" }
{
 gsub("/"IP_RE"/","")
 while(match($0, IP_RE)) {
    fnd=substr($0,RSTART,RLENGTH);
    if(!have[fnd]++) print fnd;
    $0=substr($0,RSTART+RLENGTH+1);
 }
}' "$tempFileName")

# This sed expression removes IP-like sequences if they are surrounded by a / at each end.
# e.g. src="http://web.com/libs/1.6.1.0/file.js" would have the "/1.6.1.0/" bit removed.
# A basic regex is used to remain Posix compatible and because the Free BSD / Apple OS X
# seds use -E to specify extended regex, while GNU sed uses -r (both may accept both?).
# The use of 'xxx' as the replacement string, instead of using the expty string, is to
# prevent inadvertidly creating a valid IP address with digits on either side of the
# removed section. e.g. http://web.com/libs/v.15.5/15.5.2.1/.23.12/file.js

local sedRemoveEncInSlashes="s:\/[0-9]*\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]*\/:xxx:g"

# This sed expression removes number sequences longer than 3 digits. The awk described by
# Posix does not include interval expressions, e.g. {1,3}, GNU Awk 3.1.8 for instance
# requires the --re-interval command option to allow their use. By removing any number
# sequences longer than 3 digits with sed, it is not necessary to check for them in the
# awk code below. 'xxx' is used as the replacement string for the same reason as above.

local sedRemoveNumDigitsOutOfRange="s:[0-9]\{4,\}:xxx:g"

# Combine the two sed removal expressions, so sed can do them with one call.
# Clarity of what the expressions do above, efficiency in processing them below.

local sedRemove="$sedRemoveEncInSlashes;$sedRemoveNumDigitsOutOfRange"

# This awk code will print all IP-like sequences which match the ipRegEx expression. It
# does not check that the numbers within any IP-like sequences are in the valid range of
# an IP address, 0..255, the function Is_Ip_Address(), called below, will do so.
# Note: RSTART and RLENGTH are awk built-in variables, after the match() function has
# been called they will contain the start position and length of the matched string.

local awkExtract='BEGIN { ipRegEx = "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+" }    \
                  {                                                          \
                       line = $0;                                            \
                       while (match(line, ipRegEx))                          \
                       {                                                     \
                            ip = substr(line, RSTART, RLENGTH);              \
                            line = substr(line, RSTART + RLENGTH + 1);       \
                            print ip;                                        \
                       }                                                     \
                  }'

# Extract IP-like sequences from the temp file. sort -u removes duplicates.

local ipLikeAddressMatches=$(sed "$sedRemove" < "$tempFileName" | \
                             awk "$awkExtract" | sort -u)

    local awkExtractIPAddresses='                                                        \
    BEGIN                                                                                \
    {                                                                                    \
        ipSequence = "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+";                                \
        digitSequenceTooLongNotIP = "[0-9][0-9][0-9][0-9][0-9]*";                        \
        encInFwdSlashesNotIP = "[/][0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+[/]";                \
        versioningNotIP = "(version|ver|v)+[ \\.]*[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+";    \
    }                                                                                    \
    {                                                                                    \
        line = tolower($0);                                                              \
        gsub(digitSequenceTooLongNotIP, "xxx", line);                                    \
        gsub(encInFwdSlashesNotIP, "xxx", line);                                         \
        gsub(versioningNotIP, "xxx", line);                                              \
        while (match(line, ipSequence))                                                  \
        {                                                                                \
            ip = substr(line, RSTART, RLENGTH);                                          \
            ipUnique[ip] = ip;                                                           \
            line = substr(line, RSTART + RLENGTH + 1);                                   \
        }                                                                                \
    }                                                                                    \
    END                                                                                  \
    {                                                                                    \
        ipRangeMin = 0;                                                                  \
        ipRangeMax = 255;                                                                \
        ipNumSegments = 4;                                                               \
        ipDelimiter = ".";                                                               \
        for (ip in ipUnique)                                                             \
        {                                                                                \
            numSegments = split(ip, ipSegments, ipDelimiter);                            \
            if (numSegments == ipNumSegments &&                                          \
                ipSegments[1] >= ipRangeMin && ipSegments[1] <= ipRangeMax &&            \
                ipSegments[2] >= ipRangeMin && ipSegments[2] <= ipRangeMax &&            \
                ipSegments[3] >= ipRangeMin && ipSegments[3] <= ipRangeMax &&            \
                ipSegments[4] >= ipRangeMin && ipSegments[4] <= ipRangeMax)              \
            {                                                                            \
                print ip;                                                                \
            }                                                                            \
        }                                                                                \
    }'

    local ipAddressMatches=$(awk "$awkExtractIPAddresses" < "$tempFileName")