Dear All,

I have executed a awk script in linux box which consists of 21 Million records.And i have two mapping files of 500 and 5200 records.To my surprise i found an error

awk: cmd. line:19: (FILENAME=/home/FILE FNR=21031272) fatal: Memory exhausted.

Is there any limitation for records in awk?

regards,
cskumar.

show your code

Also if possible post sample input and output

Possibly there could be a better logic for that

Hi gurus,

awk -F"|" 'BEGIN {

while(getline<"'${LOOKUPDIR}'/EAST_N_EXPANSION.lst")
{
gsub(" ",""); N[$1] = $2;
}

while(getline<"'${LOOKUPDIR}'/C_E_US.lst")
{
gsub(" ",""); U[$1] = $1;
}
}
function trim(str)
{
sub("^[ ]*", "", str);
sub("[ ]*$", "", str);
return str;
}
{
gsub ("\r","");
if (trim($5) == "")
{
W=substr($1,1,10)
}
else
{
W=N[substr($5,1,1)]substr($5,2,7)
}

HASH_W=W;
while ( length (HASH_W) < 20 )
{
HASH_W=("#"HASH_W)
}

HASH_B=substr($1,1,10);
while ( length (HASH_B) < 20 )
{
HASH_B=("#"HASH_B)
}

if ( W==substr($1,1,10))
{

LINESTATUS="PRIMARY"
}
else
{

LINESTATUS="ADDITIONAL"
}
if (trim($12) != "")
{
SID=$12
}
if (trim($11) != "")
{
SID=$11
}
if ((trim($12) == "") && (trim($11) == ""))
{
SID=" "
}
if ( substr(W,4,3) < 200 )
{
print $0 > "'${BASEPATH}'/'${REGION}'/READY/'${FILENAME}'_'${FDATE}'_'${REGION}'_ALL_SVO_ALL_INXX"
}
else
{
if ( trim($3) == "L" && trim($13) == U[trim($13)] )
{
printf ( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%020s|%s|%s|%s|ESVO|1\n",$1,$2,$3,$4,$5 ,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,HASH_W,substr($1,1,14),substr($1,15,3),LINESTAT US,SID ) > "ACTRECORDS"
}

if ( trim($3) == "L" && ( trim($13) == "NMO6A" || trim($13) == "NMO6P" || trim($13) == "NMO6Q" || trim($13) == "NMO6Z" ))
{

printf ( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|ESVO|1\n",$1,$2,$3,$4,$5,$6,$7 ,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,HASH_W,HASH_B,substr($1,15,3),LINESTATUS ) > "SNRECORDS"
}
if ( trim($3) == "L" && ( $16 == "0236" || $17 == "0236" ) && !(trim($13) == "NPL" || trim($13) == "RTY" || trim($13) == "SEA" || trim($13) == "SCG3X" ))
{
printf ( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|ESVO|1\n",$1,$2,$3,$4,$5,$6,$7 ,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,HASH_W,HASH_B,substr($1,15,3),LINESTATUS ) > "LDRECORDS"
}
if ( trim($3) == "L" && trim($13) ~/^XRE/ && ( trim($19) != "" || trim($19) != "?" ) )
{
match ($19,/[1-9]/)
{
printf ( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%020s|%s|%s|ESVO|1|%020s\n",$1,$2,$3,$4 ,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,W,substr($1,1,10),substr($1,15,3),LINESTATUS ,substr(trim($19),RSTART) ) > "TVRECORDS"
}
}
if ( trim($3) == "L" && ( trim($13) == "WAL" || trim($13) == "WRL" || trim($13) == "WAB" || trim($13) == "WBL" ) )
{
printf ( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%020s|%s|%s|ESVO|1\n",$1,$2,$3,$4,$5,$6 ,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,HASH_W,substr($1,1,10)substr($1,15,3),substr($1,15 ,3),LINESTATUS ) > "2KRECORDS"
}
}
}' ${INPUTDIR}/${FILENAME}

Input(sample)
---------
701A180008 008| |L |I8006426261|16426261 |01C|900822|WCT1S| |W| | |8L9S2|010|20060701|0288|5123| | |
701A180032 067| |L | |FA180032 |01C|990519|XDV6X| |)| | |NP3 |010|20070118| | | | |
701A180032 067| |L | |FA180032 |01C|990519|XDV6X| |)| | |MOM5G|010|19990519| | | | |
701A180548 857| |L |I8009328869|09328869 |01C|801204|WCTCS| |W| | |TWASK|010|20060701| | | | |
701A182760 184| |L |I8004722649|34722649 |01C|000000|WCT1S| |W| | |8L9SQ|010|20060701|0288|5123| | |
EAST_N_EXPANSION.lst(sample)
-----------------------------
A|641
B|563
C|319
D|515
E|712
F|701
R|320
S|651
T|763
U|952

C_E_US.lst(sample)
--------------------
AAO
AB3
ACB3X
ADV
AD53D
AD83B
AD83C
AD83D
AD93B
AD93C


Please help...


regards,
cskumar.