Shell Programming and Scripting

$ wc -l huge_data.txt
5970387 huge_data.txt

$ awk -vLN=500000 '{print > ("huge_data.txt" 12-(NR>LN))}' huge_data.txt
$
$ ls -lh
total 6.5G
-rw-rw-r-- 1 appusr appusr 3.3G Dec 16 17:04 huge_data.txt
-rw-rw-r-- 1 appusr appusr 3.0G Dec 19 11:45 huge_data.txt11
-rw-rw-r-- 1 appusr appusr 276M Dec 19 11:45 huge_data.txt12
$
$
$ wc -l huge_data.txt11
5470387 huge_data.txt11
$
$ wc -l huge_data.txt12
500000 huge_data.txt12
$

awk -v LN=500000 '
!((NR - 1) % LN) {
	if(NR > 1) close(f)
	f = sprintf("huge_data%03d.txt", 1 + int((NR - 1) / LN))
}
{	print > f
}' huge_data.txt

-bash-3.2$ ls -l test*
-rw-r-----   1 e415243  other    4037689 Dec 19 14:35 test_file_01.txt
-bash-3.2$ wc -l test*
  139186 test_file_01.txt
-bash-3.2$ split -l 14000 test_file_01.txt
-bash-3.2$ ls x*
xaa  xab  xac  xad  xae  xaf  xag  xah  xai  xaj
-bash-3.2$ cat xa* >> test_file_02.txt
-bash-3.2$ diff test_file_01.txt test_file_02.txt
-bash-3.2$