|
|
awk to create link, download, and extract in sub-directory
03-05-2019
.xxx_006 19-0000_xxx-yyy-aaa xxx_007 19-0001_zzz-bbb-ccc R_2019_02_28_00_xx_yy_user_S5-0271-00-Medexome yyyy_0287 19-0v02-xxx yyyy_0289 19-0v31-xxxx yyyy_0293 19-0v05-xxxx R_2019_02_15_11_56_40_user_S5-0271-00-v5.6_Oncomine_Childhood_Cancer_Research_DNA_and_Fusions
https://xx.yy.zz/path/to/file.zip 19-0v05-xxx_000_001 cc112233 https://xx.yy.zz/path/to/download/file.zip 19-0v31-xxx-001-000 bb4456784 https://xx.yy.zz/path/to/file.zip 19-0v02-xxx_000_001 aaa331232
awk 'NR==FNR { for(i=2; i<NF; i+=2) a[substr($i,1,7)] = $NF; next } ## start loop and iterate over each first 6 digits in $2 of line 2 in file1
{ k = substr($0, 1, 7) } ## store value extracted in k
{ for(i=1; i<NF; i+=1) a[substr($i,1,7)] = $NF; next } ## start loop and iterate over each previous matching line of file1
{ l = ($0) } ## store value extracted in l (grab the link in line 1)
k in a { cmd = sprintf("mkdir -p %s/%s", a[k], $0); system(cmd); } ## for each k in file2 make a directory with sub-directory as k
l in a { cmd = sprintf("curl -O -v -k -X GET "https://xxx/path/to/download/.zip" -H "Content-Type:application/x-www-form-urlencoded" -H "Authorization:xxx"", a[k], $0); system(cmd); && unzip } ## for each l in file2 make a directory with sub-directory as k and download l in it
' RS= file1 RS='\n' file2 ## files to useR_2019_02_15_11_56_40_user_S5-0271-00-v5.6_Oncomine_Childhood_Cancer_Research_DNA_and_Fusions --- directory 19-0v02-xxx_000_001 --- sub folder 19-0v05-xxx_000_001 --- sub-folder 19-0v31-xxx-001-000 --- sub-folder
R_2019_02_15_11_56_40_user_S5-0271-00-v5.6_Oncomine_Childhood_Cancer_Research_DNA_and_Fusions --- directory
19-0v02-xxx_000_001 --- sub folder
https://xx.yy.zz/path/to/file.zip --- zip and extracted downloaded to sub-folder
19-0v05-xxx_000_001 --- sub-folder
https://xx.yy.zz/path/to/file.zip --- zip and extracted downloaded to sub-folder
19-0v31-xxx-001-000 --- sub-folder
https://xx.yy.zz/path/to/file.zip --- zip and extracted downloaded to sub-folder|
|