|
|
Removing repeating lines from a data frame (AWK)
07-19-2011
#!/bin/bash
# input files from each day of data in june and combine into one big file
find /u/Picarro/DataLog/2011/june -type f -name "*dat" -exec cat {} > june.dat \;
#use new combined data as input file
IN_ALL='/u/Picarro/DataLog/2011/june/june.dat'
# the csv file to create for all data called 'june.csv' in the respective directory
OUT_all='/u/Picarro/DataLog/2011/Awk/june.csv'
# gawk files to create csv file
GAWK='/u/Picarro/DataLog/2011/Awk/Format_trial.csv.awk'
#produce the OUT file from the IN file(s)
$GAWK $IN_all > $OUT_all#!/bin/gawk -f
# This file is to restructure the picarro data into the correct .csv columns for R
# create a header with same headings as variable in table
# also set other variables before parsing data
BEGIN {
OFS="," # tells awk that the output separator is a comma
ORS="" # tells awk to not print newline after each print command so all records are
# on the same line until we want a new line "\n"
getline} # removes 1st line of input file ie header so we can replace it with correct one
# rearrange the yyyy-mm-dd | hh:mm:ss date and time to single date column of yyyy/mm/dd hh:mm:ss needed for openair
{print substr($1,9,2) "/" substr($1,6,2) "/" substr($1,1,4) " " substr($2,1,5)}
# print the rest of variables as columns
{print (" ", $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
}
$1 $2 != prev {
{print "\n"} # newline after each 5 seconds of data has been parsed
; prev=$1 $2}
|
|
|
|