Programming

Thanks fo your valuable advice. This is the output I got, but I wonder how its working in linux very fastly [with in 20 sec and in HP 1.06 minutes]. Can you help me in finding the problem responsible for this.

$ prof cpp_sw_gsmconverter
%Time Seconds Cumsecs #Calls msec/call Name

46.0 27.22 27.22 __lseek_sys
10.9 6.47 33.69 1 6469.92 parseBinaryFile
6.2 3.64 37.33 _read_sys
5.4 3.17 40.50 _mcount
3.8 2.23 42.7350779408 0.00 strcasecmp
2.5 1.50 44.2318239584 0.00 _ftell
1.7 0.99 45.22 __strlen20
1.6 0.93 46.1518771675 0.00 _lseek
1.4 0.82 46.9714242692 0.00 _strncasecmp
1.3 0.75 47.7219630595 0.00 _strlen
1.0 0.57 48.29 673710 0.00 getValues
0.7 0.44 48.73 4274805 0.00 strcat
0.7 0.39 49.12 $$mulU
0.6 0.37 49.49 916432 0.00 getHEXString
0.6 0.36 49.86 _monstartup
0.5 0.32 50.17 6003397 0.00 __thread_mutex_lock
0.5 0.31 50.49 4670622 0.00 _doprnt
0.5 0.30 50.79 4371040 0.00 sprintf
0.5 0.27 51.06 594491 0.00 power
0.5 0.27 51.33 1443280 0.00 _fread
0.4 0.24 51.57 6003397 0.00 __thread_mutex_unlock
0.4 0.22 51.79 728632 0.00 setBlockSize
0.3 0.19 51.98 3001697 0.00 malloc
0.3 0.18 52.16 3000911 0.00 free
0.3 0.16 52.32 3000907 0.00 calloc
0.3 0.15 52.47 762634 0.00 strtoul
0.2 0.14 52.61 428734 0.00 hex_dex
0.2 0.11 52.73 _ioctl_sys
0.2 0.11 52.84 __memset20
0.2 0.10 52.94 532091 0.00 fseek
0.1 0.07 53.01 3000907 0.00 _memset
0.1 0.06 53.07 1 65.00 _start
0.1 0.05 53.12 532102 0.00 __filbuf
0.1 0.05 53.17 _brk
0.1 0.03 53.20 532102 0.00 _read
0.1 0.03 53.23 $$mulI
0.1 0.03 53.26 __strcpy20
0.0 0.02 53.28 671126 0.00 strcpy
0.0 0.02 53.30 299580 0.00 fprintf
0.0 0.02 53.32 34386 0.00 memcpy
0.0 0.01 53.33 1 12.50 _setvbuf
0.0 0.01 53.35 __strcmp20
0.0 0.01 53.36 299582 0.00 _wrtchk
0.0 0.01 53.37 $$divide_by_constant
0.0 0.01 53.38 __doprnt_wide
0.0 0.00 53.38 __thread_mutex_trylock
0.0 0.00 53.39 _open_sys
0.0 0.00 53.39 __locale_init
0.0 0.00 53.39 malloc_usable_size
0.0 0.00 53.39 54923 0.00 printPipedOutput
0.0 0.00 53.39 34002 0.00 getBlockSize
0.0 0.00 53.39 26050 0.00 getStream
0.0 0.00 53.39 20611 0.00 strcmp
0.0 0.00 53.39 6370 0.00 strpbrk
0.0 0.00 53.39 6370 0.00 strspn
0.0 0.00 53.39 6370 0.00 strtok
0.0 0.00 53.39 792 0.00 memchr
0.0 0.00 53.39 790 0.00 _memccpy
0.0 0.00 53.39 789 0.00 _sbrk
0.0 0.00 53.39 783 0.00 __fgets_unlocked
0.0 0.00 53.39 783 0.00 fgets
0.0 0.00 53.39 783 0.00 loadstruct
0.0 0.00 53.39 783 0.00 numpts
0.0 0.00 53.39 340 0.00 write
0.0 0.00 53.39 339 0.00 _xflsbuf
0.0 0.00 53.39 335 0.00 __fwrite_unlocked
0.0 0.00 53.39 335 0.00 _bufsync
0.0 0.00 53.39 105 0.00 createStream
0.0 0.00 53.39 100 0.00 strtol
0.0 0.00 53.39 25 0.00 __errno
0.0 0.00 53.39 6 0.00 open
0.0 0.00 53.39 5 0.00 __fflush_unlocked
0.0 0.00 53.39 5 0.00 _findiop
0.0 0.00 53.39 5 0.00 _getenv
0.0 0.00 53.39 5 0.00 close
0.0 0.00 53.39 5 0.00 fclose
0.0 0.00 53.39 5 0.00 fopen
0.0 0.00 53.39 4 0.00 _findbuf
0.0 0.00 53.39 4 0.00 isatty
0.0 0.00 53.39 4 0.00 strncpy
0.0 0.00 53.39 3 0.00 __gmtime_r_posix
0.0 0.00 53.39 3 0.00 __syscall_err
0.0 0.00 53.39 3 0.00 _asctime
0.0 0.00 53.39 3 0.00 _strncmp
0.0 0.00 53.39 3 0.00 _tzset
0.0 0.00 53.39 3 0.00 asctime_r
0.0 0.00 53.39 3 0.00 curr_time
0.0 0.00 53.39 3 0.00 localtime
0.0 0.00 53.39 3 0.00 localtime_r
0.0 0.00 53.39 3 0.00 ltoa
0.0 0.00 53.39 2 0.00 __toupper
0.0 0.00 53.39 2 0.00 printf
0.0 0.00 53.39 1 0.00 ___exit
0.0 0.00 53.39 1 0.00 __creat64
0.0 0.00 53.39 1 0.00 __q3private
0.0 0.00 53.39 1 0.00 _sysconf
0.0 0.00 53.39 1 0.00 closeAllStreams
0.0 0.00 53.39 1 0.00 main
0.0 0.00 53.39 1 0.00 monitor

It looks like you are trying to do random access on a file, because you're spending a lot of time repositioning the disk heads (__lseek_sys).

This is probably okay on desktop Linux boxes. This is a no-no on a multiuser system.

Here's why. Suppose you want to make 20 separate deposit transactions at the bank.
If you go to the teller and do all 20 at one time, it's fast. If you are limited to one transaction, and then you have to go to the back of the line and wait your turn again, it takes a LOT longer. Your code is in line and having to wait for everybody else's I/O requests to complete everytime you ask the disk to reposition the head. Plus you end up re-reading a lot more off disk (assuming a disk hardware cache) than exists in the file.

50% of your time is disk I/O. You're I/O bound. Big time.

Read the entire file sequentially into memory with one call. Put the file into an array of binary records, then access them using random methods. try something like this:

Code:

#include <stddef.h>
#include <sys/types.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <errno.h>
#include <assert.h>

#define ck(x) \
if( (x) == (-1) ){ perror("");exit(EXIT_FAILURE);}

/* read a buffer from a file */
ssize_t readall(int fd, void *buf, size_t *bytes){
     ssize_t nread = 0, n=0;
     size_t nbyte = *bytes;

     do {
         if ((n = read(fd, &((char *)buf)[nread], nbyte - nread)) == -1) {
             if (errno == EINTR)
                 continue;
             else
                 return (-1);
         }
         if (n == 0)
             return nread;
         nread += n;
     } while (nread < nbyte);
     return nread;
}

/* read control */
void readfile(char *fname, char *buffer, size_t *size, mode_t *mode)
{
   int fd=0;   
   struct stat st;
   
   ck(fd=open(fname,O_RDONLY) );
   ck(fstat(fd,&st) );
   *size=st.st_size;   
   *mode=st.st_mode;
   buffer=calloc(1,*size+1);
   ck(readall(fd, buffer, size) );    
   ck(close(fd) );
}


int main(int argc, char *argv[])
{
   char *buffer=NULL;
   size_t size;
   mode_t mode;
 
   readfile(argv[1],buffer,&size,&mode);
   /* play with buffer here  */
   free(buffer);
   return 0;  
}

Hi jim mcnamara,

Thank you very much for your wonderful reply. Can you please explain me why file buffers[set with setvbuf() ] does not come to rescue when we issue seek command[ for 1 or 2 bytes forward or backward], that trys to fetch data which may be in buffer only. Can you provide links to valuable resources that helps in accomplishing a faster I/O in C. Once again thanks for your great reply.

The default value for BUFFSIZE is usually 4096, which is what the stdio C functions like fgets or fread read from. You should never set the value to a small number except for very special cases. If you have a copy of 'Advanced Programming in the UNIX Environment' look on pages 68-70 for I/O efficiency. BUFFSIZE of 4096 is really optimum.

Anytime the seek goes beyond the buffer boundary, you have to go back into kernel mode to get another buffer. Which means you get to wait. And because you're on a multi-user system, it's possible the hard drive cache has been flushed by somebody else. So you get to wait for I/O.

If you are really moving the file pointer ahead or back by 1-2 bytes, you REALLY should have read the whole file into memory. seek is not meant to do that thousands of times. It's like using ungetc which can cause the same problem. I would seriously consider a pointer instead. Here is a starter version, it compiles.

Code:

/****************************
*
*char *buffer is the whole file, read into memory 
*buflen is the size of the buffer 
*char *fileptr is the current "file pointer"  
*
*****************************/
#include <errno.h>
#include <stdio.h>
extern char *buffer;
extern size_t buflen;

int myseek(char **fileptr, long int offset, int whence)
{
    int retval=0;
	switch(whence)
	{
		case SEEK_CUR:
		    *fileptr+=offset;
			break;
		case SEEK_END:
		    *fileptr=buffer;
		    *fileptr+=buflen;
		    *fileptr+=offset;
			break;
		case SEEK_SET:
		    *fileptr=buffer;
		    *fileptr+=offset;
			break;
		default:
		    errno=EINVAL;
		    retval=(-1);
		    break;
	}
	if (*fileptr > (buffer+buflen) || *fileptr < buffer )
	{
		retval=(-1);
		errno=EINVAL;
	}
    return retval;
}

HI jim mcnamara,

Again hats off to your wonderful reply. What I noticed now is subsequent calls to ftell() command is taking more time [40-50% of time] and when I simulated a file position counter , the c program is taking only 44 seconds. When sample programs were prepared with more file reads and file seeks in HP UX, its working quite fastly but including ftell() to get file position is slowing the program.

Profile of last c program when custom file position counter is implemented::

$ prof cpp_sw_gsmconverter_shailesh5 [without ftell() - 1 time]

%Time Seconds Cumsecs #Calls msec/call Name

27.4 10.20 10.20 1 10199.87 parseBinaryFile
12.1 4.52 14.72 __strlen20
11.2 4.17 18.8996142248 0.00 strcasecmp
11.0 4.08 22.98 _mcount
7.4 2.77 25.7581449783 0.00 _strlen
2.9 1.06 26.8119130580 0.00 _strncasecmp
2.0 0.76 27.57 673710 0.00 getValues
1.2 0.46 28.03 $$mulU
0.9 0.34 28.38 _monstartup
0.9 0.34 28.72 916432 0.00 getHEXString
0.9 0.33 29.05 3663231 0.00 strcat
0.8 0.29 29.34 594491 0.00 power
0.6 0.21 29.55 1443280 0.00 _fread
0.5 0.19 29.74 3000907 0.00 calloc
0.5 0.19 29.93 4059057 0.00 _doprnt
0.5 0.18 30.11 728632 0.00 setBlockSize
0.5 0.18 30.29 __strcpy20
0.5 0.18 30.47 3001698 0.00 malloc
0.4 0.16 30.63 6003401 0.00 __thread_mutex_lock
0.4 0.16 30.79 3759466 0.00 sprintf
0.4 0.15 30.94 6003401 0.00 __thread_mutex_unlock
0.4 0.14 31.09 762634 0.00 _strtoul
0.3 0.12 31.21 1 125.00 _start
0.3 0.11 31.32 3000912 0.00 free
0.3 0.11 31.43 3000907 0.00 memset
0.3 0.10 31.53 428734 0.00 hex_dex
0.2 0.09 31.62 __memset20
0.2 0.07 31.69 _read_sys
0.2 0.06 31.75 671126 0.00 strcpy
0.1 0.05 31.80 299580 0.00 _fprintf
0.1 0.05 31.85 _brk
0.1 0.05 31.90 $$mulI
0.1 0.02 31.92 34002 0.00 getBlockSize
0.1 0.02 31.94 _write_sys
0.0 0.02 31.96 54922 0.00 printPipedOutput
0.0 0.01 31.97 __doprnt_wide
0.0 0.01 31.98 20611 0.00 _strcmp
0.0 0.01 31.99 $$divide_by_constant
0.0 0.01 32.00 11 0.68 printf
0.0 0.01 32.00 __locale_init
0.0 0.00 32.01 __thread_mutex_trylock
0.0 0.00 32.01 299591 0.00 _wrtchk
0.0 0.00 32.01 34386 0.00 _memcpy
0.0 0.00 32.01 26050 0.00 getStream
0.0 0.00 32.01 6370 0.00 _strspn
0.0 0.00 32.01 6370 0.00 _strtok
0.0 0.00 32.01 6370 0.00 strpbrk
0.0 0.00 32.01 3455 0.00 __filbuf
0.0 0.00 32.01 3455 0.00 _read
0.0 0.00 32.01 3441 0.00 lseek
0.0 0.00 32.01 3439 0.00 _fseek
0.0 0.00 32.01 801 0.00 _memchr
0.0 0.00 32.01 791 0.00 sbrk
0.0 0.00 32.01 790 0.00 _memccpy
0.0 0.00 32.01 783 0.00 __fgets_unlocked
0.0 0.00 32.01 783 0.00 _fgets
0.0 0.00 32.01 783 0.00 loadstruct
0.0 0.00 32.01 783 0.00 numpts
0.0 0.00 32.01 349 0.00 _write
0.0 0.00 32.01 348 0.00 _xflsbuf
0.0 0.00 32.01 335 0.00 __fwrite_unlocked
0.0 0.00 32.01 335 0.00 _bufsync
0.0 0.00 32.01 105 0.00 createStream
0.0 0.00 32.01 100 0.00 _strtol
0.0 0.00 32.01 31 0.00 __errno
0.0 0.00 32.01 6 0.00 open
0.0 0.00 32.01 5 0.00 __fflush_unlocked
0.0 0.00 32.01 5 0.00 _findbuf
0.0 0.00 32.01 5 0.00 _findiop
0.0 0.00 32.01 5 0.00 close
0.0 0.00 32.01 5 0.00 fclose
0.0 0.00 32.01 5 0.00 fopen
0.0 0.00 32.01 5 0.00 getenv
0.0 0.00 32.01 5 0.00 isatty
0.0 0.00 32.01 4 0.00 __syscall_err
0.0 0.00 32.01 4 0.00 strncpy
0.0 0.00 32.01 3 0.00 __gmtime_r_posix
0.0 0.00 32.01 3 0.00 _localtime
0.0 0.00 32.01 3 0.00 _ltoa
0.0 0.00 32.01 3 0.00 asctime
0.0 0.00 32.01 3 0.00 asctime_r
0.0 0.00 32.01 3 0.00 curr_time
0.0 0.00 32.01 3 0.00 localtime_r
0.0 0.00 32.01 3 0.00 strncmp
0.0 0.00 32.01 3 0.00 tzset
0.0 0.00 32.01 2 0.00 __toupper
0.0 0.00 32.01 2 0.00 _ftell
0.0 0.00 32.01 1 0.00 __creat64
0.0 0.00 32.01 1 0.00 __q3private
0.0 0.00 32.01 1 0.00 _monitor
0.0 0.00 32.01 1 0.00 closeAllStreams
0.0 0.00 32.01 1 0.00 exit
0.0 0.00 32.01 1 0.00 main
0.0 0.00 32.01 1 0.00 sysconf

Process Start Time - Thu Sep 8 18:36:36 2005
filepos = 4190249, ftell(fhex) = 4190249
fseek count = 3439
fseekeof count = 3439
fseeknodefound count = 0
fseeknodenotfound count = 0
fseekuntag forward count = 0
read count = 1632401
Process End Time - Thu Sep 8 18:37:20 2005