[PVFS2-developers] PVFS2 Request question

Murali Vilayannur vilayann at mcs.anl.gov
Thu Jul 15 17:30:33 EDT 2004


Hi Walt, Rob,

Here are the results of what I have been able to test thus far.
Attached are modified versions of the same program, one of which tests
noncontiguity on files and the other on memory. For what it is worth, this 
could be added to the test sub-directory or something.

a) the test that does non-contiguous memory + contiguous on file WORKS 
fine always. i.e file_contig-test.c seems to work no matter what random 
numbers I throw at it. I did not even have to reduce the # of tuples 
xferred in a call as Walt had suggested. So we are ok on this one.

b) the test that does contiguous memory + non-contiguous on file does 
_not_ _work_ unless the # of tuples is set to 1. This is the test program mem_contig-test.c.
The size of the file that is created seems to be incorrect.
I also think that PVFS_Request_commit has a memory leak.
I think there should be atleast a  free(*req) just before *req = region if 
not more.

> 
> Murali, getting back to our discussion, when you call the PVFS_Request_indexed
> and have count set to PVFS_REQ_LIMIT_PINT_REQUEST_NUM the resulting type
> will have MORE than that many records, and will be too big.
> 
> Try this, run the same program, but set the number of items in your calls
> to index to a nice small number, like 10.  See what that does.  Also,
> try replacing the calls with call to PVFS_Request_vector - this will
> always be a compact representation, but you can do the same kind of thing
> with it.  Does that work?

The program has an option -r <# of tuples> with which I can test whether 
small xfers work. As I said test program in (b) does not work regardless.
I will get back to you on what happens if I use PVFS_Request_vector 
instead of indexed later on today.

> 
> Finally, Rob's question remains - is it a memory type problem or a file type
> problem?  it would help to know what the erroneous data in the file actually
> looks like - is it just shifted over, or totally munged, or what?

I think it is a file type problem, unless of course I have managed to 
introduce newer bugs into my code :).
The file size is not correct. If you wish me to take a look at the 
contents of file, then I could do that as well and report back later 
today.

thanks
Murali
-------------- next part --------------
/*
 * (C) 2001 Clemson University and The University of Chicago
 *
 * See COPYING in top-level directory.
 */

#include <unistd.h>
#include <assert.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/time.h>
#include <time.h>
#include <stdlib.h>

#include "pvfs2.h"
#include "str-utils.h"

#ifndef PVFS_REQ_LIMIT_PINT_REQUEST_NUM
#define PVFS_REQ_LIMIT_PINT_REQUEST_NUM 100
#endif

#ifndef PVFS2_VERSION
#define PVFS2_VERSION "Unknown"
#endif

#ifndef FNAME
#define FNAME "/mnt/pvfs2/testfile"
#endif

#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif

#ifndef MAX_REGIONS
#define MAX_REGIONS 2
#endif

#ifndef MAX_REGIONS
#define MAX_REGIONS 2
#endif

#ifndef MAX_OFFSET
#define MAX_OFFSET 32768
#endif

#ifndef MAX_BUF_SIZE
#define MAX_BUF_SIZE 32768
#endif

static int max_regions = MAX_REGIONS;
static int max_offset  = MAX_OFFSET;

static void usage(int argc, char** argv);
static double Wtime(void);
typedef int32_t cm_count_t;
typedef int64_t cm_offset_t;
typedef int32_t cm_size_t;

typedef struct {
    cm_count_t	 cf_valid_count;
    cm_offset_t *cf_valid_start;
    cm_size_t   *cf_valid_size;
    void	*wr_buffer;
    void	*rd_buffer;
    int		 buf_size;
} cm_frame_t;

static int fillup_buffer(cm_frame_t *frame)
{
    int c, size = sizeof(double), should_be = 0, chunk_size = 0;

    srand(time(NULL));
    if (frame->buf_size < 0 || frame->buf_size % size != 0)
    {
	fprintf(stderr, "buffer size [%d] must be a multiple of %d\n",
		frame->buf_size, size);
	return -1;
    }
    frame->wr_buffer = (char *) calloc(frame->buf_size, sizeof(char));
    assert(frame->wr_buffer);
    frame->rd_buffer = (char *) calloc(frame->buf_size, sizeof(char));
    assert(frame->rd_buffer);

    for (c = 0; c < frame->buf_size / size; c++)
    {
	*((int *) frame->wr_buffer + c) = c;
    }
    frame->cf_valid_count = (rand() % max_regions) + 1;
    frame->cf_valid_start = (cm_offset_t *) calloc(sizeof(cm_offset_t), frame->cf_valid_count);
    frame->cf_valid_size = (cm_size_t *) calloc(sizeof(cm_size_t), frame->cf_valid_count);
    assert(frame->cf_valid_start && frame->cf_valid_size);
    chunk_size = frame->buf_size / frame->cf_valid_count;

    printf("Buffer size is %d bytes\n", frame->buf_size);
    printf("Generating %d valid regions\n", frame->cf_valid_count);
    for (c = 0; c < frame->cf_valid_count; c++)
    {
	int tmp_start;

	tmp_start = rand() % chunk_size;
	frame->cf_valid_start[c] = c * chunk_size + tmp_start;
	frame->cf_valid_size[c] = (rand() % (chunk_size - tmp_start)) + 1;
	assert(frame->cf_valid_start[c] + frame->cf_valid_size[c] <= frame->buf_size);
	
	printf("(%d): valid_start: %Ld, valid_size: %d\n", c, frame->cf_valid_start[c],
		frame->cf_valid_size[c]);
	
	should_be += frame->cf_valid_size[c];
    }
    printf("PVFS_sys_write should write %d bytes\n", should_be);
    return should_be;
}

int find_size(cm_frame_t *frame, int count)
{
    int i, tot = 0;

    assert(count >= 0 && count < frame->cf_valid_count);
    for (i = count; i < MIN(count + 100, frame->cf_valid_count); i++)
    {
	tot += frame->cf_valid_size[i];
    }
    return tot;
}

int main(int argc, char **argv)
{
    int ret = -1;
    char *wptr = NULL;
    loff_t offset;
    char str_buf[PVFS_NAME_MAX] = {0}, *fname = NULL;
    char pvfs_path[PVFS_NAME_MAX] = {0};
    PVFS_fs_id cur_fs;
    PVFS_sysresp_lookup resp_lookup;
    PVFS_sysresp_create resp_create;
    PVFS_sysresp_io resp_io;
    PVFS_sysresp_getattr resp_getattr;
    int64_t total_written = 0, total_read = 0;
    double time1, time2;
    char* entry_name;
    PVFS_object_ref parent_ref;
    PVFS_sys_attr attr;
    PVFS_credentials credentials;
    PVFS_object_ref ref;
    PVFS_Request file_req;
    PVFS_Request mem_req;
    int c, count = 0, should_be = 0;
    cm_frame_t    frame;

    frame.cf_valid_count = 0;
    frame.cf_valid_start = NULL;
    frame.cf_valid_size = NULL;
    frame.wr_buffer   = NULL;
    frame.rd_buffer = NULL;
    frame.buf_size = 32768;

    while ((c = getopt(argc, argv, "o:r:b:f:h")) != EOF)
    {
	switch (c)
	{
	    case 'o':
		max_offset = atoi(optarg);
		break;
	    case 'r':
		max_regions = atoi(optarg);
		break;
	    case 'f':
		fname = optarg;
		break;
	    case 'b':
		frame.buf_size = atoi(optarg);
		break;
	    case 'h':
	    default:
		usage(argc, argv);
		exit(1);
	}
    }

    if (fname == NULL)
    {
	fname = FNAME;
    }
    if ((should_be = fillup_buffer(&frame)) < 0)
    {
	usage(argc, argv);
	exit(1);
    }
    offset = rand() % max_offset;

    ret = PVFS_util_init_defaults();
    if(ret < 0)
    {
	PVFS_perror("PVFS_util_init_defaults", ret);
	return(-1);
    }

    /* translate local path into pvfs2 relative path */
    ret = PVFS_util_resolve(fname,
	&cur_fs, pvfs_path, PVFS_NAME_MAX);
    if(ret < 0)
    {
	PVFS_perror("PVFS_util_resolve", ret);
	ret = -1;
	goto main_out;
    }
    PVFS_util_gen_credentials(&credentials);

    entry_name = str_buf;
    attr.owner = credentials.uid; 
    attr.group = credentials.gid;
    attr.perms = PVFS_U_WRITE|PVFS_U_READ;
    attr.atime = time(NULL);
    attr.mtime = attr.atime;
    attr.ctime = attr.atime;
    attr.mask = (PVFS_ATTR_SYS_ALL_SETABLE);
    attr.dfile_count = -1;

    if (strcmp(pvfs_path,"/") == 0)
    {
        char *segp = NULL, *prev_segp = NULL;
        void *segstate = NULL;

        memset(&resp_lookup, 0, sizeof(PVFS_sysresp_lookup));
        ret = PVFS_sys_lookup(cur_fs, pvfs_path,
                              &credentials, &resp_lookup,
                              PVFS2_LOOKUP_LINK_FOLLOW);
        if (ret < 0)
        {
            PVFS_perror("PVFS_sys_lookup", ret);
            ret = -1;
            goto main_out;
        }
        parent_ref.handle = resp_lookup.ref.handle;
        parent_ref.fs_id = resp_lookup.ref.fs_id;

        while(!PINT_string_next_segment(
                  fname, &segp, &segstate))
        {
            prev_segp = segp;
        }
        entry_name = prev_segp;
    }
    else
    {
        /* get the absolute path on the pvfs2 file system */
#ifdef OLD_PVFS2
        if (PVFS_util_remove_base_dir(pvfs_path, str_buf, PVFS_NAME_MAX))
#else
        if (PINT_remove_base_dir(pvfs_path, str_buf, PVFS_NAME_MAX))
#endif
        {
            if (pvfs_path[0] != '/')
            {
                fprintf(stderr, "Error: poorly formatted path.\n");
            }
            fprintf(stderr, "Error: cannot retrieve entry name for "
                    "creation on %s\n", pvfs_path);
            ret = -1;
            goto main_out;
        }

#ifdef OLD_PVFS2
        ret = PVFS_util_lookup_parent(pvfs_path, cur_fs, &credentials, &parent_ref.handle);
#else
        ret = PINT_lookup_parent(pvfs_path, cur_fs, &credentials, &parent_ref.handle);
#endif
	
        if(ret < 0)
        {
            PVFS_perror("PVFS_util_lookup_parent", ret);
            ret = -1;
            goto main_out;
        }
        else
        {
            int len = strlen(pvfs_path);
            if (pvfs_path[len - 1] == '/')
            {
                char *segp = NULL, *prev_segp = NULL;
                void *segstate = NULL;

                while(!PINT_string_next_segment(
                          fname, &segp, &segstate))
                {
                    prev_segp = segp;
                }
                strncat(pvfs_path, prev_segp, PVFS_NAME_MAX);
                entry_name = prev_segp;
            }
            parent_ref.fs_id = cur_fs;
        }
    }

    memset(&resp_lookup, 0, sizeof(PVFS_sysresp_lookup));
    ret = PVFS_sys_ref_lookup(parent_ref.fs_id, entry_name,
                              parent_ref, &credentials, &resp_lookup,
                              PVFS2_LOOKUP_LINK_NO_FOLLOW);
    if (ret == 0)
    {
        fprintf(stderr, "Target file %s already exists! Deleting.\n", entry_name);
	if (PVFS_sys_remove(entry_name,
		    parent_ref, &credentials) < 0)
	{
	    fprintf(stderr, "Could not unlink?\n");
	}
	else
	{
	    fprintf(stdout, "Unlinked successfully!\n");
	}
        ret = 0;
    }

    memset(&resp_create, 0, sizeof(PVFS_sysresp_create));
    ret = PVFS_sys_create(entry_name, parent_ref, attr,
                          &credentials, NULL, &resp_create);
    if (ret < 0)
    {
	PVFS_perror("PVFS_sys_create", ret);
	ret = -1;
	goto main_out;
    }

    ref = resp_create.ref;
    c = frame.cf_valid_count;
    count = 0;
    total_written = 0;
    wptr = frame.wr_buffer;
    printf("Writing at offset %Ld\n", offset);
    time1 = Wtime();
    while (c > 0)
    {
	int tmp_valid_count;
	
	tmp_valid_count = MIN(c, PVFS_REQ_LIMIT_PINT_REQUEST_NUM);
	printf("mem_req contiguous size is %d\n", find_size(&frame, count));
	ret = PVFS_Request_contiguous(find_size(&frame, count), PVFS_BYTE, &mem_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_contiguous", ret);
	    ret = -1;
	    goto main_out;
	}
	ret = PVFS_Request_commit(&mem_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_commit", ret);
	    ret = -1;
	    goto main_out;
	}
	ret = PVFS_Request_hindexed(tmp_valid_count, &frame.cf_valid_size[count],
		&frame.cf_valid_start[count], PVFS_BYTE, &file_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_hindexed", ret);
	    ret = -1;
	    goto main_out;
	}
	ret = PVFS_Request_commit(&file_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_commit", ret);
	    ret = -1;
	    goto main_out;
	}
	/* write out the data */
	ret = PVFS_sys_write(ref, file_req,
                             offset, wptr, mem_req, 
                             &credentials, &resp_io);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_sys_write", ret);
	    ret = -1;
	    goto main_out;
	}
	wptr += resp_io.total_completed;
	total_written += resp_io.total_completed;
	assert(resp_io.total_completed == find_size(&frame, count));

	/* need to free the request descriptions */
	PVFS_Request_free(&mem_req);
	PVFS_Request_free(&file_req);
	
	c -= tmp_valid_count;
	count += tmp_valid_count;
    }
    time2 = Wtime();
    /* sanity check */
    if(should_be != total_written)
    {
	fprintf(stderr, "Error: short write!\n");
	fprintf(stderr, "Tried to write %d bytes at offset %Ld.\n", 
	    should_be, offset);
	fprintf(stderr, "Only got %Ld bytes.\n",
		total_written);
	ret = -1;
	goto main_out;
    }

    /* print some statistics */
    printf("********************************************************\n");
    printf("PVFS2 Hindexed Test Write Statistics:\n");
    printf("********************************************************\n");
    printf("Bytes written: %Ld\n", Ld(total_written));
    printf("Elapsed time: %f seconds\n", (time2-time1));
    printf("Bandwidth: %f MB/second\n",
	(((double)total_written)/((double)(1024*1024))/(time2-time1)));
    printf("********************************************************\n\n");

    ret = PVFS_sys_getattr(ref, PVFS_ATTR_SYS_ALL,
	    &credentials, &resp_getattr);
    if (ret < 0)
    {
	PVFS_perror("Getattr failed", ret);
	ret = -1;
	goto main_out;
    }
    printf("Measured file size is %Ld\n", resp_getattr.attr.size);

    /* try to reason the size of the file */
    if (resp_getattr.attr.size != (offset + frame.cf_valid_start[frame.cf_valid_count - 1]
		+ frame.cf_valid_size[frame.cf_valid_count - 1]))
    {
	printf("file size %Ld is not equal to (%Ld + %Ld + %d)\n",
		resp_getattr.attr.size, offset, frame.cf_valid_start[frame.cf_valid_count - 1], 
		frame.cf_valid_size[frame.cf_valid_count - 1]);
	ret = -1;
	goto main_out;
    }

    /* now read it back from the file and make sure we have the correct data */
    time1 = Wtime();
    printf("Reading back %Ld contiguous bytes for verification\n", resp_getattr.attr.size - offset);
    assert(resp_getattr.attr.size - offset <= MAX_BUF_SIZE);
    ret = PVFS_Request_contiguous(resp_getattr.attr.size - offset, PVFS_BYTE, &mem_req);
    if(ret < 0)
    {
	PVFS_perror("PVFS_Request_contiguous", ret);
	ret = -1;
	goto main_out;
    }
    ret = PVFS_Request_commit(&mem_req);
    if (ret < 0)
    {
	PVFS_perror("PVFS_Request_commit", ret);
	ret = -1;
	goto main_out;
    }
    file_req = PVFS_BYTE;
    total_read = 0;
    /* read back the data */
    ret = PVFS_sys_read(ref, file_req, offset, 
	    frame.rd_buffer, mem_req, &credentials,
	    &resp_io);
    if(ret < 0)
    {
	PVFS_perror("PVFS_sys_read", ret);
	ret = -1;
	goto main_out;
    }
    total_read = resp_io.total_completed;
    /* need to free the request descriptions */
    PVFS_Request_free(&mem_req);

    /* sanity check */
    if(resp_getattr.attr.size - offset != total_read)
    {
	fprintf(stderr, "Error: short reads!\n");
	fprintf(stderr, "Tried to read %Ld bytes at offset %Ld.\n", 
	   resp_getattr.attr.size - offset, offset);
	fprintf(stderr, "Only got %Ld bytes.\n",
	   total_read);
	ret = -1;
	goto main_out;
    }
    time2 = Wtime();

    /* print some statistics */
    printf("\n********************************************************\n");
    printf("PVFS2 Hindexed Test Read Statistics:\n");
    printf("********************************************************\n");
    printf("Bytes read: %Ld\n", Ld(total_read));
    printf("Elapsed time: %f seconds\n", (time2-time1));
    printf("Bandwidth: %f MB/second\n",
	(((double)total_read)/((double)(1024*1024))/(time2-time1)));
    printf("********************************************************\n");


    ret = 0;
    wptr = frame.wr_buffer;
    /* now compare the relevant portions */
    for (c = 0; c < frame.cf_valid_count; c++)
    {
	if (memcmp(frame.rd_buffer + frame.cf_valid_start[c],
		    wptr,
		    frame.cf_valid_size[c]))
	{
	    fprintf(stderr, "(%d) -> Read buffer did not match with write buffer from [%Ld upto %d bytes]\n",
		    c, frame.cf_valid_start[c], frame.cf_valid_size[c]);
	    ret = -1;
	}
	wptr += frame.cf_valid_size[c];
    }
    if (ret == 0)
    {
	fprintf(stdout, "Test passed!\n");
    }
    else
    {
	fprintf(stdout, "Test failed!\n");
    }

main_out:

    PVFS_sys_finalize();

    if(frame.rd_buffer)
	free(frame.rd_buffer);
    if(frame.wr_buffer)
	free(frame.wr_buffer);
    if(frame.cf_valid_start)
	free(frame.cf_valid_start);
    if(frame.cf_valid_size)
	free(frame.cf_valid_size);

    return(ret);
}

static void usage(int argc, char** argv)
{
    fprintf(stderr, "Usage: %s -f [filename]\n", argv[0]);
    fprintf(stderr, "	       -b [max. buffer size] default %d bytes.\n", MAX_BUF_SIZE);
    fprintf(stderr, "	       -o [max. file offset] default %d bytes.\n", MAX_OFFSET);
    fprintf(stderr, "	       -r [max. valid regions] default %d regions\n", MAX_REGIONS);
    return;
}

static double Wtime(void)
{
    struct timeval t;
    gettimeofday(&t, NULL);
    return((double)t.tv_sec + (double)(t.tv_usec) / 1000000);
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 noexpandtab
 */

-------------- next part --------------
/*
 * (C) 2001 Clemson University and The University of Chicago
 *
 * See COPYING in top-level directory.
 */

#include <unistd.h>
#include <assert.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/time.h>
#include <time.h>
#include <stdlib.h>

#include "pvfs2.h"
#include "str-utils.h"

#ifndef PVFS_REQ_LIMIT_PINT_REQUEST_NUM
#define PVFS_REQ_LIMIT_PINT_REQUEST_NUM 100
#endif

#ifndef PVFS2_VERSION
#define PVFS2_VERSION "Unknown"
#endif

#ifndef FNAME
#define FNAME "/mnt/pvfs2/testfile"
#endif

#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif

#ifndef MAX_REGIONS
#define MAX_REGIONS 2
#endif

#ifndef MAX_REGIONS
#define MAX_REGIONS 2
#endif

#ifndef MAX_OFFSET
#define MAX_OFFSET 32768
#endif

#ifndef MAX_BUF_SIZE
#define MAX_BUF_SIZE 32768
#endif

static int max_regions = MAX_REGIONS;
static int max_offset  = MAX_OFFSET;

static void usage(int argc, char** argv);
static double Wtime(void);
typedef int32_t cm_count_t;
typedef int64_t cm_offset_t;
typedef int32_t cm_size_t;

typedef struct {
    cm_count_t	 cf_valid_count;
    cm_offset_t *cf_valid_start;
    cm_size_t   *cf_valid_size;
    void	*wr_buffer;
    void	*rd_buffer;
    int		 buf_size;
} cm_frame_t;

static int fillup_buffer(cm_frame_t *frame)
{
    int c, size = sizeof(double), should_be = 0, chunk_size = 0;

    srand(time(NULL));
    if (frame->buf_size < 0 || frame->buf_size % size != 0)
    {
	fprintf(stderr, "buffer size [%d] must be a multiple of %d\n",
		frame->buf_size, size);
	return -1;
    }
    frame->wr_buffer = (char *) calloc(frame->buf_size, sizeof(char));
    assert(frame->wr_buffer);
    frame->rd_buffer = (char *) calloc(frame->buf_size, sizeof(char));
    assert(frame->rd_buffer);

    for (c = 0; c < frame->buf_size / size; c++)
    {
	*((int *) frame->wr_buffer + c) = c;
    }
    frame->cf_valid_count = (rand() % max_regions) + 1;
    frame->cf_valid_start = (cm_offset_t *) calloc(sizeof(cm_offset_t), frame->cf_valid_count);
    frame->cf_valid_size = (cm_size_t *) calloc(sizeof(cm_size_t), frame->cf_valid_count);
    assert(frame->cf_valid_start && frame->cf_valid_size);
    chunk_size = frame->buf_size / frame->cf_valid_count;

    printf("Buffer size is %d bytes\n", frame->buf_size);
    printf("Generating %d valid regions\n", frame->cf_valid_count);
    for (c = 0; c < frame->cf_valid_count; c++)
    {
	int tmp_start;

	tmp_start = rand() % chunk_size;
	frame->cf_valid_start[c] = c * chunk_size + tmp_start;
	frame->cf_valid_size[c] = (rand() % (chunk_size - tmp_start)) + 1;
	assert(frame->cf_valid_start[c] + frame->cf_valid_size[c] <= frame->buf_size);
	
	printf("(%d): valid_start: %Ld, valid_size: %d\n", c, frame->cf_valid_start[c],
		frame->cf_valid_size[c]);
	
	should_be += frame->cf_valid_size[c];
    }
    printf("PVFS_sys_write should write %d bytes\n", should_be);
    return should_be;
}

int find_size(cm_frame_t *frame, int count)
{
    int i, tot = 0;

    assert(count >= 0 && count < frame->cf_valid_count);
    for (i = count; i < MIN(count + 100, frame->cf_valid_count); i++)
    {
	tot += frame->cf_valid_size[i];
    }
    return tot;
}

int main(int argc, char **argv)
{
    int ret = -1;
    void *rptr = NULL;
    loff_t offset, tmp_offset;
    char str_buf[PVFS_NAME_MAX] = {0}, *fname = NULL;
    char pvfs_path[PVFS_NAME_MAX] = {0};
    PVFS_fs_id cur_fs;
    PVFS_sysresp_lookup resp_lookup;
    PVFS_sysresp_create resp_create;
    PVFS_sysresp_io resp_io;
    PVFS_sysresp_getattr resp_getattr;
    int64_t total_written = 0, total_read = 0;
    double time1, time2;
    char* entry_name;
    PVFS_object_ref parent_ref;
    PVFS_sys_attr attr;
    PVFS_credentials credentials;
    PVFS_object_ref ref;
    PVFS_Request file_req;
    PVFS_Request mem_req;
    int c, count = 0, should_be = 0;
    cm_frame_t    frame;

    frame.cf_valid_count = 0;
    frame.cf_valid_start = NULL;
    frame.cf_valid_size = NULL;
    frame.wr_buffer   = NULL;
    frame.rd_buffer = NULL;
    frame.buf_size = 32768;

    while ((c = getopt(argc, argv, "o:r:b:f:h")) != EOF)
    {
	switch (c)
	{
	    case 'o':
		max_offset = atoi(optarg);
		break;
	    case 'r':
		max_regions = atoi(optarg);
		break;
	    case 'f':
		fname = optarg;
		break;
	    case 'b':
		frame.buf_size = atoi(optarg);
		break;
	    case 'h':
	    default:
		usage(argc, argv);
		exit(1);
	}
    }

    if (fname == NULL)
    {
	fname = FNAME;
    }
    if ((should_be = fillup_buffer(&frame)) < 0)
    {
	usage(argc, argv);
	exit(1);
    }
    offset = rand() % max_offset;

    ret = PVFS_util_init_defaults();
    if(ret < 0)
    {
	PVFS_perror("PVFS_util_init_defaults", ret);
	return(-1);
    }

    /* translate local path into pvfs2 relative path */
    ret = PVFS_util_resolve(fname,
	&cur_fs, pvfs_path, PVFS_NAME_MAX);
    if(ret < 0)
    {
	PVFS_perror("PVFS_util_resolve", ret);
	ret = -1;
	goto main_out;
    }
    PVFS_util_gen_credentials(&credentials);

    entry_name = str_buf;
    attr.owner = credentials.uid; 
    attr.group = credentials.gid;
    attr.perms = PVFS_U_WRITE|PVFS_U_READ;
    attr.atime = time(NULL);
    attr.mtime = attr.atime;
    attr.ctime = attr.atime;
    attr.mask = (PVFS_ATTR_SYS_ALL_SETABLE);
    attr.dfile_count = -1;

    if (strcmp(pvfs_path,"/") == 0)
    {
        char *segp = NULL, *prev_segp = NULL;
        void *segstate = NULL;

        memset(&resp_lookup, 0, sizeof(PVFS_sysresp_lookup));
        ret = PVFS_sys_lookup(cur_fs, pvfs_path,
                              &credentials, &resp_lookup,
                              PVFS2_LOOKUP_LINK_FOLLOW);
        if (ret < 0)
        {
            PVFS_perror("PVFS_sys_lookup", ret);
            ret = -1;
            goto main_out;
        }
        parent_ref.handle = resp_lookup.ref.handle;
        parent_ref.fs_id = resp_lookup.ref.fs_id;

        while(!PINT_string_next_segment(
                  fname, &segp, &segstate))
        {
            prev_segp = segp;
        }
        entry_name = prev_segp;
    }
    else
    {
        /* get the absolute path on the pvfs2 file system */
#ifdef OLD_PVFS2
        if (PVFS_util_remove_base_dir(pvfs_path, str_buf, PVFS_NAME_MAX))
#else
        if (PINT_remove_base_dir(pvfs_path, str_buf, PVFS_NAME_MAX))
#endif
        {
            if (pvfs_path[0] != '/')
            {
                fprintf(stderr, "Error: poorly formatted path.\n");
            }
            fprintf(stderr, "Error: cannot retrieve entry name for "
                    "creation on %s\n", pvfs_path);
            ret = -1;
            goto main_out;
        }

#ifdef OLD_PVFS2
        ret = PVFS_util_lookup_parent(pvfs_path, cur_fs, &credentials, &parent_ref.handle);
#else
        ret = PINT_lookup_parent(pvfs_path, cur_fs, &credentials, &parent_ref.handle);
#endif
	
        if(ret < 0)
        {
            PVFS_perror("PVFS_util_lookup_parent", ret);
            ret = -1;
            goto main_out;
        }
        else
        {
            int len = strlen(pvfs_path);
            if (pvfs_path[len - 1] == '/')
            {
                char *segp = NULL, *prev_segp = NULL;
                void *segstate = NULL;

                while(!PINT_string_next_segment(
                          fname, &segp, &segstate))
                {
                    prev_segp = segp;
                }
                strncat(pvfs_path, prev_segp, PVFS_NAME_MAX);
                entry_name = prev_segp;
            }
            parent_ref.fs_id = cur_fs;
        }
    }

    memset(&resp_lookup, 0, sizeof(PVFS_sysresp_lookup));
    ret = PVFS_sys_ref_lookup(parent_ref.fs_id, entry_name,
                              parent_ref, &credentials, &resp_lookup,
                              PVFS2_LOOKUP_LINK_NO_FOLLOW);
    if (ret == 0)
    {
        fprintf(stderr, "Target file %s already exists! Deleting.\n", entry_name);
	if (PVFS_sys_remove(entry_name,
		    parent_ref, &credentials) < 0)
	{
	    fprintf(stderr, "Could not unlink?\n");
	}
	else
	{
	    fprintf(stdout, "Unlinked successfully!\n");
	}
        ret = 0;
    }

    memset(&resp_create, 0, sizeof(PVFS_sysresp_create));
    ret = PVFS_sys_create(entry_name, parent_ref, attr,
                          &credentials, NULL, &resp_create);
    if (ret < 0)
    {
	PVFS_perror("PVFS_sys_create", ret);
	ret = -1;
	goto main_out;
    }

    ref = resp_create.ref;
    c = frame.cf_valid_count;
    count = 0;
    total_written = 0;
    printf("Writing at offset %Ld\n", offset);
    time1 = Wtime();
    tmp_offset = offset;
    while (c > 0)
    {
	int tmp_valid_count;
	
	tmp_valid_count = MIN(c, PVFS_REQ_LIMIT_PINT_REQUEST_NUM);
	ret = PVFS_Request_hindexed(tmp_valid_count, &frame.cf_valid_size[count],
		&frame.cf_valid_start[count], PVFS_BYTE, &mem_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_hindexed", ret);
	    ret = -1;
	    goto main_out;
	}
	ret = PVFS_Request_commit(&mem_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_commit", ret);
	    ret = -1;
	    goto main_out;
	}
	printf("file_req contiguous size is %d\n", find_size(&frame, count));
	ret = PVFS_Request_contiguous(find_size(&frame, count), PVFS_BYTE, &file_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_contiguous", ret);
	    ret = -1;
	    goto main_out;
	}
	ret = PVFS_Request_commit(&file_req);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_Request_commit", ret);
	    ret = -1;
	    goto main_out;
	}
	/* write out the data */
	ret = PVFS_sys_write(ref, file_req,
                             tmp_offset, frame.wr_buffer, mem_req, 
                             &credentials, &resp_io);
	if(ret < 0)
	{
	    PVFS_perror("PVFS_sys_write", ret);
	    ret = -1;
	    goto main_out;
	}
	total_written += resp_io.total_completed;
	assert(resp_io.total_completed == find_size(&frame, count));
	tmp_offset += resp_io.total_completed;

	/* need to free the request descriptions */
	PVFS_Request_free(&mem_req);
	PVFS_Request_free(&file_req);
	
	c -= tmp_valid_count;
	count += tmp_valid_count;
    }
    time2 = Wtime();
    /* sanity check */
    if(should_be != total_written)
    {
	fprintf(stderr, "Error: short write!\n");
	fprintf(stderr, "Tried to write %d bytes at offset %Ld.\n", 
	    should_be, offset);
	fprintf(stderr, "Only got %Ld bytes.\n",
		total_written);
	ret = -1;
	goto main_out;
    }

    /* print some statistics */
    printf("********************************************************\n");
    printf("PVFS2 Hindexed Test Write Statistics:\n");
    printf("********************************************************\n");
    printf("Bytes written: %Ld\n", Ld(total_written));
    printf("Elapsed time: %f seconds\n", (time2-time1));
    printf("Bandwidth: %f MB/second\n",
	(((double)total_written)/((double)(1024*1024))/(time2-time1)));
    printf("********************************************************\n\n");

    ret = PVFS_sys_getattr(ref, PVFS_ATTR_SYS_ALL,
	    &credentials, &resp_getattr);
    if (ret < 0)
    {
	PVFS_perror("Getattr failed", ret);
	ret = -1;
	goto main_out;
    }
    printf("Measured file size is %Ld\n", resp_getattr.attr.size);

    /* try to reason the size of the file */
    if (resp_getattr.attr.size != (offset + should_be))
    {
	printf("file size %Ld is not equal to (%Ld + %d)\n",
		resp_getattr.attr.size, offset, should_be);
	ret = -1;
	goto main_out;
    }

    /* now read it back from the file and make sure we have the correct data */
    time1 = Wtime();
    ret = PVFS_Request_contiguous(resp_getattr.attr.size - offset, PVFS_BYTE, &mem_req);
    if(ret < 0)
    {
	PVFS_perror("PVFS_Request_contiguous", ret);
	ret = -1;
	goto main_out;
    }
    ret = PVFS_Request_commit(&mem_req);
    if (ret < 0)
    {
	PVFS_perror("PVFS_Request_commit", ret);
	ret = -1;
	goto main_out;
    }
    file_req = PVFS_BYTE;
    total_read = 0;
    /* read back the data */
    ret = PVFS_sys_read(ref, file_req, offset, 
	    frame.rd_buffer, mem_req, &credentials,
	    &resp_io);
    if(ret < 0)
    {
	PVFS_perror("PVFS_sys_read", ret);
	ret = -1;
	goto main_out;
    }
    total_read = resp_io.total_completed;
    /* need to free the request descriptions */
    PVFS_Request_free(&mem_req);

    /* sanity check */
    if(resp_getattr.attr.size - offset != total_read)
    {
	fprintf(stderr, "Error: short reads!\n");
	fprintf(stderr, "Tried to read %Ld bytes at offset %Ld.\n", 
	   resp_getattr.attr.size - offset, offset);
	fprintf(stderr, "Only got %Ld bytes.\n",
	   total_read);
	ret = -1;
	goto main_out;
    }
    time2 = Wtime();

    /* print some statistics */
    printf("\n********************************************************\n");
    printf("PVFS2 Hindexed Test Read Statistics:\n");
    printf("********************************************************\n");
    printf("Bytes read: %Ld\n", Ld(total_read));
    printf("Elapsed time: %f seconds\n", (time2-time1));
    printf("Bandwidth: %f MB/second\n",
	(((double)total_read)/((double)(1024*1024))/(time2-time1)));
    printf("********************************************************\n");


    ret = 0;
    rptr = frame.rd_buffer;
    /* now compare the relevant portions */
    for (c = 0; c < frame.cf_valid_count; c++)
    {
	if (memcmp(rptr,
		    frame.wr_buffer + frame.cf_valid_start[c],
		    frame.cf_valid_size[c]))
	{
	    fprintf(stderr, "(%d) -> Read buffer did not match with write buffer from [%Ld upto %d bytes]\n",
		    c, frame.cf_valid_start[c], frame.cf_valid_size[c]);
	    ret = -1;
	}
	rptr += frame.cf_valid_size[c];
    }
    if (ret == 0)
    {
	fprintf(stdout, "Test passed!\n");
    }
    else
    {
	fprintf(stdout, "Test failed!\n");
    }

main_out:

    PVFS_sys_finalize();

    if(frame.rd_buffer)
	free(frame.rd_buffer);
    if(frame.wr_buffer)
	free(frame.wr_buffer);
    if(frame.cf_valid_start)
	free(frame.cf_valid_start);
    if(frame.cf_valid_size)
	free(frame.cf_valid_size);

    return(ret);
}

static void usage(int argc, char** argv)
{
    fprintf(stderr, "Usage: %s -f [filename]\n", argv[0]);
    fprintf(stderr, "	       -b [max. buffer size] default %d bytes.\n", MAX_BUF_SIZE);
    fprintf(stderr, "	       -o [max. file offset] default %d bytes.\n", MAX_OFFSET);
    fprintf(stderr, "	       -r [max. valid regions] default %d regions\n", MAX_REGIONS);
    return;
}

static double Wtime(void)
{
    struct timeval t;
    gettimeofday(&t, NULL);
    return((double)t.tv_sec + (double)(t.tv_usec) / 1000000);
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 noexpandtab
 */



More information about the PVFS2-developers mailing list