[Pvfs2-cvs] commit by slang in pvfs2/src/kernel/linux-2.6:
devpvfs2-req.c file.c pvfs2-bufmap.c pvfs2-bufmap.h super.c
CVS commit program
cvs at parl.clemson.edu
Mon Dec 4 01:18:25 EST 2006
Update of /projects/cvsroot/pvfs2/src/kernel/linux-2.6
In directory parlweb1:/tmp/cvs-serv2372/src/kernel/linux-2.6
Modified Files:
devpvfs2-req.c file.c pvfs2-bufmap.c pvfs2-bufmap.h super.c
Log Message:
merge of murali's kernel buffer size tuning options to HEAD.
Index: devpvfs2-req.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/kernel/linux-2.6/devpvfs2-req.c,v
diff -p -u -r1.67 -r1.68
--- devpvfs2-req.c 3 Dec 2006 03:33:20 -0000 1.67
+++ devpvfs2-req.c 4 Dec 2006 06:18:25 -0000 1.68
@@ -787,7 +787,9 @@ static int pvfs2_devreq_ioctl(
struct PVFS_dev_map_desc32
{
compat_uptr_t ptr;
+ int32_t total_size;
int32_t size;
+ int32_t count;
};
#ifndef PVFS2_LINUX_KERNEL_2_4
@@ -806,9 +808,13 @@ static unsigned long translate_dev_map26
/* try to put that into a 64-bit layout */
if (put_user(compat_ptr(addr), &p->ptr))
goto err;
- /* copy the remaining field */
+ /* copy the remaining fields */
+ if (copy_in_user(&p->total_size, &p32->total_size, sizeof(int32_t)))
+ goto err;
if (copy_in_user(&p->size, &p32->size, sizeof(int32_t)))
goto err;
+ if (copy_in_user(&p->count, &p32->count, sizeof(int32_t)))
+ goto err;
return (unsigned long) p;
err:
*error = -EFAULT;
@@ -819,17 +825,23 @@ static unsigned long translate_dev_map24
unsigned long args, struct PVFS_dev_map_desc *p, long *error)
{
struct PVFS_dev_map_desc32 __user *p32 = (void __user *) args;
- u32 addr, size;
+ u32 addr, size, total_size, count;
*error = 0;
/* get the ptr from the 32 bit user-space */
if (get_user(addr, &p32->ptr))
goto err;
p->ptr = compat_ptr(addr);
- /* copy the size */
+ /* copy the remaining fields */
+ if (get_user(total_size, &p32->total_size))
+ goto err;
if (get_user(size, &p32->size))
goto err;
+ if (get_user(count, &p32->count))
+ goto err;
+ p->total_size = total_size;
p->size = size;
+ p->count = count;
return 0;
err:
*error = -EFAULT;
Index: file.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/kernel/linux-2.6/file.c,v
diff -p -u -r1.133 -r1.134
--- file.c 3 Dec 2006 03:33:20 -0000 1.133
+++ file.c 4 Dec 2006 06:18:25 -0000 1.134
@@ -311,7 +311,9 @@ out:
/*
* The reason we need to do this is to be able to support
* readv and writev that are
- * larger than PVFS_DEFAULT_DESC_SIZE (4 MB). What that means is that
+ * larger than (pvfs_bufmap_size_query())
+ * Default is PVFS2_BUFMAP_DEFAULT_DESC_SIZE MB.
+ * What that means is that
* we will create a new io vec descriptor for those memory addresses that
* go beyond the limit
* Return value for this routine is -ve in case of errors
@@ -319,7 +321,7 @@ out:
* Further, the new_nr_segs pointer is updated to hold the new value
* of number of iovecs, the new_vec pointer is updated to hold the pointer
* to the new split iovec, and the size array is an array of integers holding
- * the number of iovecs that straddle PVFS_DEFAULT_DESC_SIZE.
+ * the number of iovecs that straddle pvfs_bufmap_size_query().
* The max_new_nr_segs value is computed by the caller and returned.
* (It will be (count of all iov_len/ block_size) + 1).
*/
@@ -870,7 +872,9 @@ static int construct_file_offset_trailer
/*
* The reason we need to do this is to be able to support readx() and writex()
- * of larger than PVFS_DEFAULT_DESC_SIZE (4 MB). What that means is that
+ * of larger than (pvfs_bufmap_size_query())
+ * (default is PVFS2_BUFMAP_DEFAULT_DESC_SIZE MB).
+ * What that means is that
* we will create a new xtvec descriptor for those file offsets that
* go beyond the limit
* Return value for this routine is -ve in case of errors
@@ -878,7 +882,7 @@ static int construct_file_offset_trailer
* Further, the new_nr_segs pointer is updated to hold the new value
* of number of xtvecs, the new_xtvec pointer is updated to hold the pointer
* to the new split xtvec, and the size array is an array of integers holding
- * the number of xtvecs that straddle PVFS_DEFAULT_DESC_SIZE.
+ * the number of xtvecs that straddle (pvfs_bufmap_size_query()).
* The max_new_nr_segs value is computed by the caller and passed in.
* (It will be (count of all xtv_len/ block_size) + 1).
*/
Index: pvfs2-bufmap.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/kernel/linux-2.6/pvfs2-bufmap.c,v
diff -p -u -r1.47 -r1.48
--- pvfs2-bufmap.c 1 Dec 2006 09:53:34 -0000 1.47
+++ pvfs2-bufmap.c 4 Dec 2006 06:18:25 -0000 1.48
@@ -7,23 +7,93 @@
#include "pvfs2-bufmap.h"
#include "pint-dev-shared.h"
-#define BUFMAP_PAGE_COUNT (PVFS2_BUFMAP_TOTAL_SIZE/PAGE_SIZE)
-#define PAGES_PER_DESC (PVFS2_BUFMAP_DEFAULT_DESC_SIZE/PAGE_SIZE)
+
+static int bufmap_page_count, pages_per_desc;
+
+static int32_t pvfs2_bufmap_desc_size, pvfs2_bufmap_desc_shift,
+ pvfs2_bufmap_desc_count, pvfs2_bufmap_total_size;
+
+inline int pvfs_bufmap_size_query(void)
+{
+ return pvfs2_bufmap_desc_size;
+}
+
+inline int pvfs_bufmap_shift_query(void)
+{
+ return pvfs2_bufmap_desc_shift;
+}
static int bufmap_init = 0;
static struct page **bufmap_page_array = NULL;
+
/* array to track usage of buffer descriptors */
-static int buffer_index_array[PVFS2_BUFMAP_DESC_COUNT] = {0};
+static int *buffer_index_array = NULL;
static spinlock_t buffer_index_lock = SPIN_LOCK_UNLOCKED;
/* array to track usage of buffer descriptors for readdir/readdirplus */
-int readdir_index_array[PVFS2_READDIR_DESC_COUNT] = {0};
+static int readdir_index_array[PVFS2_READDIR_DEFAULT_DESC_COUNT] = {0};
static spinlock_t readdir_index_lock = SPIN_LOCK_UNLOCKED;
-static struct pvfs_bufmap_desc desc_array[PVFS2_BUFMAP_DESC_COUNT];
+static struct pvfs_bufmap_desc *desc_array = NULL;
+
static DECLARE_WAIT_QUEUE_HEAD(bufmap_waitq);
static DECLARE_WAIT_QUEUE_HEAD(readdir_waitq);
+static int initialize_bufmap_descriptors(int ndescs)
+{
+ int err;
+
+ err = -EINVAL;
+ if (ndescs < 0)
+ {
+ gossip_err("pvfs2: ndescs (%d) cannot be < 0.\n", ndescs);
+ goto out;
+ }
+ err = -ENOMEM;
+ buffer_index_array = (int *) kmalloc(ndescs * sizeof(int),
+ PVFS2_BUFMAP_GFP_FLAGS);
+ if (buffer_index_array == NULL)
+ {
+ gossip_err("pvfs2: could not allocate %d bytes\n",
+ ndescs * sizeof(int));
+ goto out;
+ }
+ memset(buffer_index_array, 0, ndescs * sizeof(int));
+
+ desc_array = (struct pvfs_bufmap_desc *)
+ kmalloc(ndescs * sizeof(struct pvfs_bufmap_desc),
+ PVFS2_BUFMAP_GFP_FLAGS);
+ if (desc_array == NULL)
+ {
+ gossip_err("pvfs2: could not allocate %d bytes\n",
+ ndescs * sizeof(struct pvfs_bufmap_desc));
+ goto out1;
+ }
+ err = 0;
+ goto out;
+
+out1:
+ kfree(buffer_index_array);
+ buffer_index_array = NULL;
+out:
+ return err;
+}
+
+static void finalize_bufmap_descriptors(void)
+{
+ if (buffer_index_array != NULL)
+ {
+ kfree(buffer_index_array);
+ buffer_index_array = NULL;
+ }
+ if (desc_array != NULL)
+ {
+ kfree(desc_array);
+ desc_array = NULL;
+ }
+ return;
+}
+
/* pvfs_bufmap_initialize()
*
* initializes the mapped buffer interface
@@ -36,7 +106,9 @@ int pvfs_bufmap_initialize(struct PVFS_d
int i = 0;
int offset = 0;
- gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2_bufmap_initialize: called\n");
+ gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2_bufmap_initialize: called "
+ "(ptr (%p) sz (%d) cnt(%d).\n",
+ user_desc->ptr, user_desc->size, user_desc->count);
if (bufmap_init == 1)
{
@@ -51,34 +123,50 @@ int pvfs_bufmap_initialize(struct PVFS_d
if (PAGE_ALIGN((unsigned long)user_desc->ptr) !=
(unsigned long)user_desc->ptr)
{
- gossip_err("pvfs2 error: memory alignment (front).\n");
+ gossip_err("pvfs2 error: memory alignment (front). %p\n",
+ user_desc->ptr);
goto init_failure;
}
- if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->size)) !=
- (unsigned long)(user_desc->ptr + user_desc->size))
+ if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->total_size)) !=
+ (unsigned long)(user_desc->ptr + user_desc->total_size))
{
- gossip_err("pvfs2 error: memory alignment (back).\n");
+ gossip_err("pvfs2 error: memory alignment (back).(%p + %d)\n",
+ user_desc->ptr, user_desc->total_size);
goto init_failure;
}
- if (user_desc->size != PVFS2_BUFMAP_TOTAL_SIZE)
+ if (user_desc->total_size != (user_desc->size * user_desc->count))
{
gossip_err("pvfs2 error: user provided an oddly "
- "sized buffer...\n");
+ "sized buffer...(%d, %d, %d)\n",
+ user_desc->total_size, user_desc->size, user_desc->count);
goto init_failure;
}
- if ((PVFS2_BUFMAP_DEFAULT_DESC_SIZE % PAGE_SIZE) != 0)
+ if ((user_desc->size % PAGE_SIZE) != 0)
{
gossip_err("pvfs2 error: bufmap size not page size "
- "divisible.\n");
+ "divisible (%d).\n", user_desc->size);
+ goto init_failure;
+ }
+ /* Initialize critical variables */
+ pvfs2_bufmap_total_size = user_desc->total_size;
+ pvfs2_bufmap_desc_count = user_desc->count;
+ pvfs2_bufmap_desc_size = user_desc->size;
+ pvfs2_bufmap_desc_shift = LOG2(pvfs2_bufmap_desc_size);
+ bufmap_page_count = pvfs2_bufmap_total_size / PAGE_SIZE;
+ pages_per_desc = pvfs2_bufmap_desc_size / PAGE_SIZE;
+ /* Initialize descriptor arrays */
+ if ((ret = initialize_bufmap_descriptors(pvfs2_bufmap_desc_count)) < 0)
+ {
goto init_failure;
}
/* allocate storage to track our page mappings */
- bufmap_page_array = (struct page **)kmalloc(
- BUFMAP_PAGE_COUNT*sizeof(struct page *), PVFS2_BUFMAP_GFP_FLAGS);
+ bufmap_page_array = (struct page **)
+ kmalloc(bufmap_page_count * sizeof(struct page *),
+ PVFS2_BUFMAP_GFP_FLAGS);
if (!bufmap_page_array)
{
ret = -ENOMEM;
@@ -90,7 +178,7 @@ int pvfs_bufmap_initialize(struct PVFS_d
ret = get_user_pages(
current, current->mm, (unsigned long)user_desc->ptr,
- BUFMAP_PAGE_COUNT, 1, 0, bufmap_page_array, NULL);
+ bufmap_page_count, 1, 0, bufmap_page_array, NULL);
up_read(¤t->mm->mmap_sem);
@@ -104,10 +192,10 @@ int pvfs_bufmap_initialize(struct PVFS_d
in theory we could run with what we got, but I will just treat
it as an error for simplicity's sake right now
*/
- if (ret != BUFMAP_PAGE_COUNT)
+ if (ret != bufmap_page_count)
{
gossip_err("pvfs2 error: asked for %d pages, only got %d.\n",
- (int)BUFMAP_PAGE_COUNT, ret);
+ (int) bufmap_page_count, ret);
for(i = 0; i < ret; i++)
{
@@ -129,37 +217,43 @@ int pvfs_bufmap_initialize(struct PVFS_d
it's worth. in 2.4.x, marking the pages does what's expected
and doesn't try to swap out our pages
*/
- for(i = 0; i < BUFMAP_PAGE_COUNT; i++)
+ for(i = 0; i < bufmap_page_count; i++)
{
flush_dcache_page(bufmap_page_array[i]);
pvfs2_set_page_reserved(bufmap_page_array[i]);
}
/* build a list of available descriptors */
- for(offset = 0, i = 0; i < PVFS2_BUFMAP_DESC_COUNT; i++)
+ for(offset = 0, i = 0; i < pvfs2_bufmap_desc_count; i++)
{
desc_array[i].page_array = &bufmap_page_array[offset];
- desc_array[i].array_count = PAGES_PER_DESC;
+ desc_array[i].array_count = pages_per_desc;
desc_array[i].uaddr =
- (user_desc->ptr + (i * PAGES_PER_DESC * PAGE_SIZE));
- offset += PAGES_PER_DESC;
+ (user_desc->ptr + (i * pages_per_desc * PAGE_SIZE));
+ offset += pages_per_desc;
}
/* clear any previously used buffer indices */
spin_lock(&buffer_index_lock);
- for(i = 0; i < PVFS2_BUFMAP_DESC_COUNT; i++)
+ for(i = 0; i < pvfs2_bufmap_desc_count; i++)
{
buffer_index_array[i] = 0;
}
spin_unlock(&buffer_index_lock);
+ spin_lock(&readdir_index_lock);
+ for (i = 0; i < PVFS2_READDIR_DEFAULT_DESC_COUNT; i++)
+ {
+ readdir_index_array[i] = 0;
+ }
+ spin_unlock(&readdir_index_lock);
bufmap_init = 1;
gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2_bufmap_initialize: exiting normally\n");
return 0;
- init_failure:
-
+init_failure:
+ finalize_bufmap_descriptors();
return ret;
}
@@ -183,7 +277,7 @@ void pvfs_bufmap_finalize(void)
return;
}
- for(i = 0; i < BUFMAP_PAGE_COUNT; i++)
+ for(i = 0; i < bufmap_page_count; i++)
{
pvfs2_clear_page_reserved(bufmap_page_array[i]);
page_cache_release(bufmap_page_array[i]);
@@ -192,6 +286,7 @@ void pvfs_bufmap_finalize(void)
bufmap_init = 0;
+ finalize_bufmap_descriptors();
gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2_bufmap_finalize: exiting normally\n");
}
@@ -281,7 +376,7 @@ int pvfs_bufmap_get(int *buffer_index)
{
struct slot_args slargs;
- slargs.slot_count = PVFS2_BUFMAP_DESC_COUNT;
+ slargs.slot_count = pvfs2_bufmap_desc_count;
slargs.slot_array = buffer_index_array;
slargs.slot_lock = &buffer_index_lock;
slargs.slot_wq = &bufmap_waitq;
@@ -298,7 +393,7 @@ void pvfs_bufmap_put(int buffer_index)
{
struct slot_args slargs;
- slargs.slot_count = PVFS2_BUFMAP_DESC_COUNT;
+ slargs.slot_count = pvfs2_bufmap_desc_count;
slargs.slot_array = buffer_index_array;
slargs.slot_lock = &buffer_index_lock;
slargs.slot_wq = &bufmap_waitq;
@@ -320,7 +415,7 @@ int readdir_index_get(int *buffer_index)
{
struct slot_args slargs;
- slargs.slot_count = PVFS2_READDIR_DESC_COUNT;
+ slargs.slot_count = PVFS2_READDIR_DEFAULT_DESC_COUNT;
slargs.slot_array = readdir_index_array;
slargs.slot_lock = &readdir_index_lock;
slargs.slot_wq = &readdir_waitq;
@@ -331,7 +426,7 @@ void readdir_index_put(int buffer_index)
{
struct slot_args slargs;
- slargs.slot_count = PVFS2_READDIR_DESC_COUNT;
+ slargs.slot_count = PVFS2_READDIR_DEFAULT_DESC_COUNT;
slargs.slot_array = readdir_index_array;
slargs.slot_lock = &readdir_index_lock;
slargs.slot_wq = &readdir_waitq;
Index: pvfs2-bufmap.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/kernel/linux-2.6/pvfs2-bufmap.h,v
diff -p -u -r1.16 -r1.17
--- pvfs2-bufmap.h 28 Sep 2006 05:13:41 -0000 1.16
+++ pvfs2-bufmap.h 4 Dec 2006 06:18:25 -0000 1.17
@@ -18,8 +18,11 @@ struct pvfs_bufmap_desc
struct list_head list_link;
};
-/* this would be a function call if the buffer sizes weren't hard coded */
-#define pvfs_bufmap_size_query() PVFS2_BUFMAP_DEFAULT_DESC_SIZE
+/* pvfs_bufmap_size_query is now an inline function because buffer
+ sizes are not hardcoded */
+int pvfs_bufmap_size_query(void);
+
+int pvfs_bufmap_shift_query(void);
int pvfs_bufmap_initialize(
struct PVFS_dev_map_desc *user_desc);
Index: super.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/kernel/linux-2.6/super.c,v
diff -p -u -r1.88 -r1.89
--- super.c 11 Nov 2006 18:49:22 -0000 1.88
+++ super.c 4 Dec 2006 06:18:25 -0000 1.89
@@ -5,6 +5,7 @@
*/
#include "pvfs2-kernel.h"
+#include "pvfs2-bufmap.h"
#include "pvfs2-internal.h"
/* list for storing pvfs2 specific superblocks in use */
@@ -949,8 +950,8 @@ struct super_block* pvfs2_get_sb(
sb->s_op = &pvfs2_s_ops;
sb->s_type = &pvfs2_fs_type;
- sb->s_blocksize = PVFS2_BUFMAP_DEFAULT_DESC_SIZE;
- sb->s_blocksize_bits = PVFS2_BUFMAP_DEFAULT_DESC_SHIFT;
+ sb->s_blocksize = pvfs2_bufmap_desc_size;
+ sb->s_blocksize_bits = pvfs2_bufmap_desc_shift;
sb->s_maxbytes = MAX_LFS_FILESIZE;
root_object.handle = PVFS2_SB(sb)->root_handle;
@@ -1070,8 +1071,8 @@ int pvfs2_fill_sb(
sb->s_op = &pvfs2_s_ops;
sb->s_type = &pvfs2_fs_type;
- sb->s_blocksize = PVFS2_BUFMAP_DEFAULT_DESC_SIZE;
- sb->s_blocksize_bits = PVFS2_BUFMAP_DEFAULT_DESC_SHIFT;
+ sb->s_blocksize = pvfs_bufmap_size_query();
+ sb->s_blocksize_bits = pvfs_bufmap_shift_query();
sb->s_maxbytes = MAX_LFS_FILESIZE;
root_object.handle = PVFS2_SB(sb)->root_handle;
More information about the Pvfs2-cvs
mailing list