[Pvfs2-cvs] commit by kunkel in pvfs2/src/io/dev: module.mk.in
pint-dev-shared.h pint-dev.c pint-dev.h
CVS commit program
cvs at parl.clemson.edu
Sat Feb 17 06:17:07 EST 2007
Update of /projects/cvsroot/pvfs2/src/io/dev
In directory parlweb1:/tmp/cvs-serv2872/src/io/dev
Modified Files:
Tag: kunkel-migration-branch
module.mk.in pint-dev-shared.h pint-dev.c pint-dev.h
Log Message:
Update migration branch to current CVS version
Index: module.mk.in
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/dev/module.mk.in,v
diff -p -u -r1.1 -r1.1.46.1
--- module.mk.in 20 Jun 2003 00:18:55 -0000 1.1
+++ module.mk.in 17 Feb 2007 11:17:07 -0000 1.1.46.1
@@ -3,3 +3,6 @@ LIBSRC += \
$(DIR)/pint-dev.c
SERVERSRC += \
$(DIR)/pint-dev.c
+
+MODCFLAGS_$(DIR)/pint-dev.c = \
+ -I$(srcdir)/src/kernel/linux-2.6
Index: pint-dev-shared.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/dev/pint-dev-shared.h,v
diff -p -u -r1.17 -r1.17.4.1
--- pint-dev-shared.h 10 Aug 2006 19:02:23 -0000 1.17
+++ pint-dev-shared.h 17 Feb 2007 11:17:07 -0000 1.17.4.1
@@ -11,6 +11,7 @@
#ifndef __PINT_DEV_SHARED_H
#define __PINT_DEV_SHARED_H
+
#ifdef __KERNEL__
#include <linux/ioctl.h>
#else
@@ -23,11 +24,11 @@
#define PVFS_KERNEL_PROTO_VERSION ((PVFS2_VERSION_MAJOR * 10000) + \
(PVFS2_VERSION_MINOR * 100) + PVFS2_VERSION_SUB)
-/* This is the number of discrete buffers we will break the mapped I/O
+/* This is the default number of discrete buffers we will break the mapped I/O
* region into. In some sense it governs the number of concurrent I/O
* operations that we will allow
*/
-#define PVFS2_BUFMAP_DESC_COUNT 5
+#define PVFS2_BUFMAP_DEFAULT_DESC_COUNT 5
/*
by default, we assume each description size is 4MB; this value
@@ -43,8 +44,32 @@
#define PVFS2_BUFMAP_DEFAULT_DESC_SHIFT 22 /* NOTE: 2^22 == 4MB */
/* size of mapped buffer region to use for I/O transfers (in bytes) */
-#define PVFS2_BUFMAP_TOTAL_SIZE \
-(PVFS2_BUFMAP_DESC_COUNT * PVFS2_BUFMAP_DEFAULT_DESC_SIZE)
+#define PVFS2_BUFMAP_DEFAULT_TOTAL_SIZE \
+(PVFS2_BUFMAP_DEFAULT_DESC_COUNT * PVFS2_BUFMAP_DEFAULT_DESC_SIZE)
+
+/* Sane maximum values for these parameters (128 MB) */
+#define PVFS2_BUFMAP_MAX_TOTAL_SIZE (128ULL * (1024 * 1024))
+
+/* log to base 2 when we know that number is a power of 2 */
+static inline int LOG2(int number)
+{
+ int count = 0;
+ if (number == 0 || (number & (number - 1)))
+ {
+ return -1;
+ }
+ while (number >>= 1)
+ {
+ count++;
+ }
+ return count;
+}
+
+#define PVFS2_READDIR_DEFAULT_DESC_COUNT 5
+#define PVFS2_READDIR_DEFAULT_DESC_SIZE (128 * 1024)
+#define PVFS2_READDIR_DEFAULT_DESC_SHIFT 17
+#define PVFS2_READDIR_DEFAULT_TOTAL_SIZE \
+(PVFS2_READDIR_DEFAULT_DESC_COUNT * PVFS2_READDIR_DEFAULT_DESC_SIZE)
/* pvfs2-client-core can cache readahead data up to this size in bytes */
#define PVFS2_MMAP_RACACHE_MAX_SIZE ((loff_t)(8 * (1024 * 1024)))
@@ -59,7 +84,9 @@
struct PVFS_dev_map_desc
{
void *ptr;
- int32_t size; /* Changed to an int32_t for fixed size structure */
+ int32_t total_size;
+ int32_t size;
+ int32_t count;
};
#define PVFS_DEV_MAGIC 'k'
Index: pint-dev.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/dev/pint-dev.c,v
diff -p -u -r1.30 -r1.30.4.1
--- pint-dev.c 10 Aug 2006 19:02:23 -0000 1.30
+++ pint-dev.c 17 Feb 2007 11:17:07 -0000 1.30.4.1
@@ -26,6 +26,8 @@
#include "pvfs2-debug.h"
#include "gossip.h"
#include "pint-dev.h"
+#include "pvfs2-dev-proto.h"
+#include "pvfs2-internal.h"
static int setup_dev_entry(
const char *dev_name);
@@ -40,6 +42,9 @@ static int32_t pdev_magic;
static int32_t pdev_max_upsize;
static int32_t pdev_max_downsize;
+int32_t pvfs2_bufmap_total_size, pvfs2_bufmap_desc_size;
+int32_t pvfs2_bufmap_desc_count, pvfs2_bufmap_desc_shift;
+
/* PINT_dev_initialize()
*
* initializes the device management interface
@@ -139,79 +144,140 @@ void PINT_dev_finalize(void)
}
}
-/* PINT_dev_get_mapped_region()
+/* PINT_dev_get_mapped_regions()
*
- * creates a memory buffer that is shared between user space and
+ * creates a set of memory buffers that are shared between user space and
* kernel space
*
* returns 0 on success, -PVFS_error on failure
*/
-int PINT_dev_get_mapped_region(struct PVFS_dev_map_desc *desc, int size)
+int PINT_dev_get_mapped_regions(int ndesc, struct PVFS_dev_map_desc *desc,
+ struct PINT_dev_params *params)
{
- int ret;
- int page_count = 0;
- long page_size = sysconf(_SC_PAGE_SIZE);
+ int i, ret = -1;
+ uint64_t page_size = sysconf(_SC_PAGE_SIZE), total_size;
void *ptr = NULL;
+ int ioctl_cmd[2] = {PVFS_DEV_MAP, 0};
- /* we would like to use a memaligned region that is a multiple
- * of the system page size
- */
- page_count = (int)(size / page_size);
- if ((size % page_size) != 0)
+ for (i = 0; i < ndesc; i++)
{
- page_count++;
- }
+ total_size = params[i].dev_buffer_size * params[i].dev_buffer_count;
+ if (total_size < 0)
+ {
+ gossip_err("Error:please provide sane values for device parameters.\n");
+ break;
+ }
+ if (total_size % page_size != 0)
+ {
+ gossip_err("Error: total device buffer size must be a multiple of system page size.\n");
+ break;
+ }
+ if (total_size >= PVFS2_BUFMAP_MAX_TOTAL_SIZE)
+ {
+ gossip_err(
+ "Error: total size (%llu) of device "
+ "buffer must be < %llu MB.\n",
+ llu(total_size), llu(PVFS2_BUFMAP_MAX_TOTAL_SIZE));
+ break;
+ }
+ if (params[i].dev_buffer_size & (params[i].dev_buffer_size - 1))
+ {
+ gossip_err("Error: descriptor size must be a power of 2 (%llu)\n",
+ llu(params[i].dev_buffer_size));
+ break;
+ }
+ /* we would like to use a memaligned region that is a multiple
+ * of the system page size
+ */
+ ptr = PINT_mem_aligned_alloc(total_size, page_size);
+ if (!ptr)
+ {
+ desc[i].ptr = NULL;
+ break;
+ }
+ desc[i].ptr = ptr;
+ desc[i].total_size = total_size;
+ desc[i].size = params[i].dev_buffer_size;
+ desc[i].count = params[i].dev_buffer_count;
- ptr = PINT_mem_aligned_alloc(
- (page_count * page_size), page_size);
- if (!ptr)
- {
- return -(PVFS_ENOMEM|PVFS_ERROR_DEV);
+ /* ioctl to ask driver to map pages if needed */
+ if (ioctl_cmd[i] != 0)
+ {
+ ret = ioctl(pdev_fd, ioctl_cmd[i], &desc[i]);
+ if (ret < 0)
+ {
+ break;
+ }
+ pvfs2_bufmap_desc_count = params[i].dev_buffer_count;
+ pvfs2_bufmap_desc_size = params[i].dev_buffer_size;
+ pvfs2_bufmap_total_size = total_size;
+ pvfs2_bufmap_desc_shift = LOG2(pvfs2_bufmap_desc_size);
+ }
}
- desc->ptr = ptr;
- desc->size = (page_count * page_size);
-
- /* ioctl to ask driver to map pages */
- ret = ioctl(pdev_fd, PVFS_DEV_MAP, desc);
- if (ret < 0)
- {
- free(ptr);
+ if (i != ndesc) {
+ int j;
+ for (j = 0; j < i; j++) {
+ if (desc[j].ptr) {
+ PINT_mem_aligned_free(desc[j].ptr);
+ desc[j].ptr = NULL;
+ }
+ }
return -(PVFS_ENOMEM|PVFS_ERROR_DEV);
}
return 0;
}
-/* PINT_dev_put_mapped_region()
+/* PINT_dev_put_mapped_regions()
*
- * frees the memory buffer that was shared between user space and
+ * frees the set of memory buffers that were shared between user space and
* kernel space. MUST be called only after device is closed
* (i.e. PINT_dev_finalize)
*/
-void PINT_dev_put_mapped_region(struct PVFS_dev_map_desc *desc)
+void PINT_dev_put_mapped_regions(int ndesc, struct PVFS_dev_map_desc *desc)
{
- void *ptr = (void *) desc->ptr;
+ void *ptr;
+ int i;
+
assert(desc);
- assert(ptr);
- PINT_mem_aligned_free(ptr);
+ for (i = 0; i < ndesc; i++)
+ {
+ ptr = (void *) desc[i].ptr;
+ assert(ptr);
+
+ PINT_mem_aligned_free(ptr);
+ }
}
/* PINT_dev_get_mapped_buffer()
*
- * returns a memory buffer of size PVFS2_BUFMAP_DEFAULT_DESC_SIZE
+ * returns a memory buffer of size (pvfs2_bufmap_desc_size)
* matching the specified buffer_index given a PVFS_dev_map_desc
*
* returns a valid desc addr on success, NULL on failure
*/
void *PINT_dev_get_mapped_buffer(
+ enum pvfs_bufmap_type bm_type,
struct PVFS_dev_map_desc *desc,
int buffer_index)
{
- char *ptr = (char *) desc->ptr;
+ char *ptr;
+ int desc_count, desc_size;
+
+ if (bm_type != BM_IO && bm_type != BM_READDIR)
+ return NULL;
+
+ desc_count = (bm_type == BM_IO) ?
+ pvfs2_bufmap_desc_count :
+ PVFS2_READDIR_DEFAULT_DESC_COUNT;
+ desc_size = (bm_type == BM_IO) ?
+ pvfs2_bufmap_desc_size :
+ PVFS2_READDIR_DEFAULT_DESC_SIZE;
+ ptr = (char *) desc[bm_type].ptr;
return ((desc && ptr &&
((buffer_index > -1) &&
- (buffer_index < PVFS2_BUFMAP_DESC_COUNT))) ?
- (ptr + (buffer_index * PVFS2_BUFMAP_DEFAULT_DESC_SIZE)) :
+ (buffer_index < desc_count))) ?
+ (ptr + (buffer_index * desc_size)) :
NULL);
}
@@ -234,6 +300,7 @@ int PINT_dev_test_unexpected(
int32_t *proto_ver = NULL;
uint64_t *tag = NULL;
void *buffer = NULL;
+ pvfs2_upcall_t *upc = NULL;
/* prepare to read max upcall size (magic nr and tag included) */
int read_size = pdev_max_upsize;
@@ -243,6 +310,10 @@ int PINT_dev_test_unexpected(
pfd.fd = pdev_fd;
pfd.events = POLLIN;
+ gossip_debug(GOSSIP_DEV_DEBUG,
+ "[DEV]: Entered %s: incount: %d, timeout: %d\n",
+ __func__, incount, max_idle_time);
+
do
{
/*
@@ -278,6 +349,9 @@ int PINT_dev_test_unexpected(
/* device is emptied */
if (avail == 0)
{
+ gossip_debug(GOSSIP_DEV_DEBUG,
+ "[DEV]: Exiting %s: incount: %d, device empty!\n",
+ __func__, incount);
return ((*outcount > 0) ? 1 : 0);
}
@@ -351,7 +425,8 @@ int PINT_dev_test_unexpected(
if(*proto_ver != PVFS_KERNEL_PROTO_VERSION)
{
gossip_err("Error: protocol versions do not match.\n");
- gossip_err("Please check that your pvfs2 module and pvfs2-client versions are consistent.\n");
+ gossip_err("Please check that your pvfs2 module "
+ "and pvfs2-client versions are consistent.\n");
ret = -(PVFS_EPROTO|PVFS_ERROR_DEV);
goto dev_test_unexp_error;
}
@@ -364,6 +439,24 @@ int PINT_dev_test_unexpected(
((unsigned long)buffer + 2*sizeof(int32_t) + sizeof(uint64_t));
info_array[*outcount].tag = *tag;
+ upc = (pvfs2_upcall_t *) info_array[*outcount].buffer;
+ /* if there is a trailer, allocate a buffer and issue another read */
+ if (upc->trailer_size > 0)
+ {
+ upc->trailer_buf = malloc(upc->trailer_size);
+ if (upc->trailer_buf == NULL)
+ {
+ ret = -(PVFS_ENOMEM|PVFS_ERROR_DEV);
+ goto dev_test_unexp_error;
+ }
+ ret = read(pdev_fd, upc->trailer_buf, upc->trailer_size);
+ if (ret < 0)
+ {
+ ret = -(PVFS_EIO|PVFS_ERROR_DEV);
+ goto dev_test_unexp_error;
+ }
+ }
+
(*outcount)++;
/*
@@ -373,6 +466,11 @@ int PINT_dev_test_unexpected(
} while((*outcount < incount) && avail);
+ gossip_debug(GOSSIP_DEV_DEBUG,
+ "[DEV]: %s Exit: "
+ "incount: %d, outcount: %d, bytes available: %d\n",
+ __func__, incount, *outcount, avail);
+
return ((*outcount > 0) ? 1 : 0);
dev_test_unexp_error:
@@ -380,6 +478,11 @@ dev_test_unexp_error:
/* release resources we created up to this point */
for(i = 0; i < *outcount; i++)
{
+ upc = (pvfs2_upcall_t *) info_array[i].buffer;
+ if (upc->trailer_buf)
+ {
+ free(upc->trailer_buf);
+ }
if (buffer)
{
free(buffer);
@@ -438,15 +541,21 @@ int PINT_dev_write_list(
/* lets be reasonable about list size :) */
/* two vecs are taken up by magic nr and tag */
- assert(list_count < 7);
+ if (list_count > 7)
+ {
+ return (-(PVFS_EINVAL|PVFS_ERROR_DEV));
+ }
/* even though we are ignoring the buffer_type for now,
* make sure that the caller set it to a sane value
*/
- assert(buffer_type == PINT_DEV_EXT_ALLOC ||
- buffer_type == PINT_DEV_PRE_ALLOC);
+ if (buffer_type != PINT_DEV_EXT_ALLOC &&
+ buffer_type != PINT_DEV_PRE_ALLOC)
+ {
+ return (-(PVFS_EINVAL|PVFS_ERROR_DEV));
+ }
- if (total_size > pdev_max_downsize)
+ if (size_list[0] > pdev_max_downsize)
{
return(-(PVFS_EMSGSIZE|PVFS_ERROR_DEV));
}
Index: pint-dev.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/dev/pint-dev.h,v
diff -p -u -r1.13 -r1.13.42.1
--- pint-dev.h 28 Jul 2004 14:32:41 -0000 1.13
+++ pint-dev.h 17 Feb 2007 11:17:07 -0000 1.13.42.1
@@ -3,13 +3,17 @@
*
* See COPYING in top-level directory.
*/
-
#ifndef __PINT_DEV_H
#define __PINT_DEV_H
#include "pvfs2-types.h"
#include "pint-dev-shared.h"
+enum pvfs_bufmap_type {
+ BM_IO = 0,
+ BM_READDIR = 1,
+};
+
/* describes unexpected messages coming out of the device */
struct PINT_dev_unexp_info
{
@@ -25,18 +29,27 @@ enum PINT_dev_buffer_type
PINT_DEV_EXT_ALLOC = 2
};
+struct PINT_dev_params
+{
+ uint32_t dev_buffer_count;
+ uint64_t dev_buffer_size;
+};
+
int PINT_dev_initialize(
const char* dev_name,
int flags);
-int PINT_dev_get_mapped_region(
+int PINT_dev_get_mapped_regions(
+ int ndesc,
struct PVFS_dev_map_desc *desc,
- int size);
+ struct PINT_dev_params *params);
-void PINT_dev_put_mapped_region(
+void PINT_dev_put_mapped_regions(
+ int ndesc,
struct PVFS_dev_map_desc *desc);
void *PINT_dev_get_mapped_buffer(
+ enum pvfs_bufmap_type bm_type,
struct PVFS_dev_map_desc *desc,
int buffer_index);
More information about the Pvfs2-cvs
mailing list