[Pvfs2-cvs] commit by slang in pvfs2/src/io/bmi/bmi_ib: README ib.c
ib.h mem.c module.mk.in openib.c util.c vapi.c
CVS commit program
cvs at parl.clemson.edu
Thu Oct 19 18:16:55 EDT 2006
Update of /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib
In directory parlweb1:/tmp/cvs-serv5758/src/io/bmi/bmi_ib
Modified Files:
Tag: WALT3
README ib.c ib.h mem.c module.mk.in openib.c util.c vapi.c
Log Message:
reverse merge of HEAD to WALT3 branch.
Index: README
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/README,v
diff -p -u -r1.5.4.1 -r1.5.4.2
--- README 18 Sep 2006 15:05:12 -0000 1.5.4.1
+++ README 19 Oct 2006 22:16:54 -0000 1.5.4.2
@@ -2,7 +2,7 @@ Notes on the BMI InfiniBand implementati
Copyright (C) 2003-6 Pete Wyckoff <pw at osc.edu>
-$Id: README,v 1.5.4.1 2006/09/18 15:05:12 vilayann Exp $
+$Id: README,v 1.5.4.2 2006/10/19 22:16:54 slang Exp $
InifiniBand programming interface
---------------------------------
Index: ib.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/ib.c,v
diff -p -u -r1.32.2.1 -r1.32.2.2
--- ib.c 18 Sep 2006 15:05:12 -0000 1.32.2.1
+++ ib.c 19 Oct 2006 22:16:54 -0000 1.32.2.2
@@ -6,7 +6,7 @@
*
* See COPYING in top-level directory.
*
- * $Id: ib.c,v 1.32.2.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: ib.c,v 1.32.2.2 2006/10/19 22:16:54 slang Exp $
*/
#include <stdio.h>
#include <stdlib.h>
@@ -52,6 +52,7 @@ ib_device_t *ib_device __hidden = NULL;
#define post_sr_rdmaw ib_device->func.post_sr_rdmaw
#define check_cq ib_device->func.check_cq
#define prepare_cq_block ib_device->func.prepare_cq_block
+#define ack_cq_completion_event ib_device->func.ack_cq_completion_event
#define wc_status_string ib_device->func.wc_status_string
#define mem_register ib_device->func.mem_register
#define mem_deregister ib_device->func.mem_deregister
@@ -78,7 +79,6 @@ static void encourage_recv_incoming(ib_c
u_int32_t byte_len);
static void encourage_recv_incoming_cts_ack(ib_recv_t *rq);
static int send_cts(ib_recv_t *rq);
-static void maybe_free_connection(ib_connection_t *c);
static void ib_close_connection(ib_connection_t *c);
#ifndef __PVFS2_SERVER__
static int ib_tcp_client_connect(ib_method_addr_t *ibmap,
@@ -1015,7 +1015,7 @@ test_sq(ib_send_t *sq, bmi_op_id_t *outi
free(sq->mop);
free(sq);
--c->refcnt;
- if (c->closed)
+ if (c->closed || c->cancelled)
ib_close_connection(c);
return 1;
}
@@ -1037,8 +1037,7 @@ test_sq(ib_send_t *sq, bmi_op_id_t *outi
free(sq->mop);
free(sq);
--c->refcnt;
- maybe_free_connection(c);
- if (c->closed)
+ if (c->closed || c->cancelled)
ib_close_connection(c);
return 1;
} else {
@@ -1080,7 +1079,7 @@ test_rq(ib_recv_t *rq, bmi_op_id_t *outi
c = rq->c;
free(rq);
--c->refcnt;
- if (c->closed)
+ if (c->closed || c->cancelled)
ib_close_connection(c);
return 1;
}
@@ -1108,9 +1107,8 @@ test_rq(ib_recv_t *rq, bmi_op_id_t *outi
qlist_del(&rq->list);
c = rq->c;
free(rq);
- maybe_free_connection(c);
--c->refcnt;
- if (c->closed)
+ if (c->closed || c->cancelled)
ib_close_connection(c);
return 1;
} else {
@@ -1326,7 +1324,7 @@ BMI_ib_testunexpected(int incount __unus
c = rq->c;
free(rq);
--c->refcnt;
- if (c->closed)
+ if (c->closed || c->cancelled)
ib_close_connection(c);
goto out;
}
@@ -1443,28 +1441,6 @@ BMI_ib_cancel(bmi_op_id_t id, bmi_contex
return 0;
}
-/*
- * For connections that are being cancelled, maybe delete them if no
- * more send or recvq entries remain.
- */
-static void
-maybe_free_connection(ib_connection_t *c)
-{
- list_t *l;
-
- if (!c->cancelled)
- return;
- qlist_for_each(l, &ib_device->sendq) {
- ib_send_t *sq = qlist_upcast(l);
- if (sq->c == c) return;
- }
- qlist_for_each(l, &ib_device->recvq) {
- ib_recv_t *rq = qlist_upcast(l);
- if (rq->c == c) return;
- }
- ib_close_connection(c);
-}
-
static const char *
BMI_ib_rev_lookup(struct method_addr *meth)
{
@@ -1611,6 +1587,10 @@ static ib_connection_t *ib_new_connectio
return c;
}
+/*
+ * Try to close and free a connection, but only do it if refcnt has
+ * gone to zero.
+ */
static void ib_close_connection(ib_connection_t *c)
{
ib_method_addr_t *ibmap;
@@ -1791,7 +1771,10 @@ static int ib_block_for_activity(int tim
}
ret = poll(pfd, numfd, timeout_ms);
debug(4, "%s: ret %d rev0 0x%x", __func__, ret, pfd[0].revents);
- if (ret < 0) {
+ if (ret > 0) {
+ if (pfd[0].revents == POLLIN)
+ ack_cq_completion_event();
+ } else if (ret < 0) {
if (errno == EINTR) /* normal, ignore but break */
ret = 0;
else
@@ -2018,6 +2001,6 @@ const struct bmi_method_ops bmi_ib_ops =
.BMI_meth_close_context = BMI_ib_close_context,
.BMI_meth_cancel = BMI_ib_cancel,
.BMI_meth_rev_lookup_unexpected = BMI_ib_rev_lookup,
- .BMI_meth_query_addr_range = NULL,
+ .BMI_meth_query_addr_range = NULL,
};
Index: ib.h
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/ib.h,v
diff -p -u -r1.18.2.1 -r1.18.2.2
--- ib.h 18 Sep 2006 15:05:12 -0000 1.18.2.1
+++ ib.h 19 Oct 2006 22:16:55 -0000 1.18.2.2
@@ -5,13 +5,14 @@
*
* See COPYING in top-level directory.
*
- * $Id: ib.h,v 1.18.2.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: ib.h,v 1.18.2.2 2006/10/19 22:16:55 slang Exp $
*/
#ifndef __ib_h
#define __ib_h
#include <src/io/bmi/bmi-types.h>
#include <src/common/quicklist/quicklist.h>
+#include <src/common/gossip/gossip.h>
#ifdef __GNUC__
/* # define __hidden __attribute__((visibility("hidden"))) */
@@ -332,6 +333,7 @@ struct ib_device_func {
void (*post_sr_rdmaw)(ib_send_t *sq, msg_header_cts_t *mh_cts,
void *mh_cts_buf);
int (*prepare_cq_block)(void);
+ void (*ack_cq_completion_event)(void);
int (*check_cq)(struct bmi_ib_wc *wc);
const char *(*wc_status_string)(int status);
void (*mem_register)(memcache_entry_t *c);
@@ -435,7 +437,7 @@ void memcache_shutdown(void *md);
#define debug(lvl,fmt,args...) \
do { \
if (lvl <= DEBUG_LEVEL) \
- info(fmt,##args); \
+ gossip_debug(GOSSIP_BMI_DEBUG_IB, fmt ".\n", ##args); \
} while (0)
#else
# define debug(lvl,fmt,...) do { } while (0)
Index: mem.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/mem.c,v
diff -p -u -r1.7.8.1 -r1.7.8.2
--- mem.c 18 Sep 2006 15:05:12 -0000 1.7.8.1
+++ mem.c 19 Oct 2006 22:16:55 -0000 1.7.8.2
@@ -5,7 +5,7 @@
*
* See COPYING in top-level directory.
*
- * $Id: mem.c,v 1.7.8.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: mem.c,v 1.7.8.2 2006/10/19 22:16:55 slang Exp $
*/
#include <src/common/gen-locks/gen-locks.h>
#include "pvfs2-internal.h"
Index: module.mk.in
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/module.mk.in,v
diff -p -u -r1.11.4.1 -r1.11.4.2
--- module.mk.in 18 Sep 2006 15:05:12 -0000 1.11.4.1
+++ module.mk.in 19 Oct 2006 22:16:55 -0000 1.11.4.2
@@ -5,7 +5,7 @@
#
# See COPYING in top-level directory.
#
-# $Id: module.mk.in,v 1.11.4.1 2006/09/18 15:05:12 vilayann Exp $
+# $Id: module.mk.in,v 1.11.4.2 2006/10/19 22:16:55 slang Exp $
#
# only do any of this if configure decided to use IB on OpenIB
@@ -39,13 +39,7 @@ SERVERSRC += $(src)
#
cflags :=
ifdef GNUC
-cflags += -fno-common
-cflags += -Wall -W -Wpointer-arith
-cflags += -Wcast-align -Wcast-qual -Wbad-function-cast
-cflags += -Wmissing-prototypes -Wmissing-declarations
-cflags += -Wnested-externs
-cflags += -Wshadow -Wstrict-prototypes -Wredundant-decls
-cflags += -Wundef -Wwrite-strings
+cflags += -W -Wcast-qual -Wshadow -Wwrite-strings
endif
#
Index: openib.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/openib.c,v
diff -p -u -r1.2.6.1 -r1.2.6.2
--- openib.c 18 Sep 2006 15:05:12 -0000 1.2.6.1
+++ openib.c 19 Oct 2006 22:16:55 -0000 1.2.6.2
@@ -6,10 +6,11 @@
*
* See COPYING in top-level directory.
*
- * $Id: openib.c,v 1.2.6.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: openib.c,v 1.2.6.2 2006/10/19 22:16:55 slang Exp $
*/
#include <string.h>
#include <errno.h>
+#include <unistd.h>
#include <fcntl.h>
#define __PINT_REQPROTO_ENCODE_FUNCS_C /* include definitions */
#include <src/io/bmi/bmi-byteswap.h> /* bmitoh64 */
@@ -781,6 +782,23 @@ static int openib_prepare_cq_block(void)
}
/*
+ * As poll says there is something to read, get the event, but
+ * ignore the contents as we only have one CQ. But ack it
+ * so that the count is correct and the CQ can be shutdown later.
+ */
+static void openib_ack_cq_completion_event(void)
+{
+ struct openib_device_priv *od = ib_device->priv;
+ struct ibv_cq *cq;
+ void *cq_context;
+ int ret;
+
+ ret = ibv_get_cq_event(od->channel, &cq, &cq_context);
+ if (ret == 0)
+ ibv_ack_cq_events(cq, 1);
+}
+
+/*
* Return string form of work completion status field.
*/
#define CASE(e) case e: s = #e; break
@@ -998,6 +1016,7 @@ int openib_ib_initialize(void)
ib_device->func.post_sr_rdmaw = openib_post_sr_rdmaw;
ib_device->func.check_cq = openib_check_cq;
ib_device->func.prepare_cq_block = openib_prepare_cq_block;
+ ib_device->func.ack_cq_completion_event = openib_ack_cq_completion_event;
ib_device->func.wc_status_string = openib_wc_status_string;
ib_device->func.mem_register = openib_mem_register;
ib_device->func.mem_deregister = openib_mem_deregister;
@@ -1047,12 +1066,18 @@ int openib_ib_initialize(void)
if (!od->nic_cq)
error("%s: ibv_create_cq failed", __func__);
- /* use non-blocking IO on the async fd */
+ /* use non-blocking IO on the async fd and completion fd */
flags = fcntl(ctx->async_fd, F_GETFL);
if (flags < 0)
error_errno("%s: get async fd flags", __func__);
if (fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK) < 0)
error_errno("%s: set async fd nonblocking", __func__);
+
+ flags = fcntl(od->channel->fd, F_GETFL);
+ if (flags < 0)
+ error_errno("%s: get completion fd flags", __func__);
+ if (fcntl(od->channel->fd, F_SETFL, flags | O_NONBLOCK) < 0)
+ error_errno("%s: set completion fd nonblocking", __func__);
/* will be set on first connection */
od->sg_tmp_array = 0;
Index: util.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/util.c,v
diff -p -u -r1.7.4.1 -r1.7.4.2
--- util.c 18 Sep 2006 15:05:12 -0000 1.7.4.1
+++ util.c 19 Oct 2006 22:16:55 -0000 1.7.4.2
@@ -5,14 +5,13 @@
*
* See COPYING in top-level directory.
*
- * $Id: util.c,v 1.7.4.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: util.c,v 1.7.4.2 2006/10/19 22:16:55 slang Exp $
*/
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
#include <errno.h>
#include <unistd.h>
-#include <src/common/gossip/gossip.h>
#define __util_c
#include "ib.h"
@@ -84,18 +83,6 @@ warning_errno(const char *fmt, ...)
vsprintf(s, fmt, ap);
va_end(ap);
gossip_err("Warning: %s: %s.\n", s, strerror(errno));
-}
-
-void __attribute__((format(printf,1,2))) __hidden
-info(const char *fmt, ...)
-{
- char s[2048];
- va_list ap;
-
- va_start(ap, fmt);
- vsprintf(s, fmt, ap);
- va_end(ap);
- gossip_debug(GOSSIP_BMI_DEBUG_IB, "%s.\n", s);
}
void * __attribute__((malloc)) __hidden
Index: vapi.c
===================================================================
RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_ib/vapi.c,v
diff -p -u -r1.2.6.1 -r1.2.6.2
--- vapi.c 18 Sep 2006 15:05:12 -0000 1.2.6.1
+++ vapi.c 19 Oct 2006 22:16:55 -0000 1.2.6.2
@@ -5,7 +5,7 @@
*
* See COPYING in top-level directory.
*
- * $Id: vapi.c,v 1.2.6.1 2006/09/18 15:05:12 vilayann Exp $
+ * $Id: vapi.c,v 1.2.6.2 2006/10/19 22:16:55 slang Exp $
*/
#include <stdio.h>
#include <string.h>
@@ -17,7 +17,6 @@
#include <src/io/bmi/bmi-method-support.h> /* struct method_addr */
#include <src/common/misc/pvfs2-internal.h>
#include <src/io/bmi/bmi-byteswap.h> /* bmitoh64 */
-#include <src/common/gossip/gossip.h>
#include "pvfs2-config.h" /* HAVE_IB_WRAP_COMMON_H configure symbol */
@@ -767,20 +766,29 @@ static int vapi_prepare_cq_block(void)
{
struct vapi_device_priv *vd = ib_device->priv;
int ret;
- char buf[16];
-
/* ask for the next notfication */
ret = VAPI_req_comp_notif(vd->nic_handle, vd->nic_cq, VAPI_NEXT_COMP);
if (ret < 0)
error_verrno(ret, "%s: VAPI_req_comp_notif", __func__);
- /* flush the pipe of any outstanding items, no need to be exact */
- (void) read(vd->cq_event_pipe[0], buf, sizeof(buf));
-
/* return the fd that can be fed to poll() */
return vd->cq_event_pipe[0];
}
+/*
+ * Read an event that tells us there is some action on the CQ. In
+ * reality, just read the int from the pipe that connects us to the
+ * event handler thread.
+ */
+static void vapi_ack_cq_completion_event(void)
+{
+ struct vapi_device_priv *vd = ib_device->priv;
+ int i, ret;
+
+ ret = read(vd->cq_event_pipe[0], &i, sizeof(i));
+ if (ret != sizeof(i))
+ error_errno("%s: read cq event pipe", __func__);
+}
/*
* Return string form of work completion status field.
@@ -1042,6 +1050,7 @@ int vapi_ib_initialize(void)
ib_device->func.post_sr_rdmaw = vapi_post_sr_rdmaw;
ib_device->func.check_cq = vapi_check_cq;
ib_device->func.prepare_cq_block = vapi_prepare_cq_block;
+ ib_device->func.ack_cq_completion_event = vapi_ack_cq_completion_event;
ib_device->func.wc_status_string = vapi_wc_status_string;
ib_device->func.mem_register = vapi_mem_register;
ib_device->func.mem_deregister = vapi_mem_deregister;
More information about the Pvfs2-cvs
mailing list