[Pvfs2-cvs] commit by pcarns in pvfs2-1/src/io/job: job.c thread-mgr.c

[CVS commit program]

Update of /projects/cvsroot/pvfs2-1/src/io/job
In directory parlweb1:/tmp/cvs-serv24377/src/io/job

Modified Files:
	job.c thread-mgr.c 
Log Message:
Fixed a bug in handling unexpected device jobs.  With a big enough load of
concurrent operations (particularly with threaded pvfs2-client) could cause
client-core assertion, and would likely happen repeatedly when client-core
restarted.


Index: job.c
===================================================================
RCS file: /projects/cvsroot/pvfs2-1/src/io/job/job.c,v
diff -p -u -r1.178 -r1.179
--- job.c	11 Feb 2008 17:25:28 -0000	1.178
+++ job.c	21 Feb 2008 19:24:15 -0000	1.179
@@ -4399,7 +4399,10 @@ static void dev_thread_mgr_unexp_handler
     gen_mutex_lock(&dev_unexp_mutex);
     /* remove the operation from the pending dev_unexp queue */
     tmp_desc = job_desc_q_shownext(dev_unexp_queue);
-    assert(tmp_desc != NULL);        /* TODO: fix this */
+    /* if the thread mgr accounting is accurate, then there _must_ be a
+     * dev_unexp job posted for us to hit this point.
+     */
+    assert(tmp_desc != NULL);
     if (tmp_desc->completed_flag == 0)
     {
         job_desc_q_remove(tmp_desc);

Index: thread-mgr.c
===================================================================
RCS file: /projects/cvsroot/pvfs2-1/src/io/job/thread-mgr.c,v
diff -p -u -r1.36 -r1.37
--- thread-mgr.c	2 Feb 2007 02:08:58 -0000	1.36
+++ thread-mgr.c	21 Feb 2008 19:24:15 -0000	1.37
@@ -53,6 +53,7 @@ static pthread_t dev_thread_id;
 
 static pthread_cond_t bmi_test_cond = PTHREAD_COND_INITIALIZER;
 static pthread_cond_t trove_test_cond = PTHREAD_COND_INITIALIZER;
+static pthread_cond_t dev_unexp_test_cond = PTHREAD_COND_INITIALIZER;
 #endif /* __PVFS2_JOB_THREADED__ */
 
 /* used to indicate that a bmi testcontext is in progress; we can't simply
@@ -299,8 +300,21 @@ static void *dev_thread_function(void *p
     {
 	gen_mutex_lock(&dev_mutex);
 	incount = dev_unexp_count;
+        while(incount == 0)
+        {
+            /* we need to wait until more unexp dev operations are posted */
+#ifdef __PVFS2_JOB_THREADED__
+            pthread_cond_wait(&dev_unexp_test_cond, &dev_mutex);
+            incount = dev_unexp_count;
+#else
+            gen_mutex_unlock(&dev_mutex);
+            return(NULL);
+#endif
+        }
 	if(incount > THREAD_MGR_TEST_COUNT)
+        {
 	    incount = THREAD_MGR_TEST_COUNT;
+        }
 	gen_mutex_unlock(&dev_mutex);
 
 	ret = PINT_dev_test_unexpected(
@@ -747,6 +761,13 @@ int PINT_thread_mgr_dev_unexp_handler(
     }
     dev_unexp_fn = fn;
     dev_unexp_count++;
+    if(dev_unexp_count == 1)
+    {
+        /* signal worker thread that may have been waiting for more ops */
+#ifdef __PVFS2_JOB_THREADED__
+        pthread_cond_signal(&dev_unexp_test_cond);
+#endif
+    }
     gen_mutex_unlock(&dev_mutex);
     return(0);
 }