Index: include/pvfs2-types.h =================================================================== RCS file: /projects/cvsroot/pvfs2/include/pvfs2-types.h,v retrieving revision 1.145 diff -u -a -p -r1.145 pvfs2-types.h --- include/pvfs2-types.h 18 Oct 2007 15:55:01 -0000 1.145 +++ include/pvfs2-types.h 7 Nov 2007 00:11:24 -0000 @@ -224,6 +224,7 @@ typedef struct PVFS_sys_layout_s /* predefined special values for types */ #define PVFS_HANDLE_NULL ((PVFS_handle)0) #define PVFS_FS_ID_NULL ((PVFS_fs_id)0) +#define PVFS_BMI_ADDR_NULL ((PVFS_BMI_addr_t)0) #define PVFS_OP_NULL ((id_gen_t)0) #define PVFS_ITERATE_START (INT32_MAX - 1) #define PVFS_ITERATE_END (INT32_MAX - 2) Index: src/client/sysint/sys-io.sm =================================================================== RCS file: /projects/cvsroot/pvfs2/src/client/sysint/sys-io.sm,v retrieving revision 1.158 diff -u -a -p -r1.158 sys-io.sm --- src/client/sysint/sys-io.sm 30 Aug 2007 00:13:42 -0000 1.158 +++ src/client/sysint/sys-io.sm 7 Nov 2007 00:11:24 -0000 @@ -931,6 +931,7 @@ static PINT_sm_action io_datafile_comple /* if recv failed, probably have to do the send again too */ cur_ctx->msg_send_has_been_posted = 0; cur_ctx->msg_recv_has_been_posted = 0; + BMI_endpoint_fail_current_addr(cur_ctx->msg.svr_addr); goto check_next_step; } @@ -1038,6 +1039,7 @@ static PINT_sm_action io_datafile_comple "%s: flow failed, retrying from msgpair\n", __func__); cur_ctx->msg_send_has_been_posted = 0; cur_ctx->msg_recv_has_been_posted = 0; + BMI_endpoint_fail_current_addr(cur_ctx->msg.svr_addr); } else { @@ -1077,6 +1079,7 @@ static PINT_sm_action io_datafile_comple "%s: write-ack failed, retrying from msgpair\n", __func__); cur_ctx->msg_send_has_been_posted = 0; cur_ctx->msg_recv_has_been_posted = 0; + BMI_endpoint_fail_current_addr(cur_ctx->msg.svr_addr); } else { Index: src/common/misc/msgpairarray.sm =================================================================== RCS file: /projects/cvsroot/pvfs2/src/common/misc/msgpairarray.sm,v retrieving revision 1.44 diff -u -a -p -r1.44 msgpairarray.sm --- src/common/misc/msgpairarray.sm 30 Aug 2007 00:13:43 -0000 1.44 +++ src/common/misc/msgpairarray.sm 7 Nov 2007 00:11:24 -0000 @@ -637,6 +637,8 @@ static PINT_sm_action msgpairarray_compl js_p->error_code = MSGPAIRS_RETRY; } + BMI_endpoint_fail_current_addr(msg_p->svr_addr); + } else { char s[1024]; server_config = PINT_server_config_mgr_get_config(msg_p->fs_id); Index: src/common/misc/server-config.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/common/misc/server-config.c,v retrieving revision 1.105 diff -u -a -p -r1.105 server-config.c --- src/common/misc/server-config.c 25 Sep 2007 17:27:28 -0000 1.105 +++ src/common/misc/server-config.c 7 Nov 2007 00:11:24 -0000 @@ -2015,7 +2015,7 @@ DOTCONF_CB(get_attr_cache_keywords_list) int token_count, j; token_count = PINT_split_string_list( - &tokens, fs_conf->attr_cache_keywords); + &tokens, fs_conf->attr_cache_keywords, ','); for(j = 0; j < token_count; ++j) { @@ -2039,7 +2039,7 @@ DOTCONF_CB(get_attr_cache_keywords_list) int token_count, j; token_count = PINT_split_string_list( - &tokens, cmd->data.list[i]); + &tokens, cmd->data.list[i], ','); for(j = 0; j < token_count; ++j) { Index: src/common/misc/str-utils.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/common/misc/str-utils.c,v retrieving revision 1.23 diff -u -a -p -r1.23 str-utils.c --- src/common/misc/str-utils.c 17 Aug 2007 19:06:57 -0000 1.23 +++ src/common/misc/str-utils.c 7 Nov 2007 00:11:24 -0000 @@ -344,7 +344,7 @@ int PINT_get_next_path(char *path, char * * returns the number of strings successfully parsed */ -int PINT_split_string_list(char ***tokens, const char *comma_list) +int PINT_split_string_list(char ***tokens, const char *comma_list, char delim) { const char *holder = NULL; @@ -360,7 +360,7 @@ int PINT_split_string_list(char ***token /* count how many commas we have first */ holder = comma_list; - while ((holder = index(holder, ','))) + while ((holder = index(holder, delim))) { tokencount++; holder++; @@ -398,7 +398,7 @@ int PINT_split_string_list(char ***token end = comma_list + strlen(comma_list); for (i = 0; i < tokencount && holder; i++) { - holder2 = index(holder, ','); + holder2 = index(holder, delim); if (!holder2) { holder2 = end; Index: src/common/misc/str-utils.h =================================================================== RCS file: /projects/cvsroot/pvfs2/src/common/misc/str-utils.h,v retrieving revision 1.17 diff -u -a -p -r1.17 str-utils.h --- src/common/misc/str-utils.h 16 Sep 2006 21:47:31 -0000 1.17 +++ src/common/misc/str-utils.h 7 Nov 2007 00:11:24 -0000 @@ -35,7 +35,8 @@ int PINT_get_next_path( int skip); int PINT_split_string_list( char ***tokens, - const char *comma_list); + const char *comma_list, + char delim); void PINT_free_string_list( char ** list, int len); Index: src/io/bmi/bmi-addr.c =================================================================== RCS file: src/io/bmi/bmi-addr.c diff -N src/io/bmi/bmi-addr.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/io/bmi/bmi-addr.c 7 Nov 2007 00:11:24 -0000 @@ -0,0 +1,794 @@ +/* + * (C) 2007 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#include +#include +#include +#include + +#include "bmi-addr.h" +#include "gossip.h" +#include "id-generator.h" +#include "src/common/quickhash/quickhash.h" +#include "gen-locks.h" +#include "str-utils.h" +#include "pvfs2-internal.h" + +struct bmi_endpoint_ref_s; + +/** + * Represents an individual address, + * a tuple of (protocol, hostname/ip, [board,], port). + */ +struct bmi_addr_ref_s +{ + char *addr_string; + struct bmi_method_addr *method_addr; + struct bmi_endpoint_ref_s *endpoint_ref; + struct qhash_head strlink; + struct qhash_head methlink; +}; + +/** + * Represents an endpoint, which is a grouping of addresses intended + * to refer to the same storage location. The grouping is specified + * as an ordered list of addresses that provide failover, and so addresses + * within the same group will likely point to different physical hosts, + * although they may just be different protocols on the same host. + */ +struct bmi_endpoint_ref_s +{ + char *endpoint_string; + PVFS_BMI_addr_t bmi_endpoint; + struct bmi_addr_ref_s *addrs; + int count; + int current; + struct qlist_head link; + int refcount; +}; + +struct bmi_addr_map_s +{ + struct qhash_table *addr_table; + struct qhash_table *method_table; + struct qlist_head refs; + gen_mutex_t mutex; +}; + +static int bmi_addr_ref_compare(void *key, struct qhash_head *link); +static int bmi_method_addr_compare(void *key, struct qhash_head *link); + +/** + * Construct a new bmi address mapping table. This just initializes and + * returns an opaque reference to an empty structure. All proceeding bmi_addr + * calls take a reference to this structure. + * + * @return the new address map reference. + */ +bmi_addr_map_p bmi_addr_map_new(int expected_addr_count) +{ + struct bmi_addr_map_s *map; + + id_gen_safe_initialize(); + + map = (struct bmi_addr_map_s *) malloc(sizeof(*map)); + if(!map) + { + return NULL; + } + + /* construct tables with a fill factor of 80%. This table + * holds all unique address references */ + map->addr_table = qhash_init(bmi_addr_ref_compare, + quickhash_string_hash, + (int)(expected_addr_count * 1.2)); + if(!map->addr_table) + { + free(map); + return NULL; + } + + map->method_table = qhash_init(bmi_method_addr_compare, + quickhash_voidp_hash, + (int)(expected_addr_count * 1.2)); + if(!map->method_table) + { + qhash_finalize(map->addr_table); + free(map); + return NULL; + } + + INIT_QLIST_HEAD(&map->refs); + + gen_mutex_init(&map->mutex); + + return map; +} + +/** + * Destroy the mapping table that manages BMI addresses. This + * call removes all entries for bmi addresses, and then does teardown + * of the actual structure. Any references to BMI addrs managed + * by this map will be invalid after this call. In most cases there + * won't be any outstanding references to BMI addrs before calling destroy, + * but because BMI address references are handed to users, its possible + * that there may be. The references are stored in the safe id-gen interface, + * so those invalid references can be caught in later functions. + * + * @param map The mapping table to destroy + * + * @return 0 on success, -BMI_error on failure. + */ +int bmi_addr_map_destroy(bmi_addr_map_p map) +{ + struct bmi_endpoint_ref_s *ref, *tmpref; + int ret; + + gen_mutex_lock(&map->mutex); + + qhash_finalize(map->addr_table); + qhash_finalize(map->method_table); + + qlist_for_each_entry_safe(ref, tmpref, &map->refs, link) + { + gen_mutex_unlock(&map->mutex); + ret = bmi_addr_map_remove(map, ref->bmi_endpoint); + gen_mutex_lock(&map->mutex); + if(ret < 0) + { + return ret; + } + } + + gen_mutex_unlock(&map->mutex); + gen_mutex_destroy(&map->mutex); + + id_gen_safe_finalize(); + + return 0; +} + +/** + * Add a new endpoint to the mapping table. This splits the address + * string into seperate addresses that refer to the same endpoint, + * and adds them as part of the same BMI address. Separate addresses that + * refer to the same endpoint might be for different protocols, or failover + * addresses. The BMI address reference returned by this function will + * point to the first address (leftmost) specified in the comma separated + * address list. Calls to + * @ref bmi_addr_current_id,bmi_addr_current_method_addr + * will return values based on this first address. + * The BMI address reference can be changed to point to the next address + * in the list using the @ref bmi_addr_next function. + * + * @param map the initializing mapping table reference + * @param addrs_string comma separated list of complete addresses + * (://:[]:) referring to the same endpoint. + * @param new_addr an opaque bmi addr reference for the new endpoint. + * + * @return 0 on success, negative -BMI_error on error. + */ +int bmi_addr_add_endpoint(bmi_addr_map_p map, + const char *addrs_string, + PVFS_BMI_addr_t *new_addr) +{ + struct bmi_endpoint_ref_s *newref; + struct bmi_method_ops *ops; + struct bmi_method_ops *prev_ops = NULL; + int addrcount; + char **addrs = NULL; + int i, j, n, ret; + PVFS_BMI_addr_t found_addr; + char *proto; + + assert(addrs_string != NULL); + addrcount = PINT_split_string_list(&addrs, addrs_string, ';'); + if(0 == addrcount) + { + gossip_err("%s: No addresses found in address string: %s\n", + __func__, addrs_string); + return -BMI_EINVAL; + } + + /* look for a pre-existing endpoint */ + for(i = 0; i < addrcount; ++i) + { + found_addr = bmi_addr_id_search(map, addrs[i]); + if(found_addr) + { + break; + } + } + + if(found_addr) + { + newref = id_gen_safe_lookup(found_addr); + + /* add all the new addresses to pre-existing endpoint */ + for(i = 0; i < addrcount; ++i) + { + for(j = 0; j < newref->count; ++j) + { + if(!strcmp(newref->addrs[j].addr_string, addrs[i])) + { + break; + } + } + + if(j == newref->count) + { + /* couldn't find the address in the pre-existing endpoint + * reference, so we add it. + */ + newref->addrs = realloc( + newref->addrs, + (newref->count + 1) * sizeof(struct bmi_addr_ref_s)); + if(!newref->addrs) + { + PINT_free_string_list(addrs, addrcount); + + for(n = 0; newref->count; ++n) + { + qhash_search_and_remove( + map->addr_table, newref->addrs[n].addr_string); + qhash_search_and_remove( + map->method_table, newref->addrs[n].method_addr); + qlist_del(&(newref->link)); + return -BMI_ENOMEM; + } + } + newref->addrs[j].addr_string = addrs[i]; + + /* grab the characters of the address string + * that specify the protocol + */ + ops = bmi_method_find(addrs[i]); + if(!ops) + { + /* try to activate and get the method again */ + ret = bmi_method_activate_from_addr(addrs[i]); + if(ret < 0) + { + return ret; + } + ops = bmi_method_find(addrs[i]); + } + + if(!ops) + { + gossip_err( + "%s: failed to find a BMI method for the protocol: %s\n", + __func__, proto); + free(proto); + PINT_free_string_list(addrs, addrcount); + free(newref); + return -PVFS_EINVAL; + } + + if(prev_ops && prev_ops != ops) + { + gossip_err("%s: can't add multiple addresses for the same " + "endpoint on different protocols: " + "new proto: %s, old proto: %s\n", + __func__, ops->method_name, prev_ops->method_name); + return -PVFS_EINVAL; + } + prev_ops = ops; + + newref->addrs[j].endpoint_ref = newref; + newref->count++; + } + } + } + + /* haven't seen any of these addresses before, add a new endpoint */ + newref = malloc(sizeof(*newref)); + if(!newref) + { + PINT_free_string_list(addrs, addrcount); + return -BMI_ENOMEM; + } + memset(newref, 0, sizeof(*newref)); + + newref->count = addrcount; + newref->addrs = malloc(sizeof(*newref->addrs) * addrcount); + if(!newref->addrs) + { + PINT_free_string_list(addrs, addrcount); + free(newref); + return -BMI_ENOMEM; + } + + for(i = 0; i < addrcount; ++i) + { + /* we don't strdup here because we can reuse the allocated pointer + * from the split call */ + newref->addrs[i].addr_string = addrs[i]; + + /* grab the characters of the address string that specify the protocol + */ + ops = bmi_method_find(addrs[i]); + if(!ops) + { + /* try to activate and get the method again */ + ret = bmi_method_activate_from_addr(addrs[i]); + if(ret < 0) + { + return ret; + } + ops = bmi_method_find(addrs[i]); + } + + if(!ops) + { + gossip_err("%s: failed to find a BMI method for the protocol: %s\n", + __func__, proto); + free(proto); + PINT_free_string_list(addrs, addrcount); + free(newref); + return -PVFS_EINVAL; + } + + if(prev_ops && prev_ops != ops) + { + gossip_err("%s: can't add multiple addresses for the same " + "endpoint on different protocols: " + "new proto: %s, old proto: %s\n", + __func__, ops->method_name, prev_ops->method_name); + return -PVFS_EINVAL; + } + prev_ops = ops; + + newref->addrs[i].endpoint_ref = newref; + + if(i == 0) + { + newref->addrs[i].method_addr = ops->method_addr_lookup(addrs[i]); + } + } + + /* get rid of the array of address strings from the split */ + free(addrs); + + /* add the endpoint reference to the list of references */ + gen_mutex_lock(&map->mutex); + qlist_add(&(newref->link), &map->refs); + + /* add the first address to the hashtables */ + qhash_add(map->addr_table, + newref->addrs[0].addr_string, + &newref->addrs[0].strlink); + qhash_add(map->method_table, + newref->addrs[0].method_addr, + &newref->addrs[0].methlink); + gen_mutex_unlock(&map->mutex); + + id_gen_safe_register(&newref->bmi_endpoint, newref); + *new_addr = newref->bmi_endpoint; + return 0; +} + +/** + * Add a pre-created method addr to the mapping table. + * + * @param map the initialized mapping table reference + * @param addrs_string comma separated list of complete addresses + * (://:[]:) referring to the same endpoint. + * @param new_addr an opaque bmi addr reference for the new endpoint. + * + * @return 0 on success, negative -BMI_error on error. + */ +int bmi_addr_add_method(bmi_addr_map_p map, + bmi_method_addr_p method_addr, + PVFS_BMI_addr_t *new_addr) +{ + int ret = 0; + struct bmi_endpoint_ref_s *newref; + + newref = malloc(sizeof(*newref)); + if(!newref) + { + return -BMI_ENOMEM; + } + memset(newref, 0, sizeof(*newref)); + + newref->count = 1; + newref->addrs = malloc(sizeof(*newref->addrs)); + if(!newref->addrs) + { + free(newref); + return -BMI_ENOMEM; + } + + newref->addrs[0].method_addr = method_addr; + newref->addrs[0].endpoint_ref = newref; + + /* add the endpoint reference to the list of references */ + gen_mutex_lock(&map->mutex); + qlist_add(&(newref->link), &map->refs); + gen_mutex_unlock(&map->mutex); + + ret = id_gen_safe_register(&newref->bmi_endpoint, newref); + *new_addr = newref->bmi_endpoint; + return ret; +} + +/** + * Search for a BMI address based on the address string + * (://:[:]). The reference count is incremented + * if a valid BMI address is returned. The @ref bmi_addr_release function + * should be called when the address is no longer in use. + * + * @param map The map that manages all the addresses. + * @param addr_string The address string to lookup. + * @param addr The output parameter for the BMI address if one is found. + * string. + * + * @return the BMI address on success, PVFS_BMI_ADDR_NULL if not found. + */ +PVFS_BMI_addr_t bmi_addr_id_search(bmi_addr_map_p map, + const char *addr_string) +{ + struct bmi_addr_ref_s *ref; + struct qhash_head *link; + + gen_mutex_lock(&map->mutex); + link = qhash_search(map->addr_table, (char *)addr_string); + gen_mutex_unlock(&map->mutex); + if(!link) + { + return -BMI_ENOENT; + } + + ref = qhash_entry(link, struct bmi_addr_ref_s, strlink); + + return ref->endpoint_ref->bmi_endpoint; +} + +/** + * Search for a BMI address based on the method address pointer. + * The reference count is incremented if a valid BMI address is returned. + * The @ref bmi_addr_release function should be called when the address + * is no longer in use. + * + * @param map The map that manages all the addresses. + * @param method_addr The method address pointer to lookup. + * @param addr The output parameter for the BMI address if one is found. + * + * @return the BMI address on success, PVFS_BMI_ADDR_NULL if not found. + */ +PVFS_BMI_addr_t bmi_addr_method_search(bmi_addr_map_p map, + bmi_method_addr_p method_addr) +{ + struct bmi_addr_ref_s *ref; + struct qhash_head *link; + + gen_mutex_lock(&map->mutex); + link = qhash_search(map->addr_table, method_addr); + gen_mutex_unlock(&map->mutex); + if(!link) + { + return PVFS_BMI_ADDR_NULL; + } + + ref = qhash_entry(link, struct bmi_addr_ref_s, methlink); + return ref->endpoint_ref->bmi_endpoint; +} + +/** + * Remove the BMI address and all its contents from the mapping table. + * + * @param map The mapping table to remove the address from + * @param addr The BMI address to remove + * + * @return 0 on success, -BMI_error on error. If the address is not + * found in the mapping table, -BMI_ENOENT is returned. + */ +int bmi_addr_map_remove(bmi_addr_map_p map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + struct qhash_head *strlink; + struct qhash_head *methlink; + int i; + + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return -BMI_EINVAL; + } + + id_gen_safe_unregister(addr); + + gen_mutex_lock(&map->mutex); + for(i = 0; i < ref->count; ++i) + { + strlink = qhash_search_and_remove( + map->addr_table, ref->addrs[i].addr_string); + if(!strlink) + { + /* The address string isn't in the hashtable! */ + return -BMI_EINVAL; + } + + methlink = qhash_search_and_remove( + map->method_table, ref->addrs[i].method_addr); + if(!methlink) + { + /* The method address isn't in the hashtable! */ + return -BMI_EINVAL; + } + + qlist_del(&ref->link); + } + gen_mutex_unlock(&map->mutex); + + for(i = 0; i < ref->count; ++i) + { + free(ref->addrs[i].addr_string); + } + free(ref->addrs); + free(ref); + + return 0; +} + +/** + * Get the id string of the current address pointed to by the + * BMI address reference. + * + * @param map The map maintaining the BMI address reference + * @param addr The BMI address reference to get the id string of + * + * @return The current id string of the address. If no address was found + * in the map, NULL is returned. The string returned from this function + * does not need to be freed by the caller. It remains in scope until + * the entire BMI address reference is removed from the mapping table. + */ +inline char * +bmi_addr_current_id(PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return NULL; + } + + return ref->addrs[ref->current].addr_string; + +} + +/** + * Get the method address of the current address pointed to by the + * BMI address reference. + * + * @param map The map maintaining the BMI address reference + * @param addr The BMI address reference to get the method address of + * + * @return The current method address of the BMI address. + * If no address was found + * in the map, NULL is returned. The method address returned from this function + * does not need to be freed by the caller. It remains in scope until + * the entire BMI address reference is removed from the mapping table. + */ +inline struct bmi_method_addr * +bmi_addr_current_method_addr(PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return NULL; + } + + return ref->addrs[ref->current].method_addr; +} + +int bmi_addr_refcount_increment(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return -BMI_EINVAL; + } + ref->refcount++; + return 0; +} + +int bmi_addr_refcount_decrement(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return -BMI_EINVAL; + } + + if(--ref->refcount == 0) + { + bmi_addr_drop_endpoint(addr_map, addr, 0); + } + return 0; +} + +static int bmi_addr_refcount(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + assert(ref != 0); + + return ref->refcount; +} + +/** + * Shift the current address this BMI address refers to the next + * address in the list. Future calls to @ref bmi_addr_current_id, + * bmi_addr_current_method_address will + * return updated values based on the new current address. + * + * This list of addresses is circular. Once the last address in the + * list is reached by this function, the BMI address is updated to + * refer to the first address in the list. We drop the previous address + * to prevent multiple connections for the same endpoint. + * + * @param map The mapping table maintaining the address + * @param addr The BMI address reference to update + * + * @return 0 on success, -BMI_error on failure. + */ +int bmi_addr_next(bmi_addr_map_p map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref; + ref = (struct bmi_endpoint_ref_s *)id_gen_safe_lookup(addr); + if(!ref) + { + return -BMI_EINVAL; + } + + gen_mutex_lock(&map->mutex); + bmi_addr_drop_current_addr(map, addr); + qhash_search_and_remove(map->addr_table, + ref->addrs[ref->current].addr_string); + qhash_search_and_remove(map->method_table, + ref->addrs[ref->current].method_addr); + ref->current = (ref->current + 1) % ref->count; + qhash_add(map->addr_table, + ref->addrs[ref->current].addr_string, + &ref->addrs[ref->current].strlink); + qhash_add(map->method_table, + ref->addrs[ref->current].method_addr, + &ref->addrs[ref->current].methlink); + gen_mutex_unlock(&map->mutex); + return 0; +} + +/* bmi_addr_drop + * + * Destroys a complete BMI address, including asking the method to clean up + * its portion. Will query the method for permission before proceeding + * + * NOTE: must be called with ref list mutex held + * + * Returns 0 on successful drop of the addr, or -PVFS_EINVAL if the reference + * count for the address is non-zero. + */ +int bmi_addr_drop_endpoint(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr, + int force) +{ + struct method_drop_addr_query query; + query.response = 0; + query.addr = bmi_addr_current_method_addr(addr); + int ret = 0; + + /* check that refcount of endpoint is zero. If non-zero, + * we just return EINVAL. This may not be an error, but can + * be checked by the caller. + */ + if(bmi_addr_refcount(addr_map, addr) != 0) + { + return -PVFS_EINVAL; + } + + if(force) + { + /* don't ask the method if the connection is dead, drop anyway */ + query.addr->ops->set_info(BMI_DROP_ADDR, query.addr); + bmi_addr_map_remove(addr_map, addr); + return 0; + } + + /* reference count is zero; ask module if it wants us to discard + * the address; TCP will tell us to drop addresses for which the + * socket has died with no possibility of reconnect + */ + ret = query.addr->ops->get_info(BMI_DROP_ADDR_QUERY, &query); + if(ret == 0 && query.response == 1) + { + /* kill the address */ + gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, + "[BMI CONTROL]: %s: bmi discarding address: %llu\n", + __func__, llu(addr)); + /* NOTE: this triggers request to module to free underlying + * resources if it wants to + */ + if (query.addr) + { + query.addr->ops->set_info(BMI_DROP_ADDR, query.addr); + } + bmi_addr_map_remove(addr_map, addr); + } + return 0; +} + +int bmi_addr_drop_current_addr(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr) +{ + struct bmi_endpoint_ref_s *ref = id_gen_safe_lookup(addr); + struct bmi_addr_ref_s *addr_ref = &ref->addrs[ref->current]; + + if(!ref) + { + return -PVFS_EINVAL; + } + + /* kill the address */ + gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, + "[BMI CONTROL]: %s: bmi discarding address: %s\n", + __func__, addr_ref->addr_string); + /* NOTE: this triggers request to module to free underlying + * resources if it wants to + */ + addr_ref->method_addr->ops->set_info(BMI_DROP_ADDR, addr_ref->method_addr); + + return 0; +} + +struct bmi_method_ops *bmi_addr_get_method(PVFS_BMI_addr_t addr) +{ + struct bmi_method_addr *meth_addr; + + meth_addr = bmi_addr_current_method_addr(addr); + if(meth_addr) + { + return meth_addr->ops; + } + return NULL; +} + +static int bmi_addr_ref_compare(void *key, struct qhash_head *link) +{ + const char *str = (char *)key; + struct bmi_addr_ref_s *ref = + qhash_entry(link, struct bmi_addr_ref_s, strlink); + + if(!strcmp(ref->addr_string, str)) + { + return 1; + } + return 0; +} + +static int bmi_method_addr_compare(void *key, struct qhash_head *link) +{ + struct bmi_method_addr *meth_addr = (struct bmi_method_addr *)key; + struct bmi_addr_ref_s *addr_ref = + qhash_entry(link, struct bmi_addr_ref_s, methlink); + + if(addr_ref->method_addr == meth_addr) + { + return 1; + } + return 0; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ts=8 sts=4 sw=4 expandtab + */ Index: src/io/bmi/bmi-addr.h =================================================================== RCS file: src/io/bmi/bmi-addr.h diff -N src/io/bmi/bmi-addr.h --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ src/io/bmi/bmi-addr.h 7 Nov 2007 00:11:24 -0000 @@ -0,0 +1,68 @@ +/* + * (C) 2007 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + + +/* + * Header for address mapping from strings and method address structs to + * BMI address pointers. + */ + +#ifndef __BMI_ADDR_H +#define __BMI_ADDR_H + +#include "bmi-types.h" +#include "bmi-method-support.h" +#include "quicklist.h" + +typedef struct bmi_addr_map_s *bmi_addr_map_p; + +bmi_addr_map_p bmi_addr_map_new(int expected_addr_count); + +int bmi_addr_map_destroy(bmi_addr_map_p map); + +int bmi_addr_add_endpoint(bmi_addr_map_p map, + const char *addrs_string, + PVFS_BMI_addr_t *new_addr); + +int bmi_addr_add_method(bmi_addr_map_p map, + bmi_method_addr_p method_addr, + PVFS_BMI_addr_t *new_addr); + +PVFS_BMI_addr_t bmi_addr_method_search(bmi_addr_map_p map, + bmi_method_addr_p method_addr); + +PVFS_BMI_addr_t bmi_addr_id_search(bmi_addr_map_p map, + const char *addr_string); + +int bmi_addr_map_remove(bmi_addr_map_p map, PVFS_BMI_addr_t addr); + +inline char * +bmi_addr_current_id(PVFS_BMI_addr_t addr); + +inline struct bmi_method_addr * +bmi_addr_current_method_addr(PVFS_BMI_addr_t addr); + +int bmi_addr_next(bmi_addr_map_p map, PVFS_BMI_addr_t addr); + +int bmi_addr_drop_endpoint(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr, + int force); +int bmi_addr_drop_current_addr(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr); + +struct bmi_method_ops *bmi_addr_get_method(PVFS_BMI_addr_t addr); + +int bmi_addr_refcount_increment(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr); +int bmi_addr_refcount_decrement(bmi_addr_map_p addr_map, PVFS_BMI_addr_t addr); + +#endif /* __BMI_ADDR_H */ + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ts=8 sts=4 sw=4 expandtab + */ Index: src/io/bmi/bmi-method-support.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi-method-support.c,v retrieving revision 1.12 diff -u -a -p -r1.12 bmi-method-support.c --- src/io/bmi/bmi-method-support.c 6 Nov 2007 23:08:33 -0000 1.12 +++ src/io/bmi/bmi-method-support.c 7 Nov 2007 00:11:24 -0000 @@ -13,11 +13,82 @@ #include "bmi-types.h" #include "bmi-method-support.h" #include "id-generator.h" -#include "reference-list.h" +#include "gen-locks.h" +#include "str-utils.h" +#include "gossip.h" + +/* + * List of "known" BMI methods. This is dynamic, starting with + * just the static ones above, and perhaps adding more if we turn + * back on dynamic module loading. + */ +static int known_method_count = 0; +static struct bmi_method_ops **known_method_table = 0; + +int bmi_active_method_count = 0; +gen_mutex_t bmi_active_method_count_mutex = GEN_MUTEX_INITIALIZER; + +/* + * List of active BMI methods. These are the ones that will be + * dealt with for a test call, for example. On a client, known methods + * become active only when someone calls BMI_addr_lookup(). On + * a server, all possibly active methods are known at startup time + * because we listen on them for the duration. + */ +struct bmi_method_ops **bmi_active_method_table = NULL; + +static int activate_method(const char *name, const char *listen_addr, + int flags); + +/* + * Static list of defined BMI methods. These are pre-compiled into + * the client libraries and into the server. + */ +#ifdef __STATIC_METHOD_BMI_TCP__ +extern struct bmi_method_ops bmi_tcp_ops; +#endif +#ifdef __STATIC_METHOD_BMI_GM__ +extern struct bmi_method_ops bmi_gm_ops; +#endif +#ifdef __STATIC_METHOD_BMI_MX__ +extern struct bmi_method_ops bmi_mx_ops; +#endif +#ifdef __STATIC_METHOD_BMI_IB__ +extern struct bmi_method_ops bmi_ib_ops; +#endif +#ifdef __STATIC_METHOD_BMI_PORTALS__ +extern struct bmi_method_ops bmi_portals_ops; +#endif + +static struct bmi_method_ops *const static_methods[] = { +#ifdef __STATIC_METHOD_BMI_TCP__ + &bmi_tcp_ops, +#endif +#ifdef __STATIC_METHOD_BMI_GM__ + &bmi_gm_ops, +#endif +#ifdef __STATIC_METHOD_BMI_MX__ + &bmi_mx_ops, +#endif +#ifdef __STATIC_METHOD_BMI_IB__ + &bmi_ib_ops, +#endif +#ifdef __STATIC_METHOD_BMI_PORTALS__ + &bmi_portals_ops, +#endif + NULL +}; + +struct bmi_method_usage *method_usage = NULL; +static const int usage_iters_starvation = 100000; +static const int usage_iters_active = 10000; + +extern int context_array[BMI_MAX_CONTEXTS]; +extern gen_mutex_t context_mutex; /* - * alloc_method_op() + * bmi_alloc_method_op() * * allocates storage for an operation info struct. * @@ -57,7 +128,7 @@ method_op_p bmi_alloc_method_op(bmi_size } /* - * dealloc_method_op() + * bmi_dealloc_method_op() * * frees the memory allocated to an method_op structure * @@ -71,9 +142,8 @@ void bmi_dealloc_method_op(method_op_p o return; } - /* - * alloc_method_addr() + * bmi_alloc_method_addr() * * alloc_method_addr is used to generate a basic method_addr struct and * initialize it correctly. @@ -107,7 +177,7 @@ struct bmi_method_addr *bmi_alloc_method } /* - * dealloc_method_addr() + * bmi_dealloc_method_addr() * * used to deallocate a method_addr structure safely. mainly used by * list management functions. MAKE SURE that any method specific @@ -123,86 +193,417 @@ void bmi_dealloc_method_addr(bmi_method_ return; } +int bmi_methods_initialize(const char *method_list, + const char *listen_addr, + int flags) +{ + int ret; + char **requested_methods = NULL; + char **listen_addrs = NULL; + char *this_addr = NULL; + char *proto = NULL; + int addr_count = 0; + int i, j; + + /* initialize the known method list from the null-terminated static list */ + known_method_count = sizeof(static_methods) / sizeof(static_methods[0]) - 1; + known_method_table = malloc( + known_method_count * sizeof(*known_method_table)); + if (!known_method_table) + return bmi_errno_to_pvfs(-ENOMEM); + memcpy(known_method_table, static_methods, + known_method_count * sizeof(*known_method_table)); + + gen_mutex_lock(&bmi_active_method_count_mutex); + if (!method_list) { + /* nothing active until lookup */ + bmi_active_method_count = 0; + } else { + /* split and initialize the requested method list */ + int numreq = PINT_split_string_list(&requested_methods, method_list, ','); + if (numreq < 1) + { + gossip_lerr("Error: bad method list.\n"); + ret = bmi_errno_to_pvfs(-EINVAL); + gen_mutex_unlock(&bmi_active_method_count_mutex); + goto init_failure; + } + + addr_count = PINT_split_string_list(&listen_addrs, listen_addr, ','); + + for (i=0; i, and find the + * part + */ + proto = strstr(requested_methods[i], "bmi_"); + if(!proto) + { + gossip_err("%s: Invalid method name: %s. Method names " + "must start with 'bmi_'\n", + __func__, requested_methods[i]); + ret = -PVFS_EINVAL; + gen_mutex_unlock(&bmi_active_method_count_mutex); + goto init_failure; + } + proto += 4; + + /* match the proper listen addr to the method */ + for(j=0; j:// and ends with a null terminator. - * The function strips all whitespace, commas, and address fields which do - * not match the key. - * - * Boy, I sure do hate writing code to parse strings... - * - * returns a pointer to the new string on success, NULL on failure. - */ -char *string_key(const char *key, - const char *id_string) +init_failure: + gen_mutex_lock(&bmi_active_method_count_mutex); + /* look for loaded methods and shut down */ + if (bmi_active_method_table) + { + for (i = 0; i < bmi_active_method_count; i++) + { + if (bmi_active_method_table[i]) + { + bmi_active_method_table[i]->finalize(); + } + } + free(bmi_active_method_table); + } + + if (known_method_table) { + free(known_method_table); + known_method_count = 0; + } + + /* get rid of method string list */ + if (requested_methods) + { + for (i = 0; i < bmi_active_method_count; i++) + { + if (requested_methods[i]) + { + free(requested_methods[i]); + } + } + free(requested_methods); + } + + if(listen_addrs) + { + PINT_free_string_list(listen_addrs, addr_count); + } + + bmi_active_method_count = 0; + gen_mutex_unlock(&bmi_active_method_count_mutex); + return ret; +} + +int bmi_methods_finalize(void) { + int i; - const char *holder = NULL; - const char *end = NULL; - char *newkey = NULL; - char *retstring = NULL; - int keysize = 0; - int strsize = 0; - int retsize = 0; + gen_mutex_lock(&bmi_active_method_count_mutex); + /* attempt to shut down active methods */ + for (i = 0; i < bmi_active_method_count; i++) + { + bmi_active_method_table[i]->finalize(); + } + bmi_active_method_count = 0; + free(bmi_active_method_table); + gen_mutex_unlock(&bmi_active_method_count_mutex); + + free(known_method_table); + known_method_count = 0; + + if (method_usage) + free(method_usage); + + return 0; +} + +struct bmi_method_ops *bmi_method_find(const char *protocol) +{ + int i = 0; + const char *proto; - if ((!id_string) || (!key)) + gen_mutex_lock(&bmi_active_method_count_mutex); + for (i = 0; i < bmi_active_method_count; i++) { - return (NULL); + proto = bmi_active_method_table[i]->method_name + 4; + if(strncmp(proto, protocol, strlen(proto)) == 0) + { + gen_mutex_unlock(&bmi_active_method_count_mutex); + return bmi_active_method_table[i]; + } } - keysize = strlen(key); - strsize = strlen(id_string); + gen_mutex_unlock(&bmi_active_method_count_mutex); + return NULL; +} - /* create a new key of the form :// */ - if ((newkey = (char *) malloc(keysize + 4)) == NULL) +int bmi_method_activate_from_addr(const char *id_string) +{ + int i, ret; + gen_mutex_lock(&bmi_active_method_count_mutex); + for (i=0; i "bmi_x" */ + name = known_method_table[i]->method_name + 4; + if (!strncmp(id_string, name, strlen(name))) { + ret = activate_method(known_method_table[i]->method_name, 0, 0); + if (ret < 0) { + break; + } + } + } + gen_mutex_unlock(&bmi_active_method_count_mutex); + if(ret == 0 && i == known_method_count) { - return (NULL); + return -BMI_ENOPROTOOPT; } - strcpy(newkey, key); - strcat(newkey, "://"); + return ret; +} - holder = id_string; +int bmi_methods_open_context(int context_index) +{ + int i, ret; - holder = strstr(holder, newkey); - /* first match */ - if (holder) - { - end = strpbrk(holder, ", \t\n"); - if (end) - { - end = end; /* stop on terminator */ - } - else - { - end = id_string + strsize; /* go to the end of the id string (\0) */ - } - /* move holder so it doesn't include the opening key and deliminator */ - holder = holder + keysize + 3; + gen_mutex_lock(&bmi_active_method_count_mutex); + /* tell all of the modules about the new context */ + for (i = 0; i < bmi_active_method_count; i++) + { + ret = bmi_active_method_table[i]->open_context(context_index); + if(ret < 0) + { + /* + one of them failed; kill this context in the previous + modules + */ + --i; + while (i >= 0) + { + bmi_active_method_table[i]->close_context( + context_index); + --i; + } + } } - else + gen_mutex_unlock(&bmi_active_method_count_mutex); + return ret; +} + +int bmi_methods_close_context(int context_index) +{ + int i; + + /* tell all of the modules to get rid of this context */ + gen_mutex_lock(&bmi_active_method_count_mutex); + for (i = 0; i < bmi_active_method_count; i++) { - /* no match */ - free(newkey); - return (NULL); + bmi_active_method_table[i]->close_context(context_index); } + gen_mutex_unlock(&bmi_active_method_count_mutex); + + return 0; +} - /* figure out how long our substring is */ - retsize = (end - holder); - if ((retstring = (char *) malloc(retsize + 1)) == NULL) +/* + * If some method was recently active, poll it again for speed, + * but be sure not to starve any method. If multiple active, + * poll them all. Return idle_time per method too. + */ +void bmi_methods_construct_poll_plan(struct bmi_method_usage **musage, + int *idle_time_ms) +{ + int i, numplan; + int tmp_active_method_count; + + numplan = 0; + + gen_mutex_lock(&bmi_active_method_count_mutex); + tmp_active_method_count = bmi_active_method_count; + gen_mutex_unlock(&bmi_active_method_count_mutex); + + for (i=0; i= usage_iters_starvation) { + /* starving, time to poke this one */ + if (0) gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, + "%s: polling starving meth %d: %d / %d\n", __func__, i, + method_usage[i].iters_polled, usage_iters_starvation); + method_usage[i].plan = 1; + ++numplan; + } + } + + /* if nothing is starving or busy, poll everybody */ + if (numplan == 0) { + for (i=0; imethod_name, name)) break; + if (i < bmi_active_method_count) { - free(newkey); - return (NULL); + return 0; + } + + /* is the method known? */ + for (i=0; imethod_name, name)) break; + if (i == known_method_count) { + gossip_lerr("Error: no method available for %s.\n", name); + return -ENOPROTOOPT; + } + meth = known_method_table[i]; + + /* + * Later: try to load a dynamic module, growing the known method + * table and search it again. + */ + + /* toss it into the active table */ + x = bmi_active_method_table; + bmi_active_method_table = malloc( + (bmi_active_method_count + 1) * sizeof(*bmi_active_method_table)); + if (!bmi_active_method_table) { + bmi_active_method_table = x; + return -ENOMEM; + } + if (bmi_active_method_count) { + memcpy(bmi_active_method_table, x, + bmi_active_method_count * sizeof(*bmi_active_method_table)); + free(x); + } + bmi_active_method_table[bmi_active_method_count] = meth; + + x = method_usage; + method_usage = malloc((bmi_active_method_count + 1) * sizeof(*method_usage)); + if (!method_usage) { + method_usage = x; + return -ENOMEM; + } + if (bmi_active_method_count) { + memcpy(method_usage, x, bmi_active_method_count * sizeof(*method_usage)); + free(x); + } + memset(&method_usage[bmi_active_method_count], 0, sizeof(*method_usage)); + + ++bmi_active_method_count; + + /* initialize it */ + new_addr = 0; + if (listen_addr) { + new_addr = meth->method_addr_lookup(listen_addr); + if (!new_addr) { + gossip_err( + "Error: failed to lookup listen address %s for method %s.\n", + listen_addr, name); + --bmi_active_method_count; + return -EINVAL; + } + /* this is a bit of a hack */ + new_addr->method_type = bmi_active_method_count - 1; + } + ret = meth->initialize(new_addr, bmi_active_method_count - 1, flags); + if (ret < 0) { + gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, + "failed to initialize method %s.\n", name); + --bmi_active_method_count; + return ret; } - /* copy it out */ - strncpy(retstring, holder, retsize); - retstring[retsize] = '\0'; + /* tell it about any open contexts */ + for (i=0; iopen_context(i); + if (ret < 0) + break; + } - free(newkey); - return (retstring); + return ret; } /* Index: src/io/bmi/bmi-method-support.h =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi-method-support.h,v retrieving revision 1.30 diff -u -a -p -r1.30 bmi-method-support.h --- src/io/bmi/bmi-method-support.h 6 Nov 2007 23:08:33 -0000 1.30 +++ src/io/bmi/bmi-method-support.h 7 Nov 2007 00:11:24 -0000 @@ -210,6 +210,12 @@ struct method_drop_addr_query int response; }; +struct bmi_method_usage { + int iters_polled; /* how many iterations since this method was polled */ + int iters_active; /* how many iterations since this method had action */ + int plan; +}; + /*********************************************************** * utility functions provided for use by the network methods */ @@ -220,12 +226,22 @@ void bmi_dealloc_method_op(method_op_p o /* These functions can be used to manage generic address structures */ bmi_method_addr_p bmi_alloc_method_addr(int method_type, - bmi_size_t payload_size); + bmi_size_t payload_size); void bmi_dealloc_method_addr(bmi_method_addr_p old_method_addr); -/* string parsing utilities */ -char *string_key(const char *key, - const char *id_string); +int bmi_methods_initialize( + const char *method_list, const char *listen_addr, int flags); +int bmi_methods_finalize(void); + +int bmi_method_activate_from_addr(const char *id_string); + +int bmi_methods_open_context(int context_index); +int bmi_methods_close_context(int context_index); + +void bmi_methods_construct_poll_plan(struct bmi_method_usage **musage, + int *idle_time_ms); + +struct bmi_method_ops *bmi_method_find(const char *protocol); #endif /* __BMI_METHOD_SUPPORT_H */ Index: src/io/bmi/bmi.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi.c,v retrieving revision 1.89 diff -u -a -p -r1.89 bmi.c --- src/io/bmi/bmi.c 6 Nov 2007 23:08:33 -0000 1.89 +++ src/io/bmi/bmi.c 7 Nov 2007 00:11:24 -0000 @@ -21,22 +21,37 @@ #include "bmi-method-support.h" #include "bmi-method-callback.h" #include "gossip.h" -#include "reference-list.h" +#include "bmi-addr.h" #include "op-list.h" #include "gen-locks.h" #include "str-utils.h" #include "id-generator.h" #include "pvfs2-internal.h" +/* Addresses are stored in a hashtable, with each client address added to + * the hashtable for a given server. This value should be about 20% larger + * than the max number of clients expected, so we set it to 100K for now. + */ +#define BMI_CLIENT_ADDR_COUNT (1024 * 128) + +/* For clients, we only need to keep track of the server addresses, which + * will be fewer. Assume no more than a few hundred servers for now. + */ +#define BMI_SERVER_ADDR_COUNT (1024) + /* - * List of BMI addrs currently managed. + * Address map of BMI addrs currently managed. */ -static ref_list_p cur_ref_list = NULL; +static bmi_addr_map_p bmi_addrs_map = NULL; + +extern int bmi_active_method_count; +extern gen_mutex_t bmi_active_method_count_mutex; + +extern struct bmi_method_ops **bmi_active_method_table; /* array to keep up with active contexts */ -static int context_array[BMI_MAX_CONTEXTS] = { 0 }; -static gen_mutex_t context_mutex = GEN_MUTEX_INITIALIZER; -static gen_mutex_t ref_mutex = GEN_MUTEX_INITIALIZER; +int context_array[BMI_MAX_CONTEXTS] = { 0 }; +gen_mutex_t context_mutex = GEN_MUTEX_INITIALIZER; static QLIST_HEAD(forget_list); static gen_mutex_t forget_list_mutex = GEN_MUTEX_INITIALIZER; @@ -47,75 +62,7 @@ struct forget_item PVFS_BMI_addr_t addr; }; -/* - * Static list of defined BMI methods. These are pre-compiled into - * the client libraries and into the server. - */ -#ifdef __STATIC_METHOD_BMI_TCP__ -extern struct bmi_method_ops bmi_tcp_ops; -#endif -#ifdef __STATIC_METHOD_BMI_GM__ -extern struct bmi_method_ops bmi_gm_ops; -#endif -#ifdef __STATIC_METHOD_BMI_MX__ -extern struct bmi_method_ops bmi_mx_ops; -#endif -#ifdef __STATIC_METHOD_BMI_IB__ -extern struct bmi_method_ops bmi_ib_ops; -#endif -#ifdef __STATIC_METHOD_BMI_PORTALS__ -extern struct bmi_method_ops bmi_portals_ops; -#endif - -static struct bmi_method_ops *const static_methods[] = { -#ifdef __STATIC_METHOD_BMI_TCP__ - &bmi_tcp_ops, -#endif -#ifdef __STATIC_METHOD_BMI_GM__ - &bmi_gm_ops, -#endif -#ifdef __STATIC_METHOD_BMI_MX__ - &bmi_mx_ops, -#endif -#ifdef __STATIC_METHOD_BMI_IB__ - &bmi_ib_ops, -#endif -#ifdef __STATIC_METHOD_BMI_PORTALS__ - &bmi_portals_ops, -#endif - NULL -}; - -/* - * List of "known" BMI methods. This is dynamic, starting with - * just the static ones above, and perhaps adding more if we turn - * back on dynamic module loading. - */ -static int known_method_count = 0; -static struct bmi_method_ops **known_method_table = 0; -/* - * List of active BMI methods. These are the ones that will be - * dealt with for a test call, for example. On a client, known methods - * become active only when someone calls BMI_addr_lookup(). On - * a server, all possibly active methods are known at startup time - * because we listen on them for the duration. - */ -static int active_method_count = 0; -static gen_mutex_t active_method_count_mutex = GEN_MUTEX_INITIALIZER; - -static struct bmi_method_ops **active_method_table = NULL; -static struct { - int iters_polled; /* how many iterations since this method was polled */ - int iters_active; /* how many iterations since this method had action */ - int plan; -} *method_usage = NULL; -static const int usage_iters_starvation = 100000; -static const int usage_iters_active = 10000; - -static int activate_method(const char *name, const char *listen_addr, - int flags); -static void bmi_addr_drop(ref_st_p tmp_ref); static void bmi_check_forget_list(void); /** Initializes the BMI layer. Must be called before any other BMI @@ -134,12 +81,6 @@ int BMI_initialize(const char *method_li int flags) { int ret = -1; - int i = 0, j = 0; - char **requested_methods = NULL; - char **listen_addrs = NULL; - char *this_addr = NULL; - char *proto = NULL; - int addr_count = 0; /* server must specify method list at startup, optional for client */ if (flags & BMI_INIT_SERVER) { @@ -153,311 +94,30 @@ int BMI_initialize(const char *method_li } } - /* make a new reference list */ - cur_ref_list = ref_list_new(); - if (!cur_ref_list) - { - ret = bmi_errno_to_pvfs(-ENOMEM); - goto bmi_initialize_failure; - } - - /* initialize the known method list from the null-terminated static list */ - known_method_count = sizeof(static_methods) / sizeof(static_methods[0]) - 1; - known_method_table = malloc( - known_method_count * sizeof(*known_method_table)); - if (!known_method_table) - return bmi_errno_to_pvfs(-ENOMEM); - memcpy(known_method_table, static_methods, - known_method_count * sizeof(*known_method_table)); - - gen_mutex_lock(&active_method_count_mutex); - if (!method_list) { - /* nothing active until lookup */ - active_method_count = 0; - } else { - /* split and initialize the requested method list */ - int numreq = PINT_split_string_list(&requested_methods, method_list); - if (numreq < 1) - { - gossip_lerr("Error: bad method list.\n"); - ret = bmi_errno_to_pvfs(-EINVAL); - gen_mutex_unlock(&active_method_count_mutex); - goto bmi_initialize_failure; - } - - /* Today is that day! */ - addr_count = PINT_split_string_list(&listen_addrs, listen_addr); - - for (i=0; i, and find the - * part - */ - proto = strstr(requested_methods[i], "bmi_"); - if(!proto) - { - gossip_err("%s: Invalid method name: %s. Method names " - "must start with 'bmi_'\n", - __func__, requested_methods[i]); - ret = -PVFS_EINVAL; - gen_mutex_unlock(&active_method_count_mutex); - goto bmi_initialize_failure; - } - proto += 4; - - /* match the proper listen addr to the method */ - for(j=0; jfinalize(); - } - } - free(active_method_table); - } - - if (known_method_table) { - free(known_method_table); - known_method_count = 0; - } - - /* get rid of method string list */ - if (requested_methods) - { - for (i = 0; i < active_method_count; i++) - { - if (requested_methods[i]) - { - free(requested_methods[i]); - } - } - free(requested_methods); - } - - if(listen_addrs) - { - PINT_free_string_list(listen_addrs, addr_count); - } - - active_method_count = 0; - gen_mutex_unlock(&active_method_count_mutex); - - return (ret); -} - -/* the following is the old BMI_initialize() function that used dl to - * pull in method modules dynamically. Just hanging around as an - * example... - */ -#if 0 -/* BMI_initialize() - * - * Initializes the BMI layer. Must be called before any other BMI - * functions. module_string is a comma separated list of BMI modules to - * use, listen_addr is a comma separated list of addresses to listen on - * for each module (if needed), and flags are initialization flags. - * - * returns 0 on success, -errno on failure - */ -int BMI_initialize(const char *module_string, - const char *listen_addr, - int flags) -{ - - int ret = -1; - int i = 0; - char **modules = NULL; - void *meth_mod = NULL; - char *mod_error = NULL; - method_addr_p new_addr = NULL; - op_list_p olp = NULL; - - /* TODO: this is a hack to make sure we get all of the symbols loaded - * into the library... is there a better way? - */ - olp = op_list_new(); - op_list_cleanup(olp); - - if (((flags & BMI_INIT_SERVER) && (!listen_addr)) || !module_string) - { - return (bmi_errno_to_pvfs(-EINVAL)); - } - - /* separate out the module list */ - active_method_count = PINT_split_string_list( - &modules, module_string); - if (active_method_count < 1) - { - gossip_lerr("Error: bad module list.\n"); - ret = bmi_errno_to_pvfs(-EINVAL); - goto bmi_initialize_failure; - } - - /* create a table to keep up with the method modules */ - active_method_table = (struct bmi_method_ops **)malloc( - active_method_count * sizeof(struct bmi_method_ops *)); - if (!active_method_table) - { - ret = bmi_errno_to_pvfs(-ENOMEM); - goto bmi_initialize_failure; - } - - /* iterate through each method in the list and load its module */ - for (i = 0; i < active_method_count; i++) - { - meth_mod = dlopen(modules[i], RTLD_NOW); - if (!meth_mod) - { - gossip_lerr("Error: could not open module: %s\n", dlerror()); - ret = bmi_errno_to_pvfs(-EINVAL); - goto bmi_initialize_failure; - } - dlerror(); - - active_method_table[i] = (struct bmi_method_ops *) - dlsym(meth_mod, "method_interface"); - mod_error = dlerror(); - if (mod_error) - { - gossip_lerr("Error: module load: %s\n", mod_error); - ret = bmi_errno_to_pvfs(-EINVAL); - goto bmi_initialize_failure; - } - } + bmi_methods_initialize(method_list, listen_addr, flags); /* make a new reference list */ - cur_ref_list = ref_list_new(); - if (!cur_ref_list) + bmi_addrs_map = bmi_addr_map_new( + (flags & BMI_INIT_SERVER) ? + BMI_CLIENT_ADDR_COUNT : BMI_SERVER_ADDR_COUNT); + if (!bmi_addrs_map) { ret = bmi_errno_to_pvfs(-ENOMEM); goto bmi_initialize_failure; } - /* initialize methods */ - for (i = 0; i < active_method_count; i++) - { - if (flags & BMI_INIT_SERVER) - { - if ((new_addr = - active_method_table[i]-> - BMI_meth_method_addr_lookup(listen_addr)) != NULL) - { - /* this is a bit of a hack */ - new_addr->method_type = i; - ret = active_method_table[i]->BMI_meth_initialize( - new_addr, i, flags); - } - else - { - ret = -1; - } - } - else - { - ret = active_method_table[i]->BMI_meth_initialize( - NULL, i, flags); - } - if (ret < 0) - { - gossip_lerr("Error: initializing module: %s\n", modules[i]); - goto bmi_initialize_failure; - } - } - return (0); bmi_initialize_failure: - /* kill reference list */ - if (cur_ref_list) + if (bmi_addrs_map) { - ref_list_cleanup(cur_ref_list); - } - - /* look for loaded methods and shut down */ - if (active_method_table) - { - for (i = 0; i < active_method_count; i++) - { - if (active_method_table[i]) - { - active_method_table[i]->BMI_meth_finalize(); - } - } - free(active_method_table); - } - - /* get rid of method string list */ - if (modules) - { - for (i = 0; i < active_method_count; i++) - { - if (modules[i]) - { - free(modules[i]); - } - } - free(modules); + bmi_addr_map_destroy(bmi_addrs_map); + bmi_addrs_map = NULL; } return (ret); } -#endif /* 0 */ /** Shuts down the BMI layer. * @@ -465,28 +125,10 @@ int BMI_initialize(const char *module_st */ int BMI_finalize(void) { - int i = -1; - - gen_mutex_lock(&active_method_count_mutex); - /* attempt to shut down active methods */ - for (i = 0; i < active_method_count; i++) - { - active_method_table[i]->finalize(); - } - active_method_count = 0; - free(active_method_table); - gen_mutex_unlock(&active_method_count_mutex); - - free(known_method_table); - known_method_count = 0; - - if (method_usage) - free(method_usage); - /* destroy the reference list */ /* (side effect: destroys all method addresses as well) */ - ref_list_cleanup(cur_ref_list); - + bmi_addr_map_destroy(bmi_addrs_map); + bmi_methods_finalize(); return (0); } @@ -499,7 +141,6 @@ int BMI_finalize(void) int BMI_open_context(bmi_context_id* context_id) { int context_index; - int i; int ret = 0; gen_mutex_lock(&context_mutex); @@ -520,29 +161,11 @@ int BMI_open_context(bmi_context_id* con return(bmi_errno_to_pvfs(-EBUSY)); } - gen_mutex_lock(&active_method_count_mutex); - /* tell all of the modules about the new context */ - for (i = 0; i < active_method_count; i++) - { - ret = active_method_table[i]->open_context( - context_index); - if(ret < 0) - { - /* - one of them failed; kill this context in the previous - modules - */ - --i; - while (i >= 0) - { - active_method_table[i]->close_context( - context_index); - --i; - } - goto out; - } + ret = bmi_methods_open_context(context_index); + if(ret < 0) + { + goto out; } - gen_mutex_unlock(&active_method_count_mutex); context_array[context_index] = 1; *context_id = context_index; @@ -553,13 +176,10 @@ out: return(ret); } - /** Destroys a context previous generated with BMI_open_context(). */ void BMI_close_context(bmi_context_id context_id) { - int i; - gen_mutex_lock(&context_mutex); if(!context_array[context_id]) @@ -568,15 +188,8 @@ void BMI_close_context(bmi_context_id co return; } - /* tell all of the modules to get rid of this context */ - gen_mutex_lock(&active_method_count_mutex); - for (i = 0; i < active_method_count; i++) - { - active_method_table[i]->close_context(context_id); - } + bmi_methods_close_context(context_id); context_array[context_id] = 0; - gen_mutex_unlock(&active_method_count_mutex); - gen_mutex_unlock(&context_mutex); return; } @@ -596,7 +209,7 @@ int BMI_post_recv(bmi_op_id_t * id, void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; + struct bmi_method_addr *method_addr; int ret = -1; gossip_debug(GOSSIP_BMI_DEBUG_OFFSETS, @@ -605,17 +218,13 @@ int BMI_post_recv(bmi_op_id_t * id, *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, src); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(src); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); + return -BMI_EINVAL; } - gen_mutex_unlock(&ref_mutex); - - ret = tmp_ref->interface->post_recv( - id, tmp_ref->method_addr, buffer, expected_size, actual_size, + ret = method_addr->ops->post_recv( + id, method_addr, buffer, expected_size, actual_size, buffer_type, tag, user_ptr, context_id); return (ret); } @@ -634,7 +243,7 @@ int BMI_post_send(bmi_op_id_t * id, void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; + struct bmi_method_addr *method_addr; int ret = -1; gossip_debug(GOSSIP_BMI_DEBUG_OFFSETS, @@ -643,17 +252,14 @@ int BMI_post_send(bmi_op_id_t * id, *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, dest); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(dest); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); + return -BMI_EINVAL; } - gen_mutex_unlock(&ref_mutex); - ret = tmp_ref->interface->post_send( - id, tmp_ref->method_addr, buffer, size, buffer_type, tag, + ret = method_addr->ops->post_send( + id, method_addr, buffer, size, buffer_type, tag, user_ptr, context_id); return (ret); } @@ -672,7 +278,7 @@ int BMI_post_sendunexpected(bmi_op_id_t void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; + struct bmi_method_addr *method_addr; int ret = -1; gossip_debug(GOSSIP_BMI_DEBUG_OFFSETS, @@ -681,17 +287,14 @@ int BMI_post_sendunexpected(bmi_op_id_t *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, dest); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(dest); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); + return -BMI_EINVAL; } - gen_mutex_unlock(&ref_mutex); - ret = tmp_ref->interface->post_sendunexpected( - id, tmp_ref->method_addr, buffer, size, buffer_type, tag, + ret = method_addr->ops->post_sendunexpected( + id, method_addr, buffer, size, buffer_type, tag, user_ptr, context_id); return (ret); } @@ -720,8 +323,7 @@ int BMI_test(bmi_op_id_t id, target_op = id_gen_fast_lookup(id); assert(target_op->op_id == id); - ret = active_method_table[ - target_op->addr->method_type]->test( + ret = target_op->addr->ops->test( id, outcount, error_code, actual_size, user_ptr, max_idle_time_ms, context_id); @@ -744,14 +346,14 @@ int BMI_test(bmi_op_id_t id, * if it becomes used again. */ int BMI_testsome(int incount, - bmi_op_id_t * id_array, - int *outcount, - int *index_array, - bmi_error_code_t * error_code_array, - bmi_size_t * actual_size_array, - void **user_ptr_array, - int max_idle_time_ms, - bmi_context_id context_id) + bmi_op_id_t * id_array, + int *outcount, + int *index_array, + bmi_error_code_t * error_code_array, + bmi_size_t * actual_size_array, + void **user_ptr_array, + int max_idle_time_ms, + bmi_context_id context_id) { int ret = 0; int idle_per_method = 0; @@ -762,9 +364,9 @@ int BMI_testsome(int incount, int tmp_outcount; int tmp_active_method_count; - gen_mutex_lock(&active_method_count_mutex); - tmp_active_method_count = active_method_count; - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_lock(&bmi_active_method_count_mutex); + tmp_active_method_count = bmi_active_method_count; + gen_mutex_unlock(&bmi_active_method_count_mutex); if (max_idle_time_ms < 0) return (bmi_errno_to_pvfs(-EINVAL)); @@ -773,7 +375,7 @@ int BMI_testsome(int incount, if (tmp_active_method_count == 1) { /* shortcircuit for perhaps common case of only one method */ - ret = active_method_table[0]->testsome( + ret = bmi_active_method_table[0]->testsome( incount, id_array, outcount, index_array, error_code_array, actual_size_array, user_ptr_array, max_idle_time_ms, context_id); @@ -823,7 +425,7 @@ int BMI_testsome(int incount, if(need_to_test) { tmp_outcount = 0; - ret = active_method_table[i]->testsome( + ret = bmi_active_method_table[i]->testsome( need_to_test, tmp_id_array, &tmp_outcount, &(index_array[*outcount]), &(error_code_array[*outcount]), @@ -850,60 +452,6 @@ int BMI_testsome(int incount, return(0); } - -/* - * If some method was recently active, poll it again for speed, - * but be sure not to starve any method. If multiple active, - * poll them all. Return idle_time per method too. - */ -static void -construct_poll_plan(int nmeth, int *idle_time_ms) -{ - int i, numplan; - - numplan = 0; - for (i=0; i= usage_iters_starvation) { - /* starving, time to poke this one */ - if (0) gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, - "%s: polling starving meth %d: %d / %d\n", __func__, i, - method_usage[i].iters_polled, usage_iters_starvation); - method_usage[i].plan = 1; - ++numplan; - } - } - - /* if nothing is starving or busy, poll everybody */ - if (numplan == 0) { - for (i=0; itestunexpected( + ret = bmi_active_method_table[i]->testunexpected( (incount - position), &tmp_outcount, (&(sub_info[position])), max_idle_time_ms); if (ret < 0) @@ -962,18 +506,8 @@ int BMI_testunexpected(int incount, info_array[i].buffer = sub_info[i].buffer; info_array[i].size = sub_info[i].size; info_array[i].tag = sub_info[i].tag; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_method_addr( - cur_ref_list, sub_info[i].addr); - if (!tmp_ref) - { - /* yeah, right */ - gossip_lerr("Error: critical BMI_testunexpected failure.\n"); - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); - } - gen_mutex_unlock(&ref_mutex); - info_array[i].addr = tmp_ref->bmi_addr; + info_array[i].addr = bmi_addr_method_search( + bmi_addrs_map, sub_info[i].addr); } /* return 1 if anything completed */ if (ret == 0 && *outcount > 0) @@ -1004,10 +538,11 @@ int BMI_testcontext(int incount, int tmp_outcount = 0; int tmp_active_method_count = 0; struct timespec ts; + struct bmi_method_usage *method_usage; - gen_mutex_lock(&active_method_count_mutex); - tmp_active_method_count = active_method_count; - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_lock(&bmi_active_method_count_mutex); + tmp_active_method_count = bmi_active_method_count; + gen_mutex_unlock(&bmi_active_method_count_mutex); if (max_idle_time_ms < 0) return (bmi_errno_to_pvfs(-EINVAL)); @@ -1026,12 +561,12 @@ int BMI_testcontext(int incount, return(0); } - construct_poll_plan(tmp_active_method_count, &max_idle_time_ms); + bmi_methods_construct_poll_plan(&method_usage, &max_idle_time_ms); while (position < incount && i < tmp_active_method_count) { if (method_usage[i].plan) { - ret = active_method_table[i]->testcontext( + ret = bmi_active_method_table[i]->testcontext( incount - position, &out_id_array[position], &tmp_outcount, @@ -1079,22 +614,8 @@ int BMI_testcontext(int incount, */ const char* BMI_addr_rev_lookup(PVFS_BMI_addr_t addr) { - ref_st_p tmp_ref = NULL; - char* tmp_str = NULL; - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (NULL); - } - gen_mutex_unlock(&ref_mutex); - - tmp_str = tmp_ref->id_string; - - return(tmp_str); + return bmi_addr_current_id(addr); } /** Performs a reverse lookup, returning a string @@ -1108,25 +629,15 @@ const char* BMI_addr_rev_lookup(PVFS_BMI */ const char* BMI_addr_rev_lookup_unexpected(PVFS_BMI_addr_t addr) { - ref_st_p tmp_ref = NULL; + bmi_method_addr_p method_addr; - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(addr); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return ("UNKNOWN"); - } - gen_mutex_unlock(&ref_mutex); - - if(!tmp_ref->interface->rev_lookup_unexpected) - { - return("UNKNOWN"); + return "UNKNOWN"; } - return(tmp_ref->interface->rev_lookup_unexpected( - tmp_ref->method_addr)); + return method_addr->ops->rev_lookup_unexpected(method_addr); } @@ -1139,20 +650,16 @@ void *BMI_memalloc(PVFS_BMI_addr_t addr, enum bmi_op_type send_recv) { void *new_buffer = NULL; - ref_st_p tmp_ref = NULL; + bmi_method_addr_p method_addr; - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(addr); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (NULL); + return NULL; } - gen_mutex_unlock(&ref_mutex); /* allocate the buffer using the method's mechanism */ - new_buffer = tmp_ref->interface->memalloc(size, send_recv); + new_buffer = method_addr->ops->memalloc(size, send_recv); return (new_buffer); } @@ -1166,23 +673,16 @@ int BMI_memfree(PVFS_BMI_addr_t addr, bmi_size_t size, enum bmi_op_type send_recv) { - ref_st_p tmp_ref = NULL; - int ret = -1; + bmi_method_addr_p method_addr; - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(addr); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EINVAL)); + return -PVFS_EINVAL; } - gen_mutex_unlock(&ref_mutex); /* free the memory */ - ret = tmp_ref->interface->memfree(buffer, size, send_recv); - - return (ret); + return method_addr->ops->memfree(buffer, size, send_recv); } /** Acknowledge that an unexpected message has been @@ -1191,29 +691,24 @@ int BMI_memfree(PVFS_BMI_addr_t addr, * \return 0 on success, -errno on failure. */ int BMI_unexpected_free(PVFS_BMI_addr_t addr, - void *buffer) + void *buffer) { - ref_st_p tmp_ref = NULL; - int ret = -1; + bmi_method_addr_p method_addr; + int ret; - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(addr); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EINVAL)); + return -PVFS_EINVAL; } - gen_mutex_unlock(&ref_mutex); - if (!tmp_ref->interface->unexpected_free) + if (method_addr->ops->unexpected_free) { gossip_err("unimplemented unexpected_free callback\n"); return bmi_errno_to_pvfs(-EOPNOTSUPP); } /* free the memory */ - ret = tmp_ref->interface->unexpected_free(buffer); - + ret = method_addr->ops->unexpected_free(buffer); return (ret); } @@ -1227,7 +722,6 @@ int BMI_set_info(PVFS_BMI_addr_t addr, { int ret = -1; int i = 0; - ref_st_p tmp_ref = NULL; gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, "[BMI CONTROL]: %s: set_info: %llu option: %d\n", @@ -1237,90 +731,63 @@ int BMI_set_info(PVFS_BMI_addr_t addr, */ if (!addr) { - if (!active_method_table) + if (!bmi_active_method_table) { return (bmi_errno_to_pvfs(-EINVAL)); } - gen_mutex_lock(&active_method_count_mutex); - for (i = 0; i < active_method_count; i++) + gen_mutex_lock(&bmi_active_method_count_mutex); + for (i = 0; i < bmi_active_method_count; i++) { - ret = active_method_table[i]->set_info( - option, inout_parameter); + ret = bmi_active_method_table[i]->set_info(option, inout_parameter); /* we bail out if even a single set_info fails */ if (ret < 0) { gossip_lerr( "Error: failure on set_info to method: %d\n", i); - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_unlock(&bmi_active_method_count_mutex); return (ret); } } - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_unlock(&bmi_active_method_count_mutex); return (0); } gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, "[BMI CONTROL]: %s: searching for ref %llu\n", __func__, llu(addr)); - /* find a reference that matches this address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EINVAL)); - } /* shortcut address reference counting */ if(option == BMI_INC_ADDR_REF) { - tmp_ref->ref_count++; - gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, - "[BMI CONTROL]: %s: incremented ref %llu to: %d\n", - __func__, llu(addr), tmp_ref->ref_count); - gen_mutex_unlock(&ref_mutex); - return(0); + bmi_addr_refcount_increment(bmi_addrs_map, addr); + return(0); } if(option == BMI_DEC_ADDR_REF) { - tmp_ref->ref_count--; - gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, - "[BMI CONTROL]: %s: decremented ref %llu to: %d\n", - __func__, llu(addr), tmp_ref->ref_count); - assert(tmp_ref->ref_count >= 0); - - if(tmp_ref->ref_count == 0) - { - bmi_addr_drop(tmp_ref); - } - gen_mutex_unlock(&ref_mutex); - return(0); + bmi_addr_refcount_decrement(bmi_addrs_map, addr); + return(0); } /* if the caller requests a TCP specific close socket action */ if (option == BMI_TCP_CLOSE_SOCKET) { /* check to see if the address is in fact a tcp address */ - if(strcmp(tmp_ref->interface->method_name, "bmi_tcp") == 0) + if(strcmp(bmi_addr_get_method(addr)->method_name, "bmi_tcp") == 0) { /* take the same action as in the BMI_DEC_ADDR_REF case to clean * out the entire address structure and anything linked to it so * that the next addr_lookup starts from scratch */ - gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, + gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, "[BMI CONTROL]: %s: Closing bmi_tcp " "connection at caller's request.\n", __func__); - ref_list_rem(cur_ref_list, addr); - dealloc_ref_st(tmp_ref); + bmi_addr_drop_endpoint(bmi_addrs_map, addr, 0); } - gen_mutex_unlock(&ref_mutex); return 0; } - gen_mutex_unlock(&ref_mutex); - - ret = tmp_ref->interface->set_info(option, inout_parameter); + ret = bmi_addr_get_method(addr)->set_info(option, inout_parameter); return (ret); } @@ -1337,28 +804,28 @@ int BMI_get_info(PVFS_BMI_addr_t addr, int maxsize = 0; int tmp_maxsize; int ret = 0; - ref_st_p tmp_ref = NULL; + struct bmi_method_ops *ops; switch (option) { /* check to see if the interface is initialized */ case BMI_CHECK_INIT: - gen_mutex_lock(&active_method_count_mutex); - if (active_method_count > 0) + gen_mutex_lock(&bmi_active_method_count_mutex); + if (bmi_active_method_count > 0) { - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_unlock(&bmi_active_method_count_mutex); return (0); } else { - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_unlock(&bmi_active_method_count_mutex); return (bmi_errno_to_pvfs(-ENETDOWN)); } case BMI_CHECK_MAXSIZE: - gen_mutex_lock(&active_method_count_mutex); - for (i = 0; i < active_method_count; i++) + gen_mutex_lock(&bmi_active_method_count_mutex); + for (i = 0; i < bmi_active_method_count; i++) { - ret = active_method_table[i]->get_info( + ret = bmi_active_method_table[i]->get_info( option, &tmp_maxsize); if (ret < 0) { @@ -1375,30 +842,11 @@ int BMI_get_info(PVFS_BMI_addr_t addr, } *((int *) inout_parameter) = maxsize; } - gen_mutex_unlock(&active_method_count_mutex); - break; - case BMI_GET_METH_ADDR: - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if(!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EINVAL)); - } - gen_mutex_unlock(&ref_mutex); - *((void**) inout_parameter) = tmp_ref->method_addr; + gen_mutex_unlock(&bmi_active_method_count_mutex); break; case BMI_GET_UNEXP_SIZE: - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if(!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EINVAL)); - } - gen_mutex_unlock(&ref_mutex); - ret = tmp_ref->interface->get_info( - option, inout_parameter); + ops = bmi_addr_get_method(addr); + ret = ops->get_info(option, inout_parameter); if(ret < 0) { return ret; @@ -1406,7 +854,7 @@ int BMI_get_info(PVFS_BMI_addr_t addr, break; default: - return (bmi_errno_to_pvfs(-ENOSYS)); + return (bmi_errno_to_pvfs(-ENOSYS)); } return (0); } @@ -1423,21 +871,18 @@ int BMI_query_addr_range (PVFS_BMI_addr_ int i = 0, failed = 1; int provided_method_length = 0; char *ptr, *provided_method_name = NULL; - ref_st_p tmp_ref = NULL; + struct bmi_method_addr *method_addr; if((strlen(id_string)+1) > BMI_MAX_ADDR_LEN) { return(bmi_errno_to_pvfs(-ENAMETOOLONG)); } /* lookup the provided address */ - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, addr); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(addr); + if(!method_addr) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); + return -PVFS_EINVAL; } - gen_mutex_unlock(&ref_mutex); ptr = strchr(id_string, ':'); if (ptr == NULL) @@ -1457,16 +902,16 @@ int BMI_query_addr_range (PVFS_BMI_addr_ * matches the specified wildcard address. */ i = 0; - gen_mutex_lock(&active_method_count_mutex); - while (i < active_method_count) + gen_mutex_lock(&bmi_active_method_count_mutex); + while (i < bmi_active_method_count) { - const char *active_method_name = active_method_table[i]->method_name + 4; + const char *active_method_name = bmi_active_method_table[i]->method_name + 4; /* provided name matches this interface */ if (!strncmp(active_method_name, provided_method_name, provided_method_length)) { int (*meth_fnptr)(bmi_method_addr_p, const char *, int); failed = 0; - if ((meth_fnptr = active_method_table[i]->query_addr_range) == NULL) + if ((meth_fnptr = bmi_active_method_table[i]->query_addr_range) == NULL) { ret = -ENOSYS; gossip_lerr("Error: method doesn't implement querying address range/wildcards! Cannot implement FS export options!\n"); @@ -1474,20 +919,25 @@ int BMI_query_addr_range (PVFS_BMI_addr_ break; } /* pass it into the specific bmi layer */ - ret = meth_fnptr(tmp_ref->method_addr, id_string, netmask); + ret = meth_fnptr(method_addr, id_string, netmask); if (ret < 0) failed = 1; break; } i++; } - gen_mutex_unlock(&active_method_count_mutex); + gen_mutex_unlock(&bmi_active_method_count_mutex); free(provided_method_name); if (failed) return bmi_errno_to_pvfs(ret); return ret; } +int BMI_endpoint_fail_current_addr(PVFS_BMI_addr_t addr) +{ + return bmi_addr_next(bmi_addrs_map, addr); +} + /** Resolves the string representation of a host address into a BMI * address handle. * @@ -1496,128 +946,41 @@ int BMI_query_addr_range (PVFS_BMI_addr_ int BMI_addr_lookup(PVFS_BMI_addr_t * new_addr, const char *id_string) { - - ref_st_p new_ref = NULL; bmi_method_addr_p meth_addr = NULL; int ret = -1; int i = 0; - int failed; if((strlen(id_string)+1) > BMI_MAX_ADDR_LEN) { return(bmi_errno_to_pvfs(-ENAMETOOLONG)); } - /* set the addr to zero in case we fail */ - *new_addr = 0; - /* First we want to check to see if this host has already been * discovered! */ - gen_mutex_lock(&ref_mutex); - new_ref = ref_list_search_str(cur_ref_list, id_string); - gen_mutex_unlock(&ref_mutex); - - if (new_ref) - { - /* we found it. */ - *new_addr = new_ref->bmi_addr; - return (0); - } - - /* Now we will run through each method looking for one that - * responds successfully. It is assumed that they are already - * listed in order of preference - */ - i = 0; - gen_mutex_lock(&active_method_count_mutex); - while ((i < active_method_count) && - !(meth_addr = active_method_table[i]->method_addr_lookup(id_string))) - { - i++; - } - - /* if not found, try to bring it up now */ - failed = 0; - if (!meth_addr) { - for (i=0; i "bmi_x" */ - name = known_method_table[i]->method_name + 4; - if (!strncmp(id_string, name, strlen(name))) { - ret = activate_method(known_method_table[i]->method_name, 0, 0); - if (ret < 0) { - failed = 1; - break; - } - meth_addr = known_method_table[i]-> - method_addr_lookup(id_string + strlen(name) + 3); - i = active_method_count - 1; /* point at the new one */ - break; - } - } - } - gen_mutex_unlock(&active_method_count_mutex); - if (failed) - return bmi_errno_to_pvfs(ret); - - /* make sure one was successful */ - if (!meth_addr) + *new_addr = bmi_addr_id_search(bmi_addrs_map, id_string); + if(*new_addr != 0) { - return bmi_errno_to_pvfs(-ENOPROTOOPT); - } - - /* create a new reference for the addr */ - new_ref = alloc_ref_st(); - if (!new_ref) - { - ret = bmi_errno_to_pvfs(-ENOMEM); - goto bmi_addr_lookup_failure; + return 0; } - /* fill in the details */ - new_ref->method_addr = meth_addr; - new_ref->id_string = (char *) malloc(strlen(id_string) + 1); - if (!new_ref->id_string) + ret = bmi_addr_add_endpoint(bmi_addrs_map, id_string, new_addr); + if(ret != 0) { - ret = bmi_errno_to_pvfs(errno); - goto bmi_addr_lookup_failure; + goto bmi_addr_lookup_failure; } - strcpy(new_ref->id_string, id_string); - new_ref->interface = active_method_table[i]; - /* keep up with the reference and we are done */ - gen_mutex_lock(&ref_mutex); - ref_list_add(cur_ref_list, new_ref); - gen_mutex_unlock(&ref_mutex); - - *new_addr = new_ref->bmi_addr; return (0); bmi_addr_lookup_failure: if (meth_addr) { - active_method_table[i]->set_info( - BMI_DROP_ADDR, meth_addr); - } - - if (new_ref) - { - dealloc_ref_st(new_ref); + bmi_active_method_table[i]->set_info(BMI_DROP_ADDR, meth_addr); } return (ret); } - /** Similar to BMI_post_send(), except that the source buffer is * replaced by a list of (possibly non contiguous) buffers. * @@ -1636,8 +999,9 @@ int BMI_post_send_list(bmi_op_id_t * id, void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; int ret = -1; + struct bmi_method_addr *method_addr; + struct bmi_method_ops *ops; #ifndef GOSSIP_DISABLE_DEBUG int i; @@ -1656,19 +1020,12 @@ int BMI_post_send_list(bmi_op_id_t * id, *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, dest); - if (!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); - } - gen_mutex_unlock(&ref_mutex); - - if (tmp_ref->interface->post_send_list) + method_addr = bmi_addr_current_method_addr(dest); + ops = bmi_addr_get_method(dest); + if (ops->post_send_list) { - ret = tmp_ref->interface->post_send_list( - id, tmp_ref->method_addr, buffer_list, size_list, + ret = ops->post_send_list( + id, method_addr, buffer_list, size_list, list_count, total_size, buffer_type, tag, user_ptr, context_id); @@ -1703,8 +1060,9 @@ int BMI_post_recv_list(bmi_op_id_t * id, void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; int ret = -1; + struct bmi_method_addr *method_addr; + struct bmi_method_ops *ops; #ifndef GOSSIP_DISABLE_DEBUG int i; @@ -1723,19 +1081,12 @@ int BMI_post_recv_list(bmi_op_id_t * id, *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, src); - if (!tmp_ref) + method_addr = bmi_addr_current_method_addr(src); + ops = bmi_addr_get_method(src); + if (ops->post_recv_list) { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); - } - gen_mutex_unlock(&ref_mutex); - - if (tmp_ref->interface->post_recv_list) - { - ret = tmp_ref->interface->post_recv_list( - id, tmp_ref->method_addr, buffer_list, size_list, + ret = ops->post_recv_list( + id, method_addr, buffer_list, size_list, list_count, total_expected_size, total_actual_size, buffer_type, tag, user_ptr, context_id); @@ -1768,8 +1119,9 @@ int BMI_post_sendunexpected_list(bmi_op_ void *user_ptr, bmi_context_id context_id) { - ref_st_p tmp_ref = NULL; int ret = -1; + struct bmi_method_addr *method_addr; + struct bmi_method_ops *ops; #ifndef GOSSIP_DISABLE_DEBUG int i; @@ -1789,19 +1141,12 @@ int BMI_post_sendunexpected_list(bmi_op_ *id = 0; - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, dest); - if (!tmp_ref) - { - gen_mutex_unlock(&ref_mutex); - return (bmi_errno_to_pvfs(-EPROTO)); - } - gen_mutex_unlock(&ref_mutex); - - if (tmp_ref->interface->post_send_list) + method_addr = bmi_addr_current_method_addr(dest); + ops = bmi_addr_get_method(dest); + if (ops->post_send_list) { - ret = tmp_ref->interface->post_sendunexpected_list( - id, tmp_ref->method_addr, buffer_list, size_list, + ret = ops->post_sendunexpected_list( + id, method_addr, buffer_list, size_list, list_count, total_size, buffer_type, tag, user_ptr, context_id); @@ -1841,10 +1186,9 @@ int BMI_cancel(bmi_op_id_t id, assert(target_op->op_id == id); - if(active_method_table[target_op->addr->method_type]->cancel) + if(bmi_active_method_table[target_op->addr->method_type]->cancel) { - ret = active_method_table[ - target_op->addr->method_type]->cancel( + ret = bmi_active_method_table[target_op->addr->method_type]->cancel( id, context_id); } else @@ -1876,33 +1220,22 @@ int BMI_cancel(bmi_op_id_t id, */ PVFS_BMI_addr_t bmi_method_addr_reg_callback(bmi_method_addr_p map) { - ref_st_p new_ref = NULL; + PVFS_BMI_addr_t new_addr; + int ret; /* NOTE: we are trusting the method to make sure that we really * don't know about the address yet. No verification done here. */ - /* create a new reference structure */ - new_ref = alloc_ref_st(); - if (!new_ref) + ret = bmi_addr_add_method(bmi_addrs_map, map, &new_addr); + if(ret != 0) { - return 0; + gossip_err("%s: failed to register address in callback: %d\n", + __func__, ret); + return -1; } - /* - fill in the details; we don't have an id string for this one. - */ - new_ref->method_addr = map; - new_ref->id_string = NULL; - - /* check the method_type from the method_addr pointer to know - * which interface to use */ - new_ref->interface = active_method_table[map->method_type]; - - /* add the reference structure to the list */ - ref_list_add(cur_ref_list, new_ref); - - return new_ref->bmi_addr; + return new_addr; } int bmi_method_addr_forget_callback(PVFS_BMI_addr_t addr) @@ -1927,104 +1260,6 @@ int bmi_method_addr_forget_callback(PVFS return (0); } -/* - * Attempt to insert this name into the list of active methods, - * and bring it up. - * NOTE: assumes caller has protected active_method_count with a mutex lock - */ -static int -activate_method(const char *name, const char *listen_addr, int flags) -{ - int i, ret; - void *x; - struct bmi_method_ops *meth; - bmi_method_addr_p new_addr; - - /* already active? */ - for (i=0; imethod_name, name)) break; - if (i < active_method_count) - { - return 0; - } - - /* is the method known? */ - for (i=0; imethod_name, name)) break; - if (i == known_method_count) { - gossip_lerr("Error: no method available for %s.\n", name); - return -ENOPROTOOPT; - } - meth = known_method_table[i]; - - /* - * Later: try to load a dynamic module, growing the known method - * table and search it again. - */ - - /* toss it into the active table */ - x = active_method_table; - active_method_table = malloc( - (active_method_count + 1) * sizeof(*active_method_table)); - if (!active_method_table) { - active_method_table = x; - return -ENOMEM; - } - if (active_method_count) { - memcpy(active_method_table, x, - active_method_count * sizeof(*active_method_table)); - free(x); - } - active_method_table[active_method_count] = meth; - - x = method_usage; - method_usage = malloc((active_method_count + 1) * sizeof(*method_usage)); - if (!method_usage) { - method_usage = x; - return -ENOMEM; - } - if (active_method_count) { - memcpy(method_usage, x, active_method_count * sizeof(*method_usage)); - free(x); - } - memset(&method_usage[active_method_count], 0, sizeof(*method_usage)); - - ++active_method_count; - - /* initialize it */ - new_addr = 0; - if (listen_addr) { - new_addr = meth->method_addr_lookup(listen_addr); - if (!new_addr) { - gossip_err( - "Error: failed to lookup listen address %s for method %s.\n", - listen_addr, name); - --active_method_count; - return -EINVAL; - } - /* this is a bit of a hack */ - new_addr->method_type = active_method_count - 1; - } - ret = meth->initialize(new_addr, active_method_count - 1, flags); - if (ret < 0) { - gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, - "failed to initialize method %s.\n", name); - --active_method_count; - return ret; - } - - /* tell it about any open contexts */ - for (i=0; iopen_context(i); - if (ret < 0) - break; - } - - return ret; -} - - int bmi_errno_to_pvfs(int error) { int bmi_errno = error; @@ -2102,13 +1337,12 @@ static void bmi_check_forget_list(void) { PVFS_BMI_addr_t tmp_addr; struct forget_item* tmp_item; - ref_st_p tmp_ref = NULL; - + gen_mutex_lock(&forget_list_mutex); while(!qlist_empty(&forget_list)) { tmp_item = qlist_entry(forget_list.next, struct forget_item, - link); + link); qlist_del(&tmp_item->link); /* item is off of the list; unlock for a moment while we work on * this addr @@ -2117,53 +1351,15 @@ static void bmi_check_forget_list(void) tmp_addr = tmp_item->addr; free(tmp_item); - gen_mutex_lock(&ref_mutex); - tmp_ref = ref_list_search_addr(cur_ref_list, tmp_addr); - if(tmp_ref && tmp_ref->ref_count == 0) - { - bmi_addr_drop(tmp_ref); - } - gen_mutex_unlock(&ref_mutex); - + /* we ignore an EINVAL from addr_drop at this point, a non-zero + * ref count just means that there are still references to + * this address. + */ + bmi_addr_drop_endpoint(bmi_addrs_map, tmp_addr, 0); gen_mutex_lock(&forget_list_mutex); } gen_mutex_unlock(&forget_list_mutex); - return; -} - -/* bmi_addr_drop - * - * Destroys a complete BMI address, including asking the method to clean up - * its portion. Will query the method for permission before proceeding - * - * NOTE: must be called with ref list mutex held - */ -static void bmi_addr_drop(ref_st_p tmp_ref) -{ - struct method_drop_addr_query query; - query.response = 0; - query.addr = tmp_ref->method_addr; - int ret = 0; - - /* reference count is zero; ask module if it wants us to discard - * the address; TCP will tell us to drop addresses for which the - * socket has died with no possibility of reconnect - */ - ret = tmp_ref->interface->get_info(BMI_DROP_ADDR_QUERY, - &query); - if(ret == 0 && query.response == 1) - { - /* kill the address */ - gossip_debug(GOSSIP_BMI_DEBUG_CONTROL, - "[BMI CONTROL]: %s: bmi discarding address: %llu\n", - __func__, llu(tmp_ref->bmi_addr)); - ref_list_rem(cur_ref_list, tmp_ref->bmi_addr); - /* NOTE: this triggers request to module to free underlying - * resources if it wants to - */ - dealloc_ref_st(tmp_ref); - } return; } Index: src/io/bmi/bmi.h =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi.h,v retrieving revision 1.30 diff -u -a -p -r1.30 bmi.h --- src/io/bmi/bmi.h 20 Sep 2006 04:36:12 -0000 1.30 +++ src/io/bmi/bmi.h 7 Nov 2007 00:11:24 -0000 @@ -123,7 +123,9 @@ int BMI_get_info(PVFS_BMI_addr_t addr, void *inout_parameter); int BMI_addr_lookup(PVFS_BMI_addr_t * new_addr, - const char *id_string); + const char *id_string); + +int BMI_endpoint_fail_current_addr(PVFS_BMI_addr_t addr); const char* BMI_addr_rev_lookup(PVFS_BMI_addr_t addr); Index: src/io/bmi/module.mk.in =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/module.mk.in,v retrieving revision 1.2 diff -u -a -p -r1.2 module.mk.in --- src/io/bmi/module.mk.in 24 Jan 2003 21:52:49 -0000 1.2 +++ src/io/bmi/module.mk.in 7 Nov 2007 00:11:24 -0000 @@ -3,9 +3,9 @@ LIBSRC += \ $(DIR)/bmi.c \ $(DIR)/bmi-method-support.c \ $(DIR)/op-list.c \ - $(DIR)/reference-list.c + $(DIR)/bmi-addr.c SERVERSRC += \ $(DIR)/bmi.c \ $(DIR)/bmi-method-support.c \ $(DIR)/op-list.c \ - $(DIR)/reference-list.c + $(DIR)/bmi-addr.c Index: src/io/bmi/bmi_tcp/socket-collection.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/bmi/bmi_tcp/socket-collection.c,v retrieving revision 1.17 diff -u -a -p -r1.17 socket-collection.c --- src/io/bmi/bmi_tcp/socket-collection.c 6 Nov 2007 23:08:36 -0000 1.17 +++ src/io/bmi/bmi_tcp/socket-collection.c 7 Nov 2007 00:11:24 -0000 @@ -66,7 +66,7 @@ socket_collection_p BMI_socket_collectio return(NULL); } tmp_scp->addr_array = - (bmi_method_addr_p*)malloc(POLLFD_ARRAY_START*sizeof(bmi_method_addr_p)); + (bmi_method_addr_p*)malloc(POLLFD_ARRAY_START*sizeof(*tmp_scp->addr_array)); if(!tmp_scp->addr_array) { free(tmp_scp->pollfd_array); Index: src/io/flow/flow.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/io/flow/flow.c,v retrieving revision 1.54 diff -u -a -p -r1.54 flow.c --- src/io/flow/flow.c 1 Jun 2007 20:58:04 -0000 1.54 +++ src/io/flow/flow.c 7 Nov 2007 00:11:24 -0000 @@ -93,7 +93,7 @@ int PINT_flow_initialize( { /* seperate out the list of flowprotos to activate */ active_flowproto_count = PINT_split_string_list( - &requested_flowprotos, flowproto_list); + &requested_flowprotos, flowproto_list, ','); if (active_flowproto_count < 1) { gossip_lerr("Error: bad flow protocol list.\n"); Index: src/server/pvfs2-server.c =================================================================== RCS file: /projects/cvsroot/pvfs2/src/server/pvfs2-server.c,v retrieving revision 1.246 diff -u -a -p -r1.246 pvfs2-server.c --- src/server/pvfs2-server.c 30 Aug 2007 00:13:44 -0000 1.246 +++ src/server/pvfs2-server.c 7 Nov 2007 00:11:24 -0000 @@ -43,6 +43,7 @@ #include "pint-cached-config.h" #include "pvfs2-internal.h" #include "src/server/request-scheduler/request-scheduler.h" +#include "str-utils.h" #ifndef PVFS2_VERSION #define PVFS2_VERSION "Unknown" @@ -109,9 +110,10 @@ typedef struct int server_background; char *pidfile; char *server_alias; + int address_index; } options_t; -static options_t s_server_options = { 0, 0, 1, NULL, NULL}; +static options_t s_server_options = { 0, 0, 1, NULL, NULL, 0}; static char *fs_conf = NULL; /* each of the elements in this array consists of a string and its length. @@ -889,6 +891,8 @@ static int server_initialize_subsystems( PVFS_ds_flags init_flags = 0; int port_num = 0; int bmi_flags = BMI_INIT_SERVER; + char **host_ids; + int host_id_count; /* Initialize distributions */ ret = PINT_dist_initialize(0); @@ -918,9 +922,18 @@ static int server_initialize_subsystems( bmi_flags |= BMI_TCP_BIND_SPECIFIC; } + host_id_count = PINT_split_string_list(&host_ids, server_config.host_id, ';'); + if(s_server_options.address_index >= host_id_count) + { + PVFS_perror_gossip("Error: address index specified is too large\n", + -PVFS_EINVAL); + return ret; + } + ret = BMI_initialize(server_config.bmi_modules, - server_config.host_id, + host_ids[s_server_options.address_index], bmi_flags); + PINT_free_string_list(host_ids, host_id_count); if (ret < 0) { PVFS_perror_gossip("Error: BMI_initialize", ret); @@ -1607,10 +1620,11 @@ static int server_parse_cmd_line_args(in {"version",0,0,0}, {"pidfile",1,0,0}, {"alias",0,0,0}, + {"addrind",0,0,0}, {0,0,0,0} }; - while ((ret = getopt_long(argc, argv,"dfhrvp:a:", + while ((ret = getopt_long(argc, argv,"dfhrvp:a:i", long_opts, &option_index)) != -1) { total_arguments++; @@ -1648,6 +1662,10 @@ static int server_parse_cmd_line_args(in { goto do_alias; } + else if (strcmp("addrind", cur_option) == 0) + { + goto do_addrind; + } break; case 'v': do_version: @@ -1680,8 +1698,13 @@ static int server_parse_cmd_line_args(in total_arguments++; s_server_options.server_alias = strdup(optarg); break; + do_addrind: + total_arguments++; + s_server_options.address_index = atoi(optarg); + break; case '?': case 'h': + do_help: default: parse_cmd_line_args_failure: