[Pvfs2-cvs] commit by pcarns in pvfs2-1/src/apps/admin:
pvfs2-fs-dump.c pvfs2-fsck.c pvfs2-genconfig
CVS commit program
cvs at parl.clemson.edu
Mon Sep 8 11:42:38 EDT 2008
Update of /projects/cvsroot/pvfs2-1/src/apps/admin
In directory parlweb1:/tmp/cvs-serv32611/src/apps/admin
Modified Files:
pvfs2-fs-dump.c pvfs2-fsck.c pvfs2-genconfig
Log Message:
Merging small files branch to head. Includes server side precreation of
data files and file stuffing.
Index: pvfs2-fs-dump.c
===================================================================
RCS file: /projects/cvsroot/pvfs2-1/src/apps/admin/pvfs2-fs-dump.c,v
diff -p -u -r1.43 -r1.44
--- pvfs2-fs-dump.c 16 Jun 2006 21:01:11 -0000 1.43
+++ pvfs2-fs-dump.c 8 Sep 2008 15:42:38 -0000 1.44
@@ -348,6 +348,7 @@ int build_handlelist(PVFS_fs_id cur_fs,
position_array,
addr_array,
server_count,
+ 0,
NULL /* details */);
if (ret < 0)
{
Index: pvfs2-fsck.c
===================================================================
RCS file: /projects/cvsroot/pvfs2-1/src/apps/admin/pvfs2-fsck.c,v
diff -p -u -r1.24 -r1.25
--- pvfs2-fsck.c 10 Dec 2007 16:13:53 -0000 1.24
+++ pvfs2-fsck.c 8 Sep 2008 15:42:38 -0000 1.25
@@ -41,6 +41,9 @@ struct options *fsck_opts = NULL;
/* lost+found reference */
PVFS_object_ref laf_ref;
+static void handlelist_remove_handle_no_idx(struct handlelist *hl,
+ PVFS_handle handle);
+
int main(int argc, char **argv)
{
int ret = -1, in_admin_mode = 0;
@@ -353,6 +356,7 @@ struct handlelist *build_handlelist(PVFS
position_array,
addr_array,
server_count,
+ 0,
NULL /* details */);
if (ret < 0)
{
@@ -412,6 +416,74 @@ struct handlelist *build_handlelist(PVFS
i, total_count_array[i], used_handles);
return NULL;
}
+ }
+
+ handlelist_finished_adding_handles(hl); /* sanity check */
+
+ /* now look for reserved handles */
+ for (i=0; i < server_count; i++)
+ {
+ hcount_array[i] = HANDLE_BATCH;
+ position_array[i] = PVFS_ITERATE_START;
+ }
+
+ more_flag = 1;
+ while (more_flag)
+ {
+ ret = PVFS_mgmt_iterate_handles_list(cur_fs,
+ creds,
+ handle_matrix,
+ hcount_array,
+ position_array,
+ addr_array,
+ server_count,
+ PVFS_MGMT_RESERVED,
+ NULL /* details */);
+ if (ret < 0)
+ {
+ PVFS_perror("PVFS_mgmt_iterate_handles_list", ret);
+ PVFS_mgmt_setparam_list(cur_fs,
+ creds,
+ PVFS_SERV_PARAM_MODE,
+ (uint64_t)PVFS_SERVER_NORMAL_MODE,
+ addr_array,
+ NULL,
+ server_count,
+ NULL);
+ return NULL;
+ }
+
+ for (i=0; i < server_count; i++)
+ {
+ /* remove any reserved handles from the handlelist. These will
+ * not show up in normal objects when we walk the file system
+ * tree.
+ */
+ for (j=0; j < hcount_array[i]; j++)
+ {
+ /* we don't know the server index. Reserved handles can be
+ * reported by any server; not just the server that actually
+ * owns that handle.
+ */
+ handlelist_remove_handle_no_idx(hl,
+ handle_matrix[i][j]);
+ }
+ }
+
+ /* find out if any servers have more handles to dump */
+ more_flag = 0;
+ for (i=0; i < server_count; i++)
+ {
+ if (position_array[i] != PVFS_ITERATE_END)
+ {
+ more_flag = 1;
+ hcount_array[i] = HANDLE_BATCH;
+ }
+ }
+ }
+
+ for (i = 0; i < server_count; i++)
+ {
free(handle_matrix[i]);
}
@@ -421,7 +493,6 @@ struct handlelist *build_handlelist(PVFS
free(total_count_array);
free(position_array);
- handlelist_finished_adding_handles(hl); /* sanity check */
free(stat_array);
stat_array = NULL;
@@ -1300,6 +1371,50 @@ static int handlelist_find_handle(struct
}
return -1;
+}
+
+/* handlelist_remove_handle_no_idx()
+ *
+ * same as handlelist_remove_handle(), but will search for the correct
+ * server index
+ */
+/* TODO: we could speed this up by resolving which server the handle
+ * belongs to using the cached_config api
+ */
+static void handlelist_remove_handle_no_idx(struct handlelist *hl,
+ PVFS_handle handle)
+{
+ unsigned long i;
+ int server_idx = 0;
+ int found = 0;
+
+ for(server_idx = 0; server_idx<hl->server_ct; server_idx++)
+ {
+ for (i = 0; i < hl->used_array[server_idx]; i++)
+ {
+ if (hl->list_array[server_idx][i] == handle)
+ {
+ if (i < (hl->used_array[server_idx] - 1))
+ {
+ /* move last entry to this position before decrement */
+ hl->list_array[server_idx][i] =
+ hl->list_array[server_idx][hl->used_array[server_idx]-1];
+
+ }
+ hl->used_array[server_idx]--;
+ found = 1;
+ break;
+ }
+ }
+ if(found)
+ {
+ break;
+ }
+ }
+
+ if (!found) {
+ printf("! problem removing %llu.\n", llu(handle));
+ }
}
static void handlelist_remove_handle(struct handlelist *hl,
Index: pvfs2-genconfig
===================================================================
RCS file: /projects/cvsroot/pvfs2-1/src/apps/admin/pvfs2-genconfig,v
diff -p -u -r1.82 -r1.83
--- pvfs2-genconfig 2 Jul 2008 21:21:01 -0000 1.82
+++ pvfs2-genconfig 8 Sep 2008 15:42:38 -0000 1.83
@@ -200,6 +200,8 @@ sub emit_defaults
print $target "\tClientJobFlowTimeoutSecs $client_job_timeout\n";
print $target "\tClientRetryLimit 5\n";
print $target "\tClientRetryDelayMilliSecs 2000\n";
+ print $target "\tPrecreateBatchSize 512\n";
+ print $target "\tPrecreateLowThreshold 256\n";
if(defined($default_storage))
{
@@ -244,7 +246,7 @@ sub emit_filesystem
$default_num_dfiles, $default_flow_buffer_size, $default_flow_buffer_count) = @_;
# divide handle range space equally among servers ((2^63)-1 for now)
- my($total_num_handles_available, $start, $end, $i, $step, $num_ranges);
+ my($total_num_handles_available, $start, $end, $i, $step, $num_ranges, $stuffing);
$num_ranges = $count;
$total_num_handles_available = $last_handle->copy();
$total_num_handles_available->bsub($first_handle);
@@ -264,6 +266,23 @@ sub emit_filesystem
print $target "\tDefaultNumDFiles $default_num_dfiles\n";
}
+ # Rules for default stuffing setting: only enable it if every I/O server
+ # is also a metadata server, otherwise we would tend to unbalance by
+ # always stuffing on the subset that does metadata. User can override
+ # if desired.
+ $stuffing = "yes";
+ foreach my $alias (keys %all_endpoints)
+ {
+ if($all_endpoints{$alias}->{TYPE} & $IO_ENDPOINT)
+ {
+ if(!($all_endpoints{$alias}->{TYPE} & $META_ENDPOINT))
+ {
+ $stuffing = "no";
+ }
+ }
+ }
+ print $target "\tFileStuffing $stuffing\n";
+
print $target "\t<MetaHandleRanges>\n";
$start = $end = $first_handle->copy();
$start->bdec();
@@ -941,17 +960,36 @@ sub get_metanames
my $storage = shift;
my $logfile = shift;
my $metaline = '';
+ my @meta_hosts;
if ($opt_metaservers) {
$metaline = $opt_metaservers;
- } else {
- print $OUT "Now list the hostnames of the machines that will act as " .
- "Metadata\nservers. This list may or may not overlap " .
- "with the I/O server list.\n";
+ @meta_hosts = parse_hostlist($metaline);
+ }
+ else
+ {
+ print $OUT "Use same servers for metadata? (recommended)\n";
$metaline = prompt_word(
- "Enter hostnames [Default is localhost]: ",
- "localhost");
+ "Enter yes or no [Default is yes]: ",
+ "yes");
+ if($metaline =~ /^yes$/i)
+ {
+ foreach my $alias (keys %all_endpoints)
+ {
+ $all_endpoints{$alias}->{TYPE} |= $META_ENDPOINT;
+ }
+ }
+ else
+ {
+ print $OUT "Now list the hostnames of the machines that will act as " .
+ "Metadata\nservers. This list may or may not overlap " .
+ "with the I/O server list.\n";
+ $metaline = prompt_word(
+ "Enter hostnames [Default is localhost]: ",
+ "localhost");
+ @meta_hosts = parse_hostlist($metaline);
+ }
}
- my @meta_hosts = parse_hostlist($metaline);
+
foreach my $meta_host (@meta_hosts)
{
if(exists $all_endpoints{$meta_host})
More information about the Pvfs2-cvs
mailing list