[PVFS-users] Re: PVFS access hangs
Murali Vilayannur
vilayann at mcs.anl.gov
Fri Jul 16 16:53:40 EDT 2004
Hi Rob,
Attached is a patch that fixes some I/O errors that I get when using PVFS1
from the latest CVS. It is somewhat related to the error scenario that you
mention below.
The problem was that in function do_access() in iod/jobs.c, the variable
lost_data is not initialized to 0, hence bad things happen below, when
a_p->u.rw.off et. al are updated. the job layer function i.e do_job()'s
return value is then interpreted as socket errors and the client socket is
closed. Hence clients see EPIPE's.
In addition, the patch addresses some more cleanups that bring the
kpvfsd
code in synch. with some more recent changes that have come up recently.
Hopefully, this will fix some of the errors that Brannen is noticing.
Thanks,
Murali
> That "empty job" message occurs when a client asks for a region of a file
> and the server has no data in that region -- a common case would be trying
> to read beyond EOF.
>
> Are you sure that those messages are from the "cat" case below? If they
> are, then they imply that some data was in fact getting to the iods from
> the client node. But my first guess would be that that isn't happening.
>
> Another thing that is possible is that somehow the IP addresses of your
> iods are not quite right? For example, if you listed an address in the
> .iodtab that is accessible from the local node but not remotely, then you
> would have issues. Can you send your .iodtab to us? The pvfs-ping output
> too, in both cases?
>
> > I could use pvfs-ping, pvfs-ls, etc (forgot to try u2p though)
> > without a problem. After mounting the pvfs filesystem I could use 'ls',
> > etc. However my test program hangs while trying to write files (though it
> > can be interrupted). I also tried to 'cat' one of the files, but that hung
> > also (the last errors above are from that attempt).
>
> What does your test program do, and what interface does it use?
>
> Please do try u2p also if you have a chance, I'm guessing that it won't
> work.
>
> > I'm using the Jan patch to pvfs-1.6.2, the two patches for redhat to
> > the kernel code. Not sure what else to try - I could set up one of the
> > other two machines as the Manager and see what happens then - so far my
> > experience has been that unless a machine is the Manager node, I cannot
> > access the pvfs filesystem for read or write even though the utilities work
> > fine.
>
> I'm almost certain that you're seeing either a misconfiguration problem
> due to odd IP addresses in a PVFS config file, or a routing problem, or a
> firewall problem.
>
> Regards,
>
> Rob
> _______________________________________________
> PVFS-users mailing list
> PVFS-users at www.beowulf-underground.org
> http://www.beowulf-underground.org/mailman/listinfo/pvfs-users
>
>
-------------- next part --------------
Index: include/req.h
===================================================================
RCS file: /anoncvs/pvfs/include/req.h,v
retrieving revision 1.32
diff -u -r1.32 req.h
--- include/req.h 21 Apr 2004 20:57:44 -0000 1.32
+++ include/req.h 16 Jul 2004 19:30:22 -0000
@@ -42,7 +42,7 @@
#define MGR_NOOP 21
#define MGR_LOOKUP 22
#define MGR_CTIME 23
-#define MGR_LINK 24
+#define MGR_LINK 24
#define MGR_READLINK 25
#define MGR_STAT 26
@@ -129,8 +129,8 @@
int64_t length;
} getdents;
struct {
- fmeta meta; /* metadata for the link file */
- int32_t soft; /* Do we need a hard or soft link? */
+ fmeta meta; /* meta data of the link file */
+ int32_t soft; /* whether it is a soft/hard link? */
} link;
#if 0
/* the PGI 4.0-2 compiler does not like empty structs */
@@ -350,4 +350,4 @@
*
* vim: ts=3
* End:
- */
+ */
Index: iod/iod.c
===================================================================
RCS file: /anoncvs/pvfs/iod/iod.c,v
retrieving revision 1.72
diff -u -r1.72 iod.c
--- iod/iod.c 13 May 2004 20:54:28 -0000 1.72
+++ iod/iod.c 16 Jul 2004 19:30:22 -0000
@@ -941,8 +941,8 @@
}
if (f_p->fsize < offset + size) /* trying to go past EOF */ {
- LOG2("mmap_page: fsize = %Ld; offset + size = %Ld\n",
- (int64_t)f_p->fsize, (int64_t)(offset + size));
+ LOG4("mmap_page: fsize = %Ld; offset (%Ld) + size (%Ld) = %Ld\n",
+ (int64_t)f_p->fsize, offset, size, (int64_t)(offset + size));
return(-1);
}
else {
Index: iod/jobs.c
===================================================================
RCS file: /anoncvs/pvfs/iod/jobs.c,v
retrieving revision 1.48
diff -u -r1.48 jobs.c
--- iod/jobs.c 25 Jun 2004 15:30:38 -0000 1.48
+++ iod/jobs.c 16 Jul 2004 19:30:22 -0000
@@ -281,6 +281,7 @@
}
a_p->type = type;
a_p->u.rw.off = pl;
+ LOG2("a_p: %p, a_p->u.rw.off = %Ld\n", a_p, a_p->u.rw.off);
a_p->u.rw.file_p = f_p;
a_p->u.rw.size = 0;
/* aligned wr : start */
@@ -431,7 +432,8 @@
int sock = j_p->sock;
finfo_p f_p;
int smallsize=0, map_size=0;
- int64_t comp, lost_data;
+ int64_t comp = 0, lost_data = 0;
+ /* lost_data not equal to 0 did cause a few problems for the read path */
/* try to perform the access (read/write/ack/whatever)
*/
@@ -453,6 +455,7 @@
return 1;
}
a_p->u.rw.off += comp;
+ LOG3("a_p: %p, a_p->u.rw.off = %Ld, comp: %Ld\n", a_p, a_p->u.rw.off, comp);
return 0;
}
@@ -475,13 +478,15 @@
}
/* else partially completed */
a_p->u.rw.off += comp;
+ LOG3("a_p: %p, a_p->u.rw.off = %Ld, comp: %Ld\n", a_p, a_p->u.rw.off, comp);
return(0);
}
}
#endif
+ LOG3("map_size: %d, a_p->u.rw.off = %Ld, smallsize = %d\n",
+ map_size, a_p->u.rw.off, smallsize);
map_size=mmap_page(a_p->u.rw.file_p, a_p->u.rw.off, smallsize);
- LOG("map_size: %d\n", map_size);
if (map_size == -1 ) {
ERR("do_access: something bad happened mapping file\n");
return(-1);
@@ -550,6 +555,8 @@
}
/* else partially completed */
a_p->u.rw.off += comp + lost_data;
+ LOG4("a_p: %p, a_p->u.rw.off = %Ld, comp: %Ld, lost_data: %Ld\n",
+ a_p, a_p->u.rw.off, comp, lost_data);
return(0);
case A_ACK /* send ack to app */:
-------------- next part --------------
Index: libkpvfs/kinclude/desc.h
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/kinclude/desc.h,v
retrieving revision 1.1
diff -u -r1.1 desc.h
--- libkpvfs/kinclude/desc.h 1 Nov 2002 22:07:36 -0000 1.1
+++ libkpvfs/kinclude/desc.h 16 Jul 2004 19:31:05 -0000
@@ -52,24 +52,13 @@
#endif
struct sockaddr_in addr;
iack ack;
- int32_t sock; /* serves as an index to the data structure having the real socket fd */
+ int32_t sock; /* with single connections to iod, this field now serves as an index
+ into the iodinfo data structure which holds the socket file-descriptor.
+ Hence this is not the real socket fd in that case
+ */
int32_t __pad2; /* struct is passed as arrays, must be 64-bit */
};
-typedef struct file_dim file_dim, *file_dim_p;
-
-struct file_dim {
- int32_t dims; /* # of dimensions */
- int32_t rcc, bc;
- int32_t esize; /* size of element */
- int8_t dirty; /* flags if buffer is dirty */
- char *buffer; /* points to block buffer */
- int *buftag; /* points to current buffer tag */
- int *bsize; /* points to block size */
- int *bcount; /* points to block count */
- int *bbuff; /* points to buffering factor */
-};
-
typedef struct fdesc fdesc, *fdesc_p;
struct fdesc {
@@ -78,7 +67,6 @@
#endif
int8_t fs; /* no longer boolean -- SEE DEFINES ABOVE! */
fpart_p part_p;
- file_dim_p dim_p;
char *fn_p; /* pointer to optionally available file name -- */
/* in terms of manager's view! */
struct {
@@ -87,10 +75,9 @@
int32_t cap; /* capability */
int32_t ref; /* number of pfds referencing this structure */
int flag; /* flags used to open file */
- int32_t grp_nr; /* # in group (if using scheduled I/O) */
iod_info iod[1];
- /* NOTE THAT WE ALLOCATE ENOUGH SPACE FOR AN IOD_INFO PER
- * IOD WHEN WE ACTUALLY MALLOC() THIS */
+ /* NOTE THAT WE ALLOCATE ENOUGH SPACE FOR AN IOD_INFO PER
+ * IOD WHEN WE ACTUALLY MALLOC() THIS */
} fd;
};
Index: libkpvfs/kinclude/meta.h
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/kinclude/meta.h,v
retrieving revision 1.5
diff -u -r1.5 meta.h
--- libkpvfs/kinclude/meta.h 25 Jun 2004 12:38:15 -0000 1.5
+++ libkpvfs/kinclude/meta.h 16 Jul 2004 19:31:05 -0000
@@ -47,6 +47,9 @@
/*
* Just the useful fields from struct stat, rounded up in size to fit
* all architectures.
+ *
+ * glibc-2.3.2 introduces backwards compatibility macros for st_[amc]time,
+ * which means we can't use the names from struct stat
*/
struct pvfs_stat {
uint64_t st_size;
@@ -60,8 +63,9 @@
uint32_t __pad; /* round up to 64-bit fields */
/* unused standard stat fields: dev, nlink, rdev, blksize, blocks */
};
+
/* use this when copying FROM a stat or stat64 TO a stat or stat64 struct */
-#define COPY_STAT_TO_STAT (dest, src) do { \
+#define COPY_STAT_TO_STAT(dest, src) do { \
memset(dest, 0, sizeof(*(dest))); \
(dest)->st_size = (src)->st_size; \
(dest)->st_ino = (src)->st_ino; \
@@ -75,31 +79,31 @@
/* use this when copying FROM a pvfs_stat TO a stat or stat64 struct */
#define COPY_PSTAT_TO_STAT(dest, src) do { \
- memset(dest, 0, sizeof(*(dest))); \
- (dest)->st_size = (src)->st_size; \
- (dest)->st_ino = (src)->st_ino; \
- (dest)->st_atime = (src)->atime; \
- (dest)->st_mtime = (src)->mtime; \
- (dest)->st_ctime = (src)->ctime; \
- (dest)->st_mode = (src)->st_mode; \
- (dest)->st_uid = (src)->st_uid; \
- (dest)->st_gid = (src)->st_gid; \
+ memset(dest, 0, sizeof(*(dest))); \
+ (dest)->st_size = (src)->st_size; \
+ (dest)->st_ino = (src)->st_ino; \
+ (dest)->st_atime = (src)->atime; \
+ (dest)->st_mtime = (src)->mtime; \
+ (dest)->st_ctime = (src)->ctime; \
+ (dest)->st_mode = (src)->st_mode; \
+ (dest)->st_uid = (src)->st_uid; \
+ (dest)->st_gid = (src)->st_gid; \
} while (0);
/* use this when copying FROM a stat or stat64 struct TO a pvfs_stat struct */
#define COPY_STAT_TO_PSTAT(dest, src) do { \
- memset(dest, 0, sizeof(*(dest))); \
- (dest)->st_size = (src)->st_size; \
- (dest)->st_ino = (src)->st_ino; \
- (dest)->atime = (src)->st_atime; \
- (dest)->mtime = (src)->st_mtime; \
- (dest)->ctime = (src)->st_ctime; \
- (dest)->st_mode = (src)->st_mode;\
- (dest)->st_uid = (src)->st_uid; \
- (dest)->st_gid = (src)->st_gid; \
+ memset(dest, 0, sizeof(*(dest))); \
+ (dest)->st_size = (src)->st_size; \
+ (dest)->st_ino = (src)->st_ino; \
+ (dest)->atime = (src)->st_atime; \
+ (dest)->mtime = (src)->st_mtime; \
+ (dest)->ctime = (src)->st_ctime; \
+ (dest)->st_mode = (src)->st_mode; \
+ (dest)->st_uid = (src)->st_uid; \
+ (dest)->st_gid = (src)->st_gid; \
} while (0);
-
+
struct pvfs_filestat {
int32_t base;
int32_t pcount;
@@ -123,6 +127,7 @@
#endif
#endif
+
/*
* Local variables:
* c-indent-level: 3
@@ -132,4 +137,3 @@
* vim: ts=3
* End:
*/
-
Index: libkpvfs/kinclude/pvfs_config.h
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/kinclude/pvfs_config.h,v
retrieving revision 1.7
diff -u -r1.7 pvfs_config.h
--- libkpvfs/kinclude/pvfs_config.h 14 May 2004 18:50:47 -0000 1.7
+++ libkpvfs/kinclude/pvfs_config.h 16 Jul 2004 19:31:05 -0000
@@ -1,7 +1,7 @@
/*
* (C) 1995-2001 Clemson University and Argonne National Laboratory.
*
- * See COPYING in top-level directory.
+ * See LIBRARY_COPYING in top-level directory.
*/
@@ -36,14 +36,15 @@
/* __USE_SENDFILE__ turns on the use of sendfile() in the I/O daemon.
* We haven't seen any speedup from this yet, so it's also off by default.
+ *
+ * NOTE: THIS HAS BEEN REPLACED BY "enable_sendfile" OPTION IN IOD.CONF AND
+ * A CONFIGURE-TIME TEST. YOU NO LONGER NEED TO DEFINE THIS.
*/
-/* #undef __USE_SENDFILE__ */
/* __RANDOM_NEXTSOCK__ turns on randomization in the socket selection
* routines of nextsock(), defined in shared/sockset.c. This hasn't
* seemed to make a difference in any situations, so it's off by
- * default. This could probably never be turned on in the klibpvfs case.
- * mostly because the rand() stuff is still not in yet.
+ * default.
*/
/* #undef __RANDOM_NEXTSOCK__ */
@@ -84,6 +85,11 @@
*/
#define PVFS_NR_OPEN 1024
+/* PVFSTAB FILE LOCATION */
+#define PVFSTAB_PATH "/etc/pvfstab"
+/* PVFSTAB ENVIRONMENT VARIABLE FOR OVERRIDING IT */
+#define PVFSTAB_ENV "PVFSTAB_FILE"
+
/* OUR FS MAGIC NUMBER...returned by statfs() */
#define PVFS_SUPER_MAGIC 0x0872
@@ -109,6 +115,15 @@
REQUEST_BRECV_TIMEOUT_SECS = 60
};
+/* timeout used for breaking out of infinite selects in check_socks to
+ * manually poke the read sockets- this is a workaround for cases in
+ * which select hangs indefinitely when sockets really have data ready
+ * or have reached an error state.
+ */
+enum {
+ CHECK_SOCKS_SELECT_HANG_TIMEOUT = 20 /* seconds */
+};
+
/* PVFS_LIST_IO_CUTOFF - maximum number of noncontiguous regions to use
* in a listio request
*/
@@ -119,7 +134,7 @@
/* MANAGER STUFF */
#define MGR_REQ_PORT 3000
#define MAXIODS 512
-#define MGR_BACKLOG 64
+#define MGR_BACKLOG 256
/* default stripe size */
#define DEFAULT_SSIZE 65536
@@ -140,15 +155,10 @@
#define SOL_TCP 6
#endif
-/* MAP_ANONYMOUS - not defined in IRIX */
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS 0
-#endif
-
/* IOD STUFF */
#define IOD_REQ_PORT 7000
#define IOD_BASE_PORT IOD_REQ_PORT
-#define IOD_BACKLOG 64
+#define IOD_BACKLOG 256
/* this determines how many directories the iod splits files into;
* 101, 199, 499, 997 are all decent choices.
@@ -175,9 +185,9 @@
/* IOCTL DEFINES - COULDN'T FIND A BETTER PLACE TO PUT THEM... */
/* These are just arbitrary #s that linux doesn't seem to use. */
-#define GETPART 0x5601
-#define SETPART 0x5602
-#define GETMETA 0x5603
+#define GETPART 0x5601
+#define SETPART 0x5602
+#define GETMETA 0x5603
/* more arbitrary #s; these are flags for pvfs_open() */
/* using lowest 4 bits of highest byte in the integer */
@@ -188,6 +198,16 @@
#define O_GADD 01000000000
#define O_CONN 02000000000
+/* syslog facility to use for logging invalid access attempts. */
+#define LOG_ACC_FACILITY LOG_LOCAL7
+
+#ifdef ENABLE_TRUSTED_PORTS
+#define ACC_ADDR INADDR_NONE
+#define ACC_MASK (~(in_addr_t)0)
+#else
+#define ACC_ADDR INADDR_ANY
+#define ACC_MASK ((in_addr_t)0)
+#endif
#endif
/*
Index: libkpvfs/kinclude/pvfs_types.h
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/kinclude/pvfs_types.h,v
retrieving revision 1.1
diff -u -r1.1 pvfs_types.h
--- libkpvfs/kinclude/pvfs_types.h 1 Nov 2002 22:07:36 -0000 1.1
+++ libkpvfs/kinclude/pvfs_types.h 16 Jul 2004 19:31:05 -0000
@@ -1,6 +1,10 @@
/*
* (C) 2001 Pete Wyckoff <pw at osc.edu>
*
+ * See LIBRARY_COPYING in top-level directory.
+ */
+
+/*
* Common types shared by both pvfs and pvfs-kernel.
*/
Index: libkpvfs/kinclude/req.h
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/kinclude/req.h,v
retrieving revision 1.6
diff -u -r1.6 req.h
--- libkpvfs/kinclude/req.h 14 May 2004 19:30:08 -0000 1.6
+++ libkpvfs/kinclude/req.h 16 Jul 2004 19:31:05 -0000
@@ -130,7 +130,7 @@
} getdents;
struct {
fmeta meta; /* meta data of the link file */
- int32_t soft; /* whether it is a soft/hard link ? */
+ int32_t soft; /* whether it is a soft/hard link? */
} link;
#if 0
/* the PGI 4.0-2 compiler does not like empty structs */
Index: libkpvfs/klib/iodcomm.c
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/klib/iodcomm.c,v
retrieving revision 1.2
diff -u -r1.2 iodcomm.c
--- libkpvfs/klib/iodcomm.c 12 Nov 2002 20:43:25 -0000 1.2
+++ libkpvfs/klib/iodcomm.c 16 Jul 2004 19:31:05 -0000
@@ -319,9 +319,7 @@
}
pfds[fd]->fs = FS_RESV;
pfds[fd]->part_p = NULL;
- pfds[fd]->dim_p = NULL;
pfds[fd]->fd.ref = -1;
- pfds[fd]->fd.grp_nr = -1;
}
return(iodinfo[slot].fd);
}
Index: libkpvfs/klib/mgrcomm.c
===================================================================
RCS file: /anoncvs/pvfs-kernel/libkpvfs/klib/mgrcomm.c,v
retrieving revision 1.3
diff -u -r1.3 mgrcomm.c
--- libkpvfs/klib/mgrcomm.c 25 Jun 2004 12:42:43 -0000 1.3
+++ libkpvfs/klib/mgrcomm.c 16 Jul 2004 19:31:05 -0000
@@ -208,7 +208,7 @@
if (ack_p->eno > 1000) /* serious error -- errno's > 1000 will be interpreted as a pointer upstream - must be garbage on manager anyway */
{
- PERROR("The PVFS Manager returned an unreasonable errno [%d] on reqest type [%d] for data [%s]. Something must be wrong here.\n", ack_p->eno, req_p->type, data_p == NULL ? "NONE" : data_p);
+ PERROR("The PVFS Manager returned an unreasonable errno [%d] on reqest type [%d] for data [%s]. Something must be wrong here.\n", ack_p->eno, req_p->type, data_p == NULL ? "NONE" : (char *)data_p);
Close(fd);
if (!newconn) badmgrfd(fd);
return(-EIO);
@@ -250,9 +250,7 @@
}
pfds[fd]->fs = FS_RESV;
pfds[fd]->part_p = NULL;
- pfds[fd]->dim_p = NULL;
pfds[fd]->fd.ref = -1;
- pfds[fd]->fd.grp_nr = -1;
return(0);
}
More information about the PVFS-users
mailing list