* Sebastian Hammer, Adam Dickmeiss
*
* $Log: extract.c,v $
- * Revision 1.53 1996-04-26 12:09:43 adam
+ * Revision 1.54 1996-05-01 13:46:35 adam
+ * First work on multiple records in one file.
+ * New option, -offset, to the "unread" command in the filter module.
+ *
+ * Revision 1.53 1996/04/26 12:09:43 adam
* Added a few comments.
*
* Revision 1.52 1996/04/25 13:27:57 adam
static int records_inserted = 0;
static int records_updated = 0;
static int records_deleted = 0;
+static int records_processed = 0;
+
+static void logRecord (int showFlag)
+{
+ if (!showFlag)
+ ++records_processed;
+ if (showFlag || !(records_processed % 1000))
+ {
+ logf (LOG_LOG, "Records: %7d i/u/d %d/%d/%d",
+ records_processed, records_inserted, records_updated,
+ records_deleted);
+ }
+}
void key_open (int mem)
{
rec_close (&records);
dict_close (matchDict);
- logf (LOG_LOG, "Records inserted %6d", records_inserted);
- logf (LOG_LOG, "Records updated %6d", records_updated);
- logf (LOG_LOG, "Records deleted %6d", records_deleted);
+ logRecord (1);
return key_file_no;
}
addRecordKey (p);
}
-#define FILE_READ_BUFSIZE 4096
struct file_read_info {
- int file_noread;
+ off_t file_max;
+ off_t file_offset;
+ off_t file_moffset;
+ int file_more;
int fd;
-#if FILE_READ_BUFSIZE
- char *file_buf;
- int file_offset;
- int file_bufsize;
-#endif
};
static struct file_read_info *file_read_start (int fd)
struct file_read_info *fi = xmalloc (sizeof(*fi));
fi->fd = fd;
- fi->file_noread = 0;
-#if FILE_READ_BUFSIZE
- fi->file_offset = 0;
- fi->file_buf = xmalloc (FILE_READ_BUFSIZE);
- fi->file_bufsize = read (fd, fi->file_buf, FILE_READ_BUFSIZE);
-#endif
+ fi->file_max = 0;
+ fi->file_moffset = 0;
return fi;
}
static void file_read_stop (struct file_read_info *fi)
{
assert (fi);
-#if FILE_READ_BUFSIZE
- xfree (fi->file_buf);
- fi->file_buf = NULL;
-#endif
xfree (fi);
}
+static off_t file_seek (void *handle, off_t offset)
+{
+ struct file_read_info *p = handle;
+ p->file_offset = offset;
+ return lseek (p->fd, offset, SEEK_SET);
+}
+
static int file_read (void *handle, char *buf, size_t count)
{
struct file_read_info *p = handle;
int fd = p->fd;
-#if FILE_READ_BUFSIZE
- int l = p->file_bufsize - p->file_offset;
-
- if (count > l)
- {
- int r;
- if (l > 0)
- memcpy (buf, p->file_buf + p->file_offset, l);
- count = count-l;
- if (count > FILE_READ_BUFSIZE)
- {
- if ((r = read (fd, buf + l, count)) == -1)
- {
- logf (LOG_FATAL|LOG_ERRNO, "read");
- exit (1);
- }
- p->file_bufsize = 0;
- p->file_offset = 0;
- p->file_noread += l+r;
- return l+r;
- }
- p->file_bufsize = r = read (fd, p->file_buf, FILE_READ_BUFSIZE);
- if (r == -1)
- {
- logf (LOG_FATAL|LOG_ERRNO, "read");
- exit (1);
- }
- else if (r <= count)
- {
- p->file_offset = r;
- memcpy (buf + l, p->file_buf, r);
- p->file_noread += l+r;
- return l+r;
- }
- else
- {
- p->file_offset = count;
- memcpy (buf + l, p->file_buf, count - l);
- p->file_noread += count;
- return count;
- }
- }
- memcpy (buf, p->file_buf + p->file_offset, count);
- p->file_offset += count;
- p->file_noread += count;
- return count;
-#else
int r;
r = read (fd, buf, count);
if (r > 0)
- p->file_noread += r;
+ {
+ p->file_offset += r;
+ if (p->file_offset > p->file_max)
+ p->file_max = p->file_offset;
+ }
return r;
-#endif
+}
+
+static void file_begin (void *handle)
+{
+ struct file_read_info *p = handle;
+
+ p->file_offset = p->file_moffset;
+ if (p->file_moffset)
+ lseek (p->fd, p->file_moffset, SEEK_SET);
+ p->file_more = 0;
+}
+
+static void file_end (void *handle, off_t offset)
+{
+ struct file_read_info *p = handle;
+
+ assert (p->file_more == 0);
+ p->file_more = 1;
+ p->file_moffset = offset;
}
static int atois (const char **s)
int r;
char *matchStr;
SYSNO sysnotmp;
+ off_t recordOffset = 0;
Record rec;
struct recordLogInfo logInfo;
reckeys.buf_used = 0;
reckeys.prevAttrUse = -1;
reckeys.prevAttrSet = -1;
+
+ recordOffset = fi->file_moffset;
+ extractCtrl.offset = recordOffset;
extractCtrl.readf = file_read;
+ extractCtrl.seekf = file_seek;
+ extractCtrl.endf = file_end;
r = (*recType->extract)(&extractCtrl);
if (r)
}
logInfo.op = "add";
if (rGroup->fileVerboseFlag)
- logf (LOG_LOG, "add %s %s", rGroup->recordType, fname);
+ logf (LOG_LOG, "add %s %s %ld", rGroup->recordType,
+ fname, (long) recordOffset);
rec = rec_new (records);
*sysno = rec->sysno;
else
{
if (rGroup->fileVerboseFlag)
- logf (LOG_LOG, "delete %s %s", rGroup->recordType, fname);
+ logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
+ fname, (long) recordOffset);
records_deleted++;
if (matchStr)
dict_delete (matchDict, matchStr);
rec_del (records, &rec);
}
+ logRecord (0);
return 1;
}
else
else
{
if (rGroup->fileVerboseFlag)
- logf (LOG_LOG, "update %s %s", rGroup->recordType, fname);
+ logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
+ fname, (long) recordOffset);
flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName);
records_updated++;
}
xfree (rec->info[recInfo_storeData]);
if (rGroup->flagStoreData == 1)
{
- rec->size[recInfo_storeData] = fi->file_noread;
- rec->info[recInfo_storeData] = xmalloc (fi->file_noread);
-#if FILE_READ_BUFSIZE
- if (fi->file_noread < FILE_READ_BUFSIZE)
- memcpy (rec->info[recInfo_storeData], fi->file_buf,
- fi->file_noread);
- else
-#endif
+ rec->size[recInfo_storeData] = fi->file_max;
+ rec->info[recInfo_storeData] = xmalloc (fi->file_max);
+ if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
{
- if (lseek (fi->fd, 0L, SEEK_SET) < 0)
- {
- logf (LOG_ERRNO|LOG_FATAL, "seek to 0 in %s", fname);
- exit (1);
- }
- if (read (fi->fd, rec->info[recInfo_storeData], fi->file_noread)
- < fi->file_noread)
- {
- logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
- fi->file_noread, fname);
- exit (1);
- }
+ logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname,
+ (long) recordOffset);
+ exit (1);
+ }
+ if (read (fi->fd, rec->info[recInfo_storeData], fi->file_max)
+ < fi->file_max)
+ {
+ logf (LOG_ERRNO|LOG_FATAL, "read %d bytes of %s",
+ fi->file_max, fname);
+ exit (1);
}
}
else
rec->info[recInfo_databaseName] =
rec_strdup (rGroup->databaseName, &rec->size[recInfo_databaseName]);
+ /* update offset */
+ xfree (rec->info[recInfo_offset]);
+
+ rec->size[recInfo_offset] = sizeof(recordOffset);
+ rec->info[recInfo_offset] = xmalloc (sizeof(recordOffset));
+ memcpy (rec->info[recInfo_offset], &recordOffset, sizeof(recordOffset));
+
/* commit this record */
rec_put (records, &rec);
+ logRecord (0);
return 1;
}
}
}
fi = file_read_start (fd);
- r = recordExtract (sysno, fname, rGroup, deleteFlag, fi, recType, subType);
+ do
+ {
+ file_begin (fi);
+ r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
+ recType, subType);
+ } while (r && !sysno && fi->file_more);
log_event_start (NULL, NULL);
file_read_stop (fi);
if (fd != -1)
* Sebastian Hammer, Adam Dickmeiss
*
* $Log: trav.c,v $
- * Revision 1.24 1996-04-26 10:00:23 adam
+ * Revision 1.25 1996-05-01 13:46:37 adam
+ * First work on multiple records in one file.
+ * New option, -offset, to the "unread" command in the filter module.
+ *
+ * Revision 1.24 1996/04/26 10:00:23 adam
* Added option -V to zebraidx to display version information.
* Removed stupid warnings from file update.
*
}
-static void stdinExtractR (int deleteFlag, struct recordGroup *rGroup)
-{
- char tmppath[1024];
-
- logf (LOG_LOG, "stdinExtractR");
- while (scanf ("%s", tmppath) == 1)
- fileExtract (NULL, tmppath, rGroup, deleteFlag);
-}
-
static void fileDeleteR (struct dirs_info *di, struct dirs_entry *dst,
const char *base, char *src,
struct recordGroup *rGroup)
rGroup->recordId = res_get (common_resource, resStr);
}
-
void repositoryShow (struct recordGroup *rGroup)
{
char src[1024];
}
}
+
+static void repositoryExtract (int deleteFlag, struct recordGroup *rGroup,
+ const char *path)
+{
+ struct stat sbuf;
+ char src[1024];
+
+ assert (path);
+ strcpy (src, path);
+
+ stat (src, &sbuf);
+ if (S_ISREG(sbuf.st_mode))
+ fileExtract (NULL, src, rGroup, deleteFlag);
+ else if (S_ISDIR(sbuf.st_mode))
+ repositoryExtractR (deleteFlag, src, rGroup);
+ else
+ logf (LOG_WARN, "Cannot handle file %s", src);
+}
+
+static void repositoryExtractG (int deleteFlag, struct recordGroup *rGroup)
+{
+ if (*rGroup->path == '\0' || !strcmp(rGroup->path, "-"))
+ {
+ char src[1024];
+
+ while (scanf ("%s", src) == 1)
+ repositoryExtract (deleteFlag, rGroup, src);
+ }
+ else
+ repositoryExtract (deleteFlag, rGroup, rGroup->path);
+}
+
void repositoryUpdate (struct recordGroup *rGroup)
{
groupRes (rGroup);
dict_close (dict);
}
else
- {
- char src[1024];
-
- strcpy (src, rGroup->path);
- if (*src == '\0' || !strcmp (src, "-"))
- stdinExtractR (0, rGroup);
- else
- repositoryExtractR (0, src, rGroup);
- }
+ repositoryExtractG (0, rGroup);
}
void repositoryDelete (struct recordGroup *rGroup)
{
- char src[256];
-
- assert (rGroup->path);
- groupRes (rGroup);
- strcpy (src, rGroup->path);
- if (*src == '\0' || !strcmp(src, "-"))
- stdinExtractR (1, rGroup);
- else
- repositoryExtractR (1, src, rGroup);
+ repositoryExtractG (1, rGroup);
}
* Sebastian Hammer, Adam Dickmeiss
*
* $Log: zserver.c,v $
- * Revision 1.35 1996-03-26 16:01:14 adam
+ * Revision 1.36 1996-05-01 13:46:37 adam
+ * First work on multiple records in one file.
+ * New option, -offset, to the "unread" command in the filter module.
+ *
+ * Revision 1.35 1996/03/26 16:01:14 adam
* New setting lockPath: directory of various lock files.
*
* Revision 1.34 1996/03/20 09:36:46 adam
return &r;
}
+static int record_offset;
+
static int record_ext_read (void *fh, char *buf, size_t count)
{
return read (*((int*) fh), buf, count);
}
+static off_t record_ext_seek (void *fh, off_t offset)
+{
+ return lseek (*((int*) fh), offset + record_offset, SEEK_SET);
+}
+
static int record_int_pos;
static char *record_int_buf;
static int record_int_len;
+static off_t record_int_seek (void *fh, off_t offset)
+{
+ return (off_t) (record_int_pos = offset);
+}
+
static int record_int_read (void *fh, char *buf, size_t count)
{
int l = record_int_len - record_int_pos;
if (rec->size[recInfo_storeData] > 0)
{
retrieveCtrl.readf = record_int_read;
+ retrieveCtrl.seekf = record_int_seek;
record_int_len = rec->size[recInfo_storeData];
record_int_buf = rec->info[recInfo_storeData];
record_int_pos = 0;
rec_rm (&rec);
return 0; /* or 14: System error in presenting records */
}
+
+ memcpy (&record_offset, rec->info[recInfo_offset],
+ sizeof(record_offset));
+
retrieveCtrl.fh = &fd;
retrieveCtrl.readf = record_ext_read;
+ retrieveCtrl.seekf = record_ext_seek;
+
+ record_ext_seek (retrieveCtrl.fh, 0);
}
retrieveCtrl.subType = subType;
retrieveCtrl.localno = sysno;