/*
- * Copyright (C) 1994-1998, Index Data I/S
+ * Copyright (C) 1994-2001, Index Data
* All rights reserved.
* Sebastian Hammer, Adam Dickmeiss
*
* $Log: extract.c,v $
- * Revision 1.81 1998-03-11 11:19:04 adam
+ * Revision 1.107 2001-05-28 13:58:48 adam
+ * Call flushSortKeys when record is skipped to fix bad re-use of
+ * sort keys to whatever next record that comes in.
+ *
+ * Revision 1.106 2000/12/05 12:22:53 adam
+ * Termlist source implemented (so that we can index values of XML/SGML
+ * attributes).
+ *
+ * Revision 1.105 2000/12/05 10:01:44 adam
+ * Fixed bug regarding user-defined attribute sets.
+ *
+ * Revision 1.104 2000/09/05 14:04:05 adam
+ * Updates for prefix 'yaz_' for YAZ log functions.
+ *
+ * Revision 1.103 2000/05/18 12:01:36 adam
+ * System call times(2) used again. More 64-bit fixes.
+ *
+ * Revision 1.102 2000/05/15 15:32:33 adam
+ * Added 64 bit file input.
+ *
+ * Revision 1.101 2000/05/15 13:02:39 adam
+ * Minor change.
+ *
+ * Revision 1.100 2000/03/20 19:08:36 adam
+ * Added remote record import using Z39.50 extended services and Segment
+ * Requests.
+ *
+ * Revision 1.99 2000/02/24 10:57:02 adam
+ * Sequence number incremented after each incomplete-field.
+ *
+ * Revision 1.98 1999/09/07 07:19:21 adam
+ * Work on character mapping. Implemented replace rules.
+ *
+ * Revision 1.97 1999/07/06 12:28:04 adam
+ * Updated record index structure. Format includes version ID. Compression
+ * algorithm ID is stored for each record block.
+ *
+ * Revision 1.96 1999/05/26 07:49:13 adam
+ * C++ compilation.
+ *
+ * Revision 1.95 1999/05/21 12:00:17 adam
+ * Better diagnostics for extraction process.
+ *
+ * Revision 1.94 1999/05/20 12:57:18 adam
+ * Implemented TCL filter. Updated recctrl system.
+ *
+ * Revision 1.93 1999/05/15 14:36:38 adam
+ * Updated dictionary. Implemented "compression" of dictionary.
+ *
+ * Revision 1.92 1999/03/09 16:27:49 adam
+ * More work on SDRKit integration.
+ *
+ * Revision 1.91 1999/02/12 13:29:22 adam
+ * Implemented position-flag for registers.
+ *
+ * Revision 1.90 1999/02/02 14:50:52 adam
+ * Updated WIN32 code specific sections. Changed header.
+ *
+ * Revision 1.89 1998/10/28 10:54:38 adam
+ * SDRKit integration.
+ *
+ * Revision 1.88 1998/10/16 08:14:29 adam
+ * Updated record control system.
+ *
+ * Revision 1.87 1998/10/15 13:10:33 adam
+ * Fixed bug in Zebra that caused it to stop indexing when empty
+ * record was read.
+ *
+ * Revision 1.86 1998/10/13 20:33:53 adam
+ * Fixed one log message and change use ordinal to be an unsigned char.
+ *
+ * Revision 1.85 1998/09/22 10:03:41 adam
+ * Changed result sets to be persistent in the sense that they can
+ * be re-searched if needed.
+ * Fixed memory leak in rsm_or.
+ *
+ * Revision 1.84 1998/06/11 15:42:22 adam
+ * Changed the way use attributes are specified in the recordId
+ * specification.
+ *
+ * Revision 1.83 1998/06/08 14:43:10 adam
+ * Added suport for EXPLAIN Proxy servers - added settings databasePath
+ * and explainDatabase to facilitate this. Increased maximum number
+ * of databases and attributes in one register.
+ *
+ * Revision 1.82 1998/05/20 10:12:15 adam
+ * Implemented automatic EXPLAIN database maintenance.
+ * Modified Zebra to work with ASN.1 compiled version of YAZ.
+ *
+ * Revision 1.81 1998/03/11 11:19:04 adam
* Changed the way sequence numbers are generated.
*
* Revision 1.80 1998/03/05 08:45:11 adam
*/
#include <stdio.h>
#include <assert.h>
-#ifdef WINDOWS
+#ifdef WIN32
#include <io.h>
#else
#include <unistd.h>
#include "zinfo.h"
+#if _FILE_OFFSET_BITS == 64
+#define PRINTF_OFF_T "%Ld"
+#else
+#define PRINTF_OFF_T "%ld"
+#endif
+
+#ifndef ZEBRASDR
+#define ZEBRASDR 0
+#endif
+
+#if ZEBRASDR
+#include "zebrasdr.h"
+#endif
+
static Dict matchDict;
static Records records = NULL;
static ZebraExplainInfo zti = NULL;
+
static void logRecord (int showFlag)
{
if (!showFlag)
}
}
-int key_open (BFiles bfs, int mem, int rw, data1_handle dh)
+static int explain_extract (void *handle, Record drec, data1_node *n);
+
+int key_open (struct recordGroup *rGroup, int mem)
{
+ BFiles bfs = rGroup->bfs;
+ int rw = rGroup->flagRw;
+ data1_handle dh = rGroup->dh;
+ char *recordCompression;
+ int record_compression = REC_COMPRESS_NONE;
if (!mem)
- mem = atoi(res_get_def (common_resource, "memMax", "4"))*1024*1024;
+ mem = atoi(res_get_def (common_resource, "memMax", "16"))*1024*1024;
if (mem < 50000)
mem = 50000;
- key_buf = xmalloc (mem);
+ key_buf = (char **) xmalloc (mem);
ptr_top = mem/sizeof(char*);
ptr_i = 0;
key_buf_used = 0;
key_file_no = 0;
- if (!(matchDict = dict_open (bfs, GMATCH_DICT, 50, rw)))
+ if (!(matchDict = dict_open (bfs, GMATCH_DICT, 50, rw, 0)))
{
logf (LOG_FATAL, "dict_open fail of %s", GMATCH_DICT);
return -1;
}
assert (!records);
- records = rec_open (bfs, rw);
+ recordCompression = res_get_def (common_resource,
+ "recordCompression", "none");
+ if (!strcmp (recordCompression, "none"))
+ record_compression = REC_COMPRESS_NONE;
+ if (!strcmp (recordCompression, "bzip2"))
+ record_compression = REC_COMPRESS_BZIP2;
+ records = rec_open (bfs, rw, record_compression);
if (!records)
{
dict_close (matchDict);
return -1;
}
- zti = zebraExplain_open (records, dh, rw);
+ zti = zebraExplain_open (records, dh, common_resource,
+ rw, rGroup, explain_extract);
if (!zti)
{
rec_close (&records);
struct encode_info {
int sysno;
int seqno;
+ int cmd;
char buf[768];
};
{
i->sysno = 0;
i->seqno = 0;
+ i->cmd = -1;
}
char *encode_key_int (int d, char *bp)
i->sysno = key.sysno;
i->seqno = 0;
}
+ else if (!i->seqno && !key.seqno && i->cmd == *k)
+ return;
bp = encode_key_int (key.seqno - i->seqno, bp);
i->seqno = key.seqno;
+ i->cmd = *k;
if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
{
logf (LOG_FATAL|LOG_ERRNO, "fwrite");
logf (LOG_LOG, "sorting section %d", key_file_no);
#if !SORT_EXTRA
qsort (key_buf + ptr_top-ptr_i, ptr_i, sizeof(char*), key_qsort_compare);
- getFnameTmp (out_fname, key_file_no);
+ getFnameTmp (common_resource, out_fname, key_file_no);
if (!(outf = fopen (out_fname, "wb")))
{
key_buf_used = 0;
}
-int key_close (int rw)
+int key_close (struct recordGroup *rGroup)
{
- key_flush ();
- xfree (key_buf);
+ int rw = rGroup->flagRw;
if (rw)
zebraExplain_runNumberIncrement (zti, 1);
zebraExplain_close (zti, rw);
+ key_flush ();
+ xfree (key_buf);
rec_close (&records);
dict_close (matchDict);
sortIdx_close (sortIdx);
{
w->zebra_maps = p->zebra_maps;
w->seqnos = p->seqno;
- w->attrSet = 1;
+ w->attrSet = VAL_BIB1;
w->attrUse = 1016;
w->reg_type = 'w';
+ w->extractCtrl = p;
}
static struct sortKey {
static void addIndexString (RecWord *p, const char *string, int length)
{
char *dst;
- char attrSet;
- short attrUse;
+ unsigned char attrSet;
+ unsigned short attrUse;
int lead = 0;
int diff = 0;
int *pseqno = &p->seqnos[p->reg_type];
{
char *b;
- b = xmalloc (reckeys.buf_max += 128000);
+ b = (char *) xmalloc (reckeys.buf_max += 128000);
if (reckeys.buf_used > 0)
memcpy (b, reckeys.buf, reckeys.buf_used);
xfree (reckeys.buf);
*dst++ = lead;
+#if SU_SCHEME
+ if ((lead & 3) < 3)
+ {
+ int ch = zebraExplain_lookupSU (zti, attrSet, attrUse);
+ if (ch < 0)
+ {
+ ch = zebraExplain_addSU (zti, attrSet, attrUse);
+ }
+ assert (ch > 0);
+ memcpy (dst, &ch, sizeof(ch));
+ dst += sizeof(ch);
+ }
+#else
if (!(lead & 1))
{
memcpy (dst, &attrSet, sizeof(attrSet));
memcpy (dst, &attrUse, sizeof(attrUse));
dst += sizeof(attrUse);
}
+#endif
*dst++ = p->reg_type;
memcpy (dst, string, length);
dst += length;
dst += sizeof(*pseqno);
}
reckeys.buf_used = dst - reckeys.buf;
- (*pseqno)++;
+ if (*pseqno)
+ (*pseqno)++;
}
static void addSortString (RecWord *p, const char *string, int length)
if (sk->attrSet == p->attrSet && sk->attrUse == p->attrUse)
return;
- sk = xmalloc (sizeof(*sk));
+ sk = (struct sortKey *) xmalloc (sizeof(*sk));
sk->next = sortKeys;
sortKeys = sk;
- sk->string = xmalloc (p->length);
- sk->length = p->length;
- memcpy (sk->string, p->string, p->length);
+ sk->string = (char *) xmalloc (length);
+ sk->length = length;
+ memcpy (sk->string, string, length);
+
sk->attrSet = p->attrSet;
sk->attrUse = p->attrUse;
}
return;
addString (p, buf, i);
}
+ (p->seqnos[p->reg_type])++; /* to separate this from next one */
}
static void addCompleteField (RecWord *p)
static void addRecordKey (RecWord *p)
{
+ WRBUF wrbuf;
+ if ((wrbuf = zebra_replace(p->zebra_maps, p->reg_type, 0,
+ p->string, p->length)))
+ {
+ p->string = wrbuf_buf(wrbuf);
+ p->length = wrbuf_len(wrbuf);
+ }
if (zebra_maps_is_complete (p->zebra_maps, p->reg_type))
addCompleteField (p);
else
while (sk)
{
struct sortKey *sk_next = sk->next;
- sortIdx_type (sortIdx, sk->attrUse);
- sortIdx_add (sortIdx, sk->string, sk->length);
+ if (cmd >= 0)
+ {
+ sortIdx_type (sortIdx, sk->attrUse);
+ sortIdx_add (sortIdx, sk->string, sk->length);
+ }
xfree (sk->string);
xfree (sk);
sk = sk_next;
sortKeys = NULL;
}
-static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys,
- const char *databaseName)
+static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys)
{
- char attrSet = -1;
- short attrUse = -1;
+#if SU_SCHEME
+#else
+ unsigned char attrSet = (unsigned char) -1;
+ unsigned short attrUse = (unsigned short) -1;
+#endif
int seqno = 0;
int off = 0;
+ int ch = 0;
- if (zebraExplain_curDatabase (zti, databaseName))
- {
- if (zebraExplain_newDatabase (zti, databaseName))
- abort ();
- }
zebraExplain_recordCountIncrement (zti, cmd ? 1 : -1);
while (off < reckeys->buf_used)
{
const char *src = reckeys->buf + off;
struct it_key key;
- int lead, ch;
+ int lead;
lead = *src++;
+#if SU_SCHEME
+ if ((lead & 3) < 3)
+ {
+ memcpy (&ch, src, sizeof(ch));
+ src += sizeof(ch);
+ }
+#else
if (!(lead & 1))
{
memcpy (&attrSet, src, sizeof(attrSet));
memcpy (&attrUse, src, sizeof(attrUse));
src += sizeof(attrUse);
}
+#endif
if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*))
key_flush ();
++ptr_i;
+
key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used;
+#if SU_SCHEME
+#else
ch = zebraExplain_lookupSU (zti, attrSet, attrUse);
if (ch < 0)
+ {
ch = zebraExplain_addSU (zti, attrSet, attrUse);
+ yaz_log (LOG_LOG, "addSU cmd=%d set=%d use=%d SU=%d",
+ cmd, attrSet, attrUse, ch);
+ }
+#endif
assert (ch > 0);
- ((char*) key_buf) [key_buf_used++] = ch;
+ key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used);
+
while (*src)
((char*)key_buf) [key_buf_used++] = *src++;
src++;
int off = 0;
int startSeq = -1;
int i;
+ int seqno = 0;
+#if SU_SCHEME
+ int chS, ch;
+#else
short attrUse;
char attrSet;
- int seqno = 0;
+#endif
for (i = 0; i<32; i++)
ws[i] = NULL;
-
+
+#if SU_SCHEME
+ chS = zebraExplain_lookupSU (zti, attrSetS, attrUseS);
+ if (chS < 0)
+ return ws;
+#endif
while (off < reckeys->buf_used)
{
int lead;
lead = *src++;
-
+#if SU_SCHEME
+ if ((lead & 3)<3)
+ {
+ memcpy (&ch, src, sizeof(ch));
+ src += sizeof(ch);
+ }
+#else
if (!(lead & 1))
{
memcpy (&attrSet, src, sizeof(attrSet));
memcpy (&attrUse, src, sizeof(attrUse));
src += sizeof(attrUse);
}
+#endif
wstart = src;
while (*src++)
;
memcpy (&seqno, src, sizeof(seqno));
src += sizeof(seqno);
}
- if (attrUseS == attrUse && attrSetS == attrSet)
+ if (
+#if SU_SCHEME
+ ch == chS
+#else
+ attrUseS == attrUse && attrSetS == attrSet
+#endif
+ )
{
int woff;
}
struct file_read_info {
- off_t file_max;
- off_t file_offset;
- off_t file_moffset;
+ off_t file_max; /* maximum offset so far */
+ off_t file_offset; /* current offset */
+ off_t file_moffset; /* offset of rec/rec boundary */
int file_more;
int fd;
+ char *sdrbuf;
+ int sdrmax;
};
static struct file_read_info *file_read_start (int fd)
{
- struct file_read_info *fi = xmalloc (sizeof(*fi));
+ struct file_read_info *fi = (struct file_read_info *)
+ xmalloc (sizeof(*fi));
fi->fd = fd;
fi->file_max = 0;
fi->file_moffset = 0;
+ fi->sdrbuf = 0;
+ fi->sdrmax = 0;
return fi;
}
static void file_read_stop (struct file_read_info *fi)
{
- assert (fi);
xfree (fi);
}
static off_t file_seek (void *handle, off_t offset)
{
- struct file_read_info *p = handle;
+ struct file_read_info *p = (struct file_read_info *) handle;
p->file_offset = offset;
+ if (p->sdrbuf)
+ return offset;
return lseek (p->fd, offset, SEEK_SET);
}
static off_t file_tell (void *handle)
{
- struct file_read_info *p = handle;
+ struct file_read_info *p = (struct file_read_info *) handle;
return p->file_offset;
}
static int file_read (void *handle, char *buf, size_t count)
{
- struct file_read_info *p = handle;
+ struct file_read_info *p = (struct file_read_info *) handle;
int fd = p->fd;
int r;
- r = read (fd, buf, count);
+ if (p->sdrbuf)
+ {
+ r = count;
+ if (r > p->sdrmax - p->file_offset)
+ r = p->sdrmax - p->file_offset;
+ if (r)
+ memcpy (buf, p->sdrbuf + p->file_offset, r);
+ }
+ else
+ r = read (fd, buf, count);
if (r > 0)
{
p->file_offset += r;
static void file_begin (void *handle)
{
- struct file_read_info *p = handle;
+ struct file_read_info *p = (struct file_read_info *) handle;
p->file_offset = p->file_moffset;
- if (p->file_moffset)
+ if (!p->sdrbuf && p->file_moffset)
lseek (p->fd, p->file_moffset, SEEK_SET);
p->file_more = 0;
}
static void file_end (void *handle, off_t offset)
{
- struct file_read_info *p = handle;
+ struct file_read_info *p = (struct file_read_info *) handle;
assert (p->file_more == 0);
p->file_more = 1;
p->file_moffset = offset;
}
-static int atois (const char **s)
-{
- int val = 0, c;
- while ( (c=**s) >= '0' && c <= '9')
- {
- val = val*10 + c - '0';
- ++(*s);
- }
- return val;
-}
-
static char *fileMatchStr (struct recKeys *reckeys, struct recordGroup *rGroup,
- const char *fname,
- const char *spec)
+ const char *fname, const char *spec)
{
static char dstBuf[2048];
char *dst = dstBuf;
const char *s = spec;
static const char **w;
- int i;
while (1)
{
break;
if (*s == '(')
{
+ char attset_str[64], attname_str[64];
+ data1_attset *attset;
+ int i;
char matchFlag[32];
- int attrSet, attrUse;
+ int attSet = 1, attUse = 1;
int first = 1;
s++;
- attrSet = atois (&s);
- if (*s != ',')
- {
- logf (LOG_WARN, "Missing , in match criteria %s in group %s",
- spec, rGroup->groupName ? rGroup->groupName : "none");
- return NULL;
- }
- s++;
- attrUse = atois (&s);
- w = searchRecordKey (reckeys, attrSet, attrUse);
+ for (i = 0; *s && *s != ',' && *s != ')'; s++)
+ if (i < 63)
+ attset_str[i++] = *s;
+ attset_str[i] = '\0';
+
+ if (*s == ',')
+ {
+ s++;
+ for (i = 0; *s && *s != ')'; s++)
+ if (i < 63)
+ attname_str[i++] = *s;
+ attname_str[i] = '\0';
+ }
+
+ if ((attset = data1_get_attset (rGroup->dh, attset_str)))
+ {
+ data1_att *att;
+ attSet = attset->reference;
+ att = data1_getattbyname(rGroup->dh, attset, attname_str);
+ if (att)
+ attUse = att->value;
+ else
+ attUse = atoi (attname_str);
+ }
+ w = searchRecordKey (reckeys, attSet, attUse);
assert (w);
if (*s == ')')
if (first)
{
logf (LOG_WARN, "Record didn't contain match"
- " fields in (%d,%d)", attrSet, attrUse);
+ " fields in (%s,%s)", attset_str, attname_str);
return NULL;
}
}
static void recordLogPreamble (int level, const char *msg, void *info)
{
- struct recordLogInfo *p = info;
- FILE *outf = log_file ();
+ struct recordLogInfo *p = (struct recordLogInfo *) info;
+ FILE *outf = yaz_log_file ();
if (level & LOG_LOG)
return ;
log_event_start (NULL, NULL);
}
+void addSchema (struct recExtractCtrl *p, Odr_oid *oid)
+{
+ zebraExplain_addSchema (zti, oid);
+}
+
static int recordExtract (SYSNO *sysno, const char *fname,
struct recordGroup *rGroup, int deleteFlag,
- struct file_read_info *fi, RecType recType,
- char *subType)
+ struct file_read_info *fi,
+ RecType recType, char *subType, void *clientData)
{
- struct recExtractCtrl extractCtrl;
RecordAttr *recordAttr;
int r;
char *matchStr;
SYSNO sysnotmp;
- off_t recordOffset = 0;
Record rec;
struct recordLogInfo logInfo;
+ off_t recordOffset = 0;
if (fi->fd != -1)
{
- int i;
- /* we are going to read from a file, so prepare the extraction */
- extractCtrl.fh = fi;
- extractCtrl.subType = subType;
- extractCtrl.init = wordInit;
- extractCtrl.add = addRecordKey;
- extractCtrl.dh = rGroup->dh;
+ struct recExtractCtrl extractCtrl;
- reckeys.buf_used = 0;
- reckeys.prevAttrUse = -1;
- reckeys.prevAttrSet = -1;
- reckeys.prevSeqNo = 0;
+ /* we are going to read from a file, so prepare the extraction */
+ int i;
+ reckeys.buf_used = 0;
+ reckeys.prevAttrUse = -1;
+ reckeys.prevAttrSet = -1;
+ reckeys.prevSeqNo = 0;
+
+ recordOffset = fi->file_moffset;
+ extractCtrl.offset = fi->file_moffset;
+ extractCtrl.readf = file_read;
+ extractCtrl.seekf = file_seek;
+ extractCtrl.tellf = file_tell;
+ extractCtrl.endf = file_end;
+ extractCtrl.fh = fi;
+ extractCtrl.subType = subType;
+ extractCtrl.init = wordInit;
+ extractCtrl.tokenAdd = addRecordKey;
+ extractCtrl.schemaAdd = addSchema;
+ extractCtrl.dh = rGroup->dh;
for (i = 0; i<256; i++)
- extractCtrl.seqno[i] = 0;
- recordOffset = fi->file_moffset;
- extractCtrl.offset = recordOffset;
- extractCtrl.readf = file_read;
- extractCtrl.seekf = file_seek;
- extractCtrl.tellf = file_tell;
- extractCtrl.endf = file_end;
+ {
+ if (zebra_maps_is_positioned(rGroup->zebra_maps, i))
+ extractCtrl.seqno[i] = 1;
+ else
+ extractCtrl.seqno[i] = 0;
+ }
extractCtrl.zebra_maps = rGroup->zebra_maps;
- extractCtrl.flagShowRecords = !rGroup->flagRw;
+ extractCtrl.flagShowRecords = !rGroup->flagRw;
+
if (!rGroup->flagRw)
- printf ("File: %s %ld\n", fname, (long) recordOffset);
+ printf ("File: %s " PRINTF_OFF_T "\n", fname, recordOffset);
logInfo.fname = fname;
logInfo.recordOffset = recordOffset;
logInfo.rGroup = rGroup;
log_event_start (recordLogPreamble, &logInfo);
- r = (*recType->extract)(&extractCtrl);
+ r = (*recType->extract)(clientData, &extractCtrl);
log_event_start (NULL, NULL);
- if (r)
- {
+ if (r == RECCTRL_EXTRACT_EOF)
+ return 0;
+ else if (r == RECCTRL_EXTRACT_ERROR)
+ {
/* error occured during extraction ... */
if (rGroup->flagRw &&
records_processed < rGroup->fileVerboseLimit)
{
- logf (LOG_WARN, "fail %s %s %ld code = %d", rGroup->recordType,
- fname, (long) recordOffset, r);
+ logf (LOG_WARN, "fail %s %s " PRINTF_OFF_T, rGroup->recordType,
+ fname, recordOffset);
}
return 0;
}
is probably empty - unless flagShowRecords is in use */
if (!rGroup->flagRw)
return 1;
- logf (LOG_WARN, "No keys generated for file %s", fname);
- logf (LOG_WARN, " The file is probably empty");
- return 0;
+
+ logf (LOG_WARN, "empty %s %s " PRINTF_OFF_T, rGroup->recordType,
+ fname, recordOffset);
+ return 1;
}
}
/* new record */
if (deleteFlag)
{
- logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T, rGroup->recordType,
+ fname, recordOffset);
logf (LOG_WARN, "cannot delete record above (seems new)");
return 1;
}
if (records_processed < rGroup->fileVerboseLimit)
- logf (LOG_LOG, "add %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "add %s %s " PRINTF_OFF_T, rGroup->recordType,
+ fname, recordOffset);
rec = rec_new (records);
*sysno = rec->sysno;
{
dict_insert (matchDict, matchStr, sizeof(*sysno), sysno);
}
- flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName);
+ flushRecordKeys (*sysno, 1, &reckeys);
flushSortKeys (*sysno, 1);
records_inserted++;
if (recordAttr->runNumber == zebraExplain_runNumberIncrement (zti, 0))
{
- logf (LOG_LOG, "skipped %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType,
+ fname, recordOffset);
+ flushSortKeys (*sysno, -1);
rec_rm (&rec);
logRecord (0);
return 1;
delkeys.buf_used = rec->size[recInfo_delKeys];
delkeys.buf = rec->info[recInfo_delKeys];
flushSortKeys (*sysno, 0);
- flushRecordKeys (*sysno, 0, &delkeys, rec->info[recInfo_databaseName]);
+ flushRecordKeys (*sysno, 0, &delkeys);
if (deleteFlag)
{
/* record going to be deleted */
if (!delkeys.buf_used)
{
- logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T,
+ rGroup->recordType, fname, recordOffset);
logf (LOG_WARN, "cannot delete file above, storeKeys false");
}
else
{
if (records_processed < rGroup->fileVerboseLimit)
- logf (LOG_LOG, "delete %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "delete %s %s " PRINTF_OFF_T,
+ rGroup->recordType, fname, recordOffset);
records_deleted++;
if (matchStr)
dict_delete (matchDict, matchStr);
/* record going to be updated */
if (!delkeys.buf_used)
{
- logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
+ logf (LOG_LOG, "update %s %s " PRINTF_OFF_T,
+ rGroup->recordType, fname, recordOffset);
logf (LOG_WARN, "cannot update file above, storeKeys false");
}
else
{
if (records_processed < rGroup->fileVerboseLimit)
- logf (LOG_LOG, "update %s %s %ld", rGroup->recordType,
- fname, (long) recordOffset);
- flushRecordKeys (*sysno, 1, &reckeys, rGroup->databaseName);
+ logf (LOG_LOG, "update %s %s " PRINTF_OFF_T,
+ rGroup->recordType, fname, recordOffset);
+ flushRecordKeys (*sysno, 1, &reckeys);
records_updated++;
}
}
if (rGroup->flagStoreData == 1)
{
rec->size[recInfo_storeData] = recordAttr->recordSize;
- rec->info[recInfo_storeData] = xmalloc (recordAttr->recordSize);
+ rec->info[recInfo_storeData] = (char *)
+ xmalloc (recordAttr->recordSize);
if (lseek (fi->fd, recordOffset, SEEK_SET) < 0)
{
- logf (LOG_ERRNO|LOG_FATAL, "seek to %ld in %s", fname,
- (long) recordOffset);
+ logf (LOG_ERRNO|LOG_FATAL, "seek to " PRINTF_OFF_T " in %s",
+ recordOffset, fname);
exit (1);
}
if (read (fi->fd, rec->info[recInfo_storeData], recordAttr->recordSize)
struct recordGroup rGroupM;
struct recordGroup *rGroup = &rGroupM;
struct file_read_info *fi;
+ void *clientData;
memcpy (rGroup, rGroupP, sizeof(*rGroupP));
logf (LOG_DEBUG, "fileExtract %s", fname);
/* determine file extension */
+ *ext = '\0';
for (i = strlen(fname); --i >= 0; )
if (fname[i] == '/')
- {
- strcpy (ext, "");
break;
- }
else if (fname[i] == '.')
{
strcpy (ext, fname+i+1);
if (!(rGroup->recordType = res_get (common_resource, ext_res)))
{
sprintf (ext_res, "%srecordType", gprefix);
- if (!(rGroup->recordType = res_get (common_resource, ext_res)))
- {
- if (records_processed < rGroup->fileVerboseLimit)
- logf (LOG_LOG, "? %s", fname);
- return 0;
- }
+ rGroup->recordType = res_get (common_resource, ext_res);
}
}
if (!rGroup->recordType)
{
if (records_processed < rGroup->fileVerboseLimit)
- logf (LOG_LOG, "? record %s", fname);
+ logf (LOG_LOG, "? %s", fname);
return 0;
}
- if (!(recType = recType_byName (rGroup->recordType, subType)))
+ if (!*rGroup->recordType)
+ return 0;
+ if (!(recType =
+ recType_byName (rGroup->recTypes, rGroup->recordType, subType,
+ &clientData)))
{
logf (LOG_WARN, "No such record type: %s", rGroup->recordType);
return 0;
if (!rGroup->databaseName)
rGroup->databaseName = "Default";
+ /* determine if explain database */
+
+ sprintf (ext_res, "%sexplainDatabase", gprefix);
+ rGroup->explainDatabase =
+ atoi (res_get_def (common_resource, ext_res, "0"));
+
+ /* announce database */
+ if (zebraExplain_curDatabase (zti, rGroup->databaseName))
+ {
+ if (zebraExplain_newDatabase (zti, rGroup->databaseName,
+ rGroup->explainDatabase))
+ return 0;
+ }
+
if (rGroup->flagStoreData == -1)
{
const char *sval;
{
file_begin (fi);
r = recordExtract (sysno, fname, rGroup, deleteFlag, fi,
- recType, subType);
+ recType, subType, clientData);
} while (r && !sysno && fi->file_more);
file_read_stop (fi);
if (fd != -1)
return r;
}
+static int explain_extract (void *handle, Record rec, data1_node *n)
+{
+ struct recordGroup *rGroup = (struct recordGroup*) handle;
+ struct recExtractCtrl extractCtrl;
+ int i;
+
+ if (zebraExplain_curDatabase (zti, rec->info[recInfo_databaseName]))
+ {
+ if (zebraExplain_newDatabase (zti, rec->info[recInfo_databaseName], 0))
+ abort ();
+ }
+
+ reckeys.buf_used = 0;
+ reckeys.prevAttrUse = -1;
+ reckeys.prevAttrSet = -1;
+ reckeys.prevSeqNo = 0;
+
+ extractCtrl.init = wordInit;
+ extractCtrl.tokenAdd = addRecordKey;
+ extractCtrl.schemaAdd = addSchema;
+ extractCtrl.dh = rGroup->dh;
+ for (i = 0; i<256; i++)
+ extractCtrl.seqno[i] = 0;
+ extractCtrl.zebra_maps = rGroup->zebra_maps;
+ extractCtrl.flagShowRecords = !rGroup->flagRw;
+
+ grs_extract_tree(&extractCtrl, n);
+
+ logf (LOG_DEBUG, "flush explain record, sysno=%d", rec->sysno);
+
+ if (rec->size[recInfo_delKeys])
+ {
+ struct recKeys delkeys;
+
+ delkeys.buf_used = rec->size[recInfo_delKeys];
+ delkeys.buf = rec->info[recInfo_delKeys];
+ flushSortKeys (rec->sysno, 0);
+ flushRecordKeys (rec->sysno, 0, &delkeys);
+ }
+ flushRecordKeys (rec->sysno, 1, &reckeys);
+ flushSortKeys (rec->sysno, 1);
+
+ xfree (rec->info[recInfo_delKeys]);
+ rec->size[recInfo_delKeys] = reckeys.buf_used;
+ rec->info[recInfo_delKeys] = reckeys.buf;
+ reckeys.buf = NULL;
+ reckeys.buf_max = 0;
+
+ return 0;
+}