X-Git-Url: http://sru.miketaylor.org.uk/?a=blobdiff_plain;f=index%2Fextract.c;h=86de3b25cc75b51d67a97e9d1f7de3daf7675ade;hb=77686142af94172d1887190ebd47aeb53f704057;hp=269c21f417a35baaf11c9f6886eb43ff8d74fca3;hpb=297ba5c5c265a5f869f43a0a211bf9f48f700add;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 269c21f..86de3b2 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,30 @@ /* - * Copyright (C) 1994-1999, Index Data + * Copyright (C) 1994-2001, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.103 2000-05-18 12:01:36 adam + * Revision 1.109 2001-10-15 19:53:43 adam + * POSIX thread updates. First work on term sets. + * + * Revision 1.108 2001/06/14 11:44:56 adam + * Bug fix: default storeKeys setting wasn't read when group was specified. + * + * Revision 1.107 2001/05/28 13:58:48 adam + * Call flushSortKeys when record is skipped to fix bad re-use of + * sort keys to whatever next record that comes in. + * + * Revision 1.106 2000/12/05 12:22:53 adam + * Termlist source implemented (so that we can index values of XML/SGML + * attributes). + * + * Revision 1.105 2000/12/05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.104 2000/09/05 14:04:05 adam + * Updates for prefix 'yaz_' for YAZ log functions. + * + * Revision 1.103 2000/05/18 12:01:36 adam * System call times(2) used again. More 64-bit fixes. * * Revision 1.102 2000/05/15 15:32:33 adam @@ -416,6 +436,7 @@ static int records_processed = 0; static ZebraExplainInfo zti = NULL; + static void logRecord (int showFlag) { if (!showFlag) @@ -736,6 +757,19 @@ static void addIndexString (RecWord *p, const char *string, int length) *dst++ = lead; +#if SU_SCHEME + if ((lead & 3) < 3) + { + int ch = zebraExplain_lookupSU (zti, attrSet, attrUse); + if (ch < 0) + { + ch = zebraExplain_addSU (zti, attrSet, attrUse); + } + assert (ch > 0); + memcpy (dst, &ch, sizeof(ch)); + dst += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (dst, &attrSet, sizeof(attrSet)); @@ -746,6 +780,7 @@ static void addIndexString (RecWord *p, const char *string, int length) memcpy (dst, &attrUse, sizeof(attrUse)); dst += sizeof(attrUse); } +#endif *dst++ = p->reg_type; memcpy (dst, string, length); dst += length; @@ -904,8 +939,11 @@ static void flushSortKeys (SYSNO sysno, int cmd) while (sk) { struct sortKey *sk_next = sk->next; - sortIdx_type (sortIdx, sk->attrUse); - sortIdx_add (sortIdx, sk->string, sk->length); + if (cmd >= 0) + { + sortIdx_type (sortIdx, sk->attrUse); + sortIdx_add (sortIdx, sk->string, sk->length); + } xfree (sk->string); xfree (sk); sk = sk_next; @@ -915,20 +953,31 @@ static void flushSortKeys (SYSNO sysno, int cmd) static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) { +#if SU_SCHEME +#else unsigned char attrSet = (unsigned char) -1; unsigned short attrUse = (unsigned short) -1; +#endif int seqno = 0; int off = 0; + int ch = 0; zebraExplain_recordCountIncrement (zti, cmd ? 1 : -1); while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; struct it_key key; - int lead, ch; + int lead; lead = *src++; +#if SU_SCHEME + if ((lead & 3) < 3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -939,16 +988,25 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*)) key_flush (); ++ptr_i; + key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used; +#if SU_SCHEME +#else ch = zebraExplain_lookupSU (zti, attrSet, attrUse); if (ch < 0) + { ch = zebraExplain_addSU (zti, attrSet, attrUse); + yaz_log (LOG_LOG, "addSU cmd=%d set=%d use=%d SU=%d", + cmd, attrSet, attrUse, ch); + } +#endif assert (ch > 0); - key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used); + key_buf_used += key_SU_encode (ch, ((char*)key_buf) + key_buf_used); while (*src) ((char*)key_buf) [key_buf_used++] = *src++; @@ -979,13 +1037,22 @@ static const char **searchRecordKey (struct recKeys *reckeys, int off = 0; int startSeq = -1; int i; + int seqno = 0; +#if SU_SCHEME + int chS, ch; +#else short attrUse; char attrSet; - int seqno = 0; +#endif for (i = 0; i<32; i++) ws[i] = NULL; - + +#if SU_SCHEME + chS = zebraExplain_lookupSU (zti, attrSetS, attrUseS); + if (chS < 0) + return ws; +#endif while (off < reckeys->buf_used) { @@ -994,7 +1061,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, int lead; lead = *src++; - +#if SU_SCHEME + if ((lead & 3)<3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -1005,6 +1078,7 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif wstart = src; while (*src++) ; @@ -1015,7 +1089,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&seqno, src, sizeof(seqno)); src += sizeof(seqno); } - if (attrUseS == attrUse && attrSetS == attrSet) + if ( +#if SU_SCHEME + ch == chS +#else + attrUseS == attrUse && attrSetS == attrSet +#endif + ) { int woff; @@ -1277,7 +1357,7 @@ struct recordLogInfo { static void recordLogPreamble (int level, const char *msg, void *info) { struct recordLogInfo *p = (struct recordLogInfo *) info; - FILE *outf = log_file (); + FILE *outf = yaz_log_file (); if (level & LOG_LOG) return ; @@ -1445,6 +1525,7 @@ static int recordExtract (SYSNO *sysno, const char *fname, { logf (LOG_LOG, "skipped %s %s " PRINTF_OFF_T, rGroup->recordType, fname, recordOffset); + flushSortKeys (*sysno, -1); rec_rm (&rec); logRecord (0); return 1; @@ -1690,11 +1771,14 @@ int fileExtract (SYSNO *sysno, const char *fname, const char *sval; sprintf (ext_res, "%sstoreKeys.%s", gprefix, ext); - if (!(sval = res_get (common_resource, ext_res))) + sval = res_get (common_resource, ext_res); + if (!sval) { sprintf (ext_res, "%sstoreKeys", gprefix); sval = res_get (common_resource, ext_res); } + if (!sval) + sval = res_get (common_resource, "storeKeys"); if (sval) rGroup->flagStoreKeys = atoi (sval); }