From 1ccf2613ceef2359f589cb3dd7e72a899c618b2f Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Tue, 5 Dec 2000 10:01:44 +0000 Subject: [PATCH] Fixed bug regarding user-defined attribute sets. --- CHANGELOG | 3 +++ TODO | 11 +++++++- index/Makefile.am | 6 +++-- index/Makefile.in | 32 ++++++++++++++++------ index/extract.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++----- index/index.h | 9 +++++-- index/kdump.c | 7 +++-- index/recindxp.h | 7 +++-- index/zebraapi.c | 40 +++++++++++++++++++++++++--- index/zinfo.c | 9 ++++--- recctrl/recgrs.c | 9 +++++-- 11 files changed, 177 insertions(+), 32 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3c328f4..56b656b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,8 @@ Remote record import using Z39.50 Extended Services and Segments. +Fixed bug where updating a database with user-defined attributes +could corrupt the register (bad storeKeys). + Multi-threaded version. Fixed bug regarding proximity. diff --git a/TODO b/TODO index 34d84a2..55848cd 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,14 @@ Zebra TODO -$Id: TODO,v 1.9 2000-07-07 12:49:20 adam Exp $ +$Id: TODO,v 1.10 2000-12-05 10:01:44 adam Exp $ + +Explain category Attributedetails should include name/description +of search- attrribute. + +String indexing/search (possibly including XPATH-like queres) + +Null-value search. + +Handle XML empty node constructs Browse set. diff --git a/index/Makefile.am b/index/Makefile.am index cc2ddfc..adf2b96 100644 --- a/index/Makefile.am +++ b/index/Makefile.am @@ -1,6 +1,6 @@ -## $Id: Makefile.am,v 1.4 2000-10-17 12:37:09 adam Exp $ +## $Id: Makefile.am,v 1.5 2000-12-05 10:01:44 adam Exp $ -noinst_PROGRAMS = apitest +noinst_PROGRAMS = apitest kdump if ISZMBOL bin_PROGRAMS = zmbolidx zmbolsrv @@ -31,6 +31,8 @@ apitest_SOURCES = apitest.c kcompare.c zrpn.c zsets.c attribute.c recindex.c \ retrieve.c kinput.c \ index.h recindex.h recindxp.h zebraapi.h zinfo.h zserver.h +kdump_SOURCES=kdump.c kcompare.c + INCLUDES = -I$(srcdir)/../include $(YAZINC) $(TCL_INCLUDE) if ISZMBOL diff --git a/index/Makefile.in b/index/Makefile.in index 61c95d5..cc604a6 100644 --- a/index/Makefile.in +++ b/index/Makefile.in @@ -73,7 +73,7 @@ YAZLIB = @YAZLIB@ YAZVERSION = @YAZVERSION@ yazconfig = @yazconfig@ -noinst_PROGRAMS = apitest +noinst_PROGRAMS = apitest kdump @ISZMBOL_TRUE@bin_PROGRAMS = zmbolidx zmbolsrv @ISZMBOL_FALSE@bin_PROGRAMS = zebraidx zebrasrv @ISZMBOL_TRUE@zmbolidx_SOURCES = main.c dir.c dirs.c trav.c extract.c kinput.c kcompare.c symtab.c recindex.c recstat.c lockutil.c lockidx.c zinfo.c invstat.c sortidx.c compact.c index.h recindex.h recindxp.h zebraapi.h zinfo.h zserver.h @@ -86,6 +86,8 @@ noinst_PROGRAMS = apitest apitest_SOURCES = apitest.c kcompare.c zrpn.c zsets.c attribute.c recindex.c lockutil.c locksrv.c zinfo.c trunc.c sortidx.c rank1.c zebraapi.c retrieve.c kinput.c index.h recindex.h recindxp.h zebraapi.h zinfo.h zserver.h +kdump_SOURCES = kdump.c kcompare.c + INCLUDES = -I$(srcdir)/../include $(YAZINC) $(TCL_INCLUDE) @ISZMBOL_TRUE@zmbol_libs = ../isam/libisam.a ../isamc/libisamc.a ../isamb/libisamb.a @ISZMBOL_FALSE@zmbol_libs = @@ -171,6 +173,16 @@ apitest_LDADD = $(LDADD) @ISZMBOL_TRUE@../recctrl/librecctrl.a ../bfile/libbfile.a \ @ISZMBOL_TRUE@../dfa/libdfa.a ../util/libutil.a apitest_LDFLAGS = +kdump_OBJECTS = kdump.o kcompare.o +kdump_LDADD = $(LDADD) +@ISZMBOL_FALSE@kdump_DEPENDENCIES = ../rset/librset.a ../dict/libdict.a \ +@ISZMBOL_FALSE@../isams/libisams.a ../recctrl/librecctrl.a \ +@ISZMBOL_FALSE@../bfile/libbfile.a ../dfa/libdfa.a ../util/libutil.a +@ISZMBOL_TRUE@kdump_DEPENDENCIES = ../rset/librset.a ../dict/libdict.a \ +@ISZMBOL_TRUE@../isams/libisams.a ../isam/libisam.a ../isamc/libisamc.a \ +@ISZMBOL_TRUE@../isamb/libisamb.a ../recctrl/librecctrl.a \ +@ISZMBOL_TRUE@../bfile/libbfile.a ../dfa/libdfa.a ../util/libutil.a +kdump_LDFLAGS = CFLAGS = @CFLAGS@ COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) CCLD = $(CC) @@ -184,13 +196,13 @@ TAR = gtar GZIP_ENV = --best DEP_FILES = .deps/apitest.P .deps/attribute.P .deps/compact.P \ .deps/dir.P .deps/dirs.P .deps/extract.P .deps/invstat.P \ -.deps/kcompare.P .deps/kinput.P .deps/lockidx.P .deps/locksrv.P \ -.deps/lockutil.P .deps/main.P .deps/rank1.P .deps/recindex.P \ -.deps/recstat.P .deps/retrieve.P .deps/sortidx.P .deps/symtab.P \ -.deps/trav.P .deps/trunc.P .deps/zebraapi.P .deps/zinfo.P .deps/zrpn.P \ -.deps/zserver.P .deps/zsets.P -SOURCES = $(zmbolidx_SOURCES) $(zmbolsrv_SOURCES) $(zebraidx_SOURCES) $(zebrasrv_SOURCES) $(apitest_SOURCES) -OBJECTS = $(zmbolidx_OBJECTS) $(zmbolsrv_OBJECTS) $(zebraidx_OBJECTS) $(zebrasrv_OBJECTS) $(apitest_OBJECTS) +.deps/kcompare.P .deps/kdump.P .deps/kinput.P .deps/lockidx.P \ +.deps/locksrv.P .deps/lockutil.P .deps/main.P .deps/rank1.P \ +.deps/recindex.P .deps/recstat.P .deps/retrieve.P .deps/sortidx.P \ +.deps/symtab.P .deps/trav.P .deps/trunc.P .deps/zebraapi.P \ +.deps/zinfo.P .deps/zrpn.P .deps/zserver.P .deps/zsets.P +SOURCES = $(zmbolidx_SOURCES) $(zmbolsrv_SOURCES) $(zebraidx_SOURCES) $(zebrasrv_SOURCES) $(apitest_SOURCES) $(kdump_SOURCES) +OBJECTS = $(zmbolidx_OBJECTS) $(zmbolsrv_OBJECTS) $(zebraidx_OBJECTS) $(zebrasrv_OBJECTS) $(apitest_OBJECTS) $(kdump_OBJECTS) all: all-redirect .SUFFIXES: @@ -273,6 +285,10 @@ apitest: $(apitest_OBJECTS) $(apitest_DEPENDENCIES) @rm -f apitest $(LINK) $(apitest_LDFLAGS) $(apitest_OBJECTS) $(apitest_LDADD) $(LIBS) +kdump: $(kdump_OBJECTS) $(kdump_DEPENDENCIES) + @rm -f kdump + $(LINK) $(kdump_LDFLAGS) $(kdump_OBJECTS) $(kdump_LDADD) $(LIBS) + tags: TAGS ID: $(HEADERS) $(SOURCES) $(LISP) diff --git a/index/extract.c b/index/extract.c index ca0dc46..366c6ec 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1994-1999, Index Data + * Copyright (C) 1994-2000, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: extract.c,v $ - * Revision 1.104 2000-09-05 14:04:05 adam + * Revision 1.105 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.104 2000/09/05 14:04:05 adam * Updates for prefix 'yaz_' for YAZ log functions. * * Revision 1.103 2000/05/18 12:01:36 adam @@ -419,6 +422,7 @@ static int records_processed = 0; static ZebraExplainInfo zti = NULL; + static void logRecord (int showFlag) { if (!showFlag) @@ -739,6 +743,21 @@ static void addIndexString (RecWord *p, const char *string, int length) *dst++ = lead; +#if SU_SCHEME + if ((lead & 3) < 3) + { + int ch = zebraExplain_lookupSU (zti, attrSet, attrUse); + if (ch < 0) + { + ch = zebraExplain_addSU (zti, attrSet, attrUse); + yaz_log (LOG_LOG, "addSU set=%d use=%d SU=%d", + attrSet, attrUse, ch); + } + assert (ch > 0); + memcpy (dst, &ch, sizeof(ch)); + dst += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (dst, &attrSet, sizeof(attrSet)); @@ -749,6 +768,7 @@ static void addIndexString (RecWord *p, const char *string, int length) memcpy (dst, &attrUse, sizeof(attrUse)); dst += sizeof(attrUse); } +#endif *dst++ = p->reg_type; memcpy (dst, string, length); dst += length; @@ -918,20 +938,31 @@ static void flushSortKeys (SYSNO sysno, int cmd) static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) { +#if SU_SCHEME +#else unsigned char attrSet = (unsigned char) -1; unsigned short attrUse = (unsigned short) -1; +#endif int seqno = 0; int off = 0; + int ch = 0; zebraExplain_recordCountIncrement (zti, cmd ? 1 : -1); while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; struct it_key key; - int lead, ch; + int lead; lead = *src++; +#if SU_SCHEME + if ((lead & 3) < 3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -942,14 +973,23 @@ static void flushRecordKeys (SYSNO sysno, int cmd, struct recKeys *reckeys) memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif if (key_buf_used + 1024 > (ptr_top-ptr_i)*sizeof(char*)) key_flush (); ++ptr_i; + key_buf[ptr_top-ptr_i] = (char*)key_buf + key_buf_used; +#if SU_SCHEME +#else ch = zebraExplain_lookupSU (zti, attrSet, attrUse); if (ch < 0) + { ch = zebraExplain_addSU (zti, attrSet, attrUse); + yaz_log (LOG_LOG, "addSU cmd=%d set=%d use=%d SU=%d", + cmd, attrSet, attrUse, ch); + } +#endif assert (ch > 0); key_buf_used += key_SU_code (ch, ((char*)key_buf) + key_buf_used); @@ -982,13 +1022,22 @@ static const char **searchRecordKey (struct recKeys *reckeys, int off = 0; int startSeq = -1; int i; + int seqno = 0; +#if SU_SCHEME + int chS, ch; +#else short attrUse; char attrSet; - int seqno = 0; +#endif for (i = 0; i<32; i++) ws[i] = NULL; - + +#if SU_SCHEME + chS = zebraExplain_lookupSU (zti, attrSetS, attrUseS); + if (chS < 0) + return ws; +#endif while (off < reckeys->buf_used) { @@ -997,7 +1046,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, int lead; lead = *src++; - +#if SU_SCHEME + if ((lead & 3)<3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -1008,6 +1063,7 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif wstart = src; while (*src++) ; @@ -1018,7 +1074,13 @@ static const char **searchRecordKey (struct recKeys *reckeys, memcpy (&seqno, src, sizeof(seqno)); src += sizeof(seqno); } - if (attrUseS == attrUse && attrSetS == attrSet) + if ( +#if SU_SCHEME + ch == chS +#else + attrUseS == attrUse && attrSetS == attrSet +#endif + ) { int woff; diff --git a/index/index.h b/index/index.h index 3fabd51..b241e6a 100644 --- a/index/index.h +++ b/index/index.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 1995-1999, Index Data + * Copyright (C) 1995-0000, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss, Heikki Levanto * (log at the end) @@ -28,6 +28,8 @@ YAZ_BEGIN_CDECL +#define SU_SCHEME 1 + #define IT_MAX_WORD 256 #define IT_KEY_HAVE_SEQNO 1 #define IT_KEY_HAVE_FIELD 0 @@ -165,7 +167,10 @@ YAZ_END_CDECL #endif /* * $Log: index.h,v $ - * Revision 1.69 2000-03-20 19:08:36 adam + * Revision 1.70 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.69 2000/03/20 19:08:36 adam * Added remote record import using Z39.50 extended services and Segment * Requests. * diff --git a/index/kdump.c b/index/kdump.c index c9aae4a..a3d3901 100644 --- a/index/kdump.c +++ b/index/kdump.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: kdump.c,v $ - * Revision 1.18 1999-09-07 07:19:21 adam + * Revision 1.19 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.18 1999/09/07 07:19:21 adam * Work on character mapping. Implemented replace rules. * * Revision 1.17 1999/02/02 14:50:55 adam @@ -163,7 +166,7 @@ int main (int argc, char **argv) } else if (ret == 'v') { - log_init (log_mask_str(arg), prog, NULL); + yaz_log_init (yaz_log_mask_str(arg), prog, NULL); } else if (ret == 'c') { diff --git a/index/recindxp.h b/index/recindxp.h index a2a7f6d..24d7259 100644 --- a/index/recindxp.h +++ b/index/recindxp.h @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: recindxp.h,v $ - * Revision 1.8 2000-04-05 09:49:35 adam + * Revision 1.9 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.8 2000/04/05 09:49:35 adam * On Unix, zebra/z'mbol uses automake. * * Revision 1.7 1999/07/06 12:28:04 adam @@ -47,7 +50,7 @@ YAZ_BEGIN_CDECL #define REC_BLOCK_TYPES 2 #define REC_HEAD_MAGIC "recindex" -#define REC_VERSION 1 +#define REC_VERSION 2 struct records_info { int rw; diff --git a/index/zebraapi.c b/index/zebraapi.c index a2c2d55..d2f6d9b 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -3,7 +3,10 @@ * All rights reserved. * * $Log: zebraapi.c,v $ - * Revision 1.41 2000-12-01 17:59:08 adam + * Revision 1.42 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.41 2000/12/01 17:59:08 adam * Fixed bug regarding online updates on WIN32. * When zebra.cfg is not available the server will not abort. * @@ -882,6 +885,7 @@ static void extract_add_index_string (RecWord *p, const char *string, int diff = 0; int *pseqno = &p->seqnos[p->reg_type]; ZebraHandle zh = p->extractCtrl->handle; + ZebraExplainInfo zei = zh->service->zei; struct recKeys *keys = &zh->keys; if (keys->buf_used+1024 > keys->buf_max) @@ -917,6 +921,21 @@ static void extract_add_index_string (RecWord *p, const char *string, *dst++ = lead; +#if SU_SCHEME + if ((lead & 3) < 3) + { + int ch = zebraExplain_lookupSU (zei, attrSet, attrUse); + if (ch < 0) + { + ch = zebraExplain_addSU (zei, attrSet, attrUse); + yaz_log (LOG_LOG, "addSU set=%d use=%d SU=%d", + attrSet, attrUse, ch); + } + assert (ch > 0); + memcpy (dst, &ch, sizeof(ch)); + dst += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (dst, &attrSet, sizeof(attrSet)); @@ -927,6 +946,7 @@ static void extract_add_index_string (RecWord *p, const char *string, memcpy (dst, &attrUse, sizeof(attrUse)); dst += sizeof(attrUse); } +#endif *dst++ = p->reg_type; memcpy (dst, string, length); dst += length; @@ -1262,10 +1282,14 @@ static void extract_flushWriteKeys (ZebraHandle zh) static void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct recKeys *reckeys) { +#if SU_SCHEME +#else unsigned char attrSet = (unsigned char) -1; unsigned short attrUse = (unsigned short) -1; +#endif int seqno = 0; int off = 0; + int ch = 0; ZebraExplainInfo zei = zh->service->zei; if (!zh->key_buf) @@ -1282,10 +1306,17 @@ static void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, { const char *src = reckeys->buf + off; struct it_key key; - int lead, ch; + int lead; lead = *src++; +#if SU_SCHEME + if ((lead & 3) < 3) + { + memcpy (&ch, src, sizeof(ch)); + src += sizeof(ch); + } +#else if (!(lead & 1)) { memcpy (&attrSet, src, sizeof(attrSet)); @@ -1296,15 +1327,18 @@ static void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, memcpy (&attrUse, src, sizeof(attrUse)); src += sizeof(attrUse); } +#endif if (zh->key_buf_used + 1024 > (zh->ptr_top-zh->ptr_i)*sizeof(char*)) extract_flushWriteKeys (zh); ++(zh->ptr_i); (zh->key_buf)[zh->ptr_top - zh->ptr_i] = (char*)zh->key_buf + zh->key_buf_used; - +#if SU_SCHEME +#else ch = zebraExplain_lookupSU (zei, attrSet, attrUse); if (ch < 0) ch = zebraExplain_addSU (zei, attrSet, attrUse); +#endif assert (ch > 0); zh->key_buf_used += key_SU_code (ch,((char*)zh->key_buf) + zh->key_buf_used); diff --git a/index/zinfo.c b/index/zinfo.c index daf64f9..7224583 100644 --- a/index/zinfo.c +++ b/index/zinfo.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1994-1999, Index Data + * Copyright (C) 1994-2000, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: zinfo.c,v $ - * Revision 1.20 2000-11-29 14:24:01 adam + * Revision 1.21 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.20 2000/11/29 14:24:01 adam * Script configure uses yaz pthreads options. Added locking for * zebra_register_{lock,unlock}. * @@ -762,7 +765,7 @@ static void zebraExplain_readDatabase (ZebraExplainInfo zei, { zdi->recordCount = atoi_n (np->child->u.data.data, np->child->u.data.len); - } + } zdi->readFlag = 0; rec_rm (&rec); } diff --git a/recctrl/recgrs.c b/recctrl/recgrs.c index 9e13697..8e60bf0 100644 --- a/recctrl/recgrs.c +++ b/recctrl/recgrs.c @@ -1,10 +1,13 @@ /* - * Copyright (C) 1994-1999, Index Data + * Copyright (C) 1994-2000, Index Data * All rights reserved. * Sebastian Hammer, Adam Dickmeiss * * $Log: recgrs.c,v $ - * Revision 1.35 2000-11-29 15:21:31 adam + * Revision 1.36 2000-12-05 10:01:44 adam + * Fixed bug regarding user-defined attribute sets. + * + * Revision 1.35 2000/11/29 15:21:31 adam * Fixed problem with passwd db. * * Revision 1.34 2000/02/25 13:24:49 adam @@ -578,7 +581,9 @@ static int grs_retrieve(void *clientData, struct recRetrieveCtrl *p) nmem_destroy (mem); return 0; } +#if 0 data1_pr_tree (p->dh, node, stdout); +#endif logf (LOG_DEBUG, "grs_retrieve: size"); if ((dnew = data1_insert_taggeddata(p->dh, node, node, "size", mem))) -- 1.7.10.4