1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2011 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
37 #define KEY_SIZE (1+sizeof(struct it_key))
38 #define INP_NAME_MAX 768
42 off_t offset; /* file offset */
43 unsigned char *buf; /* buffer block */
44 size_t buf_size; /* number of read bytes in block */
45 size_t chunk; /* number of bytes allocated */
46 size_t buf_ptr; /* current position in buffer */
47 char *prev_name; /* last word read */
49 off_t length; /* length of file */
50 /* handler invoked in each read */
51 void (*readHandler)(struct key_file *keyp, void *rinfo);
60 #if PR_KEY_LOW || PR_KEY_TOP
62 static void pkey(const char *b, int mode)
64 key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d");
69 void getFnameTmp(Res res, char *fname, int no)
73 pre = res_get_def(res, "keyTmpDir", ".");
74 sprintf(fname, "%s/key%d.tmp", pre, no);
77 void extract_get_fname_tmp(ZebraHandle zh, char *fname, int no)
81 pre = res_get_def(zh->res, "keyTmpDir", ".");
82 sprintf(fname, "%s/key%d.tmp", pre, no);
85 void key_file_chunk_read(struct key_file *f)
87 int nr = 0, r = 0, fd;
89 getFnameTmp(f->res, fname, f->no);
90 fd = open(fname, O_BINARY|O_RDONLY);
96 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot open %s", fname);
101 if ((f->length = lseek(fd, 0L, SEEK_END)) == (off_t) -1)
103 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
108 if (lseek(fd, f->offset, SEEK_SET) == -1)
110 yaz_log(YLOG_WARN|YLOG_ERRNO, "cannot seek %s", fname);
114 while (f->chunk - nr > 0)
116 r = read(fd, f->buf + nr, f->chunk - nr);
123 yaz_log(YLOG_WARN|YLOG_ERRNO, "read of %s", fname);
129 (*f->readHandler)(f, f->readInfo);
133 void key_file_destroy(struct key_file *f)
135 iscz1_stop(f->decode_handle);
141 struct key_file *key_file_init(int no, int chunk, Res res)
145 f = (struct key_file *) xmalloc(sizeof(*f));
147 f->decode_handle = iscz1_start();
152 f->readHandler = NULL;
153 f->buf = (unsigned char *) xmalloc(f->chunk);
154 f->prev_name = (char *) xmalloc(INP_NAME_MAX);
155 *f->prev_name = '\0';
156 key_file_chunk_read(f);
160 int key_file_getc(struct key_file *f)
162 if (f->buf_ptr < f->buf_size)
163 return f->buf[(f->buf_ptr)++];
164 if (f->buf_size < f->chunk)
166 f->offset += f->buf_size;
167 key_file_chunk_read(f);
168 if (f->buf_ptr < f->buf_size)
169 return f->buf[(f->buf_ptr)++];
174 int key_file_read(struct key_file *f, char *key)
178 const char *src = srcbuf;
182 c = key_file_getc(f);
185 strcpy(key, f->prev_name);
194 while ((c = key_file_getc(f)))
196 if (i < INP_NAME_MAX-2)
200 strcpy(f->prev_name, key);
201 iscz1_reset(f->decode_handle);
203 c = key_file_getc(f); /* length + insert/delete combined */
206 for (j = 0; j < c; j++)
207 srcbuf[j] = key_file_getc(f);
209 iscz1_decode(f->decode_handle, &dst, &src);
216 memcpy(&k, key+i, sizeof(k));
218 yaz_log(YLOG_LOG, "00 KEY");
221 return i + sizeof(struct it_key);
226 struct key_file **file;
231 int (*cmp)(const void *p1, const void *p2);
232 struct zebra_register *reg;
241 static struct heap_info *key_heap_malloc(void)
242 { /* malloc and clear it */
243 struct heap_info *hi;
244 hi = (struct heap_info *) xmalloc(sizeof(*hi));
252 hi->no_deletions = 0;
253 hi->no_insertions = 0;
254 hi->no_iterations = 0;
258 struct heap_info *key_heap_init_file(ZebraHandle zh,
260 int (*cmp)(const void *p1, const void *p2))
262 struct heap_info *hi;
265 hi = key_heap_malloc();
267 hi->info.file = (struct key_file **)
268 xmalloc(sizeof(*hi->info.file) * (1+nkeys));
269 hi->info.buf = (char **) xmalloc(sizeof(*hi->info.buf) * (1+nkeys));
270 hi->ptr = (int *) xmalloc(sizeof(*hi->ptr) * (1+nkeys));
272 for (i = 0; i<= nkeys; i++)
275 hi->info.buf[i] = (char *) xmalloc(INP_NAME_MAX);
280 void key_heap_destroy(struct heap_info *hi, int nkeys)
283 for (i = 0; i<=nkeys; i++)
284 xfree(hi->info.buf[i]);
287 xfree(hi->info.file);
291 static void key_heap_swap(struct heap_info *hi, int i1, int i2)
296 hi->ptr[i1] = hi->ptr[i2];
301 static void key_heap_delete(struct heap_info *hi)
303 int cur = 1, child = 2;
305 assert(hi->heapnum > 0);
307 key_heap_swap(hi, 1, hi->heapnum);
309 while (child <= hi->heapnum) {
310 if (child < hi->heapnum &&
311 (*hi->cmp)(&hi->info.buf[hi->ptr[child]],
312 &hi->info.buf[hi->ptr[child+1]]) > 0)
314 if ((*hi->cmp)(&hi->info.buf[hi->ptr[cur]],
315 &hi->info.buf[hi->ptr[child]]) > 0)
317 key_heap_swap(hi, cur, child);
326 static void key_heap_insert(struct heap_info *hi, const char *buf, int nbytes,
331 cur = ++(hi->heapnum);
332 memcpy(hi->info.buf[hi->ptr[cur]], buf, nbytes);
333 hi->info.file[hi->ptr[cur]] = kf;
336 while (parent && (*hi->cmp)(&hi->info.buf[hi->ptr[parent]],
337 &hi->info.buf[hi->ptr[cur]]) > 0)
339 key_heap_swap(hi, cur, parent);
345 static int heap_read_one(struct heap_info *hi, char *name, char *key)
348 char rbuf[INP_NAME_MAX];
354 strcpy(name, hi->info.buf[n]);
355 kf = hi->info.file[n];
357 memcpy(key, hi->info.buf[n] + r+1, KEY_SIZE);
359 if ((r = key_file_read(kf, rbuf)))
360 key_heap_insert(hi, rbuf, r, kf);
366 /* for debugging only */
367 void zebra_log_dict_entry(ZebraHandle zh, const char *s)
369 char dst[INP_NAME_MAX+1];
371 int len = key_SU_decode(&ord, (const unsigned char *) s);
372 const char *index_type;
375 yaz_log(YLOG_LOG, "ord=%d", ord);
378 const char *string_index;
380 zebraExplain_lookup_ord(zh->reg->zei,
381 ord, &index_type, &db, &string_index);
383 zebra_term_untrans(zh, index_type, dst, s + len);
385 yaz_log(YLOG_LOG, "ord=%d index_type=%s index=%s term=%s",
386 ord, index_type, string_index, dst);
390 struct heap_cread_info {
391 char prev_name[INP_NAME_MAX];
392 char cur_name[INP_NAME_MAX];
397 struct heap_info *hi;
404 static int heap_cread_item(void *vp, char **dst, int *insertMode);
406 int heap_cread_item2(void *vp, char **dst, int *insertMode)
408 struct heap_cread_info *p = (struct heap_cread_info *) vp;
413 if (p->look_level > 0)
423 memcpy(*dst, p->key_1, p->sz_1);
425 yaz_log(YLOG_LOG, "DUP level=%d", p->look_level);
426 pkey(*dst, *insertMode);
431 if (p->ret == 0) /* lookahead was 0?. Return that in read next round */
436 else if (p->ret == -1) /* Must read new item ? */
438 char *dst_1 = p->key_1;
439 p->ret = heap_cread_item(vp, &dst_1, &p->mode_1);
440 p->sz_1 = dst_1 - p->key_1;
443 { /* lookahead in 2 . Now in 1. */
445 p->mode_1 = p->mode_2;
446 memcpy(p->key_1, p->key_2, p->sz_2);
449 level = 1; /* insert */
451 level = -1; /* delete */
454 char *dst_2 = p->key_2;
455 p->ret = heap_cread_item(vp, &dst_2, &p->mode_2);
463 p->sz_2 = dst_2 - p->key_2;
465 if (key_compare(p->key_1, p->key_2) == 0)
467 if (p->mode_2) /* adjust level according to deletes/inserts */
476 /* all the same. new round .. */
478 p->mode_1 = p->mode_2;
479 memcpy(p->key_1, p->key_2, p->sz_1);
481 level = 1; /* insert */
483 level = -1; /* delete */
486 /* outcome is insert (1) or delete (0) depending on final level */
497 p->look_level = level;
498 memcpy(*dst, p->key_1, p->sz_1);
500 yaz_log(YLOG_LOG, "TOP");
501 pkey(*dst, *insertMode);
507 int heap_cread_item(void *vp, char **dst, int *insertMode)
509 struct heap_cread_info *p = (struct heap_cread_info *) vp;
510 struct heap_info *hi = p->hi;
512 if (p->first_in_list)
514 *insertMode = p->key[0];
515 memcpy(*dst, p->key+1, sizeof(struct it_key));
517 zebra_log_dict_entry(hi->zh, p->cur_name);
518 pkey(*dst, *insertMode);
520 (*dst) += sizeof(struct it_key);
521 p->first_in_list = 0;
524 strcpy(p->prev_name, p->cur_name);
525 if (!(p->more = heap_read_one(hi, p->cur_name, p->key)))
527 if (*p->cur_name && strcmp(p->cur_name, p->prev_name))
529 p->first_in_list = 1;
532 *insertMode = p->key[0];
533 memcpy(*dst, p->key+1, sizeof(struct it_key));
535 zebra_log_dict_entry(hi->zh, p->cur_name);
536 pkey(*dst, *insertMode);
538 (*dst) += sizeof(struct it_key);
542 int heap_inpc(struct heap_cread_info *hci, struct heap_info *hi)
544 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
546 isamc_i->clientData = hci;
547 isamc_i->read_item = heap_cread_item2;
551 char this_name[INP_NAME_MAX];
552 ISAM_P isamc_p, isamc_p2;
555 strcpy(this_name, hci->cur_name);
556 assert(hci->cur_name[0]);
558 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
560 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
562 isamc_merge(hi->reg->isamc, &isamc_p2, isamc_i);
566 if (!dict_delete(hi->reg->dict, this_name))
572 if (isamc_p2 != isamc_p)
573 dict_insert(hi->reg->dict, this_name,
574 sizeof(ISAM_P), &isamc_p2);
580 isamc_merge(hi->reg->isamc, &isamc_p, isamc_i);
583 dict_insert(hi->reg->dict, this_name,
584 sizeof(ISAM_P), &isamc_p);
591 int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi)
593 ISAMC_I *isamc_i = (ISAMC_I *) xmalloc(sizeof(*isamc_i));
595 isamc_i->clientData = hci;
596 isamc_i->read_item = heap_cread_item2;
600 char this_name[INP_NAME_MAX];
601 ISAM_P isamc_p, isamc_p2;
604 strcpy(this_name, hci->cur_name);
605 assert(hci->cur_name[0]);
610 zebra_log_dict_entry(hi->zh, hci->cur_name);
612 if ((dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
614 memcpy(&isamc_p, dict_info+1, sizeof(ISAM_P));
616 isamb_merge(hi->reg->isamb, &isamc_p2, isamc_i);
620 if (!dict_delete(hi->reg->dict, this_name))
626 if (isamc_p2 != isamc_p)
627 dict_insert(hi->reg->dict, this_name,
628 sizeof(ISAM_P), &isamc_p2);
634 isamb_merge(hi->reg->isamb, &isamc_p, isamc_i);
637 dict_insert(hi->reg->dict, this_name,
638 sizeof(ISAM_P), &isamc_p);
645 int heap_inps(struct heap_cread_info *hci, struct heap_info *hi)
647 ISAMS_I isams_i = (ISAMS_I) xmalloc(sizeof(*isams_i));
649 isams_i->clientData = hci;
650 isams_i->read_item = heap_cread_item;
654 char this_name[INP_NAME_MAX];
658 strcpy(this_name, hci->cur_name);
659 assert(hci->cur_name[0]);
661 if (!(dict_info = dict_lookup(hi->reg->dict, hci->cur_name)))
663 isams_p = isams_merge(hi->reg->isams, isams_i);
665 dict_insert(hi->reg->dict, this_name, sizeof(ISAM_P), &isams_p);
669 yaz_log(YLOG_FATAL, "isams doesn't support this kind of update");
677 struct progressInfo {
684 void progressFunc(struct key_file *keyp, void *info)
686 struct progressInfo *p = (struct progressInfo *) info;
687 time_t now, remaining;
689 if (keyp->buf_size <= 0 || p->totalBytes <= 0)
693 if (now >= p->lastTime+10)
696 remaining = (time_t) ((now - p->startTime)*
697 ((double) p->totalBytes/p->totalOffset - 1.0));
698 if (remaining <= 130)
699 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld seconds remaining",
700 (100.0*p->totalOffset) / p->totalBytes, (long) remaining);
702 yaz_log(YLOG_LOG, "Merge %2.1f%% completed; %ld minutes remaining",
703 (100.0*p->totalOffset) / p->totalBytes, (long) remaining/60);
705 p->totalOffset += keyp->buf_size;
712 void zebra_index_merge(ZebraHandle zh)
714 struct key_file **kf = 0;
717 struct heap_info *hi;
718 struct progressInfo progressInfo;
719 int nkeys = key_block_get_no_files(zh->reg->key_block);
730 extract_get_fname_tmp (zh, fname, nkeys+1);
731 if (access(fname, R_OK) == -1)
738 kf = (struct key_file **) xmalloc((1+nkeys) * sizeof(*kf));
739 progressInfo.totalBytes = 0;
740 progressInfo.totalOffset = 0;
741 time(&progressInfo.startTime);
742 time(&progressInfo.lastTime);
743 for (i = 1; i<=nkeys; i++)
745 kf[i] = key_file_init(i, 8192, zh->res);
746 kf[i]->readHandler = progressFunc;
747 kf[i]->readInfo = &progressInfo;
748 progressInfo.totalBytes += kf[i]->length;
749 progressInfo.totalOffset += kf[i]->buf_size;
751 hi = key_heap_init_file(zh, nkeys, key_qsort_compare);
754 for (i = 1; i<=nkeys; i++)
755 if ((r = key_file_read(kf[i], rbuf)))
756 key_heap_insert(hi, rbuf, r, kf[i]);
760 struct heap_cread_info hci;
762 hci.key = (char *) xmalloc(KEY_SIZE);
763 hci.key_1 = (char *) xmalloc(KEY_SIZE);
764 hci.key_2 = (char *) xmalloc(KEY_SIZE);
766 hci.first_in_list = 1;
769 hci.more = heap_read_one(hi, hci.cur_name, hci.key);
783 for (i = 1; i<=nkeys; i++)
785 extract_get_fname_tmp (zh, rbuf, i);
788 for (i = 1; i<=nkeys; i++)
789 key_file_destroy(kf[i]);
791 if (hi->no_iterations)
792 { /* do not log if nothing happened */
793 yaz_log(YLOG_LOG, "Iterations: isam/dict "
794 ZINT_FORMAT "/" ZINT_FORMAT,
795 hi->no_iterations, hi->no_diffs);
796 yaz_log(YLOG_LOG, "Dict: inserts/updates/deletions: "
797 ZINT_FORMAT "/" ZINT_FORMAT "/" ZINT_FORMAT,
798 hi->no_insertions, hi->no_updates, hi->no_deletions);
800 key_block_destroy(&zh->reg->key_block);
801 key_heap_destroy(hi, nkeys);
806 * c-file-style: "Stroustrup"
807 * indent-tabs-mode: nil
809 * vim: shiftwidth=4 tabstop=8 expandtab