From 4e67b823f05b521f193af096a40c0f64758c5fc1 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Sat, 7 Apr 2007 22:26:27 +0000 Subject: [PATCH] Changed extract code so that it optimizes updates of records where content is almost identical to previous version of record. This makes updating of the internal explain database faster too. Also fixed memory leak that occurred for each deleted record. --- NEWS | 6 +++ index/extract.c | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 134 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 99a5fb4..9f85435 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +Fixed memory leak that occurred for each deleted record. + +Optimize updates of records where content is almost identical to previous +version of record. This makes updating of the internal explain database +faster too. + For RPN queries the index type (w,p,..) may be specified verbatim as structure attribute with string value, e.g. @attr 4=w . diff --git a/index/extract.c b/index/extract.c index b4f7ee7..2a6829d 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.254 2007-03-20 22:07:21 adam Exp $ +/* $Id: extract.c,v 1.255 2007-04-07 22:26:27 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -40,6 +40,16 @@ static int log_level_extract = 0; static int log_level_details = 0; static int log_level_initialized = 0; +/* 1 if we use eliminitate identical delete/insert keys */ +/* eventually this the 0-case code will be removed */ +#define FLUSH2 1 + +void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, + zint ins_rank, + zebra_rec_keys_t del_keys, + zint del_rank); + static void zebra_init_log_level(void) { if (!log_level_initialized) @@ -723,7 +733,6 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0"); extractCtrl.staticrank = 0; } - recordAttr->staticrank = extractCtrl.staticrank; if (matchStr) { @@ -733,8 +742,15 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, } extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys); +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, + zh->reg->keys, extractCtrl.staticrank, + 0, recordAttr->staticrank); +#else extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys, - recordAttr->staticrank); + extractCtrl.staticrank); +#endif + recordAttr->staticrank = extractCtrl.staticrank; zh->records_inserted++; } else @@ -769,11 +785,17 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, 0); extract_flush_sort_keys(zh, *sysno, 0, sortKeys); +#if !FLUSH2 extract_flush_record_keys(zh, *sysno, 0, delkeys, recordAttr->staticrank); +#endif if (action == action_delete) { /* record going to be deleted */ +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank, + delkeys, recordAttr->staticrank); +#endif if (zebra_rec_keys_empty(delkeys)) { yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType, @@ -794,6 +816,8 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, } rec_del (zh->reg->records, &rec); } + zebra_rec_keys_close(delkeys); + zebra_rec_keys_close(sortKeys); rec_free(&rec); logRecord(zh); return ZEBRA_OK; @@ -803,10 +827,17 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, if (show_progress) yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType, pr_fname, (zint) start_offset); - recordAttr->staticrank = extractCtrl.staticrank; extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys); - extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys, - recordAttr->staticrank); + +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, + zh->reg->keys, extractCtrl.staticrank, + delkeys, recordAttr->staticrank); +#else + extract_flush_record_keys(zh, *sysno, 1, + zh->reg->keys, extractCtrl.staticrank); +#endif + recordAttr->staticrank = extractCtrl.staticrank; zh->records_updated++; } zebra_rec_keys_close(delkeys); @@ -941,7 +972,13 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); +#if FLUSH2 + extract_flush_record_keys2(zh, rec->sysno, + zh->reg->keys, 0, delkeys, 0); +#else extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0); + extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); +#endif zebra_rec_keys_close(delkeys); zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], @@ -951,7 +988,14 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys); zebra_rec_keys_close(sortkeys); } - extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); + else + { +#if FLUSH2 + extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0); +#else + extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); +#endif + } extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys); xfree (rec->info[recInfo_delKeys]); @@ -1082,6 +1126,79 @@ void extract_rec_keys_adjust(ZebraHandle zh, int is_insert, } } +void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, zint ins_rank, + zebra_rec_keys_t del_keys, zint del_rank) +{ + ZebraExplainInfo zei = zh->reg->zei; + int normal = 0; + int optimized = 0; + + if (!zh->reg->key_block) + { + int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8")); + const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", "."); + int use_threads = atoi(res_get_def (zh->res, "threads", "1")); + zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads); + } + + if (ins_keys) + { + extract_rec_keys_adjust(zh, 1, ins_keys); + if (!del_keys) + zebraExplain_recordCountIncrement (zei, 1); + zebra_rec_keys_rewind(ins_keys); + } + if (del_keys) + { + extract_rec_keys_adjust(zh, 0, del_keys); + if (!ins_keys) + zebraExplain_recordCountIncrement (zei, -1); + zebra_rec_keys_rewind(del_keys); + } + + while (1) + { + size_t del_slen; + const char *del_str; + struct it_key del_key_in; + int del = 0; + + size_t ins_slen; + const char *ins_str; + struct it_key ins_key_in; + int ins = 0; + + if (del_keys) + del = zebra_rec_keys_read(del_keys, &del_str, &del_slen, + &del_key_in); + if (ins_keys) + ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen, + &ins_key_in); + + if (del && ins && ins_rank == del_rank + && !key_compare(&del_key_in, &ins_key_in) + && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen)) + { + optimized++; + continue; + } + if (!del && !ins) + break; + + normal++; + if (del) + key_block_write(zh->reg->key_block, sysno, + &del_key_in, 0, del_str, del_slen, + del_rank, zh->m_staticrank); + if (ins) + key_block_write(zh->reg->key_block, sysno, + &ins_key_in, 1, ins_str, ins_slen, + ins_rank, zh->m_staticrank); + } + yaz_log(YLOG_LOG, "normal=%d optimized=%d", normal, optimized); +} + void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, zebra_rec_keys_t reckeys, zint staticrank) @@ -1106,6 +1223,10 @@ void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd); + print_rec_keys(zh, reckeys); +#endif if (zebra_rec_keys_rewind(reckeys)) { size_t slen; @@ -1120,7 +1241,6 @@ void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, } } - ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh, zebra_rec_keys_t reckeys, zebra_snippets *snippets) -- 1.7.10.4