-/* $Id: benchindex1.c,v 1.4 2006-12-11 15:08:55 adam Exp $
- Copyright (C) 1995-2006
+/* $Id: benchindex1.c,v 1.9 2007-01-15 15:10:17 adam Exp $
+ Copyright (C) 1995-2007
Index Data ApS
This file is part of the Zebra server.
#include <yaz/nmem.h>
#include <yaz/xmalloc.h>
#include <yaz/marcdisp.h>
+#include <yaz/timing.h>
#include <it_key.h>
#include <idzebra/isamb.h>
#include <idzebra/dict.h>
int no_words = 0, no_new_words = 0;
const char *dict_info = 0;
ISAM_P isamc_p = 0;
- zebra_timing_t tim = zebra_timing_create();
+ yaz_timing_t tim_dict = 0;
+ yaz_timing_t tim_isamb = 0;
+ zint number_of_int_splits = isamb_get_int_splits(isb);
+ zint number_of_leaf_splits = isamb_get_leaf_splits(isb);
+ zint number_of_dict_splits = dict_get_no_split(dict);
b->ar = xmalloc(sizeof(*b->ar) * b->no_entries);
for (i = 0; i < b->no_entries; i++, t = t->next)
assert(!t);
qsort(b->ar, b->no_entries, sizeof(*b->ar), cmp_ar);
+ tim_dict = yaz_timing_create();
#if 0
for (i = 0; i < b->no_entries; i++)
{
}
}
dict_insert(dict, "_w", sizeof(word_id_seq), &word_id_seq);
+
+ yaz_timing_stop(tim_dict);
+ tim_isamb = yaz_timing_create();
b->current_entry = 0;
dict_insert(dict, "_i", sizeof(isamc_p), &isamc_p);
}
- yaz_log(YLOG_LOG, "Flushed %d postings, %d/%d words, %d records",
- b->no_entries, no_words, no_new_words, no_docs);
+ yaz_timing_stop(tim_isamb);
+
+ number_of_int_splits = isamb_get_int_splits(isb) - number_of_int_splits;
+ number_of_leaf_splits = isamb_get_leaf_splits(isb) - number_of_leaf_splits;
+ number_of_dict_splits = dict_get_no_split(dict) - number_of_dict_splits;
+
+ if (b->round == 0)
+ {
+ printf("# run total dict-real user sys isam-real user sys "
+ " intsp leafsp docs postings words new d-spl\n");
+ }
+ b->round++;
+ printf("%5d %9.6f %9.6f %5.2f %5.2f %9.6f %5.2f %5.2f "
+ "%6" ZINT_FORMAT0 " %6" ZINT_FORMAT0
+ " %8d %8d %6d %6d" " %5" ZINT_FORMAT0 "\n",
+ b->round,
+ yaz_timing_get_real(tim_dict) + yaz_timing_get_real(tim_isamb),
+ yaz_timing_get_real(tim_dict),
+ yaz_timing_get_user(tim_dict),
+ yaz_timing_get_sys(tim_dict),
+ yaz_timing_get_real(tim_isamb),
+ yaz_timing_get_user(tim_isamb),
+ yaz_timing_get_sys(tim_isamb),
+ number_of_int_splits,
+ number_of_leaf_splits,
+ no_docs,
+ b->no_entries,
+ no_words,
+ no_new_words,
+ number_of_dict_splits
+ );
+ fflush(stdout);
+
xfree(b->ar);
b->ar = 0;
nmem_reset(b->nmem);
b->no_entries = 0;
b->terms = 0;
- b->round++;
- zebra_timing_stop(tim);
-
- printf("%3d %8.6f %5.2f %5.2f\n",
- b->round,
- zebra_timing_get_real(tim),
- zebra_timing_get_user(tim),
- zebra_timing_get_sys(tim));
- fflush(stdout);
- zebra_timing_destroy(&tim);
+ yaz_timing_destroy(&tim_isamb);
+ yaz_timing_destroy(&tim_dict);
}
void index_block_check_flush(struct index_block *b, ISAMB isb, Dict dict,
int max = b->current_max;
if (total > max)
{
- yaz_log(YLOG_LOG, "flush to disk total=%d max=%d", total, max);
index_block_flush(b, isb, dict, no_docs);
}
}
b->no_entries++;
}
-void exit_usage(void)
-{
- fprintf(stderr, "benchindex1 [-t type] [-m mem] [-i] [inputfile]\n");
- exit(1);
-}
-
void index_term(struct index_block *b, const char *term,
zint docid, zint *seqno)
{
}
(*docid_seq)++;
no_docs++;
- index_block_check_flush(b, isb, dict, no_docs);
+ index_block_flush(b, isb, dict, no_docs);
index_block_destroy(&b);
}
index_block_destroy(&b);
}
+void exit_usage(void)
+{
+ fprintf(stderr, "benchindex1 [-t type] [-c d:i] [-m mem] [-i] [inputfile]\n");
+ exit(1);
+}
+
int main(int argc, char **argv)
{
BFiles bfs;
- ISAMB isb;
- ISAMC_M method;
+ ISAMB isb_postings;
+ ISAMC_M method_postings;
Dict dict;
int ret;
int reset = 0;
char *arg;
int memory = 5;
+ int isam_cache_size = 40;
+ int dict_cache_size = 50;
const char *fname = 0;
FILE *inf = stdin;
- zebra_timing_t tim = 0;
+ yaz_timing_t tim = 0;
zint docid_seq = 1;
const char *dict_info;
const char *type = "iso2709";
+ int int_count_enable = 1;
- while ((ret = options("im:t:", argv, argc, &arg)) != -2)
+ while ((ret = options("im:t:c:N", argv, argc, &arg)) != -2)
{
switch(ret)
{
exit_usage();
}
break;
+ case 'c':
+ if (sscanf(arg, "%d:%d", &dict_cache_size, &isam_cache_size)
+ != 2)
+ {
+ fprintf(stderr, "bad cache sizes for -c\n");
+ exit_usage();
+ }
+ break;
case 0:
fname = arg;
break;
+ case 'N':
+ int_count_enable = 0;
+ break;
default:
fprintf(stderr, "bad option.\n");
exit_usage();
exit(1);
}
}
- /* setup method (attributes) */
- method.compare_item = key_compare;
- method.log_item = key_logdump_txt;
+ printf("# benchindex1 %s %s\n", __DATE__, __TIME__);
+ printf("# isam_cache_size = %d\n", isam_cache_size);
+ printf("# dict_cache_size = %d\n", dict_cache_size);
+ printf("# int_count_enable = %d\n", int_count_enable);
+ printf("# memory = %d\n", memory);
+
+ /* setup postings isamb attributes */
+ method_postings.compare_item = key_compare;
+ method_postings.log_item = key_logdump_txt;
- method.codec.start = iscz1_start;
- method.codec.decode = iscz1_decode;
- method.codec.encode = iscz1_encode;
- method.codec.stop = iscz1_stop;
- method.codec.reset = iscz1_reset;
+ method_postings.codec.start = iscz1_start;
+ method_postings.codec.decode = iscz1_decode;
+ method_postings.codec.encode = iscz1_encode;
+ method_postings.codec.stop = iscz1_stop;
+ method_postings.codec.reset = iscz1_reset;
- method.debug = 0;
+ method_postings.debug = 0;
/* create block system */
bfs = bfs_create(0, 0);
if (reset)
bf_reset(bfs);
- tim = zebra_timing_create();
+ tim = yaz_timing_create();
/* create isam handle */
- isb = isamb_open (bfs, "isamb", 1, &method, 0);
- if (!isb)
+ isb_postings = isamb_open (bfs, "isamb", isam_cache_size ? 1 : 0,
+ &method_postings, 0);
+ if (!isb_postings)
{
yaz_log(YLOG_WARN, "isamb_open failed");
exit(2);
}
- dict = dict_open(bfs, "dict", 50, 1, 0, 4096);
+ isamb_set_cache_size(isb_postings, isam_cache_size);
+ isamb_set_int_count(isb_postings, int_count_enable);
+ dict = dict_open(bfs, "dict", dict_cache_size, 1, 0, 4096);
dict_info = dict_lookup(dict, "_s");
if (dict_info)
}
if (!strcmp(type, "iso2709"))
- index_marc_from_file(isb, dict, &docid_seq, inf, memory,
+ index_marc_from_file(isb_postings, dict, &docid_seq, inf, memory,
0 /* verbose */ , 0 /* print_offset */);
else if (!strcmp(type, "line"))
- index_marc_line_records(isb, dict, &docid_seq, inf, memory);
+ index_marc_line_records(isb_postings, dict, &docid_seq, inf, memory);
- yaz_log(YLOG_LOG, "Total " ZINT_FORMAT " documents", docid_seq);
+ printf("# Total " ZINT_FORMAT " documents\n", docid_seq);
dict_insert(dict, "_s", sizeof(docid_seq), &docid_seq);
dict_close(dict);
- isamb_close(isb);
+ isamb_close(isb_postings);
if (fname)
fclose(inf);
/* exit block system */
bfs_destroy(bfs);
- zebra_timing_stop(tim);
+ yaz_timing_stop(tim);
- yaz_log(YLOG_LOG, "Total %8.6f %5.2f %5.2f\n",
- zebra_timing_get_real(tim),
- zebra_timing_get_user(tim),
- zebra_timing_get_sys(tim));
+ printf("# Total timings real=%8.6f user=%3.2f system=%3.2f\n",
+ yaz_timing_get_real(tim),
+ yaz_timing_get_user(tim),
+ yaz_timing_get_sys(tim));
- zebra_timing_destroy(&tim);
+ yaz_timing_destroy(&tim);
exit(0);
return 0;