1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2010 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
30 #include "key_block.h"
32 #include <yaz/xmalloc.h>
34 struct zebra_key_block {
44 char **thread_key_buf;
45 size_t thread_ptr_top;
49 pthread_mutex_t mutex;
51 pthread_cond_t work_available;
53 pthread_cond_t cond_sorting;
58 #define ENCODE_BUFLEN 768
62 char buf[ENCODE_BUFLEN];
65 #define USE_SHELLSORT 0
68 static void shellsort(void *ar, int r, size_t s,
69 int (*cmp)(const void *a, const void *b))
74 static const int incs[16] = { 1391376, 463792, 198768, 86961, 33936,
75 13776, 4592, 1968, 861, 336,
76 112, 48, 21, 7, 3, 1 };
77 for ( k = 0; k < 16; k++)
78 for (h = incs[k], i = h; i < r; i++)
82 while (j > h && (*cmp)(a + s*(j-h), v) > 0)
84 memcpy (a + s*j, a + s*(j-h), s);
93 static void encode_key_init(struct encode_info *i)
95 i->encode_handle = iscz1_start();
96 i->decode_handle = iscz1_start();
99 static void encode_key_write(const char *k, struct encode_info *i, FILE *outf)
102 char *bp = i->buf, *bp0;
103 const char *src = (char *) &key;
104 size_t klen = strlen(k);
106 if (fwrite (k, klen+1, 1, outf) != 1)
108 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
109 zebra_exit("encode_key_write");
114 /* and copy & align key so we can mangle */
115 memcpy (&key, k+1, sizeof(struct it_key)); /* *k is insert/delete */
119 key_logdump_txt(YLOG_LOG, &key, *k ? "i" : "d");
121 assert(key.mem[0] >= 0);
124 iscz1_encode(i->encode_handle, &bp, &src);
126 *bp0 = (*k * 128) + bp - bp0 - 1; /* length and insert/delete combined */
127 if (fwrite (i->buf, bp - i->buf, 1, outf) != 1)
129 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fwrite");
130 zebra_exit("encode_key_write");
138 const char *src = bp0+1;
139 char *dst = (char*) &key2;
140 iscz1_decode(i->decode_handle, &dst, &src);
142 key_logdump_txt(YLOG_LOG, &key2, *k ? "i" : "d");
149 static void encode_key_flush (struct encode_info *i, FILE *outf)
151 iscz1_stop(i->encode_handle);
152 iscz1_stop(i->decode_handle);
155 void key_block_flush_int(zebra_key_block_t p,
156 char **key_buf, size_t ptr_top, size_t ptr_i);
158 #if YAZ_POSIX_THREADS
159 static void *thread_func(void *vp)
161 zebra_key_block_t p = (zebra_key_block_t) vp;
164 pthread_mutex_lock(&p->mutex);
166 while (!p->is_sorting && !p->exit_flag)
167 pthread_cond_wait(&p->work_available, &p->mutex);
172 pthread_mutex_unlock(&p->mutex);
174 key_block_flush_int(p, p->thread_key_buf,
175 p->thread_ptr_top, p->thread_ptr_i);
177 pthread_mutex_lock(&p->mutex);
179 pthread_cond_signal(&p->cond_sorting);
180 pthread_mutex_unlock(&p->mutex);
182 pthread_mutex_unlock(&p->mutex);
187 zebra_key_block_t key_block_create(int mem, const char *key_tmp_dir,
190 zebra_key_block_t p = xmalloc(sizeof(*p));
192 #if YAZ_POSIX_THREADS
193 /* we'll be making two memory areas so cut in half */
197 p->key_buf = (char**) xmalloc (mem);
198 p->ptr_top = mem/sizeof(char*);
201 p->key_tmp_dir = xstrdup(key_tmp_dir);
207 #if YAZ_POSIX_THREADS
208 p->use_threads = use_threads;
211 pthread_mutex_init(&p->mutex, 0);
212 pthread_cond_init(&p->work_available, 0);
213 pthread_cond_init(&p->cond_sorting, 0);
214 pthread_create(&p->thread_id, 0, thread_func, p);
215 p->alt_buf = (char**) xmalloc (mem);
218 yaz_log(YLOG_DEBUG, "key_block_create t=%d", p->use_threads);
222 void key_block_destroy(zebra_key_block_t *pp)
224 zebra_key_block_t p = *pp;
229 #if YAZ_POSIX_THREADS
230 pthread_mutex_lock(&p->mutex);
232 while (p->is_sorting)
233 pthread_cond_wait(&p->cond_sorting, &p->mutex);
237 pthread_cond_broadcast(&p->work_available);
239 pthread_mutex_unlock(&p->mutex);
240 pthread_join(p->thread_id, 0);
241 pthread_cond_destroy(&p->work_available);
242 pthread_cond_destroy(&p->cond_sorting);
243 pthread_mutex_destroy(&p->mutex);
249 xfree(p->key_tmp_dir);
255 void key_block_write(zebra_key_block_t p, zint sysno, struct it_key *key_in,
256 int cmd, const char *str_buf, size_t str_len,
257 zint staticrank, int static_rank_enable)
261 struct it_key key_out;
263 if (p->key_buf_used + 1024 > (p->ptr_top -p->ptr_i)*sizeof(char*))
264 key_block_flush(p, 0);
266 assert(p->ptr_i > 0);
267 (p->key_buf)[p->ptr_top - p->ptr_i] =
268 (char*)p->key_buf + p->key_buf_used;
270 /* key_in->mem[0] ord/ch */
271 /* key_in->mem[1] filter specified record ID */
273 /* encode the ordinal value (field/use/attribute) .. */
274 ch = CAST_ZINT_TO_INT(key_in->mem[0]);
276 key_SU_encode(ch, (char*)p->key_buf +
279 /* copy the 0-terminated stuff from str to output */
280 memcpy((char*)p->key_buf + p->key_buf_used, str_buf, str_len);
281 p->key_buf_used += str_len;
282 ((char*)p->key_buf)[(p->key_buf_used)++] = '\0';
284 /* the delete/insert indicator */
285 ((char*)p->key_buf)[(p->key_buf_used)++] = cmd;
287 if (static_rank_enable)
289 assert(staticrank >= 0);
290 key_out.mem[j++] = staticrank;
293 if (key_in->mem[1]) /* filter specified record ID */
294 key_out.mem[j++] = key_in->mem[1];
296 key_out.mem[j++] = sysno;
297 for (i = 2; i < key_in->len; i++)
298 key_out.mem[j++] = key_in->mem[i];
301 memcpy((char*)p->key_buf + p->key_buf_used,
302 &key_out, sizeof(key_out));
303 (p->key_buf_used) += sizeof(key_out);
307 void key_block_flush_int(zebra_key_block_t p,
308 char **key_buf, size_t ptr_top, size_t ptr_i)
313 struct encode_info encode_info;
319 yaz_log(YLOG_DEBUG, "sorting section %d", (p->key_file_no));
324 shellsort(key_buf + ptr_top - ptr_i, ptr_i,
325 sizeof(char*), key_qsort_compare);
327 qsort(key_buf + ptr_top - ptr_i, ptr_i,
328 sizeof(char*), key_qsort_compare);
330 sprintf(out_fname, "%s/key%d.tmp", p->key_tmp_dir, p->key_file_no);
332 if (!(outf = fopen (out_fname, "wb")))
334 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fopen %s", out_fname);
335 zebra_exit("key_block_flush");
337 yaz_log(YLOG_DEBUG, "writing section %d", p->key_file_no);
338 prevcp = cp = (key_buf)[ptr_top - ptr_i];
340 encode_key_init (&encode_info);
341 encode_key_write (cp, &encode_info, outf);
345 cp = (key_buf)[ptr_top - ptr_i];
346 if (strcmp (cp, prevcp))
348 encode_key_flush ( &encode_info, outf);
349 encode_key_init (&encode_info);
350 encode_key_write (cp, &encode_info, outf);
354 encode_key_write (cp + strlen(cp), &encode_info, outf);
356 encode_key_flush ( &encode_info, outf);
359 yaz_log (YLOG_FATAL|YLOG_ERRNO, "fclose %s", out_fname);
360 zebra_exit("key_block_flush");
362 yaz_log(YLOG_DEBUG, "finished section %d", p->key_file_no);
365 void key_block_flush(zebra_key_block_t p, int is_final)
372 #if YAZ_POSIX_THREADS
375 pthread_mutex_lock(&p->mutex);
377 while (p->is_sorting)
378 pthread_cond_wait(&p->cond_sorting, &p->mutex);
382 p->thread_ptr_top = p->ptr_top;
383 p->thread_ptr_i = p->ptr_i;
384 p->thread_key_buf = p->key_buf;
387 p->key_buf = p->alt_buf;
390 pthread_cond_signal(&p->work_available);
394 while (p->is_sorting)
395 pthread_cond_wait(&p->cond_sorting, &p->mutex);
397 pthread_mutex_unlock(&p->mutex);
401 key_block_flush_int(p, p->key_buf, p->ptr_top, p->ptr_i);
406 int key_block_get_no_files(zebra_key_block_t p)
409 return p->key_file_no;
416 * c-file-style: "Stroustrup"
417 * indent-tabs-mode: nil
419 * vim: shiftwidth=4 tabstop=8 expandtab