1 /* $Id: cfile.c,v 1.40 2006-11-14 12:11:48 adam Exp $
2 Copyright (C) 1995-2006
5 This file is part of the Zebra server.
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 #include <idzebra/util.h>
28 #include <yaz/yaz-util.h>
32 /* set to 1 if extra commit/shadow check is to be performed */
35 static int write_head(CFile cf)
37 int left = cf->head.hash_size * sizeof(zint);
40 const char *tab = (char*) cf->array;
44 while (left >= (int) HASH_BSIZE)
46 r = mf_write(cf->hash_mf, bno++, 0, 0, tab);
53 r = mf_write(cf->hash_mf, bno, 0, left, tab);
57 static int read_head(CFile cf)
59 int left = cf->head.hash_size * sizeof(zint);
61 char *tab = (char*) cf->array;
65 while (left >= (int) HASH_BSIZE)
67 if (mf_read(cf->hash_mf, bno++, 0, 0, tab) == -1)
74 if (mf_read(cf->hash_mf, bno, 0, left, tab) == -1)
81 CFile cf_open(MFile mf, MFile_area area, const char *fname,
82 int block_size, int wflag, int *firstp)
86 CFile cf = (CFile) xmalloc(sizeof(*cf));
89 /* avoid valgrind warnings, but set to something nasty */
90 memset(cf, 'Z', sizeof(*cf));
92 yaz_log(YLOG_DEBUG, "cf: open %s %s", cf->rmf->name,
93 wflag ? "rdwr" : "rd");
101 cf->bucket_lru_front = cf->bucket_lru_back = NULL;
102 cf->bucket_in_memory = 0;
103 cf->max_bucket_in_memory = 100;
105 cf->iobuf = (char *) xmalloc(block_size);
106 memset(cf->iobuf, 0, block_size);
114 zebra_mutex_init(&cf->mutex);
116 sprintf(path, "%s-b", fname);
117 if (!(cf->block_mf = mf_open(area, path, block_size, wflag)))
122 sprintf(path, "%s-i", fname);
123 if (!(cf->hash_mf = mf_open(area, path, HASH_BSIZE, wflag)))
128 ret = mf_read(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head);
135 if (ret == 0 || !cf->head.state)
139 cf->head.block_size = block_size;
140 cf->head.hash_size = 199;
141 hash_bytes = cf->head.hash_size * sizeof(zint);
142 cf->head.flat_bucket = cf->head.next_bucket = cf->head.first_bucket =
143 (hash_bytes+sizeof(cf->head))/HASH_BSIZE + 2;
144 cf->head.next_block = 1;
145 cf->array = (zint *) xmalloc(hash_bytes);
146 for (i = 0; i<cf->head.hash_size; i++)
150 if (mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head))
165 assert(cf->head.block_size == block_size);
166 assert(cf->head.hash_size > 2);
167 hash_bytes = cf->head.hash_size * sizeof(zint);
168 assert(cf->head.next_bucket > 0);
169 assert(cf->head.next_block > 0);
170 if (cf->head.state == 1)
171 cf->array = (zint *) xmalloc(hash_bytes);
174 if (read_head(cf) == -1)
180 if (cf->head.state == 1)
182 cf->parray = (struct CFile_hash_bucket **)
183 xmalloc(cf->head.hash_size * sizeof(*cf->parray));
184 for (i = 0; i<cf->head.hash_size; i++)
185 cf->parray[i] = NULL;
190 static int cf_hash(CFile cf, zint no)
192 return (int) (((no >> 3) % cf->head.hash_size));
195 static void release_bucket(CFile cf, struct CFile_hash_bucket *p)
198 p->lru_prev->lru_next = p->lru_next;
200 cf->bucket_lru_back = p->lru_next;
202 p->lru_next->lru_prev = p->lru_prev;
204 cf->bucket_lru_front = p->lru_prev;
206 *p->h_prev = p->h_next;
208 p->h_next->h_prev = p->h_prev;
210 --(cf->bucket_in_memory);
214 static int flush_bucket(CFile cf, int no_to_flush)
218 struct CFile_hash_bucket *p;
220 for (i = 0; i != no_to_flush; i++)
222 p = cf->bucket_lru_back;
229 if (mf_write(cf->hash_mf, p->ph.this_bucket, 0, 0, &p->ph))
234 release_bucket(cf, p);
239 static struct CFile_hash_bucket *alloc_bucket(CFile cf, zint block_no, int hno)
241 struct CFile_hash_bucket *p, **pp;
243 if (cf->bucket_in_memory == cf->max_bucket_in_memory)
245 if (flush_bucket(cf, 1))
248 assert(cf->bucket_in_memory < cf->max_bucket_in_memory);
249 ++(cf->bucket_in_memory);
250 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
253 p->lru_prev = cf->bucket_lru_front;
254 if (cf->bucket_lru_front)
255 cf->bucket_lru_front->lru_next = p;
257 cf->bucket_lru_back = p;
258 cf->bucket_lru_front = p;
260 pp = cf->parray + hno;
264 (*pp)->h_prev = &p->h_next;
269 static struct CFile_hash_bucket *get_bucket(CFile cf, zint block_no, int hno)
271 struct CFile_hash_bucket *p;
273 p = alloc_bucket(cf, block_no, hno);
277 if (mf_read(cf->hash_mf, block_no, 0, 0, &p->ph) != 1)
279 yaz_log(YLOG_FATAL, "read get_bucket");
280 release_bucket(cf, p);
283 assert(p->ph.this_bucket == block_no);
287 static struct CFile_hash_bucket *new_bucket(CFile cf, zint *block_nop, int hno)
289 struct CFile_hash_bucket *p;
293 block_no = *block_nop = cf->head.next_bucket++;
294 p = alloc_bucket(cf, block_no, hno);
299 for (i = 0; i<HASH_BUCKET; i++)
304 p->ph.next_bucket = 0;
305 p->ph.this_bucket = block_no;
309 static int cf_lookup_flat(CFile cf, zint no, zint *vno)
311 zint hno = (no*sizeof(zint))/HASH_BSIZE;
312 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
315 if (mf_read(cf->hash_mf, hno+cf->head.next_bucket, off, sizeof(zint), vno)
323 static int cf_lookup_hash(CFile cf, zint no, zint *vno)
325 int hno = cf_hash(cf, no);
326 struct CFile_hash_bucket *hb;
330 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
332 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
333 if (hb->ph.no[i] == no)
336 *vno = hb->ph.vno[i];
340 for (block_no = cf->array[hno]; block_no; block_no = hb->ph.next_bucket)
342 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
344 if (hb->ph.this_bucket == block_no)
350 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
352 if (hb->ph.this_bucket == block_no)
354 yaz_log(YLOG_FATAL, "Found hash bucket on other chain(1)");
357 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
358 if (hb->ph.no[i] == no)
360 yaz_log(YLOG_FATAL, "Found hash bucket on other chain (2)");
366 hb = get_bucket(cf, block_no, hno);
369 for (i = 0; i<HASH_BUCKET && hb->ph.vno[i]; i++)
370 if (hb->ph.no[i] == no)
372 *vno = hb->ph.vno[i];
379 static int cf_write_flat(CFile cf, zint no, zint vno)
381 zint hno = (no*sizeof(zint))/HASH_BSIZE;
382 int off = (int) ((no*sizeof(zint)) - hno*HASH_BSIZE);
384 hno += cf->head.next_bucket;
385 if (hno >= cf->head.flat_bucket)
386 cf->head.flat_bucket = hno+1;
388 return mf_write(cf->hash_mf, hno, off, sizeof(zint), &vno);
391 static int cf_moveto_flat(CFile cf)
393 struct CFile_hash_bucket *p;
397 yaz_log(YLOG_DEBUG, "cf: Moving to flat shadow: %s", cf->rmf->name);
398 yaz_log(YLOG_DEBUG, "cf: hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT " total="
400 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
401 cf->head.next_bucket - cf->head.first_bucket);
402 assert(cf->head.state == 1);
403 if (flush_bucket(cf, -1))
405 assert(cf->bucket_in_memory == 0);
406 p = (struct CFile_hash_bucket *) xmalloc(sizeof(*p));
407 for (i = cf->head.first_bucket; i < cf->head.next_bucket; i++)
409 if (mf_read(cf->hash_mf, i, 0, 0, &p->ph) != 1)
411 yaz_log(YLOG_FATAL|YLOG_ERRNO, "read bucket moveto flat");
415 for (j = 0; j < HASH_BUCKET && p->ph.vno[j]; j++)
417 if (cf_write_flat(cf, p->ph.no[j], p->ph.vno[j]))
434 static int cf_lookup(CFile cf, zint no, zint *vno)
436 if (cf->head.state > 1)
437 return cf_lookup_flat(cf, no, vno);
438 return cf_lookup_hash(cf, no, vno);
441 static zint cf_new_flat(CFile cf, zint no)
443 zint vno = (cf->head.next_block)++;
445 cf_write_flat(cf, no, vno);
449 static zint cf_new_hash(CFile cf, zint no)
451 int hno = cf_hash(cf, no);
452 struct CFile_hash_bucket *hbprev = NULL, *hb = cf->parray[hno];
453 zint *bucketpp = &cf->array[hno];
455 zint vno = (cf->head.next_block)++;
457 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
458 if (!hb->ph.vno[HASH_BUCKET-1])
459 for (i = 0; i<HASH_BUCKET; i++)
471 for (hb = cf->parray[hno]; hb; hb = hb->h_next)
472 if (hb->ph.this_bucket == *bucketpp)
474 bucketpp = &hb->ph.next_bucket;
482 for (hb = cf->bucket_lru_back; hb; hb = hb->lru_next)
484 if (hb->ph.this_bucket == *bucketpp)
486 yaz_log(YLOG_FATAL, "Found hash bucket on other chain");
492 hb = get_bucket(cf, *bucketpp, hno);
495 for (i = 0; i<HASH_BUCKET; i++)
503 bucketpp = &hb->ph.next_bucket;
508 hb = new_bucket(cf, bucketpp, hno);
517 zint cf_new(CFile cf, zint no)
519 if (cf->head.state > 1)
520 return cf_new_flat(cf, no);
521 if (cf->no_miss*2 > cf->no_hits)
523 if (cf_moveto_flat(cf))
525 assert(cf->head.state > 1);
526 return cf_new_flat(cf, no);
528 return cf_new_hash(cf, no);
532 /** \brief reads block from commit area
533 \param cf commit file
534 \param no block number
535 \param offset offset in block
536 \param nbytes number of bytes to read
537 \param buf buffer for content (if read was succesful)
538 \retval 0 block could not be fully read
539 \retval 1 block could be read
542 int cf_read(CFile cf, zint no, int offset, int nbytes, void *buf)
548 zebra_mutex_lock(&cf->mutex);
549 ret = cf_lookup(cf, no, &block);
550 zebra_mutex_unlock(&cf->mutex);
554 yaz_log(YLOG_FATAL, "cf_lookup failed");
559 /* block could not be read */
562 else if (mf_read(cf->block_mf, block, offset, nbytes, buf) != 1)
564 yaz_log(YLOG_FATAL|YLOG_ERRNO, "mf_read no=" ZINT_FORMAT " block=" ZINT_FORMAT, no, block);
570 /** \brief writes block to commit area
571 \param cf commit file
572 \param no block number
573 \param offset offset in block
574 \param nbytes number of bytes to be written
575 \param buf buffer to be written
576 \retval 0 block written
579 int cf_write(CFile cf, zint no, int offset, int nbytes, const void *buf)
585 zebra_mutex_lock(&cf->mutex);
587 ret = cf_lookup(cf, no, &block);
591 zebra_mutex_unlock(&cf->mutex);
596 block = cf_new(cf, no);
599 zebra_mutex_unlock(&cf->mutex);
602 if (offset || nbytes)
604 if (mf_read(cf->rmf, no, 0, 0, cf->iobuf) == -1)
606 memcpy(cf->iobuf + offset, buf, nbytes);
612 zebra_mutex_unlock(&cf->mutex);
613 return mf_write(cf->block_mf, block, offset, nbytes, buf);
616 int cf_close(CFile cf)
619 yaz_log(YLOG_DEBUG, "cf: close hits=%d miss=%d bucket_in_memory=" ZINT_FORMAT
620 " total=" ZINT_FORMAT,
621 cf->no_hits, cf->no_miss, cf->bucket_in_memory,
622 cf->head.next_bucket - cf->head.first_bucket);
623 if (flush_bucket(cf, -1))
629 if (mf_write(cf->hash_mf, 0, 0, sizeof(cf->head), &cf->head))
634 mf_close(cf->hash_mf);
637 mf_close(cf->block_mf);
641 zebra_mutex_destroy(&cf->mutex);
649 * indent-tabs-mode: nil
651 * vim: shiftwidth=4 tabstop=8 expandtab