scores, due to using a better estimate for term occurrences.
-/* $Id: index.h,v 1.123 2004-09-28 12:39:55 adam Exp $
+/* $Id: index.h,v 1.124 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
Index Data Aps
char *name;
void *(*create)(ZebraHandle zh);
void (*destroy)(struct zebra_register *reg, void *class_handle);
- void *(*begin)(struct zebra_register *reg, void *class_handle, RSET rset);
+ void *(*begin)(struct zebra_register *reg,
+ void *class_handle, RSET rset, NMEM nmem,
+ TERMID *terms, int numterms);
/* ### Could add parameters to begin:
* char *index; // author, title, etc.
* int dbsize; // number of records in database
*/
void (*end)(struct zebra_register *reg, void *set_handle);
int (*calc)(void *set_handle, zint sysno);
- void (*add)(void *set_handle, int seqno, int term_index);
+ void (*add)(void *set_handle, int seqno, TERMID term);
};
struct term_set_entry {
void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
ZebraSet sset, RSET rset,
Z_SortKeySpecList *sort_sequence, int *sort_status);
-void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset);
+void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem);
void resultSetInvalidate (ZebraHandle zh);
int zebra_server_lock_init (ZebraService zh);
Licensed under the Academic Free License version 1.1.
http://opensource.org/licenses/academic.php
-$Id: livcode.c,v 1.3 2004-08-20 14:44:46 heikki Exp $
+$Id: livcode.c,v 1.4 2004-10-26 15:32:11 heikki Exp $
*/
+#ifdef SKIPTHIS /* Need to fix the interface - FIXME */
+
#include <stdlib.h>
#include <stdio.h>
#ifdef WIN32
int last_pos;
int no_entries;
int no_rank_entries;
+ NMEM nmem;
struct rank_term_info *entries;
};
* each result set. The returned handle is a "set handle" and
* will be used in each of the handlers below.
*/
-static void *begin (struct zebra_register *reg, void *class_handle, RSET rset)
+static void *begin (struct zebra_register *reg, void *class_handle,
+ RSET rset, NMEM nmem)
{
struct rank_set_info *si = (struct rank_set_info *) xmalloc (sizeof(*si));
int i;
/* do about this ??? */
si->no_entries = 0; /* rset->no_rset_terms; */ /* FIXME ??? */
si->no_rank_entries = 0;
+ si->nmem=nmem;
si->entries = (struct rank_term_info *)
xmalloc (sizeof(*si->entries)*si->no_entries);
for (i = 0; i < si->no_entries; i++)
};
struct rank_control *rankliv_class = &rank_control;
+#endif
+
-/* $Id: rank1.c,v 1.17 2004-08-20 14:44:46 heikki Exp $
+/* $Id: rank1.c,v 1.18 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
Index Data Aps
#include <unistd.h>
#endif
-#define DEBUG_RANK 0
+#define DEBUG_RANK 1
#include "index.h"
int global_inv;
int rank_flag;
int rank_weight;
+ TERMID term;
+ int term_index;
};
struct rank_set_info {
int no_entries;
int no_rank_entries;
struct rank_term_info *entries;
+ NMEM nmem;
};
static int log2_int (unsigned g)
*/
static void *create (ZebraHandle zh)
{
- struct rank_class_info *ci = (struct rank_class_info *)
- xmalloc (sizeof(*ci));
+ struct rank_class_info *ci =
+ (struct rank_class_info *) xmalloc (sizeof(*ci));
yaz_log (LOG_DEBUG, "rank-1 create");
return ci;
}
-/*
+/**
* begin: Prepares beginning of "real" ranking. Called once for
* each result set. The returned handle is a "set handle" and
* will be used in each of the handlers below.
*/
-static void *begin (struct zebra_register *reg, void *class_handle, RSET rset)
+static void *begin (struct zebra_register *reg,
+ void *class_handle, RSET rset, NMEM nmem,
+ TERMID *terms, int numterms)
{
- struct rank_set_info *si = (struct rank_set_info *) xmalloc (sizeof(*si));
+ struct rank_set_info *si =
+ (struct rank_set_info *) nmem_malloc (nmem,sizeof(*si));
int i;
#if DEBUG_RANK
yaz_log (LOG_LOG, "rank-1 begin");
#endif
- si->no_entries = 0; /* rset->no_rset_terms; */ /* FIXME - what to do here*/
- /* Now that we don't count term occurrences, ranking will have to */
- /* different! */
+ si->no_entries = numterms;
si->no_rank_entries = 0;
+ si->nmem=nmem;
si->entries = (struct rank_term_info *)
- xmalloc (sizeof(*si->entries)*si->no_entries);
- for (i = 0; i < si->no_entries; i++)
+ nmem_malloc (si->nmem, sizeof(*si->entries)*numterms);
+ for (i = 0; i < numterms; i++)
{
- zint g = 0; /* rset->rset_terms[i]->nn; */ /* FIXME ??? */
+ zint g = rset_count(terms[i]->rset);
#if DEBUG_RANK
- yaz_log(LOG_LOG, "i=%d flags=%s", i, rset->rset_terms[i]->flags);
+ yaz_log(LOG_LOG, "i=%d flags=%s '%s'", i,
+ terms[i]->flags, terms[i]->name );
#endif
- if (0) /* (!strncmp (rset->rset_terms[i]->flags, "rank,", 5)) */
- /* FIXME */ /* ??? */
+ if (!strncmp (terms[i]->flags, "rank,", 5))
{
- const char *cp = "w";
- /*= strstr(rset->rset_terms[i]->flags+4, ",w=");*/
- /* FIXME ??? */
+ const char *cp = strstr(terms[i]->flags+4, ",w=");
si->entries[i].rank_flag = 1;
if (cp)
si->entries[i].rank_weight = atoi (cp+3);
else
si->entries[i].rank_weight = 34;
#if DEBUG_RANK
- yaz_log (LOG_LOG, " i=%d weight=%d", i,
- si->entries[i].rank_weight);
+ yaz_log (LOG_LOG, " i=%d weight=%d g="ZINT_FORMAT, i,
+ si->entries[i].rank_weight, g);
#endif
(si->no_rank_entries)++;
}
else
si->entries[i].rank_flag = 0;
- si->entries[i].local_occur = 0;
+ si->entries[i].local_occur = 0; /* FIXME */
si->entries[i].global_occur = g;
si->entries[i].global_inv = 32 - log2_int (g);
- yaz_log (LOG_DEBUG, " global_inv = %d g = " ZINT_FORMAT, (int) (32-log2_int (g)), g);
+ yaz_log (LOG_DEBUG, " global_inv = %d g = " ZINT_FORMAT,
+ (int) (32-log2_int (g)), g);
+ si->entries[i].term=terms[i];
+ si->entries[i].term_index=i;
+ terms[i]->rankpriv=&(si->entries[i]);
}
return si;
}
*/
static void end (struct zebra_register *reg, void *set_handle)
{
- struct rank_set_info *si = (struct rank_set_info *) set_handle;
yaz_log (LOG_DEBUG, "rank-1 end");
- xfree (si->entries);
- xfree (si);
+ /* no need to free anything, they are in nmems */
}
-/*
+
+/**
* add: Called for each word occurence in a result set. This routine
* should be as fast as possible. This routine should "incrementally"
* update the score.
*/
-static void add (void *set_handle, int seqno, int term_index)
+static void add (void *set_handle, int seqno, TERMID term)
{
struct rank_set_info *si = (struct rank_set_info *) set_handle;
+ struct rank_term_info *ti= (struct rank_term_info *) term->rankpriv;
+ assert(si);
+ assert(term);
+ assert(ti);
#if DEBUG_RANK
- yaz_log (LOG_LOG, "rank-1 add seqno=%d term_index=%d", seqno, term_index);
+ yaz_log (LOG_LOG, "rank-1 add seqno=%d term=%s", seqno, term->name);
#endif
si->last_pos = seqno;
- si->entries[term_index].local_occur++;
+ ti->local_occur++;
}
/*
for (i = 0; i < si->no_entries; i++)
{
#if DEBUG_RANK
- yaz_log(LOG_LOG, "i=%d rank_flag=%d lo=%d",
+ yaz_log(LOG_LOG, "calc: i=%d rank_flag=%d lo=%d",
i, si->entries[i].rank_flag, si->entries[i].local_occur);
#endif
if (si->entries[i].rank_flag && (lo = si->entries[i].local_occur))
divisor = si->no_rank_entries * (8+log2_int (si->last_pos/si->no_entries));
score = score / divisor;
#if DEBUG_RANK
- yaz_log (LOG_LOG, "sysno=" ZINT_FORMAT " score=%d", sysno, score);
+ yaz_log (LOG_LOG, "calc sysno=" ZINT_FORMAT " score=%d", sysno, score);
#endif
if (score > 1000)
score = 1000;
-/* $Id: zebraapi.c,v 1.136 2004-10-15 10:07:32 heikki Exp $
+/* $Id: zebraapi.c,v 1.137 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
Index Data Aps
reg->ptr_i=0;
zebraRankInstall (reg, rank1_class);
+#ifdef SKIPTHIS /* FIXME - those ranks not yet converted to new interface */
zebraRankInstall (reg, rankzv_class);
zebraRankInstall (reg, rankliv_class);
+#endif
recordCompression = res_get_def (res, "recordCompression", "none");
if (!strcmp (recordCompression, "none"))
if (zebra_begin_read (zh))
return 1;
+#ifdef SKIPTHIS /* FIXME - livcode rank not yet available */
zebra_livcode_transform(zh, query);
+#endif
resultSetAddRPN (zh, odr_extract_mem(o), query,
zh->num_basenames, zh->basenames, setname);
-/* $Id: zrpn.c,v 1.157 2004-10-15 10:07:32 heikki Exp $
+/* $Id: zrpn.c,v 1.158 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
Index Data Aps
;
sort_sequence->num_specs = i;
if (!i)
- resultSetRank (zh, sset, rset);
+ resultSetRank (zh, sset, rset, rset_nmem);
else
{
logf (LOG_DEBUG, "resultSetSortSingle in rpn_search");
-/* $Id: zsets.c,v 1.68 2004-10-22 11:33:28 heikki Exp $
+/* $Id: zsets.c,v 1.69 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
Index Data Aps
return NULL;
}
-void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
+void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem)
{
zint kno = 0;
struct it_key key;
RSFD rfd;
TERMID termid;
TERMID *terms;
+ int numterms;
int i,n;
ZebraRankClass rank_class;
struct rank_control *rc;
sort_info = zebraSet->sort_info;
sort_info->num_entries = 0;
zebraSet->hits = 0;
+ n=0;
+ rset_getterms(rset,0,0,&n);
+ terms=malloc( sizeof(*terms)*n);
+ numterms=0;
+ rset_getterms(rset,terms,n,&numterms);
+
rfd = rset_open (rset, RSETF_READ);
rank_class = zebraRankLookup (zh, rank_handler_name);
zint psysno = key.mem[0];
int score;
void *handle =
- (*rc->begin) (zh->reg, rank_class->class_handle, rset);
+ (*rc->begin) (zh->reg, rank_class->class_handle, rset, nmem,
+ terms, numterms);
(zebraSet->hits)++;
esthits=atoi(res_get_def(zh->res,"estimatehits","0"));
if (!esthits)
est=-1; /* can not do */
do
{
- zint this_sys = key.mem[0];
+ zint this_sys = key.mem[0]; /* FIXME - assumes scope==2 */
+ zint seqno = key.mem[1]; /* FIXME - assumes scope==2 */
kno++;
if (this_sys != psysno)
{
(zebraSet->hits)++;
psysno = this_sys;
}
- /* FIXME - Ranking is broken, since rsets no longer have */
- /* term lists! */
- /* (*rc->add) (handle, this_sys, term_index); */
+ (*rc->add) (handle, seqno, termid);
if ( (est==-2) && (zebraSet->hits==esthits))
{ /* time to estimate the hits */
(*rc->end) (zh->reg, handle);
}
rset_close (rfd);
- n=0;
- rset_getterms(rset,0,0,&n);
- terms=xmalloc( sizeof(*terms)*n);
- i=n;
- n=0;
- rset_getterms(rset,terms,i,&n);
- for (i = 0; i < n; i++)
+ for (i = 0; i < numterms; i++)
{
yaz_log (LOG_LOG, "term=\"%s\" "
" type=%s count=" ZINT_FORMAT,
terms[i]->flags,
rset_count(terms[i]->rset));
}
- xfree(terms);
yaz_log (LOG_DEBUG, ZINT_FORMAT " keys, "ZINT_FORMAT" distinct sysnos",
kno, zebraSet->hits);
}
-/* $Id: zvrank.c,v 1.10 2004-08-20 14:44:46 heikki Exp $
+/* $Id: zvrank.c,v 1.11 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003
Index Data Aps
** "ntc-atn", "atc-atn", etc.
*/
+#if SKIPTHIS /* FIXME - Disabled while changing the interface to ranking */
+
#include <math.h> /* for log */
#include <stdio.h>
/* structures */
-struct rank_class_info { /* now we need this */
- int dummy;
+struct rank_class_info {
char rscheme[8]; /* name of weighting scheme */
};
char rscheme[8]; /* name of weighting scheme */
/**/
int veclen;
+ NMEM nmem;
void (*d_tf_fct)(void *, void *); /* doc term frequency function */
void (*d_idf_fct)(void *, void *); /* doc idf function */
void (*d_norm_fct)(void *, void *); /* doc normalization function */
* each result set. The returned handle is a "set handle" and
* will be used in each of the handlers below.
*/
-static void *zv_begin(struct zebra_register *reg, void *class_handle, RSET rset)
+static void *zv_begin(struct zebra_register *reg, void *class_handle,
+ RSET rset, NMEM nmem)
{
struct rs_info *rs=(struct rs_info *)xmalloc(sizeof(*rs));
struct rank_class_info *ci=(struct rank_class_info *)class_handle;
/* FIXME - Now that we don't have term lists in rsets, what do */
/* we do here ??? */
zv_init(rs, ci->rscheme);
+ rs->nmem=nmem;
rs->veclen=veclen;
prn_rs(rs);
struct rank_control *rankzv_class = &rank_control_vsm;
+#endif /* SKIPTHIS */
/* EOF */
-/* $Id: rsmultiandor.c,v 1.7 2004-10-22 10:12:52 heikki Exp $
+/* $Id: rsmultiandor.c,v 1.8 2004-10-26 15:32:11 heikki Exp $
Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002
Index Data Aps
int i;
for (i=0; i<info->no_rsets; i++){
rset_pos(mrfd->items[i].fd, &cur, &tot);
- logf(LOG_DEBUG, "r_pos: %d %0.1f %0.1f", i, cur,tot);
+ /*logf(LOG_LOG, "r_pos: %d %0.1f %0.1f", i, cur,tot); */
scur += cur;
stot += tot;
}
if (stot <1.0) { /* nothing there */
*current=0;
*total=0;
+ /* logf(LOG_LOG, "r_pos: NULL %0.1f %0.1f", *current, *total);*/
return;
}
*current=mrfd->hits;
*total=*current*stot/scur;
+ /*logf(LOG_LOG, "r_pos: = %0.1f %0.1f", *current, *total);*/
}
for (i=0;i<info->no_rsets;i++)
{
rset_getterms(info->rsets[i], terms, maxterms, curterm);
- yaz_log(LOG_DEBUG,"rsmulti: getterms: i=%d *cur=%d",i,*curterm);
- /* FIXME - remove this log once we know it works */
if ( ( (*curterm) > firstterm+1 ) &&
( (*curterm) <= maxterms ) &&
( terms[(*curterm)-1] == terms[firstterm] ) )
- *curterm--; /* forget the term, seen that before */
+ (*curterm)--; /* forget the term, seen that before */
}
}