From 0aa1114a00aa7d78e6e9dd8099f4c9b3d93f5f94 Mon Sep 17 00:00:00 2001
From: Adam Dickmeiss <adam@indexdata.dk>
Date: Fri, 14 Dec 2012 17:05:23 +0100
Subject: [PATCH] Allow merge on merged content

The new configuration metadata element, limitcluster, configures
that a metadata element (name) be used as limit name for search.
Applies to the whole service (ie all targets), unlike pz:limitmap
which is configured per-target (database).
---
 doc/pazpar2_conf.xml  |   16 ++++++++
 src/client.c          |   29 ++++++++++----
 src/client.h          |    4 +-
 src/pazpar2_config.c  |  101 ++++++++++++++++++++-----------------------------
 src/pazpar2_config.h  |    1 +
 src/reclists.c        |   67 +++++++++++++++++++++-----------
 src/reclists.h        |    1 +
 src/session.c         |   86 +++++++++++++++++++++++++++++++++--------
 src/session.h         |    3 ++
 test/test_http.cfg    |    2 +-
 test/test_http.urls   |    5 ++-
 test/test_http_23.res |    2 +-
 test/test_http_6.res  |    2 +-
 test/test_http_81.res |   62 ++++++++++++++++++++++--------
 test/test_http_82.res |   27 +++++++++++++
 test/test_http_83.res |    2 +
 test/test_http_84.res |   43 +++++++++++++++++++++
 17 files changed, 327 insertions(+), 126 deletions(-)
 create mode 100644 test/test_http_82.res
 create mode 100644 test/test_http_83.res
 create mode 100644 test/test_http_84.res

diff --git a/doc/pazpar2_conf.xml b/doc/pazpar2_conf.xml
index 60cd710..a45d77d 100644
--- a/doc/pazpar2_conf.xml
+++ b/doc/pazpar2_conf.xml
@@ -361,6 +361,22 @@
 	  </varlistentry>
 	  
 	  <varlistentry>
+	   <term id="limitcluster">limitcluster</term>
+	   <listitem>
+	    <para>
+	     Allow a limit on merged metadata. The value of this attribute
+	     is the name of actual metadata content to be used for matching
+	     (most often same name as metadata name).
+	    </para>
+	    <note>
+	     <para>
+	      Requires Pazpar2 1.6.23 or later.
+	     </para>
+	    </note>
+	   </listitem>
+	  </varlistentry>
+	  
+	  <varlistentry>
 	   <term id="metadata_limitmap">limitmap</term>
 	   <listitem>
 	    <para>
diff --git a/src/client.c b/src/client.c
index 0618d3a..bbafde0 100644
--- a/src/client.c
+++ b/src/client.c
@@ -1174,7 +1174,8 @@ const char *client_get_facet_limit_local(struct client *cl,
 
 static int apply_limit(struct session_database *sdb,
                        facet_limits_t facet_limits,
-                       WRBUF w_pqf, CCL_bibset ccl_map)
+                       WRBUF w_pqf, CCL_bibset ccl_map,
+                       struct conf_service *service)
 {
     int ret = 0;
     int i = 0;
@@ -1268,8 +1269,22 @@ static int apply_limit(struct session_database *sdb,
         }
         if (!s)
         {
-            yaz_log(YLOG_WARN, "Target %s: limit %s used, but no limitmap defined",
-                    (sdb->database ? sdb->database->id : "<no id>"), name);
+            int i;
+            for (i = 0; i < service->num_metadata; i++)
+            {
+                struct conf_metadata *md = service->metadata + i;
+                if (!strcmp(md->name, name) && md->limitcluster)
+                {
+                    yaz_log(YLOG_LOG, "limitcluster in use for %s",
+                            md->name);
+                    break;
+                }
+            }
+            if (i == service->num_metadata)
+            {
+                yaz_log(YLOG_WARN, "Target %s: limit %s used, but no limitmap defined",
+                        (sdb->database ? sdb->database->id : "<no id>"), name);
+            }
         }
     }
     nmem_destroy(nmem_tmp);
@@ -1283,15 +1298,15 @@ static int apply_limit(struct session_database *sdb,
 // return -1 on query error
 // return -2 on limit error
 int client_parse_query(struct client *cl, const char *query,
-                       facet_limits_t facet_limits,
-                       CCL_bibset bibset)
+                       facet_limits_t facet_limits)
 {
     struct session *se = client_get_session(cl);
+    struct conf_service *service = se->service;
     struct session_database *sdb = client_get_database(cl);
     struct ccl_rpn_node *cn;
     int cerror, cpos;
     ODR odr_out;
-    CCL_bibset ccl_map = prepare_cclmap(cl, bibset);
+    CCL_bibset ccl_map = prepare_cclmap(cl, service->ccl_bibset);
     const char *sru = session_setting_oneval(sdb, PZ_SRU);
     const char *pqf_prefix = session_setting_oneval(sdb, PZ_PQF_PREFIX);
     const char *pqf_strftime = session_setting_oneval(sdb, PZ_PQF_STRFTIME);
@@ -1313,7 +1328,7 @@ int client_parse_query(struct client *cl, const char *query,
         wrbuf_puts(w_pqf, " ");
     }
 
-    if (apply_limit(sdb, facet_limits, w_pqf, ccl_map))
+    if (apply_limit(sdb, facet_limits, w_pqf, ccl_map, service))
     {
         ccl_qual_rm(&ccl_map);
         return -2;
diff --git a/src/client.h b/src/client.h
index 7a38ab4..422a380 100644
--- a/src/client.h
+++ b/src/client.h
@@ -89,9 +89,7 @@ int client_is_active_preferred(struct client *cl);
 struct client *client_next_in_session(struct client *cl);
 
 int client_parse_query(struct client *cl, const char *query,
-                       facet_limits_t facet_limits,
-                       //const char *startrecs, const char *maxrecs,
-                       CCL_bibset bibset);
+                       facet_limits_t facet_limits);
 Odr_int client_get_hits(struct client *cl);
 Odr_int client_get_approximation(struct client *cl);
 int client_get_num_records(struct client *cl);
diff --git a/src/pazpar2_config.c b/src/pazpar2_config.c
index 052b457..27241a6 100644
--- a/src/pazpar2_config.c
+++ b/src/pazpar2_config.c
@@ -64,54 +64,6 @@ struct service_xslt
     struct service_xslt *next;
 };
 
-static void conf_metadata_assign(NMEM nmem,
-                                 struct conf_metadata * metadata,
-                                 const char *name,
-                                 enum conf_metadata_type type,
-                                 enum conf_metadata_merge merge,
-                                 enum conf_setting_type setting,
-                                 int brief,
-                                 int termlist,
-                                 const char *rank,
-                                 int sortkey_offset,
-                                 enum conf_metadata_mergekey mt,
-                                 const char *facetrule,
-                                 const char *limitmap)
-{
-    assert(nmem && metadata && name);
-
-    metadata->name = nmem_strdup(nmem, name);
-
-    metadata->type = type;
-
-    // enforcing that type_year is always range_merge
-    if (metadata->type == Metadata_type_year)
-        metadata->merge = Metadata_merge_range;
-    else
-        metadata->merge = merge;
-
-    metadata->setting = setting;
-    metadata->brief = brief;
-    metadata->termlist = termlist;
-    metadata->rank = nmem_strdup_null(nmem, rank);
-    metadata->sortkey_offset = sortkey_offset;
-    metadata->mergekey = mt;
-    metadata->facetrule = nmem_strdup_null(nmem, facetrule);
-    metadata->limitmap = nmem_strdup_null(nmem, limitmap);
-}
-
-
-static void conf_sortkey_assign(NMEM nmem,
-                                struct conf_sortkey * sortkey,
-                                const char *name,
-                                enum conf_sortkey_type type)
-{
-    assert(nmem && sortkey && name);
-
-    sortkey->name = nmem_strdup(nmem, name);
-    sortkey->type = type;
-}
-
 struct conf_service *service_init(struct conf_server *server,
                                          int num_metadata, int num_sortkeys,
                                          const char *service_id)
@@ -186,38 +138,61 @@ static struct conf_metadata* conf_service_add_metadata(
     int sortkey_offset,
     enum conf_metadata_mergekey mt,
     const char *facetrule,
-    const char *limitmap
+    const char *limitmap,
+    const char *limitcluster
     )
 {
     struct conf_metadata * md = 0;
+    NMEM nmem = service->nmem;
 
-    if (!service || !service->metadata || !service->num_metadata
+    if (!service->metadata || !service->num_metadata
         || field_id < 0  || !(field_id < service->num_metadata))
         return 0;
 
     md = service->metadata + field_id;
-    conf_metadata_assign(service->nmem, md, name, type, merge, setting,
-                         brief, termlist, rank, sortkey_offset,
-                         mt, facetrule, limitmap);
+    assert(nmem && md && name);
+
+    md->name = nmem_strdup(nmem, name);
+
+    md->type = type;
+
+    // enforcing that type_year is always range_merge
+    if (md->type == Metadata_type_year)
+        md->merge = Metadata_merge_range;
+    else
+        md->merge = merge;
+
+    md->setting = setting;
+    md->brief = brief;
+    md->termlist = termlist;
+    md->rank = nmem_strdup_null(nmem, rank);
+    md->sortkey_offset = sortkey_offset;
+    md->mergekey = mt;
+    md->facetrule = nmem_strdup_null(nmem, facetrule);
+    md->limitmap = nmem_strdup_null(nmem, limitmap);
+    md->limitcluster = nmem_strdup_null(nmem, limitcluster);
     return md;
 }
 
-static struct conf_sortkey * conf_service_add_sortkey(
+static struct conf_sortkey *conf_service_add_sortkey(
     struct conf_service *service,
     int field_id,
     const char *name,
     enum conf_sortkey_type type)
 {
-    struct conf_sortkey * sk = 0;
+    struct conf_sortkey *sk = 0;
+    NMEM nmem = service->nmem;
 
-    if (!service || !service->sortkeys || !service->num_sortkeys
+    if (!service->sortkeys || !service->num_sortkeys
         || field_id < 0 || !(field_id < service->num_sortkeys))
         return 0;
 
-    //sk = &((service->sortkeys)[field_id]);
     sk = service->sortkeys + field_id;
-    conf_sortkey_assign(service->nmem, sk, name, type);
 
+    assert(nmem && sk && name);
+
+    sk->name = nmem_strdup(nmem, name);
+    sk->type = type;
     return sk;
 }
 
@@ -302,9 +277,13 @@ static int parse_metadata(struct conf_service *service, xmlNode *n,
     xmlChar *xml_setting = 0;
     xmlChar *xml_mergekey = 0;
     xmlChar *xml_limitmap = 0;
+    xmlChar *xml_limitcluster = 0;
     xmlChar *xml_icu_chain = 0;
 
     struct _xmlAttr *attr;
+
+    assert(service);
+
     for (attr = n->properties; attr; attr = attr->next)
     {
         if (!xmlStrcmp(attr->name, BAD_CAST "name") &&
@@ -340,6 +319,9 @@ static int parse_metadata(struct conf_service *service, xmlNode *n,
         else if (!xmlStrcmp(attr->name, BAD_CAST "limitmap") &&
                  attr->children && attr->children->type == XML_TEXT_NODE)
             xml_limitmap = attr->children->content;
+        else if (!xmlStrcmp(attr->name, BAD_CAST "limitcluster") &&
+                 attr->children && attr->children->type == XML_TEXT_NODE)
+            xml_limitcluster = attr->children->content;
         else
         {
             yaz_log(YLOG_FATAL, "Unknown metadata attribute '%s'", attr->name);
@@ -480,7 +462,8 @@ static int parse_metadata(struct conf_service *service, xmlNode *n,
                               (const char *) xml_rank, sortkey_offset,
                               mergekey_type,
                               (const char *) xml_icu_chain,
-                              (const char *) xml_limitmap);
+                              (const char *) xml_limitmap,
+                              (const char *) xml_limitcluster);
     (*md_node)++;
     return 0;
 }
diff --git a/src/pazpar2_config.h b/src/pazpar2_config.h
index 11cec52..e258dd9 100644
--- a/src/pazpar2_config.h
+++ b/src/pazpar2_config.h
@@ -84,6 +84,7 @@ struct conf_metadata
     char *facetrule;
 
     char *limitmap;  // Should be expanded into service-wide default e.g. pz:limitmap:<name>=value setting
+    char *limitcluster;
 };
 
 
diff --git a/src/reclists.c b/src/reclists.c
index c4dd4e1..7662561 100644
--- a/src/reclists.c
+++ b/src/reclists.c
@@ -38,7 +38,6 @@ struct reclist
     int num_records;
     struct reclist_bucket *sorted_list;
     struct reclist_bucket *sorted_ptr;
-    struct reclist_bucket **last;
     NMEM nmem;
     YAZ_MUTEX mutex;
 };
@@ -46,8 +45,8 @@ struct reclist
 struct reclist_bucket
 {
     struct record_cluster *record;
-    struct reclist_bucket *hnext;
-    struct reclist_bucket *snext;
+    struct reclist_bucket *hash_next;
+    struct reclist_bucket *sorted_next;
     struct reclist_sortparms *sort_parms;
 };
 
@@ -210,6 +209,35 @@ static int reclist_cmp(const void *p1, const void *p2)
     return res;
 }
 
+void reclist_limit(struct reclist *l, struct session *se)
+{
+    unsigned i;
+    int num = 0;
+    struct reclist_bucket **pp = &l->sorted_list;
+
+    reclist_enter(l);
+    for (i = 0; i < l->hash_size; i++)
+    {
+        struct reclist_bucket *p;
+        for (p = l->hashtable[i]; p; p = p->hash_next)
+        {
+            if (session_check_cluster_limit(se, p->record))
+            {
+                *pp = p;
+                pp = &p->sorted_next;
+                num++;
+            }
+            else
+            {
+                yaz_log(YLOG_LOG, "session_check_cluster returned false");
+            }
+        }
+    }
+    *pp = 0;
+    l->num_records = num;
+    reclist_leave(l);
+}
+
 void reclist_sort(struct reclist *l, struct reclist_sortparms *parms)
 {
     struct reclist_bucket **flatlist = xmalloc(sizeof(*flatlist) * l->num_records);
@@ -225,7 +253,7 @@ void reclist_sort(struct reclist *l, struct reclist_sortparms *parms)
     {
         ptr->sort_parms = parms;
         flatlist[i] = ptr;
-        ptr = ptr->snext;
+        ptr = ptr->sorted_next;
         i++;
     }
     assert(i == l->num_records);
@@ -234,10 +262,9 @@ void reclist_sort(struct reclist *l, struct reclist_sortparms *parms)
     for (i = 0; i < l->num_records; i++)
     {
         *prev = flatlist[i];
-        prev = &flatlist[i]->snext;
+        prev = &flatlist[i]->sorted_next;
     }
     *prev = 0;
-    l->last = prev;
 
     xfree(flatlist);
 
@@ -249,7 +276,7 @@ struct record_cluster *reclist_read_record(struct reclist *l)
     if (l && l->sorted_ptr)
     {
         struct record_cluster *t = l->sorted_ptr->record;
-        l->sorted_ptr = l->sorted_ptr->snext;
+        l->sorted_ptr = l->sorted_ptr->sorted_next;
         return t;
     }
     else
@@ -283,7 +310,6 @@ struct reclist *reclist_create(NMEM nmem)
 
     res->sorted_ptr = 0;
     res->sorted_list = 0;
-    res->last = &res->sorted_list;
 
     res->num_records = 0;
     res->mutex = 0;
@@ -295,12 +321,15 @@ void reclist_destroy(struct reclist *l)
 {
     if (l)
     {
-        struct reclist_bucket *rb;
-        
-        for (rb = l->sorted_list; rb; rb = rb->snext)
+        unsigned i;
+        for (i = 0; i < l->hash_size; i++)
         {
-            wrbuf_destroy(rb->record->relevance_explain1);
-            wrbuf_destroy(rb->record->relevance_explain2);
+            struct reclist_bucket *p;
+            for (p = l->hashtable[i]; p; p = p->hash_next)
+            {
+                wrbuf_destroy(p->record->relevance_explain1);
+                wrbuf_destroy(p->record->relevance_explain2);
+            }
         }
         yaz_mutex_destroy(&l->mutex);
     }
@@ -332,7 +361,7 @@ struct record_cluster *reclist_insert(struct reclist *l,
     bucket = jenkins_hash((unsigned char*) merge_key) % l->hash_size;
 
     yaz_mutex_enter(l->mutex);
-    for (p = &l->hashtable[bucket]; *p; p = &(*p)->hnext)
+    for (p = &l->hashtable[bucket]; *p; p = &(*p)->hash_next)
     {
         // We found a matching record. Merge them
         if (!strcmp(merge_key, (*p)->record->merge_key))
@@ -364,7 +393,7 @@ struct record_cluster *reclist_insert(struct reclist *l,
 
         record->next = 0;
         new->record = cluster;
-        new->hnext = 0;
+        new->hash_next = 0;
         cluster->records = record;
         cluster->merge_key = nmem_strdup(l->nmem, merge_key);
         cluster->relevance_score = 0;
@@ -385,14 +414,6 @@ struct record_cluster *reclist_insert(struct reclist *l,
         cluster->relevance_explain2 = wrbuf_alloc();
         /* attach to hash list */
         *p = new;
-
-        /* append to sorted_list */
-        *l->last = new;
-        l->last = &new->snext;
-        l->sorted_ptr = l->sorted_list;
-        new->snext = 0;
-
-        l->num_records++;
     }
     yaz_mutex_leave(l->mutex);
     return cluster;
diff --git a/src/reclists.h b/src/reclists.h
index 818aea5..6d95401 100644
--- a/src/reclists.h
+++ b/src/reclists.h
@@ -37,6 +37,7 @@ struct reclist_sortparms
 
 struct reclist *reclist_create(NMEM);
 void reclist_destroy(struct reclist *l);
+void reclist_limit(struct reclist *l, struct session *session);
 struct record_cluster *reclist_insert(struct reclist *tl,
                                       struct conf_service *service,
                                       struct record  *record,
diff --git a/src/session.c b/src/session.c
index 8deded8..250734e 100644
--- a/src/session.c
+++ b/src/session.c
@@ -734,7 +734,6 @@ enum pazpar2_error_code session_search(struct session *se,
     int no_failed_query = 0;
     int no_failed_limit = 0;
     struct client_list *l, *l0;
-    facet_limits_t facet_limits;
     int same_sort_order = 0;
 
     session_log(se, YLOG_DEBUG, "Search");
@@ -764,8 +763,9 @@ enum pazpar2_error_code session_search(struct session *se,
         return PAZPAR2_NO_TARGETS;
     }
 
-    facet_limits = facet_limits_create(limit);
-    if (!facet_limits)
+    facet_limits_destroy(se->facet_limits);
+    se->facet_limits = facet_limits_create(limit);
+    if (!se->facet_limits)
     {
         *addinfo = "limit";
         session_leave(se, "session_search");
@@ -784,7 +784,7 @@ enum pazpar2_error_code session_search(struct session *se,
         if (prepare_map(se, client_get_database(cl)) < 0)
             continue;
 
-        parse_ret = client_parse_query(cl, query, facet_limits, se->service->ccl_bibset);
+        parse_ret = client_parse_query(cl, query, se->facet_limits);
         if (parse_ret == -1)
             no_failed_query++;
         else if (parse_ret == -2)
@@ -799,7 +799,6 @@ enum pazpar2_error_code session_search(struct session *se,
             no_working++;
         }
     }
-    facet_limits_destroy(facet_limits);
     session_reset_active_clients(se, l0);
 
     if (no_working == 0)
@@ -931,6 +930,7 @@ void session_destroy(struct session *se)
         session_log(se, YLOG_DEBUG, "NMEN operation usage %zd", nmem_total(se->nmem));
     if (nmem_total(se->session_nmem))
         session_log(se, YLOG_DEBUG, "NMEN session usage %zd", nmem_total(se->session_nmem));
+    facet_limits_destroy(se->facet_limits);
     nmem_destroy(se->nmem);
     service_destroy(se->service);
     yaz_mutex_destroy(&se->session_mutex);
@@ -973,6 +973,7 @@ struct session *new_session(NMEM nmem, struct conf_service *service,
     session->nmem = nmem_create();
     session->databases = 0;
     session->sorted_results = 0;
+    session->facet_limits = 0;
 
     for (i = 0; i <= SESSION_WATCH_MAX; i++)
     {
@@ -1208,6 +1209,8 @@ struct record_cluster *show_single_start(struct session *se, const char *id,
     *next_r = 0;
     if (se->reclist)
     {
+        reclist_limit(se->reclist, se);
+
         reclist_enter(se->reclist);
         while ((r = reclist_read_record(se->reclist)))
         {
@@ -1230,6 +1233,7 @@ void show_single_stop(struct session *se, struct record_cluster *rec)
     session_leave(se, "show_single_stop");
 }
 
+
 struct record_cluster **show_range_start(struct session *se,
                                          struct reclist_sortparms *sp,
                                          int start, int *num, int *total, Odr_int *sumhits, Odr_int *approx_hits)
@@ -1254,6 +1258,8 @@ struct record_cluster **show_range_start(struct session *se,
     {
         struct client_list *l;
 
+        reclist_limit(se->reclist, se);
+
         for (spp = sp; spp; spp = spp->next)
             if (spp->type == Metadata_sortkey_relevance)
             {
@@ -1619,14 +1625,14 @@ int ingest_record(struct client *cl, const char *rec,
     return ret;
 }
 
-static int match_metadata_local(struct record *record,
-                                struct conf_service *service,
-                                int md_field_id,
+//    struct conf_metadata *ser_md = &service->metadata[md_field_id];
+//    struct record_metadata *rec_md = record->metadata[md_field_id];
+static int match_metadata_local(struct conf_metadata *ser_md,
+                                struct record_metadata *rec_md0,
                                 char **values, int num_v)
 {
     int i;
-    struct conf_metadata *ser_md = &service->metadata[md_field_id];
-    struct record_metadata *rec_md = record->metadata[md_field_id];
+    struct record_metadata *rec_md = rec_md0;
     for (i = 0; i < num_v; )
     {
         if (rec_md)
@@ -1652,13 +1658,59 @@ static int match_metadata_local(struct record *record,
         }
         else
         {
-            rec_md = record->metadata[md_field_id];
+            rec_md = rec_md0;
             i++;
         }
     }
     return i < num_v ? 1 : 0;
 }
 
+int session_check_cluster_limit(struct session *se, struct record_cluster *rec)
+{
+    int i;
+    struct conf_service *service = se->service;
+    int ret = 1;
+    const char *name;
+    const char *value;
+    NMEM nmem_tmp = nmem_create();
+
+    for (i = 0; (name = facet_limits_get(se->facet_limits, i, &value)); i++)
+    {
+        int j;
+        for (j = 0; j < service->num_metadata; j++)
+        {
+            struct conf_metadata *md = service->metadata + j;
+            if (!strcmp(md->name, name) && md->limitcluster)
+            {
+                char **values = 0;
+                int num = 0;
+                int md_field_id =
+                    conf_service_metadata_field_id(service,
+                                                   md->limitcluster);
+
+                if (md_field_id < 0)
+                {
+                    ret = 0;
+                    break;
+                }
+
+                nmem_strsplit_escape2(nmem_tmp, "|", value, &values,
+                                      &num, 1, '\\', 1);
+
+                if (!match_metadata_local(&service->metadata[md_field_id],
+                                          rec->metadata[md_field_id],
+                                          values, num))
+                {
+                    ret = 0;
+                    break;
+                }
+            }
+        }
+    }
+    nmem_destroy(nmem_tmp);
+    return ret;
+}
+
 // Skip record on non-zero
 static int check_limit_local(struct client *cl,
                              struct record *record,
@@ -1686,8 +1738,10 @@ static int check_limit_local(struct client *cl,
             for (md_field_id = 0; md_field_id < service->num_metadata;
                  md_field_id++)
             {
-                if (match_metadata_local(record, service, md_field_id,
-                                         values, num_v))
+                if (match_metadata_local(
+                        &service->metadata[md_field_id],
+                        record->metadata[md_field_id],
+                        values, num_v))
                     break;
             }
             if (md_field_id == service->num_metadata)
@@ -1701,8 +1755,10 @@ static int check_limit_local(struct client *cl,
                 skip_record = 1;
                 break;
             }
-            if (!match_metadata_local(record, service, md_field_id,
-                                      values, num_v))
+            if (!match_metadata_local(
+                    &service->metadata[md_field_id],
+                    record->metadata[md_field_id],
+                    values, num_v))
             {
                 skip_record = 1;
             }
diff --git a/src/session.h b/src/session.h
index e1f5340..55bcb34 100644
--- a/src/session.h
+++ b/src/session.h
@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <yaz/ccl.h>
 #include <yaz/yaz-ccl.h>
 
+#include "facet_limit.h"
 #include "termlists.h"
 #include "reclists.h"
 #include "http.h"
@@ -119,6 +120,7 @@ struct session {
     YAZ_MUTEX session_mutex;
     unsigned session_id;
     int settings_modified;
+    facet_limits_t facet_limits;
     struct reclist_sortparms *sorted_results;
 };
 
@@ -185,6 +187,7 @@ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem);
 void session_alert_watch(struct session *s, int what);
 void add_facet(struct session *s, const char *type, const char *value, int count);
 
+int session_check_cluster_limit(struct session *se, struct record_cluster *rec);
 
 void perform_termlist(struct http_channel *c, struct session *se, const char *name, int num, int version);
 void session_log(struct session *s, int level, const char *fmt, ...)
diff --git a/test/test_http.cfg b/test/test_http.cfg
index 2ff1540..8cfac4e 100644
--- a/test/test_http.cfg
+++ b/test/test_http.cfg
@@ -21,7 +21,7 @@
 		termlist="yes"/>
       <metadata name="author" brief="yes" termlist="yes" merge="longest"
                 rank="2 au 3" mergekey="optional" /> <!-- rank="2 au 3" -->
-      <metadata name="subject" brief="yes" merge="unique" termlist="yes" rank="3"/>
+      <metadata name="subject" brief="yes" merge="unique" termlist="yes" rank="3" limitcluster="subject"/>
       <metadata name="id"/>
       <metadata name="lccn" merge="unique"/>
       <metadata name="description" brief="yes" merge="longest" rank="3"/>
diff --git a/test/test_http.urls b/test/test_http.urls
index 7d9a1fd..7f40d59 100644
--- a/test/test_http.urls
+++ b/test/test_http.urls
@@ -77,5 +77,8 @@ http://localhost:9763/search.pz2?session=9&command=show&block=1
 http://localhost:9763/search.pz2?command=init
 http://localhost:9763/search.pz2?session=10&command=search&query=au%3dadam
 http://localhost:9763/search.pz2?session=10&command=show&block=1
-http://localhost:9763/search.pz2?session=10&command=search&query=teachers+AND+greece
+http://localhost:9763/search.pz2?session=10&command=search&query=teachers+AND+teachers
+http://localhost:9763/search.pz2?session=10&command=show&block=1
+http://localhost:9763/search.pz2?session=10&command=termlist
+http://localhost:9763/search.pz2?session=10&command=search&query=teachers&limit=subject%3DGreece
 http://localhost:9763/search.pz2?session=10&command=show&block=1
diff --git a/test/test_http_23.res b/test/test_http_23.res
index a7ee6bb..66ad286 100644
--- a/test/test_http_23.res
+++ b/test/test_http_23.res
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <record>
  <recid>content: title how to program a computer author jack collins medium book</recid>
- <nextrecid>content: title computer processing of dynamic images from an anger scintillation camera author medium book</nextrecid>
+ <nextrecid>content: title the computer bible author medium book</nextrecid>
  <activeclients>0</activeclients>
  <md-title>How to program a computer</md-title>
  <md-author>Jack Collins</md-author>
diff --git a/test/test_http_6.res b/test/test_http_6.res
index f2f5103..66ad286 100644
--- a/test/test_http_6.res
+++ b/test/test_http_6.res
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <record>
  <recid>content: title how to program a computer author jack collins medium book</recid>
- <nextrecid>content: title computer science technology author medium book</nextrecid>
+ <nextrecid>content: title the computer bible author medium book</nextrecid>
  <activeclients>0</activeclients>
  <md-title>How to program a computer</md-title>
  <md-author>Jack Collins</md-author>
diff --git a/test/test_http_81.res b/test/test_http_81.res
index e05cf1e..1d0a181 100644
--- a/test/test_http_81.res
+++ b/test/test_http_81.res
@@ -1,10 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <show><status>OK</status>
 <activeclients>0</activeclients>
-<merged>1</merged>
-<total>1</total>
+<merged>2</merged>
+<total>2</total>
 <start>0</start>
-<num>1</num>
+<num>2</num>
 <hit>
  <md-title>The religious teachers of Greece</md-title>
  <md-date>1972</md-date>
@@ -28,24 +28,56 @@
         YYYYYYYYY</md-test-usersetting-2>
  </location>
  <count>1</count>
- <relevance>374298</relevance>
+ <relevance>48655</relevance>
  <relevance_info>
 field=title content=The religious teachers of Greece.;
 teachers: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(2)));
-greece: w[2] += w(6) / (1+log2(1+lead_decay(0.000000) * length(4)));
-greece: w[2] += w[2](6) * follow(0.000000) / (1+log2(d(2));
 teachers: tf[1] += w[1](6) / length(5) (1.200000);
-greece: tf[2] += w[2](6) / length(5) (1.200000);
-field=subject content=Greece;
-greece: w[2] += w(3) / (1+log2(1+lead_decay(0.000000) * length(0)));
-greece: tf[2] += w[2](3) / length(1) (4.200000);
 relevance = 0;
-idf[1] = log(((1 + total(1))/termoccur(1));
-teachers: relevance += 100000 * tf[1](1.200000) * idf[1](0.693147) (83177);
-idf[2] = log(((1 + total(1))/termoccur(1));
-greece: relevance += 100000 * tf[2](4.200000) * idf[2](0.693147) (291121);
-score = relevance(374298);
+idf[1] = log(((1 + total(2))/termoccur(2));
+teachers: relevance += 100000 * tf[1](1.200000) * idf[1](0.405465) (48655);
+idf[2] = log(((1 + total(2))/termoccur(0));
+teachers: relevance += 100000 * tf[2](0.000000) * idf[2](0.000000) (0);
+score = relevance(48655);
  </relevance_info>
  <recid>content: title the religious teachers of greece author adam james medium book</recid>
 </hit>
+<hit>
+ <md-title>Technology programs that work</md-title>
+ <md-date>1984</md-date>
+ <md-subject>United States</md-subject>
+ <md-subject>Educational technology</md-subject>
+ <md-subject>Federal aid to education</md-subject>
+ <md-description>&quot;This directory was developed by the Technology for the National Diffusion Network Project, Teachers College, Columbia University pursuant to contract number OE-300-83-0253, U.S. Department of Education&quot;--T.p. verso</md-description>
+ <location id="z3950.indexdata.com/marc"
+    name="Index Data MARC test server" checksum="2788512872">
+  <md-title>Technology programs that work</md-title>
+  <md-date>1984</md-date>
+  <md-subject>United States</md-subject>
+  <md-subject>Educational technology</md-subject>
+  <md-subject>Federal aid to education</md-subject>
+  <md-description tag="500">&quot;Spons agency Office of Educational Research and Improvement&quot;--Doc. resume</md-description>
+  <md-description tag="500">&quot;This directory was developed by the Technology for the National Diffusion Network Project, Teachers College, Columbia University pursuant to contract number OE-300-83-0253, U.S. Department of Education&quot;--T.p. verso</md-description>
+  <md-description tag="500">Distributed to depository libraries in microfiche</md-description>
+  <md-description tag="500">&quot;December 1984.&quot;</md-description>
+  <md-description tag="500">Includes indexes</md-description>
+  <md-test-usersetting>XXXXXXXXXX</md-test-usersetting>
+  <md-test-usersetting-2>test-usersetting-2 data: 
+        YYYYYYYYY</md-test-usersetting-2>
+ </location>
+ <count>1</count>
+ <relevance>4054</relevance>
+ <relevance_info>
+field=description content=&amp;quot;This directory was developed by the Technology f ...;
+teachers: w[1] += w(3) / (1+log2(1+lead_decay(0.000000) * length(13)));
+teachers: tf[1] += w[1](3) / length(30) (0.100000);
+relevance = 0;
+idf[1] = log(((1 + total(2))/termoccur(2));
+teachers: relevance += 100000 * tf[1](0.100000) * idf[1](0.405465) (4054);
+idf[2] = log(((1 + total(2))/termoccur(0));
+teachers: relevance += 100000 * tf[2](0.000000) * idf[2](0.000000) (0);
+score = relevance(4054);
+ </relevance_info>
+ <recid>content: title technology programs that work author medium book</recid>
+</hit>
 </show>
\ No newline at end of file
diff --git a/test/test_http_82.res b/test/test_http_82.res
new file mode 100644
index 0000000..fa5a350
--- /dev/null
+++ b/test/test_http_82.res
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<termlist><activeclients>0</activeclients>
+<list name="author">
+<term><name>Adam, James,</name><frequency>1</frequency></term>
+</list>
+<list name="date">
+<term><name>1972</name><frequency>1</frequency></term>
+<term><name>1984</name><frequency>1</frequency></term>
+</list>
+<list name="subject">
+<term><name>Educational technology</name><frequency>1</frequency></term>
+<term><name>Federal aid to education</name><frequency>1</frequency></term>
+<term><name>Greece</name><frequency>1</frequency></term>
+<term><name>Greek literature</name><frequency>1</frequency></term>
+<term><name>Philosophy, Ancient.</name><frequency>1</frequency></term>
+<term><name>United States.</name><frequency>1</frequency></term>
+</list>
+<list name="xtargets">
+<term>
+<id>z3950.indexdata.com/marc</id>
+<name>Index Data MARC test server</name>
+<frequency>2</frequency>
+<state>Client_Idle</state>
+<diagnostic>0</diagnostic>
+</term>
+</list>
+</termlist>
\ No newline at end of file
diff --git a/test/test_http_83.res b/test/test_http_83.res
new file mode 100644
index 0000000..ab63fe6
--- /dev/null
+++ b/test/test_http_83.res
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<search><status>OK</status></search>
\ No newline at end of file
diff --git a/test/test_http_84.res b/test/test_http_84.res
new file mode 100644
index 0000000..b3fd478
--- /dev/null
+++ b/test/test_http_84.res
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<show><status>OK</status>
+<activeclients>0</activeclients>
+<merged>1</merged>
+<total>2</total>
+<start>0</start>
+<num>1</num>
+<hit>
+ <md-title>The religious teachers of Greece</md-title>
+ <md-date>1972</md-date>
+ <md-author>Adam, James</md-author>
+ <md-subject>Greek literature</md-subject>
+ <md-subject>Philosophy, Ancient</md-subject>
+ <md-subject>Greece</md-subject>
+ <md-description>Reprint of the 1909 ed., which was issued as the 1904-1906 Gifford lectures</md-description>
+ <location id="z3950.indexdata.com/marc"
+    name="Index Data MARC test server" checksum="2614320583">
+  <md-title>The religious teachers of Greece</md-title>
+  <md-date>1972</md-date>
+  <md-author>Adam, James</md-author>
+  <md-subject>Greek literature</md-subject>
+  <md-subject>Philosophy, Ancient</md-subject>
+  <md-subject>Greece</md-subject>
+  <md-description tag="500">Reprint of the 1909 ed., which was issued as the 1904-1906 Gifford lectures</md-description>
+  <md-description tag="504">Includes bibliographical references</md-description>
+  <md-test-usersetting>XXXXXXXXXX</md-test-usersetting>
+  <md-test-usersetting-2>test-usersetting-2 data: 
+        YYYYYYYYY</md-test-usersetting-2>
+ </location>
+ <count>1</count>
+ <relevance>48655</relevance>
+ <relevance_info>
+field=title content=The religious teachers of Greece.;
+teachers: w[1] += w(6) / (1+log2(1+lead_decay(0.000000) * length(2)));
+teachers: tf[1] += w[1](6) / length(5) (1.200000);
+relevance = 0;
+idf[1] = log(((1 + total(2))/termoccur(2));
+teachers: relevance += 100000 * tf[1](1.200000) * idf[1](0.405465) (48655);
+score = relevance(48655);
+ </relevance_info>
+ <recid>content: title the religious teachers of greece author adam james medium book</recid>
+</hit>
+</show>
\ No newline at end of file
-- 
1.7.10.4