-/* $Id: filter_multi.cpp,v 1.3 2006-01-16 15:51:56 adam Exp $
+/* $Id: filter_multi.cpp,v 1.11 2006-01-18 16:21:48 adam Exp $
Copyright (c) 2005, Index Data.
%LICENSE%
#include <yaz/otherinfo.h>
#include <yaz/diagbib1.h>
+#include <vector>
+#include <algorithm>
#include <map>
#include <iostream>
BackendPtr m_backend;
int m_count;
bool operator < (const BackendSet &k) const;
+ bool operator == (const BackendSet &k) const;
+ };
+ struct Multi::ScanTermInfo {
+ std::string m_norm_term;
+ std::string m_display_term;
+ int m_count;
+ bool operator < (const ScanTermInfo &) const;
+ bool operator == (const ScanTermInfo &) const;
+ Z_Entry *get_entry(ODR odr);
};
struct Multi::FrontendSet {
struct PresentJob {
struct Multi::Frontend {
Frontend(Rep *rep);
~Frontend();
- yp2::Session m_session;
bool m_is_multi;
bool m_in_use;
std::list<BackendPtr> m_backend_list;
void close(Package &package);
void search(Package &package, Z_APDU *apdu);
void present(Package &package, Z_APDU *apdu);
+ void scan1(Package &package, Z_APDU *apdu);
+ void scan2(Package &package, Z_APDU *apdu);
Rep *m_p;
};
struct Multi::Map {
private:
boost::mutex m_sessions_mutex;
std::map<std::string, Multi::Map>m_maps;
-
+ std::map<std::string,std::string> m_target_route;
boost::mutex m_mutex;
boost::condition m_cond_session_ready;
std::map<yp2::Session, FrontendPtr> m_clients;
m_package->move(m_route);
}
+
void yf::Multi::Frontend::close(Package &package)
{
std::list<BackendPtr>::const_iterator bit;
std::list<std::string> targets;
- int no_targets = 0;
- while (true)
- {
- const char *vhost_cstr =
- yaz_oi_get_string_oidval(&req->otherInfo, VAL_PROXY, no_targets+1,
- 0);
- if (!vhost_cstr)
- break;
- no_targets++;
- if (no_targets > 1000)
- return;
- targets.push_back(vhost_cstr);
- }
- if (no_targets < 2)
+ yp2::util::get_vhost_otherinfo(&req->otherInfo, false, targets);
+
+ if (targets.size() < 1)
{
package.move();
return;
Backend *b = new Backend;
b->m_vhost = *t_it;
+ b->m_route = m_p->m_target_route[*t_it];
// b->m_route unset
b->m_package = PackagePtr(new Package(s, package.origin()));
BackendPtr b = *bit;
Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
- yaz_oi_set_string_oidval(&init_apdu->u.initRequest->otherInfo, odr,
- VAL_PROXY, 1, b->m_vhost.c_str());
-
+ std::list<std::string>vhost_one;
+ vhost_one.push_back(b->m_vhost);
+ yp2::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
+ odr, vhost_one);
+
Z_InitRequest *req = init_apdu->u.initRequest;
ODR_MASK_SET(req->options, Z_Options_search);
ODR_MASK_SET(req->options, Z_Options_present);
ODR_MASK_SET(req->options, Z_Options_namedResultSets);
+ ODR_MASK_SET(req->options, Z_Options_scan);
ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
{
// create search request
Z_SearchRequest *req = apdu_req->u.searchRequest;
-
- // deal with piggy back (for now disable)
+
+ // save these for later
+ int smallSetUpperBound = *req->smallSetUpperBound;
+ int largeSetLowerBound = *req->largeSetLowerBound;
+ int mediumSetPresentNumber = *req->mediumSetPresentNumber;
+
+ // they are altered now - to disable piggyback
*req->smallSetUpperBound = 0;
*req->largeSetLowerBound = 1;
*req->mediumSetPresentNumber = 1;
// look at each response
FrontendSet resultSet(std::string(req->resultSetName));
- int total_count = 0;
+ int result_set_size = 0;
Z_Records *z_records_diag = 0; // no diagnostics (yet)
for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
{
BackendSet backendSet;
backendSet.m_backend = *bit;
backendSet.m_count = *b_resp->resultCount;
- total_count += *b_resp->resultCount;
+ result_set_size += *b_resp->resultCount;
resultSet.m_backend_sets.push_back(backendSet);
}
else
Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
+ *f_resp->resultCount = result_set_size;
if (z_records_diag)
{
// search error
f_resp->records = z_records_diag;
+ package.response() = f_apdu;
+ return;
}
- else
- { // assume OK
- m_sets[resultSet.m_setname] = resultSet;
+ // assume OK
+ m_sets[resultSet.m_setname] = resultSet;
+
+ int number;
+ yp2::util::piggyback(smallSetUpperBound,
+ largeSetLowerBound,
+ mediumSetPresentNumber,
+ result_set_size,
+ number);
+ Package pp(package.session(), package.origin());
+ if (number > 0)
+ {
+ pp.copy_filter(package);
+ Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
+ Z_PresentRequest *p_req = p_apdu->u.presentRequest;
+ p_req->preferredRecordSyntax = req->preferredRecordSyntax;
+ p_req->resultSetId = req->resultSetName;
+ *p_req->resultSetStartPoint = 1;
+ *p_req->numberOfRecordsRequested = number;
+ pp.request() = p_apdu;
+ present(pp, p_apdu);
+
+ if (pp.session().is_closed())
+ package.session().close();
+
+ Z_GDU *gdu = pp.response().get();
+ if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
+ Z_APDU_presentResponse)
+ {
+ Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
+ f_resp->records = p_res->records;
+ *f_resp->numberOfRecordsReturned =
+ *p_res->numberOfRecordsReturned;
+ *f_resp->nextResultSetPosition =
+ *p_res->nextResultSetPosition;
+ }
+ else
+ {
+ package.response() = pp.response();
+ return;
+ }
}
- *f_resp->resultCount = total_count;
-
- package.response() = f_apdu;
+ package.response() = f_apdu; // in this scope because of p
}
void yf::Multi::Frontend::present(Package &package, Z_APDU *apdu_req)
package.response() = f_apdu;
}
+void yf::Multi::Frontend::scan1(Package &package, Z_APDU *apdu_req)
+{
+ if (m_backend_list.size() > 1)
+ {
+ yp2::odr odr;
+ Z_APDU *f_apdu =
+ odr.create_scanResponse(
+ apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
+ package.response() = f_apdu;
+ return;
+ }
+ Z_ScanRequest *req = apdu_req->u.scanRequest;
+
+ int default_num_db = req->num_databaseNames;
+ char **default_db = req->databaseNames;
+
+ std::list<BackendPtr>::const_iterator bit;
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+ yp2::odr odr;
+
+ if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
+ &req->num_databaseNames,
+ &req->databaseNames))
+ {
+ req->num_databaseNames = default_num_db;
+ req->databaseNames = default_db;
+ }
+ p->request() = apdu_req;
+ p->copy_filter(package);
+ }
+ multi_move(m_backend_list);
+
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+
+ if (p->session().is_closed()) // if any backend closes, close frontend
+ package.session().close();
+
+ Z_GDU *gdu = p->response().get();
+ if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
+ Z_APDU_scanResponse)
+ {
+ package.response() = p->response();
+ break;
+ }
+ else
+ {
+ // if any target does not return scan response - return that
+ package.response() = p->response();
+ return;
+ }
+ }
+}
+
+bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
+{
+ return m_norm_term < k.m_norm_term;
+}
+
+bool yf::Multi::ScanTermInfo::operator == (const ScanTermInfo &k) const
+{
+ return m_norm_term == k.m_norm_term;
+}
+
+Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
+{
+ Z_Entry *e = (Z_Entry *)odr_malloc(odr, sizeof(*e));
+ e->which = Z_Entry_termInfo;
+ Z_TermInfo *t;
+ t = e->u.termInfo = (Z_TermInfo *) odr_malloc(odr, sizeof(*t));
+ t->suggestedAttributes = 0;
+ t->displayTerm = 0;
+ t->alternativeTerm = 0;
+ t->byAttributes = 0;
+ t->otherTermInfo = 0;
+ t->globalOccurrences = odr_intdup(odr, m_count);
+ t->term = (Z_Term *)
+ odr_malloc(odr, sizeof(*t->term));
+ t->term->which = Z_Term_general;
+ Odr_oct *o;
+ t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
+
+ o->len = o->size = m_norm_term.size();
+ o->buf = (unsigned char *) odr_malloc(odr, o->len);
+ memcpy(o->buf, m_norm_term.c_str(), o->len);
+ return e;
+}
+
+void yf::Multi::Frontend::scan2(Package &package, Z_APDU *apdu_req)
+{
+ Z_ScanRequest *req = apdu_req->u.scanRequest;
+
+ int default_num_db = req->num_databaseNames;
+ char **default_db = req->databaseNames;
+
+ std::list<BackendPtr>::const_iterator bit;
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+ yp2::odr odr;
+
+ if (!yp2::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
+ &req->num_databaseNames,
+ &req->databaseNames))
+ {
+ req->num_databaseNames = default_num_db;
+ req->databaseNames = default_db;
+ }
+ p->request() = apdu_req;
+ p->copy_filter(package);
+ }
+ multi_move(m_backend_list);
+
+ ScanTermInfoList entries_before;
+ ScanTermInfoList entries_after;
+ int no_before = 0;
+ int no_after = 0;
+
+ for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
+ {
+ PackagePtr p = (*bit)->m_package;
+
+ if (p->session().is_closed()) // if any backend closes, close frontend
+ package.session().close();
+
+ Z_GDU *gdu = p->response().get();
+ if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
+ Z_APDU_scanResponse)
+ {
+ Z_ScanResponse *res = gdu->u.z3950->u.scanResponse;
+
+ if (res->entries && res->entries->nonsurrogateDiagnostics)
+ {
+ // failure
+ yp2::odr odr;
+ Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
+ Z_ScanResponse *f_res = f_apdu->u.scanResponse;
+
+ f_res->entries->nonsurrogateDiagnostics =
+ res->entries->nonsurrogateDiagnostics;
+ f_res->entries->num_nonsurrogateDiagnostics =
+ res->entries->num_nonsurrogateDiagnostics;
+
+ package.response() = f_apdu;
+ return;
+ }
+
+ if (res->entries && res->entries->entries)
+ {
+ Z_Entry **entries = res->entries->entries;
+ int num_entries = res->entries->num_entries;
+ int position = 1;
+ if (req->preferredPositionInResponse)
+ position = *req->preferredPositionInResponse;
+ if (res->positionOfTerm)
+ position = *res->positionOfTerm;
+
+ // before
+ int i;
+ for (i = 0; i<position-1 && i<num_entries; i++)
+ {
+ Z_Entry *ent = entries[i];
+
+ if (ent->which == Z_Entry_termInfo)
+ {
+ ScanTermInfo my;
+
+ int *occur = ent->u.termInfo->globalOccurrences;
+ my.m_count = occur ? *occur : 0;
+
+ if (ent->u.termInfo->term->which == Z_Term_general)
+ {
+ my.m_norm_term = std::string(
+ (const char *)
+ ent->u.termInfo->term->u.general->buf,
+ ent->u.termInfo->term->u.general->len);
+ }
+ if (my.m_norm_term.length())
+ {
+ ScanTermInfoList::iterator it =
+ entries_before.begin();
+ while (it != entries_before.end() && my <*it)
+ it++;
+ if (my == *it)
+ {
+ it->m_count += my.m_count;
+ }
+ else
+ {
+ entries_before.insert(it, my);
+ no_before++;
+ }
+ }
+ }
+ }
+ // after
+ if (position <= 0)
+ i = 0;
+ else
+ i = position-1;
+ for ( ; i<num_entries; i++)
+ {
+ Z_Entry *ent = entries[i];
+
+ if (ent->which == Z_Entry_termInfo)
+ {
+ ScanTermInfo my;
+
+ int *occur = ent->u.termInfo->globalOccurrences;
+ my.m_count = occur ? *occur : 0;
+
+ if (ent->u.termInfo->term->which == Z_Term_general)
+ {
+ my.m_norm_term = std::string(
+ (const char *)
+ ent->u.termInfo->term->u.general->buf,
+ ent->u.termInfo->term->u.general->len);
+ }
+ if (my.m_norm_term.length())
+ {
+ ScanTermInfoList::iterator it =
+ entries_after.begin();
+ while (it != entries_after.end() && *it < my)
+ it++;
+ if (my == *it)
+ {
+ it->m_count += my.m_count;
+ }
+ else
+ {
+ entries_after.insert(it, my);
+ no_after++;
+ }
+ }
+ }
+ }
+
+ }
+ }
+ else
+ {
+ // if any target does not return scan response - return that
+ package.response() = p->response();
+ return;
+ }
+ }
+
+ if (true)
+ {
+ std::cout << "BEFORE\n";
+ ScanTermInfoList::iterator it = entries_before.begin();
+ for(; it != entries_before.end(); it++)
+ {
+ std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
+ }
+
+ std::cout << "AFTER\n";
+ it = entries_after.begin();
+ for(; it != entries_after.end(); it++)
+ {
+ std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
+ }
+ }
+
+ if (false)
+ {
+ yp2::odr odr;
+ Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, "not implemented");
+ package.response() = f_apdu;
+ }
+ else
+ {
+ yp2::odr odr;
+ Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
+ Z_ScanResponse *resp = f_apdu->u.scanResponse;
+
+ int number_returned = *req->numberOfTermsRequested;
+ int position_returned = *req->preferredPositionInResponse;
+
+ resp->entries->num_entries = number_returned;
+ resp->entries->entries = (Z_Entry**)
+ odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
+ int i;
+
+ int lbefore = entries_before.size();
+ if (lbefore < position_returned-1)
+ position_returned = lbefore+1;
+
+ ScanTermInfoList::iterator it = entries_before.begin();
+ for (i = 0; i<position_returned-1 && it != entries_before.end(); i++, it++)
+ {
+ resp->entries->entries[position_returned-2-i] = it->get_entry(odr);
+ }
+
+ it = entries_after.begin();
+
+ if (position_returned <= 0)
+ i = 0;
+ else
+ i = position_returned-1;
+ for (; i<number_returned && it != entries_after.end(); i++, it++)
+ {
+ resp->entries->entries[i] = it->get_entry(odr);
+ }
+
+ number_returned = i;
+
+ resp->positionOfTerm = odr_intdup(odr, position_returned);
+ resp->numberOfEntriesReturned = odr_intdup(odr, number_returned);
+ resp->entries->num_entries = number_returned;
+
+ package.response() = f_apdu;
+ }
+}
+
+
void yf::Multi::process(Package &package) const
{
FrontendPtr f = m_p->get_frontend(package);
{
f->present(package, apdu);
}
+ else if (apdu->which == Z_APDU_scanRequest)
+ {
+ f->scan2(package, apdu);
+ }
else
{
yp2::odr odr;
{
if (ptr->type != XML_ELEMENT_NODE)
continue;
- if (!strcmp((const char *) ptr->name, "virtual"))
+ if (!strcmp((const char *) ptr->name, "target"))
+ {
+ std::string route = yp2::xml::get_route(ptr);
+ std::string target = yp2::xml::get_text(ptr);
+ std::cout << "route=" << route << " target=" << target << "\n";
+ m_p->m_target_route[target] = route;
+ }
+ else if (!strcmp((const char *) ptr->name, "virtual"))
{
std::list<std::string> targets;
std::string vhost;