1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2012 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <metaproxy/filter.hpp>
24 #include <metaproxy/package.hpp>
26 #include <boost/thread/thread.hpp>
27 #include <boost/thread/mutex.hpp>
28 #include <boost/thread/condition.hpp>
29 #include <boost/shared_ptr.hpp>
31 #include <metaproxy/util.hpp>
32 #include "filter_multi.hpp"
35 #include <yaz/otherinfo.h>
36 #include <yaz/diagbib1.h>
37 #include <yaz/match_glob.h>
44 namespace mp = metaproxy_1;
45 namespace yf = mp::filter;
47 namespace metaproxy_1 {
49 enum multi_merge_type {
53 struct Multi::BackendSet {
56 bool operator < (const BackendSet &k) const;
57 bool operator == (const BackendSet &k) const;
59 struct Multi::ScanTermInfo {
60 std::string m_norm_term;
61 std::string m_display_term;
63 bool operator < (const ScanTermInfo &) const;
64 bool operator == (const ScanTermInfo &) const;
65 Z_Entry *get_entry(ODR odr);
67 struct Multi::FrontendSet {
71 int m_pos; // position for backend (1=first, 2=second,..
72 int m_start; // present request start
73 PresentJob(BackendPtr ptr, int pos) :
74 m_backend(ptr), m_pos(pos), m_start(0) {};
76 FrontendSet(std::string setname);
80 void round_robin(int pos, int number, std::list<PresentJob> &job);
81 void serve_order(int pos, int number, std::list<PresentJob> &job);
83 std::list<BackendSet> m_backend_sets;
84 std::string m_setname;
86 struct Multi::Backend {
88 std::string m_backend_database;
92 void operator() (void); // thread operation
94 struct Multi::Frontend {
99 std::list<BackendPtr> m_backend_list;
100 std::map<std::string,Multi::FrontendSet> m_sets;
102 void multi_move(std::list<BackendPtr> &blist);
103 void init(Package &package, Z_GDU *gdu);
104 void close(Package &package);
105 void search(Package &package, Z_APDU *apdu);
106 void present(Package &package, Z_APDU *apdu);
107 void scan1(Package &package, Z_APDU *apdu);
108 void scan2(Package &package, Z_APDU *apdu);
109 void relay_apdu(Package &package, Z_APDU *apdu);
113 std::string m_target_pattern;
117 Map(std::string pattern, std::string route, std::string auth) :
118 m_target_pattern(pattern), m_route(route), m_auth(auth) {};
119 bool match(const std::string target, std::string *ret,
120 std::string *auth) const {
121 if (yaz_match_glob(m_target_pattern.c_str(), target.c_str()))
132 friend struct Frontend;
135 FrontendPtr get_frontend(Package &package);
136 void release_frontend(Package &package);
138 std::list<Multi::Map> m_route_patterns;
139 boost::mutex m_mutex;
140 boost::condition m_cond_session_ready;
141 std::map<mp::Session, FrontendPtr> m_clients;
142 bool m_hide_unavailable;
144 multi_merge_type m_merge_type;
149 yf::Multi::Rep::Rep()
151 m_hide_unavailable = false;
152 m_hide_errors = false;
153 m_merge_type = round_robin;
156 bool yf::Multi::BackendSet::operator < (const BackendSet &k) const
158 return m_count < k.m_count;
161 yf::Multi::Frontend::Frontend(Rep *rep)
167 yf::Multi::Frontend::~Frontend()
171 yf::Multi::FrontendPtr yf::Multi::Rep::get_frontend(mp::Package &package)
173 boost::mutex::scoped_lock lock(m_mutex);
175 std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
179 it = m_clients.find(package.session());
180 if (it == m_clients.end())
183 if (!it->second->m_in_use)
185 it->second->m_in_use = true;
188 m_cond_session_ready.wait(lock);
190 FrontendPtr f(new Frontend(this));
191 m_clients[package.session()] = f;
196 void yf::Multi::Rep::release_frontend(mp::Package &package)
198 boost::mutex::scoped_lock lock(m_mutex);
199 std::map<mp::Session,yf::Multi::FrontendPtr>::iterator it;
201 it = m_clients.find(package.session());
202 if (it != m_clients.end())
204 if (package.session().is_closed())
206 it->second->close(package);
211 it->second->m_in_use = false;
213 m_cond_session_ready.notify_all();
217 yf::Multi::FrontendSet::FrontendSet(std::string setname)
223 yf::Multi::FrontendSet::FrontendSet()
228 yf::Multi::FrontendSet::~FrontendSet()
232 yf::Multi::Multi() : m_p(new Multi::Rep)
236 yf::Multi::~Multi() {
240 void yf::Multi::Backend::operator() (void)
242 m_package->move(m_route);
246 void yf::Multi::Frontend::close(mp::Package &package)
248 std::list<BackendPtr>::const_iterator bit;
249 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
253 b->m_package->copy_filter(package);
254 b->m_package->request() = (Z_GDU *) 0;
255 b->m_package->session().close();
256 b->m_package->move(b->m_route);
260 void yf::Multi::Frontend::multi_move(std::list<BackendPtr> &blist)
262 std::list<BackendPtr>::const_iterator bit;
263 boost::thread_group g;
264 for (bit = blist.begin(); bit != blist.end(); bit++)
266 g.add_thread(new boost::thread(**bit));
271 void yf::Multi::FrontendSet::serve_order(int start, int number,
272 std::list<PresentJob> &jobs)
275 for (i = 0; i < number; i++)
277 std::list<BackendSet>::const_iterator bsit;
279 int offset = start + i - 1;
280 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end();
283 if (offset >= voffset && offset < voffset + bsit->m_count)
285 PresentJob job(bsit->m_backend, offset - voffset + 1);
289 voffset += bsit->m_count;
294 void yf::Multi::FrontendSet::round_robin(int start, int number,
295 std::list<PresentJob> &jobs)
298 std::list<BackendSet>::const_iterator bsit;
299 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
306 // optimization step!
312 // find min count for each set which is > omin
313 for (bsit = m_backend_sets.begin(); bsit != m_backend_sets.end(); bsit++)
315 if (bsit->m_count > omin)
317 if (no_left == 0 || bsit->m_count < min)
322 if (no_left == 0) // if nothing greater than omin, bail out.
324 int skip = no_left * min;
325 if (p + skip > start) // step gets us "into" present range?
327 // Yes. skip until start.. Rounding off is deliberate!
328 min = (start-p) / no_left;
331 // update positions in each set..
332 std::list<int>::iterator psit = pos.begin();
333 for (psit = pos.begin(); psit != pos.end(); psit++)
337 // skip on each set.. before "present range"..
340 std::list<int>::iterator psit = pos.begin();
341 for (psit = pos.begin(); psit != pos.end(); psit++)
344 omin = min; // update so we consider next class (with higher count)
352 std::list<int>::iterator psit = pos.begin();
353 bsit = m_backend_sets.begin();
355 for (; bsit != m_backend_sets.end(); psit++,bsit++)
357 if (fetched >= number)
362 if (*psit <= bsit->m_count)
366 PresentJob job(bsit->m_backend, *psit);
378 void yf::Multi::Frontend::init(mp::Package &package, Z_GDU *gdu)
380 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
382 std::list<std::string> targets;
384 mp::util::get_vhost_otherinfo(req->otherInfo, targets);
386 if (targets.size() < 1)
392 std::list<std::string>::const_iterator t_it = targets.begin();
393 for (; t_it != targets.end(); t_it++)
396 Backend *b = new Backend;
399 std::list<Multi::Map>::const_iterator it =
400 m_p->m_route_patterns.begin();
401 while (it != m_p->m_route_patterns.end()) {
402 if (it->match(*t_it, &b->m_route, &b->m_auth))
406 // b->m_route = m_p->m_target_route[*t_it];
408 b->m_package = PackagePtr(new Package(s, package.origin()));
410 m_backend_list.push_back(BackendPtr(b));
414 // create init request
415 std::list<BackendPtr>::iterator bit;
416 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
420 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
422 std::list<std::string>vhost_one;
423 vhost_one.push_back(b->m_vhost);
424 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo,
428 Z_InitRequest *breq = init_apdu->u.initRequest;
430 if (b->m_auth.length())
432 breq->idAuthentication =
433 (Z_IdAuthentication *)
434 odr_malloc(odr, sizeof(*breq->idAuthentication));
435 breq->idAuthentication->which = Z_IdAuthentication_open;
436 breq->idAuthentication->u.open = odr_strdup(odr, b->m_auth.c_str());
439 breq->idAuthentication = req->idAuthentication;
441 *breq->preferredMessageSize = *req->preferredMessageSize;
442 *breq->maximumRecordSize = *req->maximumRecordSize;
444 ODR_MASK_SET(breq->options, Z_Options_search);
445 ODR_MASK_SET(breq->options, Z_Options_present);
446 ODR_MASK_SET(breq->options, Z_Options_namedResultSets);
447 ODR_MASK_SET(breq->options, Z_Options_scan);
449 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_1);
450 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_2);
451 ODR_MASK_SET(breq->protocolVersion, Z_ProtocolVersion_3);
453 b->m_package->request() = init_apdu;
455 b->m_package->copy_filter(package);
457 multi_move(m_backend_list);
459 // create the frontend init response based on each backend init response
462 Z_APDU *f_apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
463 Z_InitResponse *f_resp = f_apdu->u.initResponse;
465 ODR_MASK_SET(f_resp->options, Z_Options_search);
466 ODR_MASK_SET(f_resp->options, Z_Options_present);
467 ODR_MASK_SET(f_resp->options, Z_Options_namedResultSets);
468 ODR_MASK_SET(f_resp->options, Z_Options_scan);
470 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_1);
471 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_2);
472 ODR_MASK_SET(f_resp->protocolVersion, Z_ProtocolVersion_3);
475 int no_succeeded = 0;
477 Odr_int preferredMessageSize = *req->preferredMessageSize;
478 Odr_int maximumRecordSize = *req->maximumRecordSize;
479 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); )
481 PackagePtr p = (*bit)->m_package;
483 if (p->session().is_closed())
485 // failed. Remove from list and increment number of failed
487 bit = m_backend_list.erase(bit);
490 Z_GDU *gdu = p->response().get();
491 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
495 Z_APDU *b_apdu = gdu->u.z3950;
496 Z_InitResponse *b_resp = b_apdu->u.initResponse;
498 // common options for all backends
499 for (i = 0; i <= Z_Options_stringSchema; i++)
501 if (!ODR_MASK_GET(b_resp->options, i))
502 ODR_MASK_CLEAR(f_resp->options, i);
504 // common protocol version
505 for (i = 0; i <= Z_ProtocolVersion_3; i++)
506 if (!ODR_MASK_GET(b_resp->protocolVersion, i))
507 ODR_MASK_CLEAR(f_resp->protocolVersion, i);
511 if (preferredMessageSize > *b_resp->preferredMessageSize)
512 preferredMessageSize = *b_resp->preferredMessageSize;
513 if (maximumRecordSize > *b_resp->maximumRecordSize)
514 maximumRecordSize = *b_resp->maximumRecordSize;
523 *f_resp->preferredMessageSize = preferredMessageSize;
524 *f_resp->maximumRecordSize = maximumRecordSize;
526 if (m_p->m_hide_unavailable)
528 if (no_succeeded == 0)
531 package.session().close();
539 package.session().close();
542 package.response() = f_apdu;
545 void yf::Multi::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
547 // create search request
548 Z_SearchRequest *req = apdu_req->u.searchRequest;
550 // save these for later
551 Odr_int smallSetUpperBound = *req->smallSetUpperBound;
552 Odr_int largeSetLowerBound = *req->largeSetLowerBound;
553 Odr_int mediumSetPresentNumber = *req->mediumSetPresentNumber;
555 // they are altered now - to disable piggyback
556 *req->smallSetUpperBound = 0;
557 *req->largeSetLowerBound = 1;
558 *req->mediumSetPresentNumber = 0;
560 int default_num_db = req->num_databaseNames;
561 char **default_db = req->databaseNames;
563 std::list<BackendPtr>::const_iterator bit;
564 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
566 PackagePtr p = (*bit)->m_package;
569 if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
570 &req->num_databaseNames,
571 &req->databaseNames))
573 req->num_databaseNames = default_num_db;
574 req->databaseNames = default_db;
576 p->request() = apdu_req;
577 p->copy_filter(package);
579 multi_move(m_backend_list);
581 // look at each response
582 FrontendSet resultSet(std::string(req->resultSetName));
585 Odr_int result_set_size = 0;
586 Z_DiagRecs *z_diag = 0;
587 int no_successful = 0;
590 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
592 PackagePtr p = (*bit)->m_package;
594 // save closing package for at least one target
595 if (p->session().is_closed())
598 Z_GDU *gdu = p->response().get();
599 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
600 Z_APDU_searchResponse)
602 Z_APDU *b_apdu = gdu->u.z3950;
603 Z_SearchResponse *b_resp = b_apdu->u.searchResponse;
605 // see we get any errors (AKA diagnstics)
608 if (b_resp->records->which == Z_Records_NSD)
612 z_diag = (Z_DiagRecs *)
613 odr_malloc(odr, sizeof(*z_diag));
614 z_diag->num_diagRecs = 0;
615 z_diag->diagRecs = (Z_DiagRec**)
616 odr_malloc(odr, sizeof(*z_diag->diagRecs));
620 Z_DiagRec **n = (Z_DiagRec **)
622 (1+z_diag->num_diagRecs) * sizeof(*n));
623 memcpy(n, z_diag->diagRecs, z_diag->num_diagRecs
625 z_diag->diagRecs = n;
627 Z_DiagRec *nr = (Z_DiagRec *) odr_malloc(odr, sizeof(*nr));
628 nr->which = Z_DiagRec_defaultFormat;
629 nr->u.defaultFormat =
630 b_resp->records->u.nonSurrogateDiagnostic;
631 z_diag->diagRecs[z_diag->num_diagRecs++] = nr;
633 else if (b_resp->records->which == Z_Records_multipleNSD)
635 // we may set this multiple times (TOO BAD!)
636 z_diag = b_resp->records->u.multipleNonSurDiagnostics;
639 no_successful++; // probably piggyback
642 no_successful++; // no records and no diagnostics
643 BackendSet backendSet;
644 backendSet.m_backend = *bit;
645 backendSet.m_count = *b_resp->resultCount;
646 result_set_size += *b_resp->resultCount;
647 resultSet.m_backend_sets.push_back(backendSet);
651 Z_APDU *f_apdu = odr.create_searchResponse(apdu_req, 0, 0);
652 Z_SearchResponse *f_resp = f_apdu->u.searchResponse;
654 yaz_log(YLOG_LOG, "no_successful=%d is_closed=%s hide_errors=%s",
656 close_p ? "true" : "false",
657 m_p->m_hide_errors ? "true" : "false");
658 *f_resp->resultCount = result_set_size;
659 if (close_p && (no_successful == 0 || !m_p->m_hide_errors))
661 package.session().close();
662 package.response() = close_p->response();
665 if (z_diag && (no_successful == 0 || !m_p->m_hide_errors))
667 f_resp->records = (Z_Records *)
668 odr_malloc(odr, sizeof(*f_resp->records));
669 if (z_diag->num_diagRecs > 1)
671 f_resp->records->which = Z_Records_multipleNSD;
672 f_resp->records->u.multipleNonSurDiagnostics = z_diag;
676 f_resp->records->which = Z_Records_NSD;
677 f_resp->records->u.nonSurrogateDiagnostic =
678 z_diag->diagRecs[0]->u.defaultFormat;
682 m_sets[resultSet.m_setname] = resultSet;
685 mp::util::piggyback(smallSetUpperBound,
687 mediumSetPresentNumber,
691 Package pp(package.session(), package.origin());
692 if (z_diag == 0 && number > 0)
694 pp.copy_filter(package);
695 Z_APDU *p_apdu = zget_APDU(odr, Z_APDU_presentRequest);
696 Z_PresentRequest *p_req = p_apdu->u.presentRequest;
697 p_req->preferredRecordSyntax = req->preferredRecordSyntax;
698 p_req->resultSetId = req->resultSetName;
699 *p_req->resultSetStartPoint = 1;
700 *p_req->numberOfRecordsRequested = number;
701 pp.request() = p_apdu;
704 if (pp.session().is_closed())
705 package.session().close();
707 Z_GDU *gdu = pp.response().get();
708 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
709 Z_APDU_presentResponse)
711 Z_PresentResponse *p_res = gdu->u.z3950->u.presentResponse;
712 f_resp->records = p_res->records;
713 *f_resp->numberOfRecordsReturned =
714 *p_res->numberOfRecordsReturned;
715 *f_resp->nextResultSetPosition =
716 *p_res->nextResultSetPosition;
720 package.response() = pp.response();
724 package.response() = f_apdu; // in this scope because of p
727 void yf::Multi::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
729 // create present request
730 Z_PresentRequest *req = apdu_req->u.presentRequest;
733 it = m_sets.find(std::string(req->resultSetId));
734 if (it == m_sets.end())
738 odr.create_presentResponse(
740 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
742 package.response() = apdu;
745 std::list<Multi::FrontendSet::PresentJob> jobs;
746 int start = *req->resultSetStartPoint;
747 int number = *req->numberOfRecordsRequested;
749 if (m_p->m_merge_type == round_robin)
750 it->second.round_robin(start, number, jobs);
751 else if (m_p->m_merge_type == serve_order)
752 it->second.serve_order(start, number, jobs);
756 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
757 for (jit = jobs.begin(); jit != jobs.end(); jit++)
759 yaz_log(YLOG_LOG, "job pos=%d", jit->m_pos);
763 std::list<BackendPtr> present_backend_list;
765 std::list<BackendSet>::const_iterator bsit;
766 bsit = it->second.m_backend_sets.begin();
767 for (; bsit != it->second.m_backend_sets.end(); bsit++)
772 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
773 for (jit = jobs.begin(); jit != jobs.end(); jit++)
775 if (jit->m_backend == bsit->m_backend)
777 if (start == -1 || jit->m_pos < start)
779 if (end == -1 || jit->m_pos > end)
786 std::list<Multi::FrontendSet::PresentJob>::iterator jit;
787 for (jit = jobs.begin(); jit != jobs.end(); jit++)
789 if (jit->m_backend == bsit->m_backend)
791 if (jit->m_pos >= start && jit->m_pos <= end)
792 jit->m_start = start;
796 PackagePtr p = bsit->m_backend->m_package;
798 *req->resultSetStartPoint = start;
799 *req->numberOfRecordsRequested = end - start + 1;
801 p->request() = apdu_req;
802 p->copy_filter(package);
804 present_backend_list.push_back(bsit->m_backend);
807 multi_move(present_backend_list);
809 // look at each response
810 Z_Records *z_records_diag = 0;
812 std::list<BackendPtr>::const_iterator pbit = present_backend_list.begin();
813 for (; pbit != present_backend_list.end(); pbit++)
815 PackagePtr p = (*pbit)->m_package;
817 if (p->session().is_closed()) // if any backend closes, close frontend
818 package.session().close();
820 Z_GDU *gdu = p->response().get();
821 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
822 Z_APDU_presentResponse)
824 Z_APDU *b_apdu = gdu->u.z3950;
825 Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
827 // see we get any errors (AKA diagnstics)
830 if (b_resp->records->which != Z_Records_DBOSD)
831 z_records_diag = b_resp->records;
832 // we may set this multiple times (TOO BAD!)
837 // if any target does not return present response - return that
838 package.response() = p->response();
844 Z_APDU *f_apdu = odr.create_presentResponse(apdu_req, 0, 0);
845 Z_PresentResponse *f_resp = f_apdu->u.presentResponse;
849 f_resp->records = z_records_diag;
850 *f_resp->presentStatus = Z_PresentStatus_failure;
852 else if (number < 0 || (size_t) number > jobs.size())
855 odr.create_presentResponse(
857 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
862 f_resp->records = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
863 Z_Records * records = f_resp->records;
864 records->which = Z_Records_DBOSD;
865 records->u.databaseOrSurDiagnostics =
866 (Z_NamePlusRecordList *)
867 odr_malloc(odr, sizeof(Z_NamePlusRecordList));
868 Z_NamePlusRecordList *nprl = records->u.databaseOrSurDiagnostics;
869 nprl->num_records = jobs.size();
870 nprl->records = (Z_NamePlusRecord**)
871 odr_malloc(odr, sizeof(Z_NamePlusRecord *) * nprl->num_records);
873 std::list<Multi::FrontendSet::PresentJob>::const_iterator jit;
874 for (jit = jobs.begin(); jit != jobs.end(); jit++, i++)
876 PackagePtr p = jit->m_backend->m_package;
878 Z_GDU *gdu = p->response().get();
879 Z_APDU *b_apdu = gdu->u.z3950;
880 Z_PresentResponse *b_resp = b_apdu->u.presentResponse;
882 nprl->records[i] = (Z_NamePlusRecord*)
883 odr_malloc(odr, sizeof(Z_NamePlusRecord));
884 int inside_pos = jit->m_pos - jit->m_start;
885 if (inside_pos >= b_resp->records->
886 u.databaseOrSurDiagnostics->num_records)
888 *nprl->records[i] = *b_resp->records->
889 u.databaseOrSurDiagnostics->records[inside_pos];
890 nprl->records[i]->databaseName =
891 odr_strdup(odr, jit->m_backend->m_vhost.c_str());
893 nprl->num_records = i; // usually same as jobs.size();
894 *f_resp->nextResultSetPosition = start + i;
895 *f_resp->numberOfRecordsReturned = i;
897 package.response() = f_apdu;
900 void yf::Multi::Frontend::scan1(mp::Package &package, Z_APDU *apdu_req)
902 if (m_backend_list.size() > 1)
906 odr.create_scanResponse(
907 apdu_req, YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP, 0);
908 package.response() = f_apdu;
911 Z_ScanRequest *req = apdu_req->u.scanRequest;
913 int default_num_db = req->num_databaseNames;
914 char **default_db = req->databaseNames;
916 std::list<BackendPtr>::const_iterator bit;
917 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
919 PackagePtr p = (*bit)->m_package;
922 if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
923 &req->num_databaseNames,
924 &req->databaseNames))
926 req->num_databaseNames = default_num_db;
927 req->databaseNames = default_db;
929 p->request() = apdu_req;
930 p->copy_filter(package);
932 multi_move(m_backend_list);
934 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
936 PackagePtr p = (*bit)->m_package;
938 if (p->session().is_closed()) // if any backend closes, close frontend
939 package.session().close();
941 Z_GDU *gdu = p->response().get();
942 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
945 package.response() = p->response();
950 // if any target does not return scan response - return that
951 package.response() = p->response();
957 bool yf::Multi::ScanTermInfo::operator < (const ScanTermInfo &k) const
959 return m_norm_term < k.m_norm_term;
962 bool yf::Multi::ScanTermInfo::operator == (const ScanTermInfo &k) const
964 return m_norm_term == k.m_norm_term;
967 Z_Entry *yf::Multi::ScanTermInfo::get_entry(ODR odr)
969 Z_Entry *e = (Z_Entry *)odr_malloc(odr, sizeof(*e));
970 e->which = Z_Entry_termInfo;
972 t = e->u.termInfo = (Z_TermInfo *) odr_malloc(odr, sizeof(*t));
973 t->suggestedAttributes = 0;
975 t->alternativeTerm = 0;
977 t->otherTermInfo = 0;
978 t->globalOccurrences = odr_intdup(odr, m_count);
980 odr_malloc(odr, sizeof(*t->term));
981 t->term->which = Z_Term_general;
983 t->term->u.general = o = (Odr_oct *)odr_malloc(odr, sizeof(Odr_oct));
985 o->len = o->size = m_norm_term.size();
986 o->buf = (unsigned char *) odr_malloc(odr, o->len);
987 memcpy(o->buf, m_norm_term.c_str(), o->len);
991 void yf::Multi::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
993 std::list<BackendPtr>::const_iterator bit;
994 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
996 PackagePtr p = (*bit)->m_package;
999 p->request() = apdu_req;
1000 p->copy_filter(package);
1002 multi_move(m_backend_list);
1003 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
1005 PackagePtr p = (*bit)->m_package;
1007 if (p->session().is_closed()) // if any backend closes, close frontend
1008 package.session().close();
1010 package.response() = p->response();
1015 void yf::Multi::Frontend::scan2(mp::Package &package, Z_APDU *apdu_req)
1017 Z_ScanRequest *req = apdu_req->u.scanRequest;
1019 int default_num_db = req->num_databaseNames;
1020 char **default_db = req->databaseNames;
1022 std::list<BackendPtr>::const_iterator bit;
1023 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
1025 PackagePtr p = (*bit)->m_package;
1028 if (!mp::util::set_databases_from_zurl(odr, (*bit)->m_vhost,
1029 &req->num_databaseNames,
1030 &req->databaseNames))
1032 req->num_databaseNames = default_num_db;
1033 req->databaseNames = default_db;
1035 p->request() = apdu_req;
1036 p->copy_filter(package);
1038 multi_move(m_backend_list);
1040 ScanTermInfoList entries_before;
1041 ScanTermInfoList entries_after;
1045 for (bit = m_backend_list.begin(); bit != m_backend_list.end(); bit++)
1047 PackagePtr p = (*bit)->m_package;
1049 if (p->session().is_closed()) // if any backend closes, close frontend
1050 package.session().close();
1052 Z_GDU *gdu = p->response().get();
1053 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1054 Z_APDU_scanResponse)
1056 Z_ScanResponse *res = gdu->u.z3950->u.scanResponse;
1058 if (res->entries && res->entries->nonsurrogateDiagnostics)
1062 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, 0);
1063 Z_ScanResponse *f_res = f_apdu->u.scanResponse;
1065 f_res->entries->nonsurrogateDiagnostics =
1066 res->entries->nonsurrogateDiagnostics;
1067 f_res->entries->num_nonsurrogateDiagnostics =
1068 res->entries->num_nonsurrogateDiagnostics;
1070 package.response() = f_apdu;
1074 if (res->entries && res->entries->entries)
1076 Z_Entry **entries = res->entries->entries;
1077 int num_entries = res->entries->num_entries;
1079 if (req->preferredPositionInResponse)
1080 position = *req->preferredPositionInResponse;
1081 if (res->positionOfTerm)
1082 position = *res->positionOfTerm;
1086 for (i = 0; i<position-1 && i<num_entries; i++)
1088 Z_Entry *ent = entries[i];
1090 if (ent->which == Z_Entry_termInfo)
1094 Odr_int *occur = ent->u.termInfo->globalOccurrences;
1095 my.m_count = occur ? *occur : 0;
1097 if (ent->u.termInfo->term->which == Z_Term_general)
1099 my.m_norm_term = std::string(
1101 ent->u.termInfo->term->u.general->buf,
1102 ent->u.termInfo->term->u.general->len);
1104 if (my.m_norm_term.length())
1106 ScanTermInfoList::iterator it =
1107 entries_before.begin();
1108 while (it != entries_before.end() && my <*it)
1112 it->m_count += my.m_count;
1116 entries_before.insert(it, my);
1127 for ( ; i<num_entries; i++)
1129 Z_Entry *ent = entries[i];
1131 if (ent->which == Z_Entry_termInfo)
1135 Odr_int *occur = ent->u.termInfo->globalOccurrences;
1136 my.m_count = occur ? *occur : 0;
1138 if (ent->u.termInfo->term->which == Z_Term_general)
1140 my.m_norm_term = std::string(
1142 ent->u.termInfo->term->u.general->buf,
1143 ent->u.termInfo->term->u.general->len);
1145 if (my.m_norm_term.length())
1147 ScanTermInfoList::iterator it =
1148 entries_after.begin();
1149 while (it != entries_after.end() && *it < my)
1153 it->m_count += my.m_count;
1157 entries_after.insert(it, my);
1168 // if any target does not return scan response - return that
1169 package.response() = p->response();
1176 std::cout << "BEFORE\n";
1177 ScanTermInfoList::iterator it = entries_before.begin();
1178 for(; it != entries_before.end(); it++)
1180 std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1183 std::cout << "AFTER\n";
1184 it = entries_after.begin();
1185 for(; it != entries_after.end(); it++)
1187 std::cout << " " << it->m_norm_term << " " << it->m_count << "\n";
1194 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 1, "not implemented");
1195 package.response() = f_apdu;
1200 Z_APDU *f_apdu = odr.create_scanResponse(apdu_req, 0, 0);
1201 Z_ScanResponse *resp = f_apdu->u.scanResponse;
1203 int number_returned = *req->numberOfTermsRequested;
1204 int position_returned = *req->preferredPositionInResponse;
1206 resp->entries->num_entries = number_returned;
1207 resp->entries->entries = (Z_Entry**)
1208 odr_malloc(odr, sizeof(Z_Entry*) * number_returned);
1211 int lbefore = entries_before.size();
1212 if (lbefore < position_returned-1)
1213 position_returned = lbefore+1;
1215 ScanTermInfoList::iterator it = entries_before.begin();
1216 for (i = 0; i<position_returned-1 && it != entries_before.end(); i++, it++)
1218 resp->entries->entries[position_returned-2-i] = it->get_entry(odr);
1221 it = entries_after.begin();
1223 if (position_returned <= 0)
1226 i = position_returned-1;
1227 for (; i<number_returned && it != entries_after.end(); i++, it++)
1229 resp->entries->entries[i] = it->get_entry(odr);
1232 number_returned = i;
1234 resp->positionOfTerm = odr_intdup(odr, position_returned);
1235 resp->numberOfEntriesReturned = odr_intdup(odr, number_returned);
1236 resp->entries->num_entries = number_returned;
1238 package.response() = f_apdu;
1243 void yf::Multi::process(mp::Package &package) const
1245 FrontendPtr f = m_p->get_frontend(package);
1247 Z_GDU *gdu = package.request().get();
1249 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1250 Z_APDU_initRequest && !f->m_is_multi)
1252 f->init(package, gdu);
1254 else if (!f->m_is_multi)
1256 else if (gdu && gdu->which == Z_GDU_Z3950)
1258 Z_APDU *apdu = gdu->u.z3950;
1259 if (apdu->which == Z_APDU_initRequest)
1263 package.response() = odr.create_close(
1265 Z_Close_protocolError,
1268 package.session().close();
1270 else if (apdu->which == Z_APDU_searchRequest)
1272 f->search(package, apdu);
1274 else if (apdu->which == Z_APDU_presentRequest)
1276 f->present(package, apdu);
1278 else if (apdu->which == Z_APDU_scanRequest)
1280 f->scan2(package, apdu);
1282 else if (apdu->which == Z_APDU_close)
1284 f->relay_apdu(package, apdu);
1290 package.response() = odr.create_close(
1291 apdu, Z_Close_protocolError,
1292 "unsupported APDU in filter multi");
1294 package.session().close();
1297 m_p->release_frontend(package);
1300 void mp::filter::Multi::configure(const xmlNode * ptr, bool test_only,
1303 for (ptr = ptr->children; ptr; ptr = ptr->next)
1305 if (ptr->type != XML_ELEMENT_NODE)
1307 if (!strcmp((const char *) ptr->name, "target"))
1310 std::string route = mp::xml::get_route(ptr, auth);
1311 std::string target = mp::xml::get_text(ptr);
1312 if (target.length() == 0)
1314 m_p->m_route_patterns.push_back(Multi::Map(target, route, auth));
1316 else if (!strcmp((const char *) ptr->name, "hideunavailable"))
1318 m_p->m_hide_unavailable = true;
1320 else if (!strcmp((const char *) ptr->name, "hideerrors"))
1322 m_p->m_hide_errors = true;
1324 else if (!strcmp((const char *) ptr->name, "mergetype"))
1326 std::string mergetype = mp::xml::get_text(ptr);
1327 if (mergetype == "roundrobin")
1328 m_p->m_merge_type = round_robin;
1329 else if (mergetype == "serveorder")
1330 m_p->m_merge_type = serve_order;
1332 throw mp::filter::FilterException
1333 ("Bad mergetype " + mergetype + " in multi filter");
1338 throw mp::filter::FilterException
1340 + std::string((const char *) ptr->name)
1341 + " in multi filter");
1346 static mp::filter::Base* filter_creator()
1348 return new mp::filter::Multi;
1352 struct metaproxy_1_filter_struct metaproxy_1_filter_multi = {
1363 * c-file-style: "Stroustrup"
1364 * indent-tabs-mode: nil
1366 * vim: shiftwidth=4 tabstop=8 expandtab