1 /* This file is part of Metaproxy.
2 Copyright (C) Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "filter_virt_db.hpp"
22 #include <metaproxy/package.hpp>
24 #include <boost/thread/mutex.hpp>
25 #include <boost/thread/condition.hpp>
26 #include <boost/shared_ptr.hpp>
28 #include <metaproxy/util.hpp>
31 #include <yaz/otherinfo.h>
32 #include <yaz/diagbib1.h>
33 #include <yaz/match_glob.h>
35 #include <yaz/oid_db.h>
40 namespace mp = metaproxy_1;
41 namespace yf = mp::filter;
43 namespace metaproxy_1 {
46 struct VirtualDB::Set {
47 Set(BackendPtr b, std::string setname);
52 std::string m_setname;
54 struct VirtualDB::Map {
55 Map(std::string database, std::list<std::string> targets, std::string route);
56 Map(std::string database, std::string target, std::string route);
58 bool match(const std::string db) const;
59 std::string m_dbpattern;
60 std::list<std::string> m_targets;
63 struct VirtualDB::Backend {
64 mp::Session m_backend_session;
65 std::list<std::string> m_frontend_databases;
66 std::list<std::string> m_targets;
68 bool m_named_result_sets;
71 struct VirtualDB::Frontend {
74 mp::Session m_session;
77 yazpp_1::GDU m_init_gdu;
78 std::list<BackendPtr> m_backend_list;
79 std::map<std::string,VirtualDB::Set> m_sets;
81 void search(Package &package, Z_APDU *apdu);
82 void present(Package &package, Z_APDU *apdu);
83 void scan(Package &package, Z_APDU *apdu);
84 int relay_apdu(Package &package, Z_APDU *apdu);
86 void close(Package &package);
87 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
89 void fixup_package(Package &p, BackendPtr b);
90 void fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
92 void fixup_npr_records(ODR odr, Z_Records *records,
95 BackendPtr lookup_backend_from_databases(
96 std::list<std::string> databases);
97 BackendPtr create_backend_from_databases(
98 std::list<std::string> databases,
100 std::string &failing_database);
102 BackendPtr init_backend(std::list<std::string> database,
104 int &error_code, std::string &addinfo);
107 class VirtualDB::Rep {
108 friend class VirtualDB;
109 friend struct Frontend;
111 FrontendPtr get_frontend(Package &package);
112 void release_frontend(Package &package);
113 void refresh_torus();
115 std::list<VirtualDB::Map>m_maps;
116 typedef std::map<std::string,VirtualDB::Set>::iterator Sets_it;
117 boost::mutex m_mutex;
118 boost::condition m_cond_session_ready;
119 std::map<mp::Session, FrontendPtr> m_clients;
125 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::lookup_backend_from_databases(
126 std::list<std::string> databases)
128 std::list<BackendPtr>::const_iterator map_it;
129 map_it = m_backend_list.begin();
130 for (; map_it != m_backend_list.end(); map_it++)
131 if ((*map_it)->m_frontend_databases == databases)
137 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::create_backend_from_databases(
138 std::list<std::string> databases, int &error_code, std::string &addinfo)
140 BackendPtr b(new Backend);
141 std::list<std::string>::const_iterator db_it = databases.begin();
143 b->m_number_of_sets = 0;
144 b->m_frontend_databases = databases;
145 b->m_named_result_sets = false;
147 bool first_route = true;
149 std::map<std::string,bool> targets_dedup;
150 for (; db_it != databases.end(); db_it++)
152 std::list<VirtualDB::Map>::const_iterator map_it;
153 map_it = m_p->m_maps.begin();
154 while (map_it != m_p->m_maps.end())
156 if (map_it->match(*db_it))
161 if (map_it == m_p->m_maps.end()) // database not found
163 error_code = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
168 std::list<std::string>::const_iterator t_it =
169 map_it->m_targets.begin();
170 for (; t_it != map_it->m_targets.end(); t_it++) {
171 if (!targets_dedup[*t_it])
173 targets_dedup[*t_it] = true;
174 b->m_targets.push_back(*t_it);
178 // see if we have a route conflict.
179 if (!first_route && b->m_route != map_it->m_route)
181 // we have a conflict.. routing must be same for all
182 error_code = YAZ_BIB1_COMBI_OF_SPECIFIED_DATABASES_UNSUPP;
186 b->m_route = map_it->m_route;
192 yf::VirtualDB::BackendPtr yf::VirtualDB::Frontend::init_backend(
193 std::list<std::string> databases, mp::Package &package,
194 int &error_code, std::string &addinfo)
196 BackendPtr b = create_backend_from_databases(databases, error_code,
200 Package init_package(b->m_backend_session, package.origin());
201 init_package.copy_filter(package);
205 Z_APDU *init_apdu = zget_APDU(odr, Z_APDU_initRequest);
207 mp::util::set_vhost_otherinfo(&init_apdu->u.initRequest->otherInfo, odr,
209 Z_InitRequest *req = init_apdu->u.initRequest;
211 // copy stuff from Frontend Init Request
212 Z_GDU *org_gdu = m_init_gdu.get();
213 Z_InitRequest *org_init = org_gdu->u.z3950->u.initRequest;
216 const char *peer_name = yaz_oi_get_string_oid(
217 &org_init->otherInfo, yaz_oid_userinfo_client_ip, 1, 0);
219 yaz_oi_set_string_oid(&init_apdu->u.initRequest->otherInfo, odr,
220 yaz_oid_userinfo_client_ip, 1, peer_name);
222 req->idAuthentication = org_init->idAuthentication;
223 req->implementationId = org_init->implementationId;
224 req->implementationName = org_init->implementationName;
225 req->implementationVersion = org_init->implementationVersion;
226 *req->preferredMessageSize = *org_init->preferredMessageSize;
227 *req->maximumRecordSize = *org_init->maximumRecordSize;
229 ODR_MASK_SET(req->options, Z_Options_search);
230 ODR_MASK_SET(req->options, Z_Options_present);
231 ODR_MASK_SET(req->options, Z_Options_namedResultSets);
232 ODR_MASK_SET(req->options, Z_Options_scan);
234 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
235 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
236 ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);
238 init_package.request() = init_apdu;
240 init_package.move(b->m_route); // sending init
242 Z_GDU *gdu = init_package.response().get();
243 // we hope to get an init response
245 if (gdu && gdu->which == Z_GDU_Z3950
246 && gdu->u.z3950->which == Z_APDU_initResponse)
248 Z_InitResponse *res = gdu->u.z3950->u.initResponse;
249 if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
251 b->m_named_result_sets = true;
253 if (*res->result && !init_package.session().is_closed())
255 m_backend_list.push_back(b);
258 mp::util::get_init_diagnostics(res, error_code, addinfo);
262 std::list<std::string>::const_iterator db_it = databases.begin();
263 error_code = YAZ_BIB1_ACCESS_TO_SPECIFIED_DATABASE_DENIED;
264 if (db_it != databases.end())
267 if (!init_package.session().is_closed())
269 Package close_package(b->m_backend_session, package.origin());
270 close_package.copy_filter(package);
271 close_package.session().close();
272 close_package.move(b->m_route); // closing it
278 void yf::VirtualDB::Frontend::search(mp::Package &package, Z_APDU *apdu_req)
280 yazpp_1::GDU ngdu(apdu_req);
281 Z_SearchRequest *req = ngdu.get()->u.z3950->u.searchRequest;
283 std::string resultSetId = req->resultSetName;
286 std::list<std::string> databases;
288 for (i = 0; i<req->num_databaseNames; i++)
289 databases.push_back(req->databaseNames[i]);
291 Sets_it sets_it = m_sets.find(req->resultSetName);
292 if (sets_it != m_sets.end())
294 // result set already exist
295 // if replace indicator is off: we return diagnostic if
296 // result set already exist.
297 if (*req->replaceIndicator == 0)
300 odr.create_searchResponse(
302 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
304 package.response() = apdu;
308 sets_it->second.m_backend->m_number_of_sets--;
310 // pick up any existing database with named result sets ..
311 // or one which has no result sets.. yet.
312 BackendPtr b; // null for now
313 std::list<BackendPtr>::const_iterator map_it;
314 map_it = m_backend_list.begin();
315 for (; map_it != m_backend_list.end(); map_it++)
317 BackendPtr tmp = *map_it;
318 if (tmp->m_frontend_databases == databases &&
319 (tmp->m_named_result_sets ||
320 tmp->m_number_of_sets == 0))
326 if (!b) // no backend yet. Must create a new one
330 b = init_backend(databases, package, error_code, addinfo);
333 // did not get a backend (unavailable somehow?)
336 odr.create_searchResponse(
337 apdu_req, error_code, addinfo.c_str());
338 package.response() = apdu;
342 m_sets.erase(req->resultSetName);
343 // sending search to backend
344 Package search_package(b->m_backend_session, package.origin());
346 search_package.copy_filter(package);
348 std::string backend_setname;
349 if (b->m_named_result_sets)
351 backend_setname = std::string(req->resultSetName);
355 backend_setname = "default";
356 req->resultSetName = odr_strdup(odr, backend_setname.c_str());
359 // pick first targets spec and move the databases from it ..
360 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
361 if (t_it != b->m_targets.end())
363 mp::util::set_databases_from_zurl(odr, *t_it,
364 &req->num_databaseNames,
365 &req->databaseNames);
368 *req->replaceIndicator = 1;
370 search_package.request() = ngdu;
372 search_package.move(b->m_route);
374 if (search_package.session().is_closed())
376 package.response() = search_package.response();
377 package.session().close();
380 b->m_number_of_sets++;
382 m_sets[resultSetId] = VirtualDB::Set(b, backend_setname);
383 fixup_package(search_package, b);
384 package.response() = search_package.response();
387 yf::VirtualDB::Frontend::Frontend(Rep *rep)
390 m_is_virtual = false;
393 void yf::VirtualDB::Frontend::close(mp::Package &package)
395 std::list<BackendPtr>::const_iterator b_it;
397 for (b_it = m_backend_list.begin(); b_it != m_backend_list.end(); b_it++)
399 (*b_it)->m_backend_session.close();
400 Package close_package((*b_it)->m_backend_session, package.origin());
401 close_package.copy_filter(package);
402 close_package.move((*b_it)->m_route);
404 m_backend_list.clear();
407 yf::VirtualDB::Frontend::~Frontend()
411 yf::VirtualDB::FrontendPtr yf::VirtualDB::Rep::get_frontend(mp::Package &package)
413 boost::mutex::scoped_lock lock(m_mutex);
415 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
419 it = m_clients.find(package.session());
420 if (it == m_clients.end())
423 if (!it->second->m_in_use)
425 it->second->m_in_use = true;
428 m_cond_session_ready.wait(lock);
430 FrontendPtr f(new Frontend(this));
431 m_clients[package.session()] = f;
436 void yf::VirtualDB::Rep::release_frontend(mp::Package &package)
438 boost::mutex::scoped_lock lock(m_mutex);
439 std::map<mp::Session,yf::VirtualDB::FrontendPtr>::iterator it;
441 it = m_clients.find(package.session());
442 if (it != m_clients.end())
444 if (package.session().is_closed())
446 it->second->close(package);
451 it->second->m_in_use = false;
453 m_cond_session_ready.notify_all();
458 yf::VirtualDB::Set::Set(BackendPtr b, std::string setname)
459 : m_backend(b), m_setname(setname)
464 yf::VirtualDB::Set::Set()
469 yf::VirtualDB::Set::~Set()
473 yf::VirtualDB::Map::Map(std::string database,
474 std::list<std::string> targets, std::string route)
475 : m_dbpattern(database), m_targets(targets), m_route(route)
479 yf::VirtualDB::Map::Map(std::string database,
480 std::string target, std::string route)
481 : m_dbpattern(database), m_route(route)
483 m_targets.push_back(target);
487 yf::VirtualDB::Map::Map()
491 bool yf::VirtualDB::Map::match(const std::string db) const
493 std::string norm_db = mp::util::database_name_normalize(db);
494 if (yaz_match_glob(m_dbpattern.c_str(), norm_db.c_str()))
499 yf::VirtualDB::VirtualDB() : m_p(new VirtualDB::Rep)
501 m_p->pass_vhosts = false;
504 yf::VirtualDB::~VirtualDB() {
507 void yf::VirtualDB::Frontend::fixup_npr_record(ODR odr, Z_NamePlusRecord *npr,
510 if (npr->databaseName)
512 std::string b_database = std::string(npr->databaseName);
514 // consider each of the frontend databases..
515 std::list<std::string>::const_iterator db_it;
516 for (db_it = b->m_frontend_databases.begin();
517 db_it != b->m_frontend_databases.end(); db_it++)
519 // see which target it corresponds to.. (if any)
520 std::list<VirtualDB::Map>::const_iterator map_it =
522 while (map_it != m_p->m_maps.end())
524 if (map_it->match(*db_it))
528 if (map_it != m_p->m_maps.end())
530 std::list<std::string>::const_iterator t
531 = map_it->m_targets.begin();
532 while (t != map_it->m_targets.end())
534 if (*t == b_database)
536 npr->databaseName = odr_strdup(odr, (*db_it).c_str());
544 db_it = b->m_frontend_databases.begin();
545 if (db_it != b->m_frontend_databases.end())
547 std::string database = *db_it;
548 npr->databaseName = odr_strdup(odr, database.c_str());
553 void yf::VirtualDB::Frontend::fixup_npr_records(ODR odr, Z_Records *records,
556 if (records && records->which == Z_Records_DBOSD)
558 Z_NamePlusRecordList *nprlist = records->u.databaseOrSurDiagnostics;
560 for (i = 0; i < nprlist->num_records; i++)
562 fixup_npr_record(odr, nprlist->records[i], b);
567 void yf::VirtualDB::Frontend::fixup_package(mp::Package &p, BackendPtr b)
569 Z_GDU *gdu = p.response().get();
572 if (gdu && gdu->which == Z_GDU_Z3950)
574 Z_APDU *apdu = gdu->u.z3950;
575 if (apdu->which == Z_APDU_presentResponse)
577 fixup_npr_records(odr, apdu->u.presentResponse->records, b);
580 else if (apdu->which == Z_APDU_searchResponse)
582 fixup_npr_records(odr, apdu->u.searchResponse->records, b);
588 void yf::VirtualDB::Frontend::present(mp::Package &package, Z_APDU *apdu_req)
590 yazpp_1::GDU ngdu(apdu_req);
591 Z_PresentRequest *req = ngdu.get()->u.z3950->u.presentRequest;
592 std::string resultSetId = req->resultSetId;
595 Sets_it sets_it = m_sets.find(resultSetId);
596 if (sets_it == m_sets.end())
599 odr.create_presentResponse(
601 YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
602 resultSetId.c_str());
603 package.response() = apdu;
607 new mp::Session(sets_it->second.m_backend->m_backend_session);
609 // sending present to backend
610 Package present_package(*id, package.origin());
611 present_package.copy_filter(package);
613 req->resultSetId = odr_strdup(odr, sets_it->second.m_setname.c_str());
615 present_package.request() = ngdu;
617 present_package.move(sets_it->second.m_backend->m_route);
619 fixup_package(present_package, sets_it->second.m_backend);
621 if (present_package.session().is_closed())
623 package.response() = present_package.response();
624 package.session().close();
628 package.response() = present_package.response();
633 int yf::VirtualDB::Frontend::relay_apdu(mp::Package &package, Z_APDU *apdu_req)
636 std::list<BackendPtr>::const_iterator map_it;
637 map_it = m_backend_list.begin();
638 for (; map_it != m_backend_list.end(); map_it++)
640 BackendPtr b = *map_it;
642 Package relay_package(b->m_backend_session, package.origin());
643 relay_package.copy_filter(package);
645 relay_package.request() = yazpp_1::GDU(apdu_req);
647 relay_package.move(b->m_route);
648 package.response() = relay_package.response();
654 void yf::VirtualDB::Frontend::scan(mp::Package &package, Z_APDU *apdu_req)
656 yazpp_1::GDU ngdu(apdu_req);
657 Z_ScanRequest *req = ngdu.get()->u.z3950->u.scanRequest;
661 std::list<std::string> databases;
663 for (i = 0; i<req->num_databaseNames; i++)
664 databases.push_back(req->databaseNames[i]);
667 // pick up any existing backend with a database match
668 std::list<BackendPtr>::const_iterator map_it;
669 map_it = m_backend_list.begin();
670 for (; map_it != m_backend_list.end(); map_it++)
672 BackendPtr tmp = *map_it;
673 if (tmp->m_frontend_databases == databases)
676 if (map_it != m_backend_list.end())
678 if (!b) // no backend yet. Must create a new one
682 b = init_backend(databases, package, error_code, addinfo);
685 // did not get a backend (unavailable somehow?)
687 odr.create_scanResponse(
688 apdu_req, error_code, addinfo.c_str());
689 package.response() = apdu;
694 // sending scan to backend
695 Package scan_package(b->m_backend_session, package.origin());
697 scan_package.copy_filter(package);
699 // pick first targets spec and move the databases from it ..
700 std::list<std::string>::const_iterator t_it = b->m_targets.begin();
701 if (t_it != b->m_targets.end())
703 mp::util::set_databases_from_zurl(odr, *t_it,
704 &req->num_databaseNames,
705 &req->databaseNames);
708 scan_package.request() = ngdu;
710 scan_package.move(b->m_route);
712 if (scan_package.session().is_closed())
714 package.response() = scan_package.response();
715 package.session().close();
718 package.response() = scan_package.response();
722 void yf::VirtualDB::add_map_db2targets(std::string db,
723 std::list<std::string> targets,
726 m_p->m_maps.push_back(
727 VirtualDB::Map(mp::util::database_name_normalize(db), targets, route));
731 void yf::VirtualDB::add_map_db2target(std::string db,
736 m_p->m_maps.push_back(
737 VirtualDB::Map(mp::util::database_name_normalize(db), target, route));
740 void yf::VirtualDB::process(mp::Package &package) const
742 FrontendPtr f = m_p->get_frontend(package);
744 Z_GDU *gdu = package.request().get();
746 if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
747 Z_APDU_initRequest && !f->m_is_virtual)
749 Z_InitRequest *req = gdu->u.z3950->u.initRequest;
751 std::list<std::string> vhosts;
752 mp::util::get_vhost_otherinfo(req->otherInfo, vhosts);
754 if (vhosts.size() > 0 && m_p->pass_vhosts)
763 Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
764 Z_InitResponse *resp = apdu->u.initResponse;
767 static const int masks[] = {
770 Z_Options_namedResultSets,
774 for (i = 0; masks[i] != -1; i++)
775 if (ODR_MASK_GET(req->options, masks[i]))
776 ODR_MASK_SET(resp->options, masks[i]);
778 static const int versions[] = {
784 for (i = 0; versions[i] != -1; i++)
785 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
786 ODR_MASK_SET(resp->protocolVersion, versions[i]);
790 *resp->preferredMessageSize = *req->preferredMessageSize;
791 *resp->maximumRecordSize = *req->maximumRecordSize;
793 package.response() = apdu;
794 f->m_is_virtual = true;
797 else if (!f->m_is_virtual)
799 else if (gdu && gdu->which == Z_GDU_Z3950)
801 Z_APDU *apdu = gdu->u.z3950;
802 if (apdu->which == Z_APDU_initRequest)
806 package.response() = odr.create_close(
808 Z_Close_protocolError,
811 package.session().close();
813 else if (apdu->which == Z_APDU_searchRequest)
815 f->search(package, apdu);
817 else if (apdu->which == Z_APDU_presentRequest)
819 f->present(package, apdu);
821 else if (apdu->which == Z_APDU_scanRequest)
823 f->scan(package, apdu);
825 else if (apdu->which == Z_APDU_close)
827 if (f->relay_apdu(package, apdu) == 0)
831 package.response() = odr.create_close(
832 apdu, Z_Close_finished, "virt_db");
834 package.session().close();
841 package.response() = odr.create_close(
842 apdu, Z_Close_protocolError,
843 "unsupported APDU in filter_virt_db");
845 package.session().close();
848 m_p->release_frontend(package);
851 void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only,
854 for (ptr = ptr->children; ptr; ptr = ptr->next)
856 if (ptr->type != XML_ELEMENT_NODE)
858 if (!strcmp((const char *) ptr->name, "pass-vhosts"))
860 m_p->pass_vhosts = mp::xml::get_bool(ptr, false);
862 else if (!strcmp((const char *) ptr->name, "virtual"))
864 std::string database;
865 std::list<std::string> targets;
866 xmlNode *v_node = ptr->children;
867 for (; v_node; v_node = v_node->next)
869 if (v_node->type != XML_ELEMENT_NODE)
872 if (mp::xml::is_element_mp(v_node, "database"))
873 database = mp::xml::get_text(v_node);
874 else if (mp::xml::is_element_mp(v_node, "target"))
875 targets.push_back(mp::xml::get_text(v_node));
877 throw mp::filter::FilterException
879 + std::string((const char *) v_node->name)
880 + " in virtual section"
883 std::string route = mp::xml::get_route(ptr);
885 VirtualDB::Map vmap(mp::util::database_name_normalize(database),
887 m_p->m_maps.push_back(vmap);
891 throw mp::filter::FilterException
893 + std::string((const char *) ptr->name)
894 + " in virt_db filter");
899 static mp::filter::Base* filter_creator()
901 return new mp::filter::VirtualDB;
905 struct metaproxy_1_filter_struct metaproxy_1_filter_virt_db = {
916 * c-file-style: "Stroustrup"
917 * indent-tabs-mode: nil
919 * vim: shiftwidth=4 tabstop=8 expandtab