From 8214f56ad711f5a61213a7cf4027a8cca93fc5d1 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Fri, 19 Apr 2013 16:42:44 +0200 Subject: [PATCH] New filter present_chunk This filter allows splitting of present requests into smaller chunks. --- etc/config1.xml | 3 + src/Makefile.am | 12 +- src/factory_static.cpp | 2 + src/filter_present_chunk.cpp | 211 +++++++++++++++++++++++++++++++++++ src/filter_present_chunk.hpp | 55 +++++++++ xml/schema/filter_present_chunk.rnc | 10 ++ xml/schema/metaproxy.rnc | 2 + 7 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 src/filter_present_chunk.cpp create mode 100644 src/filter_present_chunk.hpp create mode 100644 xml/schema/filter_present_chunk.rnc diff --git a/etc/config1.xml b/etc/config1.xml index dd91ae4..d7ff628 100644 --- a/etc/config1.xml +++ b/etc/config1.xml @@ -22,6 +22,9 @@ log + + 2 + diff --git a/src/Makefile.am b/src/Makefile.am index 04e2f81..5c1a67d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -35,6 +35,7 @@ libmetaproxy_la_SOURCES = \ filter_log.cpp filter_log.hpp \ filter_multi.cpp filter_multi.hpp \ filter_query_rewrite.cpp filter_query_rewrite.hpp \ + filter_present_chunk.cpp filter_present_chunk.hpp \ filter_record_transform.cpp filter_record_transform.hpp \ filter_session_shared.cpp filter_session_shared.hpp \ filter_sort.cpp filter_sort.hpp \ @@ -57,10 +58,11 @@ libmetaproxy_la_SOURCES = \ torus.cpp torus.hpp \ url_recipe.cpp \ util.cpp \ - xmlutil.cpp + xmlutil.cpp -libmetaproxy_la_LIBADD = $(YAZPPLALIB) $(BOOST_LIB) $(BOOST_THREAD_LIB) $(BOOST_REGEX_LIB) +libmetaproxy_la_LIBADD = $(YAZPPLALIB) $(BOOST_LIB) $(BOOST_THREAD_LIB) \ + $(BOOST_REGEX_LIB) # Rules for lib @@ -159,9 +161,3 @@ test_ses_map_LDADD = $(TESTLDADD) test_thread_pool_observer_LDADD = $(TESTLDADD) test_xmlutil_LDADD = $(TESTLDADD) -# doxygen target -dox: - (cd $(top_srcdir) ; make dox) - -showdox: - (cd $(top_srcdir) ; make showdox) diff --git a/src/factory_static.cpp b/src/factory_static.cpp index 3942637..1d0e5a8 100644 --- a/src/factory_static.cpp +++ b/src/factory_static.cpp @@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "filter_load_balance.hpp" #include "filter_log.hpp" #include "filter_multi.hpp" +#include "filter_present_chunk.hpp" #include "filter_query_rewrite.hpp" #include "filter_record_transform.hpp" #include "filter_session_shared.hpp" @@ -71,6 +72,7 @@ mp::FactoryStatic::FactoryStatic() &metaproxy_1_filter_load_balance, &metaproxy_1_filter_log, &metaproxy_1_filter_multi, + &metaproxy_1_filter_present_chunk, &metaproxy_1_filter_query_rewrite, &metaproxy_1_filter_record_transform, &metaproxy_1_filter_session_shared, diff --git a/src/filter_present_chunk.cpp b/src/filter_present_chunk.cpp new file mode 100644 index 0000000..c2dc859 --- /dev/null +++ b/src/filter_present_chunk.cpp @@ -0,0 +1,211 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2013 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "config.hpp" +#include "filter_present_chunk.hpp" + +#include +#include +#include +#include +#include + +namespace mp = metaproxy_1; +namespace yf = mp::filter; + +namespace metaproxy_1 { + namespace filter { + class PresentChunk::Impl { + public: + Impl(); + ~Impl(); + void process(metaproxy_1::Package & package); + void configure(const xmlNode * ptr); + void chunk_it(metaproxy_1::Package & package, Z_APDU *apdu); + private: + Odr_int chunk_number; + }; + } +} + +yf::PresentChunk::PresentChunk() : m_p(new Impl) +{ +} + +yf::PresentChunk::~PresentChunk() +{ // must have a destructor because of boost::scoped_ptr +} + +void yf::PresentChunk::configure(const xmlNode *xmlnode, bool test_only, + const char *path) +{ + m_p->configure(xmlnode); +} + +void yf::PresentChunk::process(mp::Package &package) const +{ + m_p->process(package); +} + +yf::PresentChunk::Impl::Impl() : chunk_number(0) +{ +} + +yf::PresentChunk::Impl::~Impl() +{ +} + +void yf::PresentChunk::Impl::configure(const xmlNode *ptr) +{ + for (ptr = ptr->children; ptr; ptr = ptr->next) + { + if (ptr->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) ptr->name, "chunk")) + { + chunk_number = mp::xml::get_int(ptr, 0); + } + else + { + throw mp::filter::FilterException("Bad element " + + std::string((const char *) + ptr->name)); + } + } +} + +void yf::PresentChunk::Impl::chunk_it(mp::Package &package, + Z_APDU *apdu) +{ + mp::odr odr; + Z_PresentRequest *pr = apdu->u.presentRequest; + + Odr_int total = *pr->numberOfRecordsRequested; + Odr_int start = *pr->resultSetStartPoint; + Odr_int offset = 0; + Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *) + odr_malloc(odr, sizeof(*npl)); + npl->num_records = total; + npl->records = (Z_NamePlusRecord **) + odr_malloc(odr, sizeof(*npl->records) * total); + while (offset < total) + { + Odr_int left = total - offset; + + Package pp(package.session(), package.origin()); + + *pr->numberOfRecordsRequested = + left > chunk_number ? chunk_number : left; + + *pr->resultSetStartPoint = start + offset; + + pp.copy_filter(package); + pp.request() = apdu; + + pp.move(); + + if (pp.session().is_closed()) + { + package.session().close(); + return; + } + Z_GDU *gdu_res = pp.response().get(); + if (gdu_res && gdu_res->which == Z_GDU_Z3950 && + gdu_res->u.z3950->which == Z_APDU_presentResponse) + { + Z_PresentResponse *pres = + gdu_res->u.z3950->u.presentResponse; + if (pres->records && + pres->records->which == Z_Records_DBOSD) + { + Z_NamePlusRecordList *nprl = + pres->records->u.databaseOrSurDiagnostics; + int i; + for (i = 0; i < nprl->num_records; i++) + { + ODR o = odr; + npl->records[offset+i] = yaz_clone_z_NamePlusRecord( + nprl->records[i], o->mem); + } + offset += nprl->num_records; + } + else + { + package.response() = pp.response(); + return; + } + } + else + { + package.response() = pp.response(); + return; + } + } + + yaz_log(YLOG_LOG, "building response . %lld", offset); + + Z_APDU *a = zget_APDU(odr, Z_APDU_presentResponse); + Z_PresentResponse *pres = a->u.presentResponse; + pres->records = (Z_Records *) + odr_malloc(odr, sizeof(Z_Records)); + pres->records->which = Z_Records_DBOSD; + pres->records->u.databaseOrSurDiagnostics = npl; + npl->num_records = offset; + *pres->numberOfRecordsReturned = offset; + + package.response() = a; +} + +void yf::PresentChunk::Impl::process(mp::Package &package) +{ + Z_GDU *gdu = package.request().get(); + if (gdu && gdu->which == Z_GDU_Z3950) + { + Z_APDU *apdu = gdu->u.z3950; + if (apdu->which == Z_APDU_presentRequest && chunk_number > 0) + chunk_it(package, apdu); + else + package.move(); + } + else + package.move(); +} + +static mp::filter::Base* filter_creator() +{ + return new mp::filter::PresentChunk; +} + +extern "C" { + struct metaproxy_1_filter_struct metaproxy_1_filter_present_chunk = { + 0, + "present_chunk", + filter_creator + }; +} + + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/filter_present_chunk.hpp b/src/filter_present_chunk.hpp new file mode 100644 index 0000000..7a17b7c --- /dev/null +++ b/src/filter_present_chunk.hpp @@ -0,0 +1,55 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2013 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef FILTER_PRESENT_CHUNK_HPP +#define FILTER_PRESENT_CHUNK_HPP + +#include + +#include + +namespace metaproxy_1 { + namespace filter { + class PresentChunk : public Base { + class Impl; + boost::scoped_ptr m_p; + public: + PresentChunk(); + ~PresentChunk(); + void process(metaproxy_1::Package & package) const; + void configure(const xmlNode * ptr, bool test_only, + const char *path); + }; + } +} + +extern "C" { + extern struct metaproxy_1_filter_struct metaproxy_1_filter_present_chunk; +} + +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/xml/schema/filter_present_chunk.rnc b/xml/schema/filter_present_chunk.rnc new file mode 100644 index 0000000..877d69e --- /dev/null +++ b/xml/schema/filter_present_chunk.rnc @@ -0,0 +1,10 @@ +# Metaproxy XML config file schema + +namespace mp = "http://indexdata.com/metaproxy" + +filter_present_chunk = + attribute type { "present_chunk" }, + attribute id { xsd:NCName }?, + attribute name { xsd:NCName }?, + element mp:chunk { xsd:integer }? + diff --git a/xml/schema/metaproxy.rnc b/xml/schema/metaproxy.rnc index 39a02bd..c817401 100644 --- a/xml/schema/metaproxy.rnc +++ b/xml/schema/metaproxy.rnc @@ -34,6 +34,7 @@ include "filter_limit.rnc" include "filter_load_balance.rnc" include "filter_log.rnc" include "filter_multi.rnc" +include "filter_present_chunk.rnc" include "filter_query_rewrite.rnc" include "filter_record_transform.rnc" include "filter_session_shared.rnc" @@ -83,6 +84,7 @@ filter = | filter_load_balance | filter_log | filter_multi + | filter_present_chunk | filter_query_rewrite | filter_record_transform | filter_session_shared -- 1.7.10.4