From 67753db293cb62940e4c2db3a6e5f635cd5b78c3 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Tue, 31 May 2011 15:29:47 +0200 Subject: [PATCH] Start work on Torus extension for virt_db --- etc/Makefile.am | 1 + etc/config-torus.xml | 37 +++++++++ src/Makefile.am | 1 + src/filter_virt_db.cpp | 18 ++++- src/filter_virt_db.hpp | 1 + src/torus.cpp | 192 ++++++++++++++++++++++++++++++++++++++++++++++ src/torus.hpp | 46 +++++++++++ win/makefile | 1 + xml/schema/metaproxy.rnc | 5 +- xml/schema/metaproxy.rng | 11 ++- xml/schema/metaproxy.xsd | 8 +- 11 files changed, 316 insertions(+), 5 deletions(-) create mode 100644 etc/config-torus.xml create mode 100644 src/torus.cpp create mode 100644 src/torus.hpp diff --git a/etc/Makefile.am b/etc/Makefile.am index 5d70da1..ec83b73 100644 --- a/etc/Makefile.am +++ b/etc/Makefile.am @@ -20,6 +20,7 @@ xmlconfig = $(srcdir)/config-bytarget.xml \ $(srcdir)/config4.xml \ $(srcdir)/config5.xml \ $(srcdir)/config-cgi.xml \ + $(srcdir)/config-torus.xml \ $(srcdir)/retrieval-info.xml config = example.simple-auth example.target-auth pqf2pqf.xsl explain.xml diff --git a/etc/config-torus.xml b/etc/config-torus.xml new file mode 100644 index 0000000..1a5310c --- /dev/null +++ b/etc/config-torus.xml @@ -0,0 +1,37 @@ + + + + + + + + 10 + @:9000 + 30 + + + F + + + + + + + M + + + + roundrobin + + + B + + + + 30 + + + + + + diff --git a/src/Makefile.am b/src/Makefile.am index e7e54f0..ddd779e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -51,6 +51,7 @@ libmetaproxy_la_SOURCES = \ session.cpp \ sru_util.cpp sru_util.hpp \ thread_pool_observer.cpp thread_pool_observer.hpp \ + torus.cpp torus.hpp \ util.cpp \ xmlutil.cpp diff --git a/src/filter_virt_db.cpp b/src/filter_virt_db.cpp index 8d68ceb..72a9eb6 100644 --- a/src/filter_virt_db.cpp +++ b/src/filter_virt_db.cpp @@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #include +#include "torus.hpp" #include #include @@ -115,6 +116,7 @@ namespace metaproxy_1 { boost::condition m_cond_session_ready; std::map m_clients; bool pass_vhosts; + mp::Torus torus; }; } } @@ -821,7 +823,6 @@ void yf::VirtualDB::process(mp::Package &package) const m_p->release_frontend(package); } - void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) { for (ptr = ptr->children; ptr; ptr = ptr->next) @@ -856,6 +857,21 @@ void mp::filter::VirtualDB::configure(const xmlNode * ptr, bool test_only) std::string route = mp::xml::get_route(ptr); add_map_db2targets(database, targets, route); } + else if (!strcmp((const char *) ptr->name, "torus")) + { + std::string url; + const struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + if (!strcmp((const char *) attr->name, "url")) + url = mp::xml::get_text(attr->children); + else + throw mp::filter::FilterException( + "Bad attribute " + std::string((const char *) + attr->name)); + } + m_p->torus.read_searchables(url); + } else { throw mp::filter::FilterException diff --git a/src/filter_virt_db.hpp b/src/filter_virt_db.hpp index e4cb028..800bf7f 100644 --- a/src/filter_virt_db.hpp +++ b/src/filter_virt_db.hpp @@ -47,6 +47,7 @@ namespace metaproxy_1 { void add_map_db2target(std::string db, std::string target, std::string route); + void read_torus(); private: boost::scoped_ptr m_p; }; diff --git a/src/torus.cpp b/src/torus.cpp new file mode 100644 index 0000000..39f87da --- /dev/null +++ b/src/torus.cpp @@ -0,0 +1,192 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2011 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include + +#include +#include +#include +#include +#include + +#include "torus.hpp" + +namespace mp = metaproxy_1; + + +static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url) +{ + Z_GDU *p = z_get_HTTP_Request(odr); + const char *host = url; + const char *cp0 = strstr(host, "://"); + const char *cp1 = 0; + if (cp0) + cp0 = cp0+3; + else + cp0 = host; + + cp1 = strchr(cp0, '/'); + if (!cp1) + cp1 = cp0 + strlen(cp0); + + if (cp0 && cp1) + { + char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1); + memcpy (h, cp0, cp1 - cp0); + h[cp1-cp0] = '\0'; + z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h); + } + p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/"); + return p; +} + +static WRBUF get_url(const char *uri, WRBUF username, WRBUF password, + int *code) +{ + int number_of_redirects = 0; + WRBUF result = 0; + ODR out = odr_createmem(ODR_ENCODE); + ODR in = odr_createmem(ODR_DECODE); + + while (1) + { + Z_HTTP_Response *res = 0; + const char *location = 0; + Z_GDU *gdu = get_HTTP_Request_url(out, uri); + yaz_log(YLOG_LOG, "GET %s", uri); + gdu->u.HTTP_Request->method = odr_strdup(out, "GET"); + if (username && password) + { + z_HTTP_header_add_basic_auth(out, &gdu->u.HTTP_Request->headers, + wrbuf_cstr(username), + wrbuf_cstr(password)); + } + z_HTTP_header_add(out, &gdu->u.HTTP_Request->headers, "Accept", + "application/xml"); + if (!z_GDU(out, &gdu, 0, 0)) + { + yaz_log(YLOG_WARN, "Can not encode HTTP request URL:%s", uri); + break; + } + void *add; + COMSTACK conn = cs_create_host(uri, 1, &add); + if (!conn) + yaz_log(YLOG_WARN, "Bad address for URL:%s", uri); + else if (cs_connect(conn, add) < 0) + yaz_log(YLOG_WARN, "Can not connect to URL:%s", uri); + else + { + int len; + char *buf = odr_getbuf(out, &len, 0); + + if (cs_put(conn, buf, len) < 0) + yaz_log(YLOG_WARN, "cs_put failed URL:%s", uri); + else + { + char *netbuffer = 0; + int netlen = 0; + int cs_res = cs_get(conn, &netbuffer, &netlen); + if (cs_res <= 0) + { + yaz_log(YLOG_WARN, "cs_get failed URL:%s", uri); + } + else + { + Z_GDU *gdu; + odr_setbuf(in, netbuffer, cs_res, 0); + if (!z_GDU(in, &gdu, 0, 0) + || gdu->which != Z_GDU_HTTP_Response) + { + yaz_log(YLOG_WARN, "HTTP decoding failed " + "URL:%s", uri); + } + else + { + res = gdu->u.HTTP_Response; + } + } + xfree(netbuffer); + } + cs_close(conn); + } + if (!res) + break; // ERROR + *code = res->code; + location = z_HTTP_header_lookup(res->headers, "Location"); + if (++number_of_redirects < 10 && + location && (*code == 301 || *code == 302 || *code == 307)) + { + odr_reset(out); + uri = odr_strdup(out, location); + odr_reset(in); + } + else + { + result = wrbuf_alloc(); + wrbuf_write(result, res->content_buf, res->content_len); + break; + } + } + odr_destroy(out); + odr_destroy(in); + return result; +} + + +mp::Torus::Torus() +{ + doc = 0; +} + +mp::Torus::~Torus() +{ + if (doc) + xmlFreeDoc(doc); +} + +void mp::Torus::read_searchables(std::string url) +{ + if (url.length() == 0) + return; + + if (doc) + { + xmlFreeDoc(doc); + doc = 0; + } + + int code; + WRBUF w = get_url(url.c_str(), 0, 0, &code); + if (code == 200) + { + doc = xmlParseMemory(wrbuf_buf(w), wrbuf_len(w)); + if (doc) + yaz_log(YLOG_LOG, "xmlParseMemory OK"); + } + wrbuf_destroy(w); +} + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/torus.hpp b/src/torus.hpp new file mode 100644 index 0000000..e3a54a9 --- /dev/null +++ b/src/torus.hpp @@ -0,0 +1,46 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2011 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef METAPROXY_TORUS_HPP +#define METAPROXY_TORUS_HPP + +#include +#include +#include + +namespace metaproxy_1 { + class Torus { + public: + Torus(); + ~Torus(); + void read_searchables(std::string url); + private: + xmlDocPtr doc; + }; +} + +#endif +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/win/makefile b/win/makefile index c6b518b..d6cf870 100644 --- a/win/makefile +++ b/win/makefile @@ -248,6 +248,7 @@ PROJECT_DLL_OBJS = \ $(OBJDIR)\session.obj \ $(OBJDIR)\sru_util.obj \ $(OBJDIR)\thread_pool_observer.obj \ + $(OBJDIR)\torus.obj \ $(OBJDIR)\util.obj \ $(OBJDIR)\xmlutil.obj diff --git a/xml/schema/metaproxy.rnc b/xml/schema/metaproxy.rnc index 2eccb00..12a44b3 100644 --- a/xml/schema/metaproxy.rnc +++ b/xml/schema/metaproxy.rnc @@ -190,11 +190,14 @@ filter_virt_db = attribute id { xsd:NCName }?, attribute name { xsd:NCName }?, element mp:pass-vhosts { xsd:boolean }?, + element mp:torus { + attribute url { xsd:string } + }?, element mp:virtual { attribute route { xsd:NCName }?, element mp:database { xsd:string }, element mp:target { xsd:string }+ - }+ + }* filter_z3950_client = attribute type { "z3950_client" }, diff --git a/xml/schema/metaproxy.rng b/xml/schema/metaproxy.rng index 769fd9d..602ea75 100644 --- a/xml/schema/metaproxy.rng +++ b/xml/schema/metaproxy.rng @@ -491,7 +491,14 @@ - + + + + + + + + @@ -507,7 +514,7 @@ - + diff --git a/xml/schema/metaproxy.xsd b/xml/schema/metaproxy.xsd index 00dc21b..8fb6159 100644 --- a/xml/schema/metaproxy.xsd +++ b/xml/schema/metaproxy.xsd @@ -408,10 +408,16 @@ - + + + + + + + -- 1.7.10.4