From: Adam Dickmeiss Date: Thu, 6 Apr 2006 01:16:54 +0000 (+0000) Subject: Implement search limit (n). X-Git-Tag: YAZPROXY.1.1.0.2~16 X-Git-Url: http://sru.miketaylor.org.uk/cgi-bin?a=commitdiff_plain;h=84434e0d19b1ecd73ff09853f7cb3d96f343c237;p=yazproxy-moved-to-github.git Implement search limit (n). Implement connect limit -- which results in delay. This complements the connect max -- which blocks out a client completely. Clean up file access using docpath configuration.. Documentation updates for these. --- diff --git a/doc/installation.xml b/doc/installation.xml index 9cc450d..96ced05 100644 --- a/doc/installation.xml +++ b/doc/installation.xml @@ -1,5 +1,5 @@ - + Installation You need a C++ compiler to compile and use YAZ proxy. @@ -61,14 +61,14 @@ --with-yazpp directory - Specifies the location of yaz++-config. - The yaz++-config program is generated in + Specifies the location of yazpp-config. + The yazpp-config program is generated in the source directory of YAZ++ as well as the binaries directory when YAZ++ is installed (via make install). If you don't supply this option, configure will - look for yaz++-config in directories of the + look for yazpp-config in directories of the PATH environment - which is nearly always what you want. diff --git a/doc/reference.xml b/doc/reference.xml index 3e52ecb..28cd49f 100644 --- a/doc/reference.xml +++ b/doc/reference.xml @@ -247,7 +247,7 @@ <?xml version="1.0"?> - <proxy xmlns="http://indexdata.dk/yazproxy/schema/0.8/"> + <proxy xmlns="http://indexdata.dk/yazproxy/schema/0.9/"> <target name="server1" default="1"> <!-- description of server1 .. --> </target> @@ -335,14 +335,17 @@ The proxy records bandwidth/pdu requests during the last 60 seconds (1 minute). The limit may include the elements bandwidth, pdu, - and retrieve. The bandwidth + retrieve and search. + The bandwidth measures the number of bytes transferred within the last minute. The pdu is the number of requests in the last minute. The retrieve holds the maximum records to - be retrieved in one Present Request. + which may be retrieved in one Present Request. + The search is the maximum number of searches + within the last minute. - If a bandwidth/pdu limit is reached the proxy will postpone the + If a bandwidth/pdu/search limit is reached the proxy will postpone the requests to the target and wait one or more seconds. The idea of the limit is to ensure that clients that downloads hundreds or thousands of records do not hurt other users. @@ -696,6 +699,58 @@ + +
+ max-connect + + The element max-connect is a child of element + proxy and specifies the maximum number + of connections to be initiated within the last minute. + + + If the maximum number is reached the proxy will terminate the + just initiated session (connection terminated). + +
+ +
+ limit-connect + + The element max-connect is a child of element + proxy and specifies the limit of number + of connections to be initiated within the last minute. + + + If the maximum number is reached the proxy delay the first operatation + in the session (Thus delaying the connection). + +
+ +
+ docpath + + The element docpath is a child of element + proxy and specifies an allowed HTTP path + for local file access. Using docpath the + proxy may return static file content. + + + The value of docpath both serves as a HTTP path prefix + and as a local file prefix. + If a value of etc is used only URLs with the + prefix /etc/ results in a local file access to the + directory etc within the working directory + of yazproxy. + + + + Care has been taken to ensure that hostile URLs are rejected - including + strings such as .. and / (absolute + file system access). + + +
+
Proxy Manual Pages @@ -743,7 +798,7 @@ - + 2000000 60 100 - 3 + 3 @@ -47,7 +47,9 @@ 30 - 2 - 3 + 30 + 10 + 5 client-requests server-requests + doc diff --git a/etc/voyager.xml b/etc/voyager.xml index c28da9e..ce55a62 100644 --- a/etc/voyager.xml +++ b/etc/voyager.xml @@ -1,5 +1,5 @@ - + 200000 31 50 + 15 @@ -210,5 +211,7 @@ client-requests server-requests - + + 10 + 5 diff --git a/include/yazproxy/proxy.h b/include/yazproxy/proxy.h index fe0f2df..d50b28a 100644 --- a/include/yazproxy/proxy.h +++ b/include/yazproxy/proxy.h @@ -1,4 +1,4 @@ -/* $Id: proxy.h,v 1.29 2006-03-30 10:35:15 adam Exp $ +/* $Id: proxy.h,v 1.30 2006-04-06 01:16:55 adam Exp $ Copyright (c) 1998-2006, Index Data. This file is part of the yazproxy. @@ -106,12 +106,16 @@ class YAZ_EXPORT Yaz_Proxy : public yazpp_1::Z_Assoc { timeout_xsl } m_timeout_mode; + int m_initial_reduce; int m_connect_max; + int m_limit_connect; int m_search_max; Yaz_bw m_bw_stat; int m_pdu_max; Yaz_bw m_pdu_stat; int m_max_record_retrieve; + Yaz_bw m_search_stat; + void handle_max_record_retrieve(Z_APDU *apdu); void display_diagrecs(Z_DiagRec **pp, int num); Z_Records *create_nonSurrogateDiagnostics(ODR o, int error, diff --git a/src/proxyp.h b/src/proxyp.h index 1e62cca..c412566 100644 --- a/src/proxyp.h +++ b/src/proxyp.h @@ -1,4 +1,4 @@ -/* $Id: proxyp.h,v 1.14 2006-03-30 10:35:15 adam Exp $ +/* $Id: proxyp.h,v 1.15 2006-04-06 01:16:55 adam Exp $ Copyright (c) 1998-2006, Index Data. This file is part of the yazproxy. @@ -105,7 +105,9 @@ public: const char **default_client_query_charset); void get_generic_info(int *log_mask, int *max_clients, - int *max_connect); + int *max_connect, int *limit_connect); + + int get_file_access_info(const char *path); void get_target_info(const char *name, const char **url, int *limit_bw, int *limit_pdu, int *limit_req, diff --git a/src/yaz-proxy-config.cpp b/src/yaz-proxy-config.cpp index 219bf9a..c4c29f8 100644 --- a/src/yaz-proxy-config.cpp +++ b/src/yaz-proxy-config.cpp @@ -1,4 +1,4 @@ -/* $Id: yaz-proxy-config.cpp,v 1.26 2006-03-30 10:35:15 adam Exp $ +/* $Id: yaz-proxy-config.cpp,v 1.27 2006-04-06 01:16:55 adam Exp $ Copyright (c) 1998-2006, Index Data. This file is part of the yazproxy. @@ -981,11 +981,35 @@ int Yaz_ProxyConfigP::mycmp(const char *hay, const char *item, size_t len) return 0; } +int Yaz_ProxyConfig::get_file_access_info(const char *path) +{ +#if HAVE_XSLT + xmlNodePtr ptr; + if (!m_cp->m_proxyPtr) + return 0; + for (ptr = m_cp->m_proxyPtr->children; ptr; ptr = ptr->next) + { + if (ptr->type == XML_ELEMENT_NODE + && !strcmp((const char *) ptr->name, "docpath")) + { + const char *docpath = m_cp->get_text(ptr); + size_t docpath_len = strlen(docpath); + if (docpath_len < strlen(path) && path[docpath_len] == '/' + && !memcmp(docpath, path, docpath_len)) + return 1; + } + } +#endif + return 0; +} + void Yaz_ProxyConfig::get_generic_info(int *log_mask, int *max_clients, - int *max_connect) + int *max_connect, + int *limit_connect) { *max_connect = 0; + *limit_connect = 0; #if HAVE_XSLT xmlNodePtr ptr; if (!m_cp->m_proxyPtr) @@ -1020,7 +1044,7 @@ void Yaz_ProxyConfig::get_generic_info(int *log_mask, v = cp; } } - if (ptr->type == XML_ELEMENT_NODE && + else if (ptr->type == XML_ELEMENT_NODE && !strcmp((const char *) ptr->name, "max-clients")) { const char *t = m_cp->get_text(ptr); @@ -1031,13 +1055,31 @@ void Yaz_ProxyConfig::get_generic_info(int *log_mask, *max_clients = 1; } } - if (ptr->type == XML_ELEMENT_NODE && + else if (ptr->type == XML_ELEMENT_NODE && !strcmp((const char *) ptr->name, "max-connect")) { const char *t = m_cp->get_text(ptr); if (t) *max_connect = atoi(t); } + else if (ptr->type == XML_ELEMENT_NODE && + !strcmp((const char *) ptr->name, "limit-connect")) + { + const char *t = m_cp->get_text(ptr); + if (t) + *limit_connect = atoi(t); + } + else if (ptr->type == XML_ELEMENT_NODE && + !strcmp((const char *) ptr->name, "target")) + ; + else if (ptr->type == XML_ELEMENT_NODE && + !strcmp((const char *) ptr->name, "docpath")) + ; + else if (ptr->type == XML_ELEMENT_NODE) + { + yaz_log(YLOG_WARN, "0 Unknown element %s in yazproxy config", + ptr->name); + } } #endif } diff --git a/src/yaz-proxy.cpp b/src/yaz-proxy.cpp index 11112ed..e2195ba 100644 --- a/src/yaz-proxy.cpp +++ b/src/yaz-proxy.cpp @@ -1,4 +1,4 @@ -/* $Id: yaz-proxy.cpp,v 1.48 2006-03-30 14:22:06 adam Exp $ +/* $Id: yaz-proxy.cpp,v 1.49 2006-04-06 01:16:55 adam Exp $ Copyright (c) 1998-2006, Index Data. This file is part of the yazproxy. @@ -202,7 +202,8 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, ISocketObservable *the_socket_observable, Yaz_Proxy *parent) : - Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60) + Z_Assoc(the_PDU_Observable), + m_bw_stat(60), m_pdu_stat(60), m_search_stat(60) { m_PDU_Observable = the_PDU_Observable; m_socket_observable = the_socket_observable; @@ -232,6 +233,7 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, m_pdu_max = 0; m_search_max = 0; m_connect_max = 0; + m_limit_connect = 0; m_timeout_mode = timeout_normal; m_timeout_gdu = 0; m_max_record_retrieve = 0; @@ -291,6 +293,7 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, m_ref_count = 1; m_main_ptr_dec = false; m_peername = 0; + m_initial_reduce = 0; } void Yaz_Proxy::inc_ref() @@ -352,7 +355,7 @@ int Yaz_Proxy::set_config(const char *config) int r = m_config->read_xml(config); if (!r) m_config->get_generic_info(&m_log_mask, &m_max_clients, - &m_connect_max); + &m_connect_max, &m_limit_connect); return r; } @@ -404,7 +407,7 @@ Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure() { m_log_mask = 0; cfg->get_generic_info(&m_log_mask, &m_max_clients, - &m_connect_max); + &m_connect_max, &m_limit_connect); } } else @@ -442,6 +445,10 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, m_socket_observable, this); + + if (m_limit_connect) + new_proxy->m_initial_reduce = connect_total / m_limit_connect; + new_proxy->m_config = 0; new_proxy->m_config_fname = 0; new_proxy->timeout(m_client_idletime); @@ -987,7 +994,6 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p) void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p, const char *backend_charset) { - yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str); int sel = m_charset_converter->get_client_charset_selected(); const char *client_record_charset = m_charset_converter->get_client_query_charset(); @@ -1046,10 +1052,6 @@ void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p, yaz_iconv_close(cd); yaz_marc_destroy(mt); } - else - { - yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str); - } } void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p, @@ -1857,21 +1859,20 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len) void Yaz_Proxy::recv_GDU_reduce(GDU *gdu) { - int reduce = 0; + int reduce = m_initial_reduce; // initial reduce from connect phase.. + m_initial_reduce = 0; // reset it.. int bw_total = m_bw_stat.get_total(); int pdu_total = m_pdu_stat.get_total(); + int search_total = m_search_stat.get_total(); assert(m_timeout_mode == timeout_busy); assert(m_timeout_gdu == 0); + if (m_search_max) + reduce += search_total / m_search_max; if (m_bw_max) - { - if (bw_total > m_bw_max) - { - reduce = (bw_total/m_bw_max); - } - } + reduce += (bw_total/m_bw_max); if (m_pdu_max) { if (pdu_total > m_pdu_max) @@ -1891,9 +1892,9 @@ void Yaz_Proxy::recv_GDU_reduce(GDU *gdu) #endif if (reduce) { - yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d", - m_session_str, reduce, bw_total, pdu_total, - m_bw_max, m_pdu_max); + yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d", + m_session_str, reduce, bw_total, pdu_total, search_total, + m_bw_max, m_pdu_max, m_search_max); m_timeout_mode = timeout_reduce; m_timeout_gdu = gdu; @@ -2473,39 +2474,45 @@ int Yaz_Proxy::file_access(Z_HTTP_Request *hreq) if (strcmp(hreq->method, "GET")) return 0; if (hreq->path[0] != '/') - { - yaz_log(YLOG_WARN, "Bad path: %s", hreq->path); return 0; - } const char *cp = hreq->path; while (*cp) { if (*cp == '/' && strchr("/.", cp[1])) { - yaz_log(YLOG_WARN, "Bad path: %s", hreq->path); + yaz_log(YLOG_LOG, "%sRejecting path %s", m_session_str, + hreq->path); return 0; } cp++; } + + Yaz_ProxyConfig *cfg = check_reconfigure(); + + if (!cfg->get_file_access_info(hreq->path+1)) + return 0; + const char *fname = hreq->path+1; if (stat(fname, &sbuf)) { - yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname); + yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str, + fname); return 0; } if ((sbuf.st_mode & S_IFMT) != S_IFREG) { - yaz_log(YLOG_WARN, "%s: not a regular file", fname); + yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname); return 0; } if (sbuf.st_size > (off_t) 1000000) { - yaz_log(YLOG_WARN, "%s: too large for transfer", fname); + yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str, + fname); return 0; } ODR o = odr_encode(); - Yaz_ProxyConfig *cfg = check_reconfigure(); + const char *ctype = cfg->check_mime_type(fname); Z_GDU *gdu = z_get_HTTP_Response(o, 200); Z_HTTP_Response *hres = gdu->u.HTTP_Response; @@ -2623,6 +2630,7 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) if (srw_pdu->which == Z_SRW_searchRetrieve_request) { + Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request; const char *backend_db = srw_req->database; @@ -3033,6 +3041,9 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu) apdu = m_initRequest_apdu; // but throw an init to the target } + if (apdu->which == Z_APDU_searchRequest) + m_search_stat.add_bytes(1); + // Determine our client. Z_OtherInformation **oi; get_otherInfoAPDU(apdu, &oi);