From: Adam Dickmeiss Date: Tue, 4 Oct 2011 13:12:56 +0000 (+0200) Subject: Simplify ZURL resolving and use regular getaddrinfo X-Git-Tag: v1.6.4~5^2~30 X-Git-Url: http://sru.miketaylor.org.uk/?a=commitdiff_plain;h=d65081fd86127344075a112002adbfa32ff88f11;p=pazpar2-moved-to-github.git Simplify ZURL resolving and use regular getaddrinfo Caching of DNS entries is done better by a local caching DNS or by libc. --- diff --git a/src/Makefile.am b/src/Makefile.am index 5d69f20..2f3643d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -22,7 +22,6 @@ libpazpar2_a_SOURCES = \ database.c database.h \ eventl.c eventl.h \ facet_limit.c facet_limit.h \ - getaddrinfo.c \ host.h \ http.c http.h http_command.c \ incref.c incref.h \ diff --git a/src/client.c b/src/client.c index e0b9ee1..f2e1742 100644 --- a/src/client.c +++ b/src/client.c @@ -671,7 +671,6 @@ void client_start_search(struct client *cl) struct connection *co = client_get_connection(cl); ZOOM_connection link = connection_get_link(co); ZOOM_resultset rs; - char *databaseName = sdb->database->databases[0]; const char *opt_piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK); const char *opt_queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING); const char *opt_elements = session_setting_oneval(sdb, PZ_ELEMENTS); @@ -729,9 +728,6 @@ void client_start_search(struct client *cl) sprintf(startrecs_str, "%d", cl->startrecs); ZOOM_connection_option_set(link, "start", startrecs_str); - if (databaseName) - ZOOM_connection_option_set(link, "databaseName", databaseName); - /* TODO Verify does it break something for CQL targets(non-SOLR) ? */ /* facets definition is in PQF */ client_set_facets_request(cl, link); diff --git a/src/connection.c b/src/connection.c index 0a6f61c..219380c 100644 --- a/src/connection.c +++ b/src/connection.c @@ -183,8 +183,8 @@ static struct connection *connection_create(struct client *cl, co->state = Conn_Resolving; co->operation_timeout = operation_timeout; co->session_timeout = session_timeout; - if (host->ipport) - connection_connect(co, iochan_man); + + connection_connect(co, iochan_man); yaz_mutex_enter(host->mutex); co->next = co->host->connections; @@ -352,50 +352,6 @@ static void connection_release(struct connection *co) co->client = 0; } -void connect_resolver_host(struct host *host, iochan_man_t iochan_man) -{ - struct connection *con; - -start: - yaz_mutex_enter(host->mutex); - con = host->connections; - while (con) - { - if (con->state == Conn_Resolving) - { - if (!host->ipport) /* unresolved */ - { - remove_connection_from_host(con); - yaz_mutex_leave(host->mutex); - connection_destroy(con); - goto start; - /* start all over .. at some point it will be NULL */ - } - else if (!con->client) - { - remove_connection_from_host(con); - yaz_mutex_leave(host->mutex); - connection_destroy(con); - /* start all over .. at some point it will be NULL */ - goto start; - } - else - { - yaz_mutex_leave(host->mutex); - connection_connect(con, iochan_man); - client_start_search(con->client); - goto start; - } - } - else - { - yaz_log(YLOG_LOG, "connect_resolver_host: state=%d", con->state); - con = con->next; - } - } - yaz_mutex_leave(host->mutex); -} - static struct host *connection_get_host(struct connection *con) { return con->host; @@ -415,7 +371,6 @@ static int connection_connect(struct connection *con, iochan_man_t iochan_man) const char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY); const char *apdulog = session_setting_oneval(sdb, PZ_APDULOG); - assert(host->ipport); assert(con); ZOOM_options_set(zoptions, "async", "1"); @@ -454,10 +409,10 @@ static int connection_connect(struct connection *con, iochan_man_t iochan_man) strcat(http_hostport, host->hostport); ZOOM_connection_connect(link, http_hostport, 0); } - else if (zproxy && *zproxy) - ZOOM_connection_connect(link, host->hostport, 0); else - ZOOM_connection_connect(link, host->ipport, 0); + { + ZOOM_connection_connect(link, host->hostport, 0); + } con->link = link; con->iochan = iochan_create(-1, connection_handler, 0, "connection_socket"); diff --git a/src/database.c b/src/database.c index 08359d6..1a450af 100644 --- a/src/database.c +++ b/src/database.c @@ -65,29 +65,25 @@ struct database_criterion { struct database_criterion *next; }; - struct database_hosts { struct host *hosts; YAZ_MUTEX mutex; }; // Create a new host structure for hostport -static struct host *create_host(const char *hostport, iochan_man_t iochan_man) +static struct host *create_host(const char *hostport) { struct host *host; + char *db_comment; host = xmalloc(sizeof(struct host)); host->hostport = xstrdup(hostport); + db_comment = strchr(host->hostport, '#'); + if (db_comment) + *db_comment = '\0'; host->connections = 0; - host->ipport = 0; host->mutex = 0; - if (host_getaddrinfo(host, iochan_man)) - { - xfree(host->hostport); - xfree(host); - return 0; - } pazpar2_mutex_create(&host->mutex, "host"); yaz_cond_create(&host->cond_ready); @@ -96,7 +92,7 @@ static struct host *create_host(const char *hostport, iochan_man_t iochan_man) } static struct host *find_host(database_hosts_t hosts, - const char *hostport, iochan_man_t iochan_man) + const char *hostport) { struct host *p; yaz_mutex_enter(hosts->mutex); @@ -105,7 +101,7 @@ static struct host *find_host(database_hosts_t hosts, break; if (!p) { - p = create_host(hostport, iochan_man); + p = create_host(hostport); if (p) { p->next = hosts->hosts; @@ -121,13 +117,7 @@ int resolve_database(struct conf_service *service, struct database *db) if (db->host == 0) { struct host *host; - char *p; - char hostport[256]; - strcpy(hostport, db->url); - if ((p = strchr(hostport, '/'))) - *p = '\0'; - if (!(host = find_host(service->server->database_hosts, - hostport, service->server->iochan_man))) + if (!(host = find_host(service->server->database_hosts, db->url))) return -1; db->host = host; } @@ -144,30 +134,14 @@ void resolve_databases(struct conf_service *service) struct database *new_database(const char *id, NMEM nmem) { struct database *db; - char hostport[256]; - char *dbname; - char *db_comment; struct setting *idset; - if (strlen(id) > 255) - return 0; - strcpy(hostport, id); - if ((dbname = strchr(hostport, '/'))) - *(dbname++) = '\0'; - else - dbname = ""; - db_comment = strchr(dbname, '#'); - if (db_comment) - *db_comment = '\0'; db = nmem_malloc(nmem, sizeof(*db)); memset(db, 0, sizeof(*db)); - db->host = 0; + db->url = nmem_strdup(nmem, id); - db->databases = nmem_malloc(nmem, 2 * sizeof(char *)); - db->databases[0] = nmem_strdup(nmem, dbname); - db->databases[1] = 0; db->errors = 0; - db->explain = 0; + db->host = 0; db->num_settings = PZ_MAX_EOF; db->settings = nmem_malloc(nmem, sizeof(struct settings*) * @@ -188,10 +162,9 @@ static struct database *load_database(const char *id, struct conf_service *service) { struct database *db; - struct zr_explain *explain = 0; db = new_database(id, service->nmem); - db->explain = explain; + db->next = service->databases; service->databases = db; @@ -344,7 +317,7 @@ static int database_match_criteria(struct setting **settings, // Cycles through databases, calling a handler function on the ones for // which all criteria matched. int session_grep_databases(struct session *se, const char *filter, - void (*fun)(void *context, struct session_database *db)) + void (*fun)(struct session *se, struct session_database *db)) { struct session_database *p; NMEM nmem = nmem_create(); @@ -401,7 +374,6 @@ void database_hosts_destroy(database_hosts_t *pp) struct host *p_next = p->next; yaz_mutex_destroy(&p->mutex); yaz_cond_destroy(&p->cond_ready); - xfree(p->ipport); xfree(p->hostport); xfree(p); p = p_next; diff --git a/src/database.h b/src/database.h index 22df499..4b27826 100644 --- a/src/database.h +++ b/src/database.h @@ -26,7 +26,7 @@ struct session; struct conf_service; struct database *find_database(const char *id, struct conf_service *service); int session_grep_databases(struct session *se, const char *filter, - void (*fun)(void *context, struct session_database *db)); + void (*fun)(struct session *se, struct session_database *db)); int predef_grep_databases(void *context, struct conf_service *service, void (*fun)(void *context, struct database *db)); int match_zurl(const char *zurl, const char *pattern); @@ -35,4 +35,5 @@ struct database *new_database(const char *id, NMEM nmem); database_hosts_t database_hosts_create(void); void database_hosts_destroy(database_hosts_t *); + #endif diff --git a/src/getaddrinfo.c b/src/getaddrinfo.c deleted file mode 100644 index 52a489f..0000000 --- a/src/getaddrinfo.c +++ /dev/null @@ -1,227 +0,0 @@ -/* This file is part of Pazpar2. - Copyright (C) 2006-2011 Index Data - -Pazpar2 is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free -Software Foundation; either version 2, or (at your option) any later -version. - -Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*/ - -#if HAVE_CONFIG_H -#include -#endif - -#include "sel_thread.h" - -#if HAVE_UNISTD_H -#include -#endif -#include - -#include -#include -#if HAVE_SYS_SOCKET_H -#include -#endif -#ifdef WIN32 -#include -#endif -#if HAVE_NETDB_H -#include -#endif -#if HAVE_NETINET_IN_H -#include -#endif - -#include -#include -#include - -#include "session.h" -#include "connection.h" -#include "host.h" - -/* Only use a threaded resolver on Unix that offers getaddrinfo. - gethostbyname is NOT reentrant. - */ -#if HAVE_GETADDRINFO -#ifndef WIN32 -#define USE_THREADED_RESOLVER 1 -#endif -#endif - -struct work { - char *hostport; /* hostport to be resolved in separate thread */ - char *ipport; /* result or NULL if it could not be resolved */ - struct host *host; /* host that we're dealing with - mother thread */ - iochan_man_t iochan_man; /* iochan manager */ -}; - -static int log_level = YLOG_LOG; - -void perform_getaddrinfo(struct work *w) -{ - int res = 0; - char *port; -#if HAVE_GETADDRINFO - struct addrinfo *addrinfo, hints; -#else - struct hostent *hp; -#endif - char *hostport = xstrdup(w->hostport); - - if ((port = strchr(hostport, ':'))) - *(port++) = '\0'; - else - port = "210"; - -#if HAVE_GETADDRINFO - hints.ai_flags = 0; - hints.ai_family = PF_INET; - hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = IPPROTO_TCP; - hints.ai_addrlen = 0; - hints.ai_addr = 0; - hints.ai_canonname = 0; - hints.ai_next = 0; - // This is not robust code. It assumes that getaddrinfo always - // returns AF_INET address. - if ((res = getaddrinfo(hostport, port, &hints, &addrinfo))) - { - yaz_log(YLOG_WARN, "Failed to resolve %s: %s", - w->hostport, gai_strerror(res)); - } - else - { - char ipport[128]; - unsigned char addrbuf[4]; - assert(addrinfo->ai_family == PF_INET); - memcpy(addrbuf, - &((struct sockaddr_in*)addrinfo->ai_addr)->sin_addr.s_addr, 4); - sprintf(ipport, "%u.%u.%u.%u:%s", - addrbuf[0], addrbuf[1], addrbuf[2], addrbuf[3], port); - freeaddrinfo(addrinfo); - w->ipport = xstrdup(ipport); - yaz_log(log_level, "Resolved %s -> %s", hostport, ipport); - } -#else - hp = gethostbyname(hostport); - if (!hp) - { - yaz_log(YLOG_WARN|YLOG_ERRNO, "Failed to resolve %s", hostport); - } - else - { - char ipport[128]; - unsigned char addrbuf[4]; - - memcpy(addrbuf, *hp->h_addr_list, 4 * sizeof(unsigned char)); - sprintf(ipport, "%u.%u.%u.%u:%s", - addrbuf[0], addrbuf[1], addrbuf[2], addrbuf[3], port); - w->ipport = xstrdup(ipport); - yaz_log(log_level, "Resolved %s -> %s", hostport, ipport); - } -#endif - xfree(hostport); -} - -static void work_handler(void *vp) -{ - struct work *w = vp; - - int sec = 0; /* >0 for debugging/testing purposes */ - if (sec) - { - yaz_log(log_level, "waiting %d seconds", sec); -#if HAVE_UNISTD_H - sleep(sec); -#endif - } - perform_getaddrinfo(w); -} - -#if USE_THREADED_RESOLVER -void iochan_handler(struct iochan *i, int event) -{ - sel_thread_t p = iochan_getdata(i); - - if (event & EVENT_INPUT) - { - struct work *w = sel_thread_result(p); - w->host->ipport = w->ipport; - connect_resolver_host(w->host, w->iochan_man); - xfree(w); - } -} - -static sel_thread_t resolver_thread = 0; - -static void getaddrinfo_start(iochan_man_t iochan_man) -{ - int fd; - sel_thread_t p = resolver_thread = - sel_thread_create(work_handler, 0 /* work_destroy */, &fd, - 3 /* no of resolver threads */); - if (!p) - { - yaz_log(YLOG_FATAL|YLOG_ERRNO, "sel_create_create failed"); - exit(1); - } - else - { - IOCHAN chan = iochan_create(fd, iochan_handler, EVENT_INPUT, - "getaddrinfo_socket"); - iochan_setdata(chan, p); - iochan_add(iochan_man, chan); - } - yaz_log(log_level, "resolver start"); - resolver_thread = p; -} -#endif - -int host_getaddrinfo(struct host *host, iochan_man_t iochan_man) -{ - struct work *w = xmalloc(sizeof(*w)); - int use_thread = 0; /* =0 to disable threading entirely */ - - w->hostport = host->hostport; - w->ipport = 0; - w->host = host; - w->iochan_man = iochan_man; -#if USE_THREADED_RESOLVER - if (use_thread) - { - if (resolver_thread == 0) - getaddrinfo_start(iochan_man); - assert(resolver_thread); - sel_thread_add(resolver_thread, w); - return 0; - } -#endif - perform_getaddrinfo(w); - host->ipport = w->ipport; - xfree(w); - if (!host->ipport) - return -1; - return 0; -} - -/* - * Local variables: - * c-basic-offset: 4 - * c-file-style: "Stroustrup" - * indent-tabs-mode: nil - * End: - * vim: shiftwidth=4 tabstop=8 expandtab - */ - diff --git a/src/host.h b/src/host.h index 4ab7fe8..752ca09 100644 --- a/src/host.h +++ b/src/host.h @@ -25,7 +25,6 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA /** \brief Represents a host (irrespective of databases) */ struct host { char *hostport; - char *ipport; struct connection *connections; // All connections to this struct host *next; YAZ_MUTEX mutex; diff --git a/src/pazpar2_config.c b/src/pazpar2_config.c index 0f5ab1a..ba1e55a 100644 --- a/src/pazpar2_config.c +++ b/src/pazpar2_config.c @@ -650,7 +650,7 @@ static struct conf_server *server_create(struct conf_config *config, server->charsets = 0; server->http_server = 0; server->iochan_man = 0; - server->database_hosts = 0; + server->database_hosts = config->database_hosts; server->settings_fname = 0; if (server_id) @@ -888,7 +888,7 @@ struct conf_config *config_create(const char *fname, int verbose) config->servers = 0; config->no_threads = 0; config->iochan_man = 0; - config->database_hosts = 0; + config->database_hosts = database_hosts_create(); config->confdir = wrbuf_alloc(); if ((p = strrchr(fname, @@ -955,9 +955,8 @@ void config_destroy(struct conf_config *config) struct conf_server *s_next = server->next; server_destroy(server); server = s_next; + database_hosts_destroy(&config->database_hosts); } - database_hosts_destroy(&config->database_hosts); - wrbuf_destroy(config->confdir); nmem_destroy(config->nmem); } @@ -974,13 +973,10 @@ void config_process_events(struct conf_config *conf) { struct conf_server *ser; - conf->database_hosts = database_hosts_create(); for (ser = conf->servers; ser; ser = ser->next) { struct conf_service *s = ser->service; - ser->database_hosts = conf->database_hosts; - for (;s ; s = s->next) { resolve_databases(s); @@ -989,7 +985,7 @@ void config_process_events(struct conf_config *conf) } http_mutex_init(ser); } - iochan_man_events(conf->iochan_man); + iochan_man_events(conf->iochan_man); } int config_start_listeners(struct conf_config *conf, diff --git a/src/session.c b/src/session.c index 697df85..4478ece 100644 --- a/src/session.c +++ b/src/session.c @@ -521,9 +521,9 @@ void session_alert_watch(struct session *s, int what) } //callback for grep_databases -static void select_targets_callback(void *context, struct session_database *db) +static void select_targets_callback(struct session *se, + struct session_database *db) { - struct session *se = (struct session*) context; struct client *cl = client_create(); struct client_list *l; client_set_database(cl, db); diff --git a/src/session.h b/src/session.h index 342665e..7647e46 100644 --- a/src/session.h +++ b/src/session.h @@ -57,9 +57,7 @@ struct host; struct database { struct host *host; char *url; - char **databases; int errors; - struct zr_explain *explain; int num_settings; struct setting **settings; struct database *next; diff --git a/win/makefile b/win/makefile index 22aa704..adf52b4 100644 --- a/win/makefile +++ b/win/makefile @@ -191,7 +191,6 @@ PAZPAR2_OBJS = \ "$(OBJDIR)\normalize7bit.obj" \ "$(OBJDIR)\database.obj" \ "$(OBJDIR)\settings.obj" \ - "$(OBJDIR)\getaddrinfo.obj" \ "$(OBJDIR)\charsets.obj" \ "$(OBJDIR)\client.obj" \ "$(OBJDIR)\jenkins_hash.obj" \