1af2e04d27ae7d390665bd0584f2cd4f391b18ba
[metaproxy-moved-to-github.git] / src / filter_zoom.cpp
1 /* This file is part of Metaproxy.
2    Copyright (C) 2005-2011 Index Data
3
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #include "config.hpp"
20 #include "filter_zoom.hpp"
21 #include <yaz/zoom.h>
22 #include <yaz/yaz-version.h>
23 #include <yaz/srw.h>
24 #include <metaproxy/package.hpp>
25 #include <metaproxy/util.hpp>
26 #include <metaproxy/xmlutil.hpp>
27 #include "torus.hpp"
28
29 #include <libxslt/xsltutils.h>
30 #include <libxslt/transform.h>
31
32 #include <boost/thread/mutex.hpp>
33 #include <boost/thread/condition.hpp>
34 #include <yaz/ccl_xml.h>
35 #include <yaz/ccl.h>
36 #include <yaz/rpn2cql.h>
37 #include <yaz/rpn2solr.h>
38 #include <yaz/pquery.h>
39 #include <yaz/cql.h>
40 #include <yaz/oid_db.h>
41 #include <yaz/diagbib1.h>
42 #include <yaz/log.h>
43 #include <yaz/zgdu.h>
44 #include <yaz/querytowrbuf.h>
45
46 namespace mp = metaproxy_1;
47 namespace yf = mp::filter;
48
49 namespace metaproxy_1 {
50     namespace filter {
51         struct Zoom::Searchable : boost::noncopyable {
52             std::string authentication;
53             std::string cfAuth;
54             std::string cfProxy;
55             std::string cfSubDb;
56             std::string udb;
57             std::string target;
58             std::string query_encoding;
59             std::string sru;
60             std::string request_syntax;
61             std::string element_set;
62             std::string record_encoding;
63             std::string transform_xsl_fname;
64             std::string urlRecipe;
65             bool use_turbomarc;
66             bool piggyback;
67             CCL_bibset ccl_bibset;
68             Searchable(CCL_bibset base);
69             ~Searchable();
70         };
71         class Zoom::Backend : boost::noncopyable {
72             friend class Impl;
73             friend class Frontend;
74             std::string zurl;
75             ZOOM_connection m_connection;
76             ZOOM_resultset m_resultset;
77             std::string m_frontend_database;
78             SearchablePtr sptr;
79             xsltStylesheetPtr xsp;
80         public:
81             Backend(SearchablePtr sptr);
82             ~Backend();
83             void connect(std::string zurl, int *error, const char **addinfo);
84             void search_pqf(const char *pqf, Odr_int *hits,
85                             int *error, const char **addinfo);
86             void search_cql(const char *cql, Odr_int *hits,
87                             int *error, const char **addinfo);
88             void present(Odr_int start, Odr_int number, ZOOM_record *recs,
89                          int *error, const char **addinfo);
90             void set_option(const char *name, const char *value);
91             const char *get_option(const char *name);
92             void get_zoom_error(int *error, const char **addinfo);
93         };
94         class Zoom::Frontend : boost::noncopyable {
95             friend class Impl;
96             Impl *m_p;
97             bool m_is_virtual;
98             bool m_in_use;
99             yazpp_1::GDU m_init_gdu;
100             BackendPtr m_backend;
101             void handle_package(mp::Package &package);
102             void handle_search(mp::Package &package);
103             void handle_present(mp::Package &package);
104             BackendPtr get_backend_from_databases(std::string &database,
105                                                   int *error,
106                                                   char **addinfo,
107                                                   ODR odr);
108             Z_Records *get_records(Odr_int start,
109                                    Odr_int number_to_present,
110                                    int *error,
111                                    const char **addinfo,
112                                    Odr_int *number_of_records_returned,
113                                    ODR odr, BackendPtr b,
114                                    Odr_oid *preferredRecordSyntax,
115                                    const char *element_set_name);
116         public:
117             Frontend(Impl *impl);
118             ~Frontend();
119         };
120         class Zoom::Impl {
121             friend class Frontend;
122         public:
123             Impl();
124             ~Impl();
125             void process(metaproxy_1::Package & package);
126             void configure(const xmlNode * ptr, bool test_only,
127                            const char *path);
128         private:
129             void configure_local_records(const xmlNode * ptr, bool test_only);
130             FrontendPtr get_frontend(mp::Package &package);
131             void release_frontend(mp::Package &package);
132             SearchablePtr parse_torus_record(const xmlNode *ptr);
133             struct cql_node *convert_cql_fields(struct cql_node *cn, ODR odr);
134             std::map<mp::Session, FrontendPtr> m_clients;            
135             boost::mutex m_mutex;
136             boost::condition m_cond_session_ready;
137             std::string torus_url;
138             std::map<std::string,std::string> fieldmap;
139             std::string xsldir;
140             std::string file_path;
141             CCL_bibset bibset;
142             std::string element_transform;
143             std::string element_raw;
144             std::map<std::string,SearchablePtr> s_map;
145         };
146     }
147 }
148
149 // define Pimpl wrapper forwarding to Impl
150  
151 yf::Zoom::Zoom() : m_p(new Impl)
152 {
153 }
154
155 yf::Zoom::~Zoom()
156 {  // must have a destructor because of boost::scoped_ptr
157 }
158
159 void yf::Zoom::configure(const xmlNode *xmlnode, bool test_only,
160                          const char *path)
161 {
162     m_p->configure(xmlnode, test_only, path);
163 }
164
165 void yf::Zoom::process(mp::Package &package) const
166 {
167     m_p->process(package);
168 }
169
170
171 // define Implementation stuff
172
173 yf::Zoom::Backend::Backend(SearchablePtr ptr) : sptr(ptr)
174 {
175     m_connection = ZOOM_connection_create(0);
176     m_resultset = 0;
177     xsp = 0;
178 }
179
180 yf::Zoom::Backend::~Backend()
181 {
182     if (xsp)
183         xsltFreeStylesheet(xsp);
184     ZOOM_connection_destroy(m_connection);
185     ZOOM_resultset_destroy(m_resultset);
186 }
187
188
189 void yf::Zoom::Backend::get_zoom_error(int *error, const char **addinfo)
190 {
191     const char *msg = 0;
192     *error = ZOOM_connection_error(m_connection, &msg, addinfo);
193     if (*error)
194     {
195         if (*error >= ZOOM_ERROR_CONNECT)
196         {
197             // turn ZOOM diagnostic into a Bib-1 2: with addinfo=zoom err msg
198             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
199             if (addinfo)
200                 *addinfo = msg;
201         }
202     }
203 }
204
205 void yf::Zoom::Backend::connect(std::string zurl,
206                                 int *error, const char **addinfo)
207 {
208     ZOOM_connection_connect(m_connection, zurl.c_str(), 0);
209     get_zoom_error(error, addinfo);
210 }
211
212 void yf::Zoom::Backend::search_pqf(const char *pqf, Odr_int *hits,
213                                    int *error, const char **addinfo)
214 {
215     m_resultset = ZOOM_connection_search_pqf(m_connection, pqf);
216     get_zoom_error(error, addinfo);
217     if (*error == 0)
218         *hits = ZOOM_resultset_size(m_resultset);
219     else
220         *hits = 0;
221 }
222
223 void yf::Zoom::Backend::search_cql(const char *cql, Odr_int *hits,
224                                    int *error, const char **addinfo)
225 {
226     ZOOM_query q = ZOOM_query_create();
227
228     ZOOM_query_cql(q, cql);
229
230     m_resultset = ZOOM_connection_search(m_connection, q);
231     ZOOM_query_destroy(q);
232     get_zoom_error(error, addinfo);
233     if (*error == 0)
234         *hits = ZOOM_resultset_size(m_resultset);
235     else
236         *hits = 0;
237 }
238
239 void yf::Zoom::Backend::present(Odr_int start, Odr_int number,
240                                 ZOOM_record *recs,
241                                 int *error, const char **addinfo)
242 {
243     ZOOM_resultset_records(m_resultset, recs, start, number);
244     get_zoom_error(error, addinfo);
245 }
246
247 void yf::Zoom::Backend::set_option(const char *name, const char *value)
248 {
249     ZOOM_connection_option_set(m_connection, name, value);
250     if (m_resultset)
251         ZOOM_resultset_option_set(m_resultset, name, value);
252 }
253
254 const char *yf::Zoom::Backend::get_option(const char *name)
255 {
256     return ZOOM_connection_option_get(m_connection, name);
257 }
258
259 yf::Zoom::Searchable::Searchable(CCL_bibset base)
260 {
261     piggyback = true;
262     use_turbomarc = true;
263     ccl_bibset = ccl_qual_dup(base);
264 }
265
266 yf::Zoom::Searchable::~Searchable()
267 {
268     ccl_qual_rm(&ccl_bibset);
269 }
270
271 yf::Zoom::Frontend::Frontend(Impl *impl) : 
272     m_p(impl), m_is_virtual(false), m_in_use(true)
273 {
274 }
275
276 yf::Zoom::Frontend::~Frontend()
277 {
278 }
279
280 yf::Zoom::FrontendPtr yf::Zoom::Impl::get_frontend(mp::Package &package)
281 {
282     boost::mutex::scoped_lock lock(m_mutex);
283
284     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
285     
286     while(true)
287     {
288         it = m_clients.find(package.session());
289         if (it == m_clients.end())
290             break;
291         
292         if (!it->second->m_in_use)
293         {
294             it->second->m_in_use = true;
295             return it->second;
296         }
297         m_cond_session_ready.wait(lock);
298     }
299     FrontendPtr f(new Frontend(this));
300     m_clients[package.session()] = f;
301     f->m_in_use = true;
302     return f;
303 }
304
305 void yf::Zoom::Impl::release_frontend(mp::Package &package)
306 {
307     boost::mutex::scoped_lock lock(m_mutex);
308     std::map<mp::Session,yf::Zoom::FrontendPtr>::iterator it;
309     
310     it = m_clients.find(package.session());
311     if (it != m_clients.end())
312     {
313         if (package.session().is_closed())
314         {
315             m_clients.erase(it);
316         }
317         else
318         {
319             it->second->m_in_use = false;
320         }
321         m_cond_session_ready.notify_all();
322     }
323 }
324
325 yf::Zoom::Impl::Impl() : element_transform("pz2") , element_raw("raw")
326 {
327     bibset = ccl_qual_mk();
328 }
329
330 yf::Zoom::Impl::~Impl()
331
332     ccl_qual_rm(&bibset);
333 }
334
335 yf::Zoom::SearchablePtr yf::Zoom::Impl::parse_torus_record(const xmlNode *ptr)
336 {
337     Zoom::SearchablePtr s(new Searchable(bibset));
338     
339     for (ptr = ptr->children; ptr; ptr = ptr->next)
340     {
341         if (ptr->type != XML_ELEMENT_NODE)
342             continue;
343         if (!strcmp((const char *) ptr->name, "layer"))
344             ptr = ptr->children;
345         else if (!strcmp((const char *) ptr->name,
346                          "authentication"))
347         {
348             s->authentication = mp::xml::get_text(ptr);
349         }
350         else if (!strcmp((const char *) ptr->name,
351                          "cfAuth"))
352         {
353             s->cfAuth = mp::xml::get_text(ptr);
354         } 
355         else if (!strcmp((const char *) ptr->name,
356                          "cfProxy"))
357         {
358             s->cfProxy = mp::xml::get_text(ptr);
359         }  
360         else if (!strcmp((const char *) ptr->name,
361                          "cfSubDb"))
362         {
363             s->cfSubDb = mp::xml::get_text(ptr);
364         }  
365         else if (!strcmp((const char *) ptr->name, "udb"))
366         {
367             s->udb = mp::xml::get_text(ptr);
368         }
369         else if (!strcmp((const char *) ptr->name, "zurl"))
370         {
371             s->target = mp::xml::get_text(ptr);
372         }
373         else if (!strcmp((const char *) ptr->name, "sru"))
374         {
375             s->sru = mp::xml::get_text(ptr);
376         }
377         else if (!strcmp((const char *) ptr->name,
378                          "queryEncoding"))
379         {
380             s->query_encoding = mp::xml::get_text(ptr);
381         }
382         else if (!strcmp((const char *) ptr->name,
383                          "piggyback"))
384         {
385             s->piggyback = mp::xml::get_bool(ptr, true);
386         }
387         else if (!strcmp((const char *) ptr->name,
388                          "requestSyntax"))
389         {
390             s->request_syntax = mp::xml::get_text(ptr);
391         }
392         else if (!strcmp((const char *) ptr->name,
393                          "elementSet"))
394         {
395             s->element_set = mp::xml::get_text(ptr);
396         }
397         else if (!strcmp((const char *) ptr->name,
398                          "recordEncoding"))
399         {
400             s->record_encoding = mp::xml::get_text(ptr);
401         }
402         else if (!strcmp((const char *) ptr->name,
403                          "transform"))
404         {
405             s->transform_xsl_fname = mp::xml::get_text(ptr);
406         }
407         else if (!strcmp((const char *) ptr->name,
408                          "urlRecipe"))
409         {
410             s->urlRecipe = mp::xml::get_text(ptr);
411         }
412         else if (!strcmp((const char *) ptr->name,
413                          "useTurboMarc"))
414         {
415             ; // useTurboMarc is ignored
416         }
417         else if (!strncmp((const char *) ptr->name,
418                           "cclmap_", 7))
419         {
420             std::string value = mp::xml::get_text(ptr);
421             ccl_qual_fitem(s->ccl_bibset, value.c_str(),
422                            (const char *) ptr->name + 7);
423         }
424     }
425     return s;
426 }
427
428 void yf::Zoom::Impl::configure_local_records(const xmlNode *ptr, bool test_only)
429 {
430     while (ptr && ptr->type != XML_ELEMENT_NODE)
431         ptr = ptr->next;
432     
433     if (ptr)
434     {
435         if (!strcmp((const char *) ptr->name, "records"))
436         {
437             for (ptr = ptr->children; ptr; ptr = ptr->next)
438             {
439                 if (ptr->type != XML_ELEMENT_NODE)
440                     continue;
441                 if (!strcmp((const char *) ptr->name, "record"))
442                 {
443                     SearchablePtr s = parse_torus_record(ptr);
444                     if (s)
445                     {
446                         std::string udb = s->udb;
447                         if (udb.length())
448                             s_map[s->udb] = s;
449                         else
450                         {
451                             throw mp::filter::FilterException
452                                 ("No udb for local torus record");
453                         }
454                     }
455                 }
456                 else
457                 {
458                     throw mp::filter::FilterException
459                         ("Bad element " 
460                          + std::string((const char *) ptr->name)
461                          + " in zoom filter inside element "
462                          "<torus><records>");
463                 }
464             }
465         }
466         else
467         {
468             throw mp::filter::FilterException
469                 ("Bad element " 
470                  + std::string((const char *) ptr->name)
471                  + " in zoom filter inside element <torus>");
472         }
473     }
474 }
475
476 void yf::Zoom::Impl::configure(const xmlNode *ptr, bool test_only,
477                                const char *path)
478 {
479     if (path && *path)
480     {
481         file_path = path;
482         if (path[strlen(path)-1] != '/')
483             file_path += "/";
484     }
485     for (ptr = ptr->children; ptr; ptr = ptr->next)
486     {
487         if (ptr->type != XML_ELEMENT_NODE)
488             continue;
489         else if (!strcmp((const char *) ptr->name, "torus"))
490         {
491             const struct _xmlAttr *attr;
492             for (attr = ptr->properties; attr; attr = attr->next)
493             {
494                 if (!strcmp((const char *) attr->name, "url"))
495                     torus_url = mp::xml::get_text(attr->children);
496                 else if (!strcmp((const char *) attr->name, "xsldir"))
497                     xsldir = mp::xml::get_text(attr->children);
498                 else if (!strcmp((const char *) attr->name, "element_transform"))
499                     element_transform = mp::xml::get_text(attr->children);
500                 else if (!strcmp((const char *) attr->name, "element_raw"))
501                     element_raw = mp::xml::get_text(attr->children);
502                 else
503                     throw mp::filter::FilterException(
504                         "Bad attribute " + std::string((const char *)
505                                                        attr->name));
506             }
507             configure_local_records(ptr->children, test_only);
508         }
509         else if (!strcmp((const char *) ptr->name, "cclmap"))
510         {
511             const char *addinfo = 0;
512             ccl_xml_config(bibset, ptr, &addinfo);
513         }
514         else if (!strcmp((const char *) ptr->name, "fieldmap"))
515         {
516             const struct _xmlAttr *attr;
517             std::string ccl_field;
518             std::string cql_field;
519             for (attr = ptr->properties; attr; attr = attr->next)
520             {
521                 if (!strcmp((const char *) attr->name, "ccl"))
522                     ccl_field = mp::xml::get_text(attr->children);
523                 else if (!strcmp((const char *) attr->name, "cql"))
524                     cql_field = mp::xml::get_text(attr->children);
525                 else
526                     throw mp::filter::FilterException(
527                         "Bad attribute " + std::string((const char *)
528                                                        attr->name));
529             }
530             if (cql_field.length())
531                 fieldmap[cql_field] = ccl_field;
532         }
533         else
534         {
535             throw mp::filter::FilterException
536                 ("Bad element " 
537                  + std::string((const char *) ptr->name)
538                  + " in zoom filter");
539         }
540     }
541 }
542
543 yf::Zoom::BackendPtr yf::Zoom::Frontend::get_backend_from_databases(
544     std::string &database, int *error, char **addinfo, ODR odr)
545 {
546     std::list<BackendPtr>::const_iterator map_it;
547     if (m_backend && m_backend->m_frontend_database == database)
548         return m_backend;
549
550     const char *sru_proxy = 0;
551     std::string db_args;
552     std::string torus_db;
553     size_t db_arg_pos = database.find(',');
554     if (db_arg_pos != std::string::npos)
555     {
556         torus_db = database.substr(0, db_arg_pos);
557         db_args = database.substr(db_arg_pos + 1);
558     }
559     else
560         torus_db = database;
561  
562     SearchablePtr sptr;
563
564     std::map<std::string,SearchablePtr>::iterator it;
565     it = m_p->s_map.find(torus_db);
566     if (it != m_p->s_map.end())
567         sptr = it->second;
568     else
569     {
570         xmlDoc *doc = mp::get_searchable(m_p->torus_url, torus_db);
571         if (!doc)
572         {
573             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
574             *addinfo = odr_strdup(odr, database.c_str());
575             BackendPtr b;
576             return b;
577         }
578         const xmlNode *ptr = xmlDocGetRootElement(doc);
579         if (ptr)
580         {   // presumably ptr is a records element node
581             // parse first record in document
582             for (ptr = ptr->children; ptr; ptr = ptr->next)
583             {
584                 if (ptr->type == XML_ELEMENT_NODE
585                     && !strcmp((const char *) ptr->name, "record"))
586                 {
587                     sptr = m_p->parse_torus_record(ptr);
588                     break;
589                 }
590             }
591         }
592         xmlFreeDoc(doc);
593     }
594
595     if (!sptr)
596     {
597         *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
598         *addinfo = odr_strdup(odr, database.c_str());
599         BackendPtr b;
600         return b;
601     }
602         
603     xsltStylesheetPtr xsp = 0;
604     if (sptr->transform_xsl_fname.length())
605     {
606         std::string fname;
607
608         if (m_p->xsldir.length()) 
609             fname = m_p->xsldir + "/" + sptr->transform_xsl_fname;
610         else
611             fname = m_p->file_path + sptr->transform_xsl_fname;
612         xmlDoc *xsp_doc = xmlParseFile(fname.c_str());
613         if (!xsp_doc)
614         {
615             *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
616             *addinfo = (char *) odr_malloc(odr, 40 + strlen(fname.c_str()));
617             sprintf(*addinfo, "xmlParseFile failed. File %s", fname.c_str());
618             BackendPtr b;
619             return b;
620         }
621         xsp = xsltParseStylesheetDoc(xsp_doc);
622         if (!xsp)
623         {
624             *error = YAZ_BIB1_DATABASE_DOES_NOT_EXIST;
625             *addinfo = odr_strdup(odr, "xsltParseStylesheetDoc failed");
626             BackendPtr b;
627             xmlFreeDoc(xsp_doc);
628             return b;
629         }
630     }
631
632     m_backend.reset();
633
634     BackendPtr b(new Backend(sptr));
635
636     b->xsp = xsp;
637     b->m_frontend_database = database;
638     std::string authentication = sptr->authentication;
639         
640     b->set_option("timeout", "40");
641
642     if (sptr->query_encoding.length())
643         b->set_option("rpnCharset", sptr->query_encoding.c_str());
644
645     if (sptr->cfAuth.length())
646     {
647         // A CF target
648         b->set_option("user", sptr->cfAuth.c_str());
649         if (db_args.length() == 0)
650         {
651             if (authentication.length())
652             {
653                 // no database (auth) args specified already.. and the
654                 // Torus authentication has it.. Generate the args that CF
655                 // understands..
656                 size_t found = authentication.find('/');
657                 if (found != std::string::npos)
658                 {
659                     db_args += "user=" + mp::util::uri_encode(authentication.substr(0, found))
660                         + "&password=" + mp::util::uri_encode(authentication.substr(found+1));
661                 }
662                 else
663                     db_args += "user=" + mp::util::uri_encode(authentication);
664             }
665             if (sptr->cfProxy.length())
666             {
667                 if (db_args.length())
668                     db_args += "&";
669                 db_args += "proxy=" + mp::util::uri_encode(sptr->cfProxy);
670             }
671         }
672         if (sptr->cfSubDb.length())
673         {
674             if (db_args.length())
675                 db_args += "&";
676             db_args += "subdatabase=" + mp::util::uri_encode(sptr->cfSubDb);
677         }
678     }
679     else
680     {
681         // A non-CF target
682         if (db_args.length())
683         {
684             // user has specified backend authentication
685             const char *param_user = 0;
686             const char *param_password = 0;
687             char **names;
688             char **values;
689             int i;
690             int no_parms = yaz_uri_to_array(db_args.c_str(),
691                                             odr, &names, &values);
692             for (i = 0; i < no_parms; i++)
693             {
694                 const char *name = names[i];
695                 const char *value = values[i];
696                 if (!strcmp(name, "user"))
697                     param_user = value;
698                 else if (!strcmp(name, "password"))
699                     param_password = value;
700                 else if (!strcmp(name, "proxy"))
701                     sru_proxy = value;
702                 else
703                 {
704                     BackendPtr notfound;
705                     char *msg = (char*) odr_malloc(odr, strlen(name) + 30);
706                     *error = YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
707                     sprintf(msg, "Bad database argument: %s", name);
708                     *addinfo = msg;
709                     return notfound;
710                 }
711             }
712             if (param_user && param_password)
713             {
714                 char *auth = (char*) odr_malloc(
715                     odr, strlen(param_user) + strlen(param_password) + 2);
716                 strcpy(auth, param_user);
717                 strcat(auth, "/");
718                 strcat(auth, param_password);
719                 b->set_option("user", auth);
720             }
721             db_args.clear(); // no arguments to be passed (non-CF)
722         }
723         else
724         {
725             // use authentication from Torus, if given
726             if (authentication.length())
727                 b->set_option("user", authentication.c_str());
728         }
729     }
730
731     if (sru_proxy)
732         b->set_option("proxy", sru_proxy);
733
734     std::string url;
735     if (sptr->sru.length())
736     {
737         url = "http://" + sptr->target;
738         b->set_option("sru", sptr->sru.c_str());
739     }
740     else
741     {
742         url = sptr->target;
743     }
744     if (db_args.length())
745         url += "," + db_args;
746     yaz_log(YLOG_LOG, "url=%s", url.c_str());
747     const char *addinfo_c = 0;
748     b->connect(url, error, &addinfo_c);
749     if (addinfo_c)
750         *addinfo = odr_strdup(odr, addinfo_c);
751     if (*error == 0)
752     {
753         m_backend = b;
754     }
755     return b;
756 }
757
758 Z_Records *yf::Zoom::Frontend::get_records(Odr_int start,
759                                            Odr_int number_to_present,
760                                            int *error,
761                                            const char **addinfo,
762                                            Odr_int *number_of_records_returned,
763                                            ODR odr,
764                                            BackendPtr b,
765                                            Odr_oid *preferredRecordSyntax,
766                                            const char *element_set_name)
767 {
768     *number_of_records_returned = 0;
769     Z_Records *records = 0;
770     bool enable_pz2_retrieval = false; // whether target profile is used
771     bool enable_pz2_transform = false; // whether XSLT is used as well
772     bool assume_marc8_charset = false;
773
774     if (start < 0 || number_to_present <= 0)
775         return records;
776     
777     if (number_to_present > 10000)
778         number_to_present = 10000;
779     
780     ZOOM_record *recs = (ZOOM_record *)
781         odr_malloc(odr, number_to_present * sizeof(*recs));
782
783     char oid_name_str[OID_STR_MAX];
784     const char *syntax_name = 0;
785     
786     if (preferredRecordSyntax &&
787         !oid_oidcmp(preferredRecordSyntax, yaz_oid_recsyn_xml)
788         && element_set_name)
789     {
790         if (!strcmp(element_set_name, m_p->element_transform.c_str()))
791         {
792             enable_pz2_retrieval = true;
793             enable_pz2_transform = true;
794         }
795         else if (!strcmp(element_set_name, m_p->element_raw.c_str()))
796         {
797             enable_pz2_retrieval = true;
798         }
799     }
800     
801     if (enable_pz2_retrieval)
802     {
803         if (b->sptr->request_syntax.length())
804         {
805             syntax_name = b->sptr->request_syntax.c_str();
806             const Odr_oid *syntax_oid = 
807                 yaz_string_to_oid(yaz_oid_std(), CLASS_RECSYN, syntax_name);
808             if (!oid_oidcmp(syntax_oid, yaz_oid_recsyn_usmarc)
809                 || !oid_oidcmp(syntax_oid, yaz_oid_recsyn_opac))
810                 assume_marc8_charset = true;
811         }
812     }
813     else if (preferredRecordSyntax)
814         syntax_name =
815             yaz_oid_to_string_buf(preferredRecordSyntax, 0, oid_name_str);
816
817     b->set_option("preferredRecordSyntax", syntax_name);
818
819     if (enable_pz2_retrieval)
820     {
821         element_set_name = 0;
822         if (b->sptr->element_set.length())
823             element_set_name = b->sptr->element_set.c_str();
824     }
825
826     b->set_option("elementSetName", element_set_name);
827
828     b->present(start, number_to_present, recs, error, addinfo);
829
830     Odr_int i = 0;
831     if (!*error)
832     {
833         for (i = 0; i < number_to_present; i++)
834             if (!recs[i])
835                 break;
836     }
837     if (i > 0)
838     {  // only return records if no error and at least one record
839         char *odr_database = odr_strdup(odr,
840                                         b->m_frontend_database.c_str());
841         Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
842             odr_malloc(odr, sizeof(*npl));
843         *number_of_records_returned = i;
844         npl->num_records = i;
845         npl->records = (Z_NamePlusRecord **)
846             odr_malloc(odr, i * sizeof(*npl->records));
847         for (i = 0; i < number_to_present; i++)
848         {
849             Z_NamePlusRecord *npr = 0;
850             const char *addinfo;
851             int sur_error = ZOOM_record_error(recs[i], 0 /* msg */,
852                                               &addinfo, 0 /* diagset */);
853                 
854             if (sur_error)
855             {
856                 npr = zget_surrogateDiagRec(odr, odr_database, sur_error,
857                                             addinfo);
858             }
859             else if (enable_pz2_retrieval)
860             {
861                 char rec_type_str[100];
862                 const char *record_encoding = 0;
863
864                 if (b->sptr->record_encoding.length())
865                     record_encoding = b->sptr->record_encoding.c_str();
866                 else if (assume_marc8_charset)
867                     record_encoding = "marc8";
868
869                 strcpy(rec_type_str, b->sptr->use_turbomarc ? "txml" : "xml");
870                 if (record_encoding)
871                 {
872                     strcat(rec_type_str, "; charset=");
873                     strcat(rec_type_str, record_encoding);
874                 }
875                 
876                 int rec_len;
877                 xmlChar *xmlrec_buf = 0;
878                 const char *rec_buf = ZOOM_record_get(recs[i], rec_type_str,
879                                                       &rec_len);
880                 if (rec_buf && b->xsp && enable_pz2_transform)
881                 {
882                     xmlDoc *rec_doc = xmlParseMemory(rec_buf, rec_len);
883                     if (rec_doc)
884                     { 
885                         xmlDoc *rec_res;
886                         rec_res = xsltApplyStylesheet(b->xsp, rec_doc, 0);
887
888                         if (rec_res)
889                             xsltSaveResultToString(&xmlrec_buf, &rec_len,
890                                                    rec_res, b->xsp);
891                         rec_buf = (const char *) xmlrec_buf;
892                         xmlFreeDoc(rec_doc);
893                         xmlFreeDoc(rec_res);
894                     }
895                 }
896
897                 if (rec_buf)
898                 {
899                     xmlDoc *doc = xmlParseMemory(rec_buf, rec_len);
900                     std::string res = 
901                         mp::xml::url_recipe_handle(doc, b->sptr->urlRecipe);
902                     if (res.length())
903                     {
904                         xmlNode *ptr = xmlDocGetRootElement(doc);
905                         while (ptr && ptr->type != XML_ELEMENT_NODE)
906                             ptr = ptr->next;
907                         xmlNode *c = 
908                             xmlNewChild(ptr, 0, BAD_CAST "generated-url", 0);
909                         xmlNode * t = xmlNewText(BAD_CAST res.c_str());
910                         xmlAddChild(c, t);
911
912                         if (xmlrec_buf)
913                             xmlFree(xmlrec_buf);
914
915                         xmlDocDumpMemory(doc, &xmlrec_buf, &rec_len);
916                         rec_buf = (const char *) xmlrec_buf;
917                     }
918                     xmlFreeDoc(doc);
919                 }
920                 if (rec_buf)
921                 {
922                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
923                     npr->databaseName = odr_database;
924                     npr->which = Z_NamePlusRecord_databaseRecord;
925                     npr->u.databaseRecord =
926                         z_ext_record_xml(odr, rec_buf, rec_len);
927                 }
928                 else
929                 {
930                     npr = zget_surrogateDiagRec(
931                         odr, odr_database, 
932                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
933                         rec_type_str);
934                 }
935                 if (xmlrec_buf)
936                     xmlFree(xmlrec_buf);
937             }
938             else
939             {
940                 Z_External *ext =
941                     (Z_External *) ZOOM_record_get(recs[i], "ext", 0);
942                 if (ext)
943                 {
944                     npr = (Z_NamePlusRecord *) odr_malloc(odr, sizeof(*npr));
945                     npr->databaseName = odr_database;
946                     npr->which = Z_NamePlusRecord_databaseRecord;
947                     npr->u.databaseRecord = ext;
948                 }
949                 else
950                 {
951                     npr = zget_surrogateDiagRec(
952                         odr, odr_database, 
953                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
954                         "ZOOM_record, type ext");
955                 }
956             }
957             npl->records[i] = npr;
958         }
959         records = (Z_Records*) odr_malloc(odr, sizeof(*records));
960         records->which = Z_Records_DBOSD;
961         records->u.databaseOrSurDiagnostics = npl;
962     }
963     return records;
964 }
965     
966 struct cql_node *yf::Zoom::Impl::convert_cql_fields(struct cql_node *cn,
967                                                     ODR odr)
968 {
969     struct cql_node *r = 0;
970     if (!cn)
971         return 0;
972     switch (cn->which)
973     {
974     case CQL_NODE_ST:
975         if (cn->u.st.index)
976         {
977             std::map<std::string,std::string>::const_iterator it;
978             it = fieldmap.find(cn->u.st.index);
979             if (it == fieldmap.end())
980                 return cn;
981             if (it->second.length())
982                 cn->u.st.index = odr_strdup(odr, it->second.c_str());
983             else
984                 cn->u.st.index = 0;
985         }
986         break;
987     case CQL_NODE_BOOL:
988         r = convert_cql_fields(cn->u.boolean.left, odr);
989         if (!r)
990             r = convert_cql_fields(cn->u.boolean.right, odr);
991         break;
992     case CQL_NODE_SORT:
993         r = convert_cql_fields(cn->u.sort.search, odr);
994         break;
995     }
996     return r;
997 }
998
999 void yf::Zoom::Frontend::handle_search(mp::Package &package)
1000 {
1001     Z_GDU *gdu = package.request().get();
1002     Z_APDU *apdu_req = gdu->u.z3950;
1003     Z_APDU *apdu_res = 0;
1004     mp::odr odr;
1005     Z_SearchRequest *sr = apdu_req->u.searchRequest;
1006     if (sr->num_databaseNames != 1)
1007     {
1008         apdu_res = odr.create_searchResponse(
1009             apdu_req, YAZ_BIB1_TOO_MANY_DATABASES_SPECIFIED, 0);
1010         package.response() = apdu_res;
1011         return;
1012     }
1013
1014     int error = 0;
1015     char *addinfo_s = 0;
1016     std::string db(sr->databaseNames[0]);
1017     BackendPtr b = get_backend_from_databases(db, &error, &addinfo_s, odr);
1018     if (error)
1019     {
1020         apdu_res = 
1021             odr.create_searchResponse(
1022                 apdu_req, error, addinfo_s);
1023         package.response() = apdu_res;
1024         return;
1025     }
1026
1027     const char *addinfo_c = 0;
1028     b->set_option("setname", "default");
1029
1030     Odr_int hits = 0;
1031     Z_Query *query = sr->query;
1032     WRBUF ccl_wrbuf = 0;
1033     WRBUF pqf_wrbuf = 0;
1034
1035     if (query->which == Z_Query_type_1 || query->which == Z_Query_type_101)
1036     {
1037         // RPN
1038         pqf_wrbuf = wrbuf_alloc();
1039         yaz_rpnquery_to_wrbuf(pqf_wrbuf, query->u.type_1);
1040     }
1041     else if (query->which == Z_Query_type_2)
1042     {
1043         // CCL
1044         ccl_wrbuf = wrbuf_alloc();
1045         wrbuf_write(ccl_wrbuf, (const char *) query->u.type_2->buf,
1046                     query->u.type_2->len);
1047     }
1048     else if (query->which == Z_Query_type_104 &&
1049              query->u.type_104->which == Z_External_CQL)
1050     {
1051         // CQL
1052         const char *cql = query->u.type_104->u.cql;
1053         CQL_parser cp = cql_parser_create();
1054         int r = cql_parser_string(cp, cql);
1055         if (r)
1056         {
1057             cql_parser_destroy(cp);
1058             apdu_res = 
1059                 odr.create_searchResponse(apdu_req, 
1060                                           YAZ_BIB1_MALFORMED_QUERY,
1061                                           "CQL syntax error");
1062             package.response() = apdu_res;
1063             return;
1064         }
1065         struct cql_node *cn = cql_parser_result(cp);
1066         struct cql_node *cn_error = m_p->convert_cql_fields(cn, odr);
1067         if (cn_error)
1068         {
1069             // hopefully we are getting a ptr to a index+relation+term node
1070             addinfo_c = 0;
1071             if (cn_error->which == CQL_NODE_ST)
1072                 addinfo_c = cn_error->u.st.index;
1073
1074             apdu_res = 
1075                 odr.create_searchResponse(apdu_req, 
1076                                           YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1077                                           addinfo_c);
1078             package.response() = apdu_res;
1079             return;
1080         }
1081         char ccl_buf[1024];
1082
1083         r = cql_to_ccl_buf(cn, ccl_buf, sizeof(ccl_buf));
1084         if (r == 0)
1085         {
1086             ccl_wrbuf = wrbuf_alloc();
1087             wrbuf_puts(ccl_wrbuf, ccl_buf);
1088         }
1089         cql_parser_destroy(cp);
1090         if (r)
1091         {
1092             apdu_res = 
1093                 odr.create_searchResponse(apdu_req, 
1094                                           YAZ_BIB1_MALFORMED_QUERY,
1095                                           "CQL to CCL conversion error");
1096             package.response() = apdu_res;
1097             return;
1098         }
1099     }
1100     else
1101     {
1102         apdu_res = 
1103             odr.create_searchResponse(apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
1104         package.response() = apdu_res;
1105         return;
1106     }
1107
1108     if (ccl_wrbuf)
1109     {
1110         // CCL to PQF
1111         assert(pqf_wrbuf == 0);
1112         int cerror, cpos;
1113         struct ccl_rpn_node *cn;
1114         yaz_log(YLOG_LOG, "CCL: %s", wrbuf_cstr(ccl_wrbuf));
1115         cn = ccl_find_str(b->sptr->ccl_bibset, wrbuf_cstr(ccl_wrbuf),
1116                           &cerror, &cpos);
1117         wrbuf_destroy(ccl_wrbuf);
1118         if (!cn)
1119         {
1120             char *addinfo = odr_strdup(odr, ccl_err_msg(cerror));
1121             int z3950_diag = YAZ_BIB1_MALFORMED_QUERY;
1122
1123             switch (cerror)
1124             {
1125             case CCL_ERR_UNKNOWN_QUAL:
1126                 z3950_diag = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
1127                 break;
1128             case CCL_ERR_TRUNC_NOT_LEFT: 
1129             case CCL_ERR_TRUNC_NOT_RIGHT:
1130             case CCL_ERR_TRUNC_NOT_BOTH:
1131                 z3950_diag = YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE;
1132                 break;
1133             }
1134             apdu_res = 
1135                 odr.create_searchResponse(apdu_req, z3950_diag, addinfo);
1136             package.response() = apdu_res;
1137             return;
1138         }
1139         pqf_wrbuf = wrbuf_alloc();
1140         ccl_pquery(pqf_wrbuf, cn);
1141         ccl_rpn_delete(cn);
1142     }
1143     
1144     assert(pqf_wrbuf);
1145     if (b->get_option("sru"))
1146     {
1147         int status = 0;
1148         Z_RPNQuery *zquery;
1149         zquery = p_query_rpn(odr, wrbuf_cstr(pqf_wrbuf));
1150         WRBUF wrb = wrbuf_alloc();
1151             
1152         if (!strcmp(b->get_option("sru"), "solr"))
1153         {
1154             solr_transform_t cqlt = solr_transform_create();
1155             
1156             status = solr_transform_rpn2solr_wrbuf(cqlt, wrb, zquery);
1157             
1158             solr_transform_close(cqlt);
1159         }
1160         else
1161         {
1162             cql_transform_t cqlt = cql_transform_create();
1163             
1164             status = cql_transform_rpn2cql_wrbuf(cqlt, wrb, zquery);
1165             
1166             cql_transform_close(cqlt);
1167         }
1168         if (status == 0)
1169         {
1170             yaz_log(YLOG_LOG, "search CQL: %s", wrbuf_cstr(wrb));
1171             b->search_cql(wrbuf_cstr(wrb), &hits, &error, &addinfo_c);
1172         }
1173         
1174         wrbuf_destroy(wrb);
1175         wrbuf_destroy(pqf_wrbuf);
1176         if (status)
1177         {
1178             apdu_res = 
1179                 odr.create_searchResponse(apdu_req, YAZ_BIB1_MALFORMED_QUERY,
1180                                           "can not convert from RPN to CQL/SOLR");
1181             package.response() = apdu_res;
1182             return;
1183         }
1184     }
1185     else
1186     {
1187         yaz_log(YLOG_LOG, "search PQF: %s", wrbuf_cstr(pqf_wrbuf));
1188         b->search_pqf(wrbuf_cstr(pqf_wrbuf), &hits, &error, &addinfo_c);
1189         wrbuf_destroy(pqf_wrbuf);
1190     }
1191     
1192     
1193     const char *element_set_name = 0;
1194     Odr_int number_to_present = 0;
1195     if (!error)
1196         mp::util::piggyback_sr(sr, hits, number_to_present, &element_set_name);
1197     
1198     Odr_int number_of_records_returned = 0;
1199     Z_Records *records = get_records(
1200         0, number_to_present, &error, &addinfo_c,
1201         &number_of_records_returned, odr, b, sr->preferredRecordSyntax,
1202         element_set_name);
1203     apdu_res = odr.create_searchResponse(apdu_req, error, addinfo_c);
1204     if (records)
1205     {
1206         apdu_res->u.searchResponse->records = records;
1207         apdu_res->u.searchResponse->numberOfRecordsReturned =
1208             odr_intdup(odr, number_of_records_returned);
1209     }
1210     apdu_res->u.searchResponse->resultCount = odr_intdup(odr, hits);
1211     package.response() = apdu_res;
1212 }
1213
1214 void yf::Zoom::Frontend::handle_present(mp::Package &package)
1215 {
1216     Z_GDU *gdu = package.request().get();
1217     Z_APDU *apdu_req = gdu->u.z3950;
1218     Z_APDU *apdu_res = 0;
1219     Z_PresentRequest *pr = apdu_req->u.presentRequest;
1220
1221     mp::odr odr;
1222     if (!m_backend)
1223     {
1224         package.response() = odr.create_presentResponse(
1225             apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, 0);
1226         return;
1227     }
1228     const char *element_set_name = 0;
1229     Z_RecordComposition *comp = pr->recordComposition;
1230     if (comp && comp->which != Z_RecordComp_simple)
1231     {
1232         package.response() = odr.create_presentResponse(
1233             apdu_req, 
1234             YAZ_BIB1_PRESENT_COMP_SPEC_PARAMETER_UNSUPP, 0);
1235         return;
1236     }
1237     if (comp && comp->u.simple->which == Z_ElementSetNames_generic)
1238         element_set_name = comp->u.simple->u.generic;
1239     Odr_int number_of_records_returned = 0;
1240     int error = 0;
1241     const char *addinfo = 0;
1242     Z_Records *records = get_records(
1243         *pr->resultSetStartPoint - 1, *pr->numberOfRecordsRequested,
1244         &error, &addinfo, &number_of_records_returned, odr, m_backend,
1245         pr->preferredRecordSyntax, element_set_name);
1246
1247     apdu_res = odr.create_presentResponse(apdu_req, error, addinfo);
1248     if (records)
1249     {
1250         apdu_res->u.presentResponse->records = records;
1251         apdu_res->u.presentResponse->numberOfRecordsReturned =
1252             odr_intdup(odr, number_of_records_returned);
1253     }
1254     package.response() = apdu_res;
1255 }
1256
1257 void yf::Zoom::Frontend::handle_package(mp::Package &package)
1258 {
1259     Z_GDU *gdu = package.request().get();
1260     if (!gdu)
1261         ;
1262     else if (gdu->which == Z_GDU_Z3950)
1263     {
1264         Z_APDU *apdu_req = gdu->u.z3950;
1265         if (apdu_req->which == Z_APDU_initRequest)
1266         {
1267             mp::odr odr;
1268             package.response() = odr.create_close(
1269                 apdu_req,
1270                 Z_Close_protocolError,
1271                 "double init");
1272         }
1273         else if (apdu_req->which == Z_APDU_searchRequest)
1274         {
1275             handle_search(package);
1276         }
1277         else if (apdu_req->which == Z_APDU_presentRequest)
1278         {
1279             handle_present(package);
1280         }
1281         else
1282         {
1283             mp::odr odr;
1284             package.response() = odr.create_close(
1285                 apdu_req,
1286                 Z_Close_protocolError,
1287                 "zoom filter cannot handle this APDU");
1288             package.session().close();
1289         }
1290     }
1291     else
1292     {
1293         package.session().close();
1294     }
1295 }
1296
1297 void yf::Zoom::Impl::process(mp::Package &package)
1298 {
1299     FrontendPtr f = get_frontend(package);
1300     Z_GDU *gdu = package.request().get();
1301
1302     if (f->m_is_virtual)
1303     {
1304         f->handle_package(package);
1305     }
1306     else if (gdu && gdu->which == Z_GDU_Z3950 && gdu->u.z3950->which ==
1307              Z_APDU_initRequest)
1308     {
1309         Z_InitRequest *req = gdu->u.z3950->u.initRequest;
1310         f->m_init_gdu = gdu;
1311         
1312         mp::odr odr;
1313         Z_APDU *apdu = odr.create_initResponse(gdu->u.z3950, 0, 0);
1314         Z_InitResponse *resp = apdu->u.initResponse;
1315         
1316         int i;
1317         static const int masks[] = {
1318             Z_Options_search,
1319             Z_Options_present,
1320             -1 
1321         };
1322         for (i = 0; masks[i] != -1; i++)
1323             if (ODR_MASK_GET(req->options, masks[i]))
1324                 ODR_MASK_SET(resp->options, masks[i]);
1325         
1326         static const int versions[] = {
1327             Z_ProtocolVersion_1,
1328             Z_ProtocolVersion_2,
1329             Z_ProtocolVersion_3,
1330             -1
1331         };
1332         for (i = 0; versions[i] != -1; i++)
1333             if (ODR_MASK_GET(req->protocolVersion, versions[i]))
1334                 ODR_MASK_SET(resp->protocolVersion, versions[i]);
1335             else
1336                 break;
1337         
1338         *resp->preferredMessageSize = *req->preferredMessageSize;
1339         *resp->maximumRecordSize = *req->maximumRecordSize;
1340         
1341         package.response() = apdu;
1342         f->m_is_virtual = true;
1343     }
1344     else
1345         package.move();
1346
1347     release_frontend(package);
1348 }
1349
1350
1351 static mp::filter::Base* filter_creator()
1352 {
1353     return new mp::filter::Zoom;
1354 }
1355
1356 extern "C" {
1357     struct metaproxy_1_filter_struct metaproxy_1_filter_zoom = {
1358         0,
1359         "zoom",
1360         filter_creator
1361     };
1362 }
1363
1364
1365 /*
1366  * Local variables:
1367  * c-basic-offset: 4
1368  * c-file-style: "Stroustrup"
1369  * indent-tabs-mode: nil
1370  * End:
1371  * vim: shiftwidth=4 tabstop=8 expandtab
1372  */
1373