1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include <metaproxy/filter.hpp>
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
23 #include "filter_http_rewrite.hpp"
24 #include "html_parser.hpp"
30 #include <boost/regex.hpp>
31 #include <boost/lexical_cast.hpp>
32 #include <boost/algorithm/string.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
41 class HttpRewrite::Replace {
47 std::map<int, std::string> group_index;
49 const std::map<std::string, std::string> & vars) const;
50 void parse_groups(std::string pattern);
53 class HttpRewrite::Rule {
55 std::list<Replace> replace_list;
57 std::map<std::string, std::string> &vars,
58 std::string &txt, bool anchor);
60 class HttpRewrite::Within {
68 bool exec(std::map<std::string, std::string> &vars,
69 std::string &txt, bool anchor) const;
72 class HttpRewrite::Content {
75 boost::regex content_re;
76 std::list<Within> within_list;
77 void configure(const xmlNode *ptr,
78 std::map<std::string, RulePtr > &rules);
79 void quoted_literal(std::string &content,
80 std::map<std::string, std::string> &vars) const;
81 void parse(int verbose, std::string &content,
82 std::map<std::string, std::string> & vars) const;
84 class HttpRewrite::Phase {
88 std::list<Content> content_list;
89 void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
90 std::map<std::string, std::string> & vars) const;
91 void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
92 std::map<std::string, std::string> & vars) const;
93 void rewrite_body(mp::odr & o,
94 const char *content_type,
95 char **content_buf, int *content_len,
96 std::map<std::string, std::string> & vars) const;
98 class HttpRewrite::Event : public HTMLParserEvent {
99 void openTagStart(const char *tag, int tag_len);
100 void anyTagEnd(const char *tag, int tag_len, int close_it);
101 void attribute(const char *tag, int tag_len,
102 const char *attr, int attr_len,
103 const char *value, int val_len,
105 void closeTag(const char *tag, int tag_len);
106 void text(const char *value, int len);
107 const Content *m_content;
109 std::stack<std::list<Within>::const_iterator> s_within;
110 std::map<std::string, std::string> &m_vars;
112 Event(const Content *p, std::map<std::string, std::string> &vars);
114 const char *result();
119 yf::HttpRewrite::HttpRewrite() :
120 req_phase(new Phase), res_phase(new Phase)
124 yf::HttpRewrite::~HttpRewrite()
128 void yf::HttpRewrite::process(mp::Package & package) const
130 yaz_log(YLOG_LOG, "HttpRewrite begins....");
131 Z_GDU *gdu = package.request().get();
132 //map of request/response vars
133 std::map<std::string, std::string> vars;
134 //we have an http req
135 if (gdu && gdu->which == Z_GDU_HTTP_Request)
137 Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
139 req_phase->rewrite_reqline(o, hreq, vars);
140 yaz_log(YLOG_LOG, ">> Request headers");
141 req_phase->rewrite_headers(o, hreq->headers, vars);
142 req_phase->rewrite_body(o,
143 z_HTTP_header_lookup(hreq->headers,
145 &hreq->content_buf, &hreq->content_len,
147 package.request() = gdu;
150 gdu = package.response().get();
151 if (gdu && gdu->which == Z_GDU_HTTP_Response)
153 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
154 yaz_log(YLOG_LOG, "Response code %d", hres->code);
156 yaz_log(YLOG_LOG, "<< Respose headers");
157 res_phase->rewrite_headers(o, hres->headers, vars);
158 res_phase->rewrite_body(o,
159 z_HTTP_header_lookup(hres->headers,
161 &hres->content_buf, &hres->content_len,
163 package.response() = gdu;
167 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
168 Z_HTTP_Request *hreq,
169 std::map<std::string, std::string> & vars) const
171 //rewrite the request line
173 if (strstr(hreq->path, "http://") == hreq->path)
175 yaz_log(YLOG_LOG, "Path in the method line is absolute, "
176 "possibly a proxy request");
181 //TODO what about proto
182 const char *host = z_HTTP_header_lookup(hreq->headers, "Host");
191 std::list<Content>::const_iterator cit = content_list.begin();
192 for (; cit != content_list.end(); cit++)
193 if (cit->type == "headers")
196 if (cit == content_list.end())
199 std::list<Within>::const_iterator it = cit->within_list.begin();
200 for (; it != cit->within_list.end(); it++)
203 yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
204 if (it->exec(vars, path, true))
206 yaz_log(YLOG_LOG, "Rewritten request URL is %s", path.c_str());
207 hreq->path = odr_strdup(o, path.c_str());
212 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
213 Z_HTTP_Header *headers,
214 std::map<std::string, std::string> & vars) const
216 std::list<Content>::const_iterator cit = content_list.begin();
217 for (; cit != content_list.end(); cit++)
218 if (cit->type == "headers")
221 if (cit == content_list.end())
224 for (Z_HTTP_Header *header = headers; header; header = header->next)
226 std::list<Within>::const_iterator it = cit->within_list.begin();
227 for (; it != cit->within_list.end(); it++)
229 if (!it->header.empty() &&
230 regex_match(header->name, it->header))
232 #ifdef OLDHEADERMATCH
233 // Matches and replaces the whole header line.
234 // This is good if you want to play with the header name too,
235 // but useless for patterns that want to anchor to the beginning
236 // or end of the header value, as we want to do with host-relative
237 // links. This code should probably be removed.
238 std::string sheader(header->name);
240 sheader += header->value;
242 if (it->exec(vars, sheader, true))
244 size_t pos = sheader.find(": ");
245 if (pos == std::string::npos)
247 yaz_log(YLOG_LOG, "Header malformed during rewrite, ignoring");
250 header->name = odr_strdup(
251 o, sheader.substr(0, pos).c_str());
252 header->value = odr_strdup(
253 o, sheader.substr(pos + 2, std::string::npos).c_str());
256 // Match and replace only the header value
257 std::string hval(header->value);
258 if (it->exec(vars, hval, true))
260 header->value = odr_strdup(o, hval.c_str());
269 void yf::HttpRewrite::Phase::rewrite_body(
271 const char *content_type,
274 std::map<std::string, std::string> & vars) const
276 if (*content_len == 0)
279 yaz_log(YLOG_LOG, "rewrite_body: null content_type, can not rewrite");
282 std::list<Content>::const_iterator cit = content_list.begin();
283 for (; cit != content_list.end(); cit++)
285 yaz_log(YLOG_LOG, "rewrite_body: content_type=%s type=%s",
286 content_type, cit->type.c_str());
287 if (cit->type != "headers"
288 && regex_match(content_type, cit->content_re))
291 if (cit == content_list.end()) {
292 yaz_log(YLOG_LOG,"rewrite_body: No content rule matched %s, not rewriting",
298 for (i = 0; i < *content_len; i++)
299 if ((*content_buf)[i] == 0) {
300 yaz_log(YLOG_LOG,"rewrite_body: Looks like binary stuff, not rewriting");
301 return; // binary content. skip
304 std::string content(*content_buf, *content_len);
305 cit->parse(m_verbose, content, vars);
306 *content_buf = odr_strdup(o, content.c_str());
307 *content_len = strlen(*content_buf);
310 yf::HttpRewrite::Event::Event(const Content *p,
311 std::map<std::string, std::string> & vars
312 ) : m_content(p), m_vars(vars)
317 yf::HttpRewrite::Event::~Event()
322 const char *yf::HttpRewrite::Event::result()
324 return wrbuf_cstr(m_w);
327 void yf::HttpRewrite::Event::openTagStart(const char *tag, int tag_len)
329 wrbuf_putc(m_w, '<');
330 wrbuf_write(m_w, tag, tag_len);
332 std::string t(tag, tag_len);
333 std::list<Within>::const_iterator it = m_content->within_list.begin();
334 for (; it != m_content->within_list.end(); it++)
336 if (!it->tag.empty() && regex_match(t, it->tag))
338 if (!it->attr.empty() && regex_match("#text", it->attr))
347 void yf::HttpRewrite::Event::anyTagEnd(const char *tag, int tag_len,
352 if (!s_within.empty())
354 std::list<Within>::const_iterator it = s_within.top();
355 std::string t(tag, tag_len);
356 if (regex_match(t, it->tag))
361 wrbuf_putc(m_w, '/');
362 wrbuf_putc(m_w, '>');
365 void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
366 const char *attr, int attr_len,
367 const char *value, int val_len,
370 std::list<Within>::const_iterator it = m_content->within_list.begin();
373 for (; it != m_content->within_list.end(); it++)
375 std::string t(tag, tag_len);
376 if (it->tag.empty() || regex_match(t, it->tag))
378 std::string a(attr, attr_len);
379 if (!it->attr.empty() && regex_match(a, it->attr))
386 wrbuf_putc(m_w, ' ');
387 wrbuf_write(m_w, attr, attr_len);
390 wrbuf_puts(m_w, "=");
391 wrbuf_puts(m_w, sep);
396 std::string s(value, val_len);
397 it->exec(m_vars, s, true);
398 wrbuf_puts(m_w, s.c_str());
401 wrbuf_write(m_w, value, val_len);
402 wrbuf_puts(m_w, sep);
406 void yf::HttpRewrite::Event::closeTag(const char *tag, int tag_len)
408 if (!s_within.empty())
410 std::list<Within>::const_iterator it = s_within.top();
411 std::string t(tag, tag_len);
412 if (regex_match(t, it->tag))
415 wrbuf_puts(m_w, "</");
416 wrbuf_write(m_w, tag, tag_len);
419 void yf::HttpRewrite::Event::text(const char *value, int len)
421 std::list<Within>::const_iterator it = m_content->within_list.end();
422 if (!s_within.empty())
424 if (it != m_content->within_list.end())
426 std::string s(value, len);
427 it->exec(m_vars, s, false);
428 wrbuf_puts(m_w, s.c_str());
431 wrbuf_write(m_w, value, len);
434 static bool embed_quoted_literal(
435 std::string &content,
436 std::map<std::string, std::string> &vars,
437 mp::filter::HttpRewrite::RulePtr ruleptr,
440 bool replace = false;
442 const char *cp = content.c_str();
443 const char *cp0 = cp;
446 if (html_context && !strncmp(cp, """, 6))
449 res.append(cp0, cp - cp0);
453 if (!strncmp(cp, """, 6))
461 std::string s(cp0, cp - cp0);
462 if (ruleptr->test_patterns(vars, s, true))
467 else if (*cp == '"' || *cp == '\'')
471 res.append(cp0, cp - cp0);
475 if (cp[-1] != '\\' && *cp == m)
483 std::string s(cp0, cp - cp0);
484 if (ruleptr->test_patterns(vars, s, true))
489 else if (*cp == '/' && cp[1] == '/')
491 while (cp[1] && cp[1] != '\n')
496 res.append(cp0, cp - cp0);
501 bool yf::HttpRewrite::Within::exec(
502 std::map<std::string, std::string> & vars,
503 std::string & txt, bool anchor) const
505 if (type == "quoted-literal")
507 return embed_quoted_literal(txt, vars, rule, true);
511 return rule->test_patterns(vars, txt, anchor);
515 bool yf::HttpRewrite::Rule::test_patterns(
516 std::map<std::string, std::string> & vars,
517 std::string & txt, bool anchor)
519 bool replaces = false;
522 std::string::const_iterator start, end;
527 std::list<Replace>::iterator bit = replace_list.end();
529 std::string::const_iterator best_pos = txt.end();
530 std::list<Replace>::iterator it = replace_list.begin();
531 for (; it != replace_list.end(); it++)
533 if (it->start_anchor && !first)
535 if (regex_search(start, end, it->what, it->re))
537 if (it->what[0].first < best_pos)
539 best_pos = it->what[0].first;
544 if (bit == replace_list.end())
550 for (i = 1; i < bit->what.size(); ++i)
552 //check if the group is named
553 std::map<int, std::string>::const_iterator git
554 = bit->group_index.find(i);
555 if (git != bit->group_index.end())
557 vars[git->second] = bit->what[i];
561 //prepare replacement string
562 std::string rvalue = bit->sub_vars(vars);
563 yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
564 bit->what.str(0).c_str(), rvalue.c_str());
565 out.append(start, bit->what[0].first);
567 start = bit->what[0].second; //move search forward
569 out.append(start, end);
574 void yf::HttpRewrite::Replace::parse_groups(std::string pattern)
578 const std::string &str = pattern;
580 start_anchor = str[0] == '^';
581 yaz_log(YLOG_LOG, "Parsing groups from '%s'", str.c_str());
582 for (size_t i = 0; i < str.size(); ++i)
585 if (!esc && str[i] == '\\')
590 if (!esc && str[i] == '(') //group starts
593 if (i+1 < str.size() && str[i+1] == '?') //group with attrs
596 if (i+1 < str.size() && str[i+1] == ':') //non-capturing
598 if (gnum > 0) gnum--;
604 if (i+1 < str.size() && str[i+1] == 'P') //optional, python
606 if (i+1 < str.size() && str[i+1] == '<') //named
611 while (++i < str.size())
613 if (str[i] == '>') { term = true; break; }
614 if (!isalnum(str[i]))
615 throw mp::filter::FilterException
616 ("Only alphanumeric chars allowed, found "
620 + boost::lexical_cast<std::string>(i));
624 throw mp::filter::FilterException
625 ("Unterminated group name '" + gname
626 + " in '" + str +"'");
627 group_index[gnum] = gname;
628 yaz_log(YLOG_LOG, "Found named group '%s' at $%d",
629 gname.c_str(), gnum);
638 std::string yf::HttpRewrite::Replace::sub_vars(
639 const std::map<std::string, std::string> & vars) const
643 const std::string & in = recipe;
644 for (size_t i = 0; i < in.size(); ++i)
646 if (!esc && in[i] == '\\')
651 if (!esc && in[i] == '$') //var
653 if (i+1 < in.size() && in[i+1] == '{') //ref prefix
658 while (++i < in.size())
660 if (in[i] == '}') { term = true; break; }
663 if (!term) throw mp::filter::FilterException
664 ("Unterminated var ref in '"+in+"' at "
665 + boost::lexical_cast<std::string>(i));
666 std::map<std::string, std::string>::const_iterator it
668 if (it != vars.end())
675 throw mp::filter::FilterException
676 ("Malformed or trimmed var ref in '"
677 +in+"' at "+boost::lexical_cast<std::string>(i));
688 yf::HttpRewrite::Phase::Phase() : m_verbose(0)
692 void yf::HttpRewrite::Content::parse(
694 std::string &content,
695 std::map<std::string, std::string> &vars) const
700 Event ev(this, vars);
702 parser.set_verbose(verbose);
704 parser.parse(ev, content.c_str());
705 content = ev.result();
707 if (type == "quoted-literal")
709 quoted_literal(content, vars);
713 void yf::HttpRewrite::Content::quoted_literal(
714 std::string &content,
715 std::map<std::string, std::string> &vars) const
717 std::list<Within>::const_iterator it = within_list.begin();
718 if (it != within_list.end())
719 embed_quoted_literal(content, vars, it->rule, false);
722 void yf::HttpRewrite::Content::configure(
723 const xmlNode *ptr, std::map<std::string, RulePtr > &rules)
725 for (; ptr; ptr = ptr->next)
727 if (ptr->type != XML_ELEMENT_NODE)
729 if (!strcmp((const char *) ptr->name, "within"))
731 static const char *names[7] =
732 { "header", "attr", "tag", "rule", "reqline", "type", 0 };
733 std::string values[6];
734 mp::xml::parse_attr(ptr, names, values);
736 if (values[0].length() > 0)
737 w.header.assign(values[0], boost::regex_constants::icase);
738 if (values[1].length() > 0)
739 w.attr.assign(values[1], boost::regex_constants::icase);
740 if (values[2].length() > 0)
741 w.tag.assign(values[2], boost::regex_constants::icase);
743 std::vector<std::string> rulenames;
744 boost::split(rulenames, values[3], boost::is_any_of(","));
745 if (rulenames.size() == 0)
747 throw mp::filter::FilterException
748 ("Empty rule in '" + values[3] +
749 "' in http_rewrite filter");
751 else if (rulenames.size() == 1)
753 std::map<std::string,RulePtr>::const_iterator it =
754 rules.find(rulenames[0]);
755 if (it == rules.end())
756 throw mp::filter::FilterException
757 ("Reference to non-existing rule '" + rulenames[0] +
758 "' in http_rewrite filter");
764 RulePtr rule(new Rule);
766 for (i = 0; i < rulenames.size(); i++)
768 std::map<std::string,RulePtr>::const_iterator it =
769 rules.find(rulenames[i]);
770 if (it == rules.end())
771 throw mp::filter::FilterException
772 ("Reference to non-existing rule '" + rulenames[i] +
773 "' in http_rewrite filter");
774 RulePtr subRule = it->second;
775 std::list<Replace>::iterator rit =
776 subRule->replace_list.begin();
777 for (; rit != subRule->replace_list.end(); rit++)
778 rule->replace_list.push_back(*rit);
782 w.reqline = values[4] == "1";
784 if (w.type.empty() || w.type == "quoted-literal")
787 throw mp::filter::FilterException
788 ("within type must be quoted-literal or none in "
789 " in http_rewrite filter");
790 within_list.push_back(w);
795 void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
797 static const char *names[2] = { "verbose", 0 };
798 std::string values[1];
800 mp::xml::parse_attr(ptr, names, values);
802 phase.m_verbose = atoi(values[0].c_str());
804 std::map<std::string, RulePtr > rules;
805 for (ptr = ptr->children; ptr; ptr = ptr->next)
807 if (ptr->type != XML_ELEMENT_NODE)
809 else if (!strcmp((const char *) ptr->name, "rule"))
811 static const char *names[2] = { "name", 0 };
812 std::string values[1];
813 values[0] = "default";
814 mp::xml::parse_attr(ptr, names, values);
816 RulePtr rule(new Rule);
817 for (xmlNode *p = ptr->children; p; p = p->next)
819 if (p->type != XML_ELEMENT_NODE)
821 if (!strcmp((const char *) p->name, "rewrite"))
825 const struct _xmlAttr *attr;
826 for (attr = p->properties; attr; attr = attr->next)
828 if (!strcmp((const char *) attr->name, "from"))
829 from = mp::xml::get_text(attr->children);
830 else if (!strcmp((const char *) attr->name, "to"))
831 replace.recipe = mp::xml::get_text(attr->children);
833 throw mp::filter::FilterException
835 + std::string((const char *) attr->name)
836 + " in rewrite section of http_rewrite");
838 yaz_log(YLOG_LOG, "Found rewrite rule from '%s' to '%s'",
839 from.c_str(), replace.recipe.c_str());
842 replace.parse_groups(from);
843 rule->replace_list.push_back(replace);
847 throw mp::filter::FilterException
849 + std::string((const char *) p->name)
850 + " in http_rewrite filter");
852 rules[values[0]] = rule;
854 else if (!strcmp((const char *) ptr->name, "content"))
856 static const char *names[3] =
857 { "type", "mime", 0 };
858 std::string values[2];
859 mp::xml::parse_attr(ptr, names, values);
860 if (values[0].empty())
862 throw mp::filter::FilterException
863 ("Missing attribute, type for for element "
864 + std::string((const char *) ptr->name)
865 + " in http_rewrite filter");
870 if (!values[1].empty())
871 c.content_re.assign(values[1], boost::regex::icase);
872 c.configure(ptr->children, rules);
873 phase.content_list.push_back(c);
877 throw mp::filter::FilterException
879 + std::string((const char *) ptr->name)
880 + " in http_rewrite filter");
885 void yf::HttpRewrite::configure(const xmlNode * ptr, bool test_only,
888 for (ptr = ptr->children; ptr; ptr = ptr->next)
890 if (ptr->type != XML_ELEMENT_NODE)
892 else if (!strcmp((const char *) ptr->name, "request"))
894 configure_phase(ptr, *req_phase);
896 else if (!strcmp((const char *) ptr->name, "response"))
898 configure_phase(ptr, *res_phase);
902 throw mp::filter::FilterException
904 + std::string((const char *) ptr->name)
905 + " in http_rewrite1 filter");
910 static mp::filter::Base* filter_creator()
912 return new mp::filter::HttpRewrite;
916 struct metaproxy_1_filter_struct metaproxy_1_filter_http_rewrite = {
927 * c-file-style: "Stroustrup"
928 * indent-tabs-mode: nil
930 * vim: shiftwidth=4 tabstop=8 expandtab