1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include <metaproxy/filter.hpp>
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
23 #include "filter_http_rewrite.hpp"
24 #include "html_parser.hpp"
30 #include <boost/regex.hpp>
31 #include <boost/lexical_cast.hpp>
32 #include <boost/algorithm/string.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
41 class HttpRewrite::Replace {
46 std::map<int, std::string> group_index;
48 const std::map<std::string, std::string> & vars) const;
49 void parse_groups(std::string pattern);
52 class HttpRewrite::Rule {
54 std::list<Replace> replace_list;
56 std::map<std::string, std::string> &vars,
57 std::string &txt, bool anchor);
59 class HttpRewrite::Within {
67 bool exec(std::map<std::string, std::string> &vars,
68 std::string &txt, bool anchor) const;
71 class HttpRewrite::Content {
74 boost::regex content_re;
75 std::list<Within> within_list;
76 void configure(const xmlNode *ptr,
77 std::map<std::string, RulePtr > &rules);
78 void quoted_literal(std::string &content,
79 std::map<std::string, std::string> &vars) const;
80 void parse(int verbose, std::string &content,
81 std::map<std::string, std::string> & vars) const;
83 class HttpRewrite::Phase {
87 std::list<Content> content_list;
88 void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
89 std::map<std::string, std::string> & vars) const;
90 void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
91 std::map<std::string, std::string> & vars) const;
92 void rewrite_body(mp::odr & o,
93 const char *content_type,
94 char **content_buf, int *content_len,
95 std::map<std::string, std::string> & vars) const;
97 class HttpRewrite::Event : public HTMLParserEvent {
98 void openTagStart(const char *tag, int tag_len);
99 void anyTagEnd(const char *tag, int tag_len, int close_it);
100 void attribute(const char *tag, int tag_len,
101 const char *attr, int attr_len,
102 const char *value, int val_len,
104 void closeTag(const char *tag, int tag_len);
105 void text(const char *value, int len);
106 const Content *m_content;
108 std::stack<std::list<Within>::const_iterator> s_within;
109 std::map<std::string, std::string> &m_vars;
111 Event(const Content *p, std::map<std::string, std::string> &vars);
113 const char *result();
118 yf::HttpRewrite::HttpRewrite() :
119 req_phase(new Phase), res_phase(new Phase)
123 yf::HttpRewrite::~HttpRewrite()
127 void yf::HttpRewrite::process(mp::Package & package) const
129 yaz_log(YLOG_LOG, "HttpRewrite begins....");
130 Z_GDU *gdu = package.request().get();
131 //map of request/response vars
132 std::map<std::string, std::string> vars;
133 //we have an http req
134 if (gdu && gdu->which == Z_GDU_HTTP_Request)
136 Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
138 req_phase->rewrite_reqline(o, hreq, vars);
139 yaz_log(YLOG_LOG, ">> Request headers");
140 req_phase->rewrite_headers(o, hreq->headers, vars);
141 req_phase->rewrite_body(o,
142 z_HTTP_header_lookup(hreq->headers,
144 &hreq->content_buf, &hreq->content_len,
146 package.request() = gdu;
149 gdu = package.response().get();
150 if (gdu && gdu->which == Z_GDU_HTTP_Response)
152 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
153 yaz_log(YLOG_LOG, "Response code %d", hres->code);
155 yaz_log(YLOG_LOG, "<< Respose headers");
156 res_phase->rewrite_headers(o, hres->headers, vars);
157 res_phase->rewrite_body(o,
158 z_HTTP_header_lookup(hres->headers,
160 &hres->content_buf, &hres->content_len,
162 package.response() = gdu;
166 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
167 Z_HTTP_Request *hreq,
168 std::map<std::string, std::string> & vars) const
170 //rewrite the request line
172 if (strstr(hreq->path, "http://") == hreq->path)
174 yaz_log(YLOG_LOG, "Path in the method line is absolute, "
175 "possibly a proxy request");
180 //TODO what about proto
181 const char *host = z_HTTP_header_lookup(hreq->headers, "Host");
190 std::list<Content>::const_iterator cit = content_list.begin();
191 for (; cit != content_list.end(); cit++)
192 if (cit->type == "headers")
195 if (cit == content_list.end())
198 std::list<Within>::const_iterator it = cit->within_list.begin();
199 for (; it != cit->within_list.end(); it++)
202 yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
203 if (it->exec(vars, path, true))
205 yaz_log(YLOG_LOG, "Rewritten request URL is %s", path.c_str());
206 hreq->path = odr_strdup(o, path.c_str());
211 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
212 Z_HTTP_Header *headers,
213 std::map<std::string, std::string> & vars) const
215 std::list<Content>::const_iterator cit = content_list.begin();
216 for (; cit != content_list.end(); cit++)
217 if (cit->type == "headers")
220 if (cit == content_list.end())
223 for (Z_HTTP_Header *header = headers; header; header = header->next)
225 std::list<Within>::const_iterator it = cit->within_list.begin();
226 for (; it != cit->within_list.end(); it++)
228 if (!it->header.empty() &&
229 regex_match(header->name, it->header))
231 // Match and replace only the header value
232 std::string hval(header->value);
233 if (it->exec(vars, hval, true))
235 header->value = odr_strdup(o, hval.c_str());
242 void yf::HttpRewrite::Phase::rewrite_body(
244 const char *content_type,
247 std::map<std::string, std::string> & vars) const
249 if (*content_len == 0)
252 yaz_log(YLOG_LOG, "rewrite_body: null content_type, can not rewrite");
255 std::list<Content>::const_iterator cit = content_list.begin();
256 for (; cit != content_list.end(); cit++)
258 yaz_log(YLOG_LOG, "rewrite_body: content_type=%s type=%s",
259 content_type, cit->type.c_str());
260 if (cit->type != "headers"
261 && regex_match(content_type, cit->content_re))
264 if (cit == content_list.end()) {
265 yaz_log(YLOG_LOG,"rewrite_body: No content rule matched %s, not rewriting",
271 for (i = 0; i < *content_len; i++)
272 if ((*content_buf)[i] == 0) {
273 yaz_log(YLOG_LOG,"rewrite_body: Looks like binary stuff, not rewriting");
274 return; // binary content. skip
277 std::string content(*content_buf, *content_len);
278 cit->parse(m_verbose, content, vars);
279 *content_buf = odr_strdup(o, content.c_str());
280 *content_len = strlen(*content_buf);
283 yf::HttpRewrite::Event::Event(const Content *p,
284 std::map<std::string, std::string> & vars
285 ) : m_content(p), m_vars(vars)
290 yf::HttpRewrite::Event::~Event()
295 const char *yf::HttpRewrite::Event::result()
297 return wrbuf_cstr(m_w);
300 void yf::HttpRewrite::Event::openTagStart(const char *tag, int tag_len)
302 wrbuf_putc(m_w, '<');
303 wrbuf_write(m_w, tag, tag_len);
305 std::string t(tag, tag_len);
306 std::list<Within>::const_iterator it = m_content->within_list.begin();
307 for (; it != m_content->within_list.end(); it++)
309 if (!it->tag.empty() && regex_match(t, it->tag))
311 if (!it->attr.empty() && regex_match("#text", it->attr))
320 void yf::HttpRewrite::Event::anyTagEnd(const char *tag, int tag_len,
325 if (!s_within.empty())
327 std::list<Within>::const_iterator it = s_within.top();
328 std::string t(tag, tag_len);
329 if (regex_match(t, it->tag))
334 wrbuf_putc(m_w, '/');
335 wrbuf_putc(m_w, '>');
338 void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
339 const char *attr, int attr_len,
340 const char *value, int val_len,
343 std::list<Within>::const_iterator it = m_content->within_list.begin();
346 for (; it != m_content->within_list.end(); it++)
348 std::string t(tag, tag_len);
349 if (it->tag.empty() || regex_match(t, it->tag))
351 std::string a(attr, attr_len);
352 if (!it->attr.empty() && regex_match(a, it->attr))
359 wrbuf_putc(m_w, ' ');
360 wrbuf_write(m_w, attr, attr_len);
363 wrbuf_puts(m_w, "=");
364 wrbuf_puts(m_w, sep);
369 std::string s(value, val_len);
370 it->exec(m_vars, s, true);
371 wrbuf_puts(m_w, s.c_str());
374 wrbuf_write(m_w, value, val_len);
375 wrbuf_puts(m_w, sep);
379 void yf::HttpRewrite::Event::closeTag(const char *tag, int tag_len)
381 if (!s_within.empty())
383 std::list<Within>::const_iterator it = s_within.top();
384 std::string t(tag, tag_len);
385 if (regex_match(t, it->tag))
388 wrbuf_puts(m_w, "</");
389 wrbuf_write(m_w, tag, tag_len);
392 void yf::HttpRewrite::Event::text(const char *value, int len)
394 std::list<Within>::const_iterator it = m_content->within_list.end();
395 if (!s_within.empty())
397 if (it != m_content->within_list.end())
399 std::string s(value, len);
400 it->exec(m_vars, s, false);
401 wrbuf_puts(m_w, s.c_str());
404 wrbuf_write(m_w, value, len);
407 static bool embed_quoted_literal(
408 std::string &content,
409 std::map<std::string, std::string> &vars,
410 mp::filter::HttpRewrite::RulePtr ruleptr,
413 bool replace = false;
415 const char *cp = content.c_str();
416 const char *cp0 = cp;
419 if (html_context && !strncmp(cp, """, 6))
422 res.append(cp0, cp - cp0);
426 if (!strncmp(cp, """, 6))
434 std::string s(cp0, cp - cp0);
435 if (ruleptr->test_patterns(vars, s, true))
440 else if (*cp == '"' || *cp == '\'')
444 res.append(cp0, cp - cp0);
448 if (cp[-1] != '\\' && *cp == m)
456 std::string s(cp0, cp - cp0);
457 if (ruleptr->test_patterns(vars, s, true))
462 else if (*cp == '/' && cp[1] == '/')
464 while (cp[1] && cp[1] != '\n')
469 res.append(cp0, cp - cp0);
474 bool yf::HttpRewrite::Within::exec(
475 std::map<std::string, std::string> & vars,
476 std::string & txt, bool anchor) const
478 if (type == "quoted-literal")
480 return embed_quoted_literal(txt, vars, rule, true);
484 return rule->test_patterns(vars, txt, anchor);
488 bool yf::HttpRewrite::Rule::test_patterns(
489 std::map<std::string, std::string> & vars,
490 std::string & txt, bool anchor)
492 bool replaces = false;
495 std::string::const_iterator start, end;
500 std::list<Replace>::iterator bit = replace_list.end();
502 bool match_one = false;
504 std::list<Replace>::iterator it = replace_list.begin();
505 for (; it != replace_list.end(); it++)
507 if (it->start_anchor && !first)
510 if (regex_search(start, end, what, it->re))
512 if (!match_one || what[0].first < bwhat[0].first)
526 for (i = 1; i < bwhat.size(); ++i)
528 //check if the group is named
529 std::map<int, std::string>::const_iterator git
530 = bit->group_index.find(i);
531 if (git != bit->group_index.end())
533 vars[git->second] = bwhat[i];
537 //prepare replacement string
538 std::string rvalue = bit->sub_vars(vars);
539 yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
540 bwhat.str(0).c_str(), rvalue.c_str());
541 out.append(start, bwhat[0].first);
543 start = bwhat[0].second; //move search forward
545 out.append(start, end);
550 void yf::HttpRewrite::Replace::parse_groups(std::string pattern)
554 const std::string &str = pattern;
556 start_anchor = str[0] == '^';
557 yaz_log(YLOG_LOG, "Parsing groups from '%s'", str.c_str());
558 for (size_t i = 0; i < str.size(); ++i)
561 if (!esc && str[i] == '\\')
566 if (!esc && str[i] == '(') //group starts
569 if (i+1 < str.size() && str[i+1] == '?') //group with attrs
572 if (i+1 < str.size() && str[i+1] == ':') //non-capturing
574 if (gnum > 0) gnum--;
580 if (i+1 < str.size() && str[i+1] == 'P') //optional, python
582 if (i+1 < str.size() && str[i+1] == '<') //named
587 while (++i < str.size())
589 if (str[i] == '>') { term = true; break; }
590 if (!isalnum(str[i]))
591 throw mp::filter::FilterException
592 ("Only alphanumeric chars allowed, found "
596 + boost::lexical_cast<std::string>(i));
600 throw mp::filter::FilterException
601 ("Unterminated group name '" + gname
602 + " in '" + str +"'");
603 group_index[gnum] = gname;
604 yaz_log(YLOG_LOG, "Found named group '%s' at $%d",
605 gname.c_str(), gnum);
614 std::string yf::HttpRewrite::Replace::sub_vars(
615 const std::map<std::string, std::string> & vars) const
619 const std::string & in = recipe;
620 for (size_t i = 0; i < in.size(); ++i)
622 if (!esc && in[i] == '\\')
627 if (!esc && in[i] == '$') //var
629 if (i+1 < in.size() && in[i+1] == '{') //ref prefix
634 while (++i < in.size())
636 if (in[i] == '}') { term = true; break; }
639 if (!term) throw mp::filter::FilterException
640 ("Unterminated var ref in '"+in+"' at "
641 + boost::lexical_cast<std::string>(i));
642 std::map<std::string, std::string>::const_iterator it
644 if (it != vars.end())
651 throw mp::filter::FilterException
652 ("Malformed or trimmed var ref in '"
653 +in+"' at "+boost::lexical_cast<std::string>(i));
664 yf::HttpRewrite::Phase::Phase() : m_verbose(0)
668 void yf::HttpRewrite::Content::parse(
670 std::string &content,
671 std::map<std::string, std::string> &vars) const
676 Event ev(this, vars);
678 parser.set_verbose(verbose);
680 parser.parse(ev, content.c_str());
681 content = ev.result();
683 if (type == "quoted-literal")
685 quoted_literal(content, vars);
689 void yf::HttpRewrite::Content::quoted_literal(
690 std::string &content,
691 std::map<std::string, std::string> &vars) const
693 std::list<Within>::const_iterator it = within_list.begin();
694 if (it != within_list.end())
695 embed_quoted_literal(content, vars, it->rule, false);
698 void yf::HttpRewrite::Content::configure(
699 const xmlNode *ptr, std::map<std::string, RulePtr > &rules)
701 for (; ptr; ptr = ptr->next)
703 if (ptr->type != XML_ELEMENT_NODE)
705 if (!strcmp((const char *) ptr->name, "within"))
707 static const char *names[7] =
708 { "header", "attr", "tag", "rule", "reqline", "type", 0 };
709 std::string values[6];
710 mp::xml::parse_attr(ptr, names, values);
712 if (values[0].length() > 0)
713 w.header.assign(values[0], boost::regex_constants::icase);
714 if (values[1].length() > 0)
715 w.attr.assign(values[1], boost::regex_constants::icase);
716 if (values[2].length() > 0)
717 w.tag.assign(values[2], boost::regex_constants::icase);
719 std::vector<std::string> rulenames;
720 boost::split(rulenames, values[3], boost::is_any_of(","));
721 if (rulenames.size() == 0)
723 throw mp::filter::FilterException
724 ("Empty rule in '" + values[3] +
725 "' in http_rewrite filter");
727 else if (rulenames.size() == 1)
729 std::map<std::string,RulePtr>::const_iterator it =
730 rules.find(rulenames[0]);
731 if (it == rules.end())
732 throw mp::filter::FilterException
733 ("Reference to non-existing rule '" + rulenames[0] +
734 "' in http_rewrite filter");
740 RulePtr rule(new Rule);
742 for (i = 0; i < rulenames.size(); i++)
744 std::map<std::string,RulePtr>::const_iterator it =
745 rules.find(rulenames[i]);
746 if (it == rules.end())
747 throw mp::filter::FilterException
748 ("Reference to non-existing rule '" + rulenames[i] +
749 "' in http_rewrite filter");
750 RulePtr subRule = it->second;
751 std::list<Replace>::iterator rit =
752 subRule->replace_list.begin();
753 for (; rit != subRule->replace_list.end(); rit++)
754 rule->replace_list.push_back(*rit);
758 w.reqline = values[4] == "1";
760 if (w.type.empty() || w.type == "quoted-literal")
763 throw mp::filter::FilterException
764 ("within type must be quoted-literal or none in "
765 " in http_rewrite filter");
766 within_list.push_back(w);
771 void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
773 static const char *names[2] = { "verbose", 0 };
774 std::string values[1];
776 mp::xml::parse_attr(ptr, names, values);
778 phase.m_verbose = atoi(values[0].c_str());
780 std::map<std::string, RulePtr > rules;
781 for (ptr = ptr->children; ptr; ptr = ptr->next)
783 if (ptr->type != XML_ELEMENT_NODE)
785 else if (!strcmp((const char *) ptr->name, "rule"))
787 static const char *names[2] = { "name", 0 };
788 std::string values[1];
789 values[0] = "default";
790 mp::xml::parse_attr(ptr, names, values);
792 RulePtr rule(new Rule);
793 for (xmlNode *p = ptr->children; p; p = p->next)
795 if (p->type != XML_ELEMENT_NODE)
797 if (!strcmp((const char *) p->name, "rewrite"))
801 const struct _xmlAttr *attr;
802 for (attr = p->properties; attr; attr = attr->next)
804 if (!strcmp((const char *) attr->name, "from"))
805 from = mp::xml::get_text(attr->children);
806 else if (!strcmp((const char *) attr->name, "to"))
807 replace.recipe = mp::xml::get_text(attr->children);
809 throw mp::filter::FilterException
811 + std::string((const char *) attr->name)
812 + " in rewrite section of http_rewrite");
814 yaz_log(YLOG_LOG, "Found rewrite rule from '%s' to '%s'",
815 from.c_str(), replace.recipe.c_str());
818 replace.parse_groups(from);
819 rule->replace_list.push_back(replace);
823 throw mp::filter::FilterException
825 + std::string((const char *) p->name)
826 + " in http_rewrite filter");
828 rules[values[0]] = rule;
830 else if (!strcmp((const char *) ptr->name, "content"))
832 static const char *names[3] =
833 { "type", "mime", 0 };
834 std::string values[2];
835 mp::xml::parse_attr(ptr, names, values);
836 if (values[0].empty())
838 throw mp::filter::FilterException
839 ("Missing attribute, type for for element "
840 + std::string((const char *) ptr->name)
841 + " in http_rewrite filter");
846 if (!values[1].empty())
847 c.content_re.assign(values[1], boost::regex::icase);
848 c.configure(ptr->children, rules);
849 phase.content_list.push_back(c);
853 throw mp::filter::FilterException
855 + std::string((const char *) ptr->name)
856 + " in http_rewrite filter");
861 void yf::HttpRewrite::configure(const xmlNode * ptr, bool test_only,
864 for (ptr = ptr->children; ptr; ptr = ptr->next)
866 if (ptr->type != XML_ELEMENT_NODE)
868 else if (!strcmp((const char *) ptr->name, "request"))
870 configure_phase(ptr, *req_phase);
872 else if (!strcmp((const char *) ptr->name, "response"))
874 configure_phase(ptr, *res_phase);
878 throw mp::filter::FilterException
880 + std::string((const char *) ptr->name)
881 + " in http_rewrite1 filter");
886 static mp::filter::Base* filter_creator()
888 return new mp::filter::HttpRewrite;
892 struct metaproxy_1_filter_struct metaproxy_1_filter_http_rewrite = {
903 * c-file-style: "Stroustrup"
904 * indent-tabs-mode: nil
906 * vim: shiftwidth=4 tabstop=8 expandtab