index/zrpn.c

   1 /* $Id: zrpn.c,v 1.211 2006-05-09 10:46:56 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #ifdef WIN32
  26 #include <io.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include <yaz/diagbib1.h>
  34 #include "index.h"
  35 #include <zebra_xpath.h>
  36
  37 #include <charmap.h>
  38 #include <rset.h>
  39
  40 struct rpn_char_map_info
  41 {
  42     ZebraMaps zm;
  43     int reg_type;
  44 };
  45
  46 typedef struct
  47 {
  48     int type;
  49     int major;
  50     int minor;
  51     Z_AttributesPlusTerm *zapt;
  52 } AttrType;
  53
  54 static int log_level_set = 0;
  55 static int log_level_rpn = 0;
  56
  57 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  58 {
  59     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  60     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  61 #if 0
  62     if (out && *out)
  63     {
  64         const char *outp = *out;
  65         yaz_log(YLOG_LOG, "---");
  66         while (*outp)
  67         {
  68             yaz_log(YLOG_LOG, "%02X", *outp);
  69             outp++;
  70         }
  71     }
  72 #endif
  73     return out;
  74 }
  75
  76 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  77                                   struct rpn_char_map_info *map_info)
  78 {
  79     map_info->zm = reg->zebra_maps;
  80     map_info->reg_type = reg_type;
  81     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  82 }
  83
  84 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
  85                         const char **string_value)
  86 {
  87     int num_attributes;
  88
  89     num_attributes = src->zapt->attributes->num_attributes;
  90     while (src->major < num_attributes)
  91     {
  92         Z_AttributeElement *element;
  93
  94         element = src->zapt->attributes->attributes[src->major];
  95         if (src->type == *element->attributeType)
  96         {
  97             switch (element->which)
  98             {
  99             case Z_AttributeValue_numeric:
 100                 ++(src->major);
 101                 if (element->attributeSet && attributeSetP)
 102                 {
 103                     oident *attrset;
 104
 105                     attrset = oid_getentbyoid(element->attributeSet);
 106                     *attributeSetP = attrset->value;
 107                 }
 108                 return *element->value.numeric;
 109                 break;
 110             case Z_AttributeValue_complex:
 111                 if (src->minor >= element->value.complex->num_list)
 112                     break;
 113                 if (element->attributeSet && attributeSetP)
 114                 {
 115                     oident *attrset;
 116
 117                     attrset = oid_getentbyoid(element->attributeSet);
 118                     *attributeSetP = attrset->value;
 119                 }
 120                 if (element->value.complex->list[src->minor]->which ==
 121                     Z_StringOrNumeric_numeric)
 122                 {
 123                     ++(src->minor);
 124                     return
 125                         *element->value.complex->list[src->minor-1]->u.numeric;
 126                 }
 127                 else if (element->value.complex->list[src->minor]->which ==
 128                          Z_StringOrNumeric_string)
 129                 {
 130                     if (!string_value)
 131                         break;
 132                     ++(src->minor);
 133                     *string_value =
 134                         element->value.complex->list[src->minor-1]->u.string;
 135                     return -2;
 136                 }
 137                 else
 138                     break;
 139             default:
 140                 assert(0);
 141             }
 142         }
 143         ++(src->major);
 144     }
 145     return -1;
 146 }
 147
 148 static int attr_find(AttrType *src, oid_value *attributeSetP)
 149 {
 150     return attr_find_ex(src, attributeSetP, 0);
 151 }
 152
 153 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 154                        int type)
 155 {
 156     src->zapt = zapt;
 157     src->type = type;
 158     src->major = 0;
 159     src->minor = 0;
 160 }
 161
 162 #define TERM_COUNT
 163
 164 struct grep_info {
 165 #ifdef TERM_COUNT
 166     int *term_no;
 167 #endif
 168     ISAM_P *isam_p_buf;
 169     int isam_p_size;
 170     int isam_p_indx;
 171     ZebraHandle zh;
 172     int reg_type;
 173     ZebraSet termset;
 174 };
 175
 176 void zebra_term_untrans(ZebraHandle zh, int reg_type,
 177                         char *dst, const char *src)
 178 {
 179     int len = 0;
 180     while (*src)
 181     {
 182         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 183                                            reg_type, &src);
 184         if (!cp)
 185         {
 186             if (len < IT_MAX_WORD-1)
 187                 dst[len++] = *src;
 188             src++;
 189         }
 190         else
 191             while (*cp && len < IT_MAX_WORD-1)
 192                 dst[len++] = *cp++;
 193     }
 194     dst[len] = '\0';
 195 }
 196
 197 static void add_isam_p(const char *name, const char *info,
 198                        struct grep_info *p)
 199 {
 200     if (!log_level_set)
 201     {
 202         log_level_rpn = yaz_log_module_level("rpn");
 203         log_level_set = 1;
 204     }
 205     if (p->isam_p_indx == p->isam_p_size)
 206     {
 207         ISAM_P *new_isam_p_buf;
 208 #ifdef TERM_COUNT
 209         int *new_term_no;
 210 #endif
 211         p->isam_p_size = 2*p->isam_p_size + 100;
 212         new_isam_p_buf = (ISAM_P *) xmalloc(sizeof(*new_isam_p_buf) *
 213                                             p->isam_p_size);
 214         if (p->isam_p_buf)
 215         {
 216             memcpy(new_isam_p_buf, p->isam_p_buf,
 217                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 218             xfree(p->isam_p_buf);
 219         }
 220         p->isam_p_buf = new_isam_p_buf;
 221
 222 #ifdef TERM_COUNT
 223         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 224         if (p->term_no)
 225         {
 226             memcpy(new_term_no, p->isam_p_buf,
 227                     p->isam_p_indx * sizeof(*p->term_no));
 228             xfree(p->term_no);
 229         }
 230         p->term_no = new_term_no;
 231 #endif
 232     }
 233     assert(*info == sizeof(*p->isam_p_buf));
 234     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 235
 236 #if 1
 237     if (p->termset)
 238     {
 239         const char *db;
 240         int set, use;
 241         char term_tmp[IT_MAX_WORD];
 242         int ord = 0;
 243         int len = key_SU_decode (&ord, (const unsigned char *) name);
 244
 245         zebra_term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 246         yaz_log(log_level_rpn, "grep: %d %c %s", ord, name[len], term_tmp);
 247         zebraExplain_lookup_ord (p->zh->reg->zei,
 248                                  ord, 0 /* index_type */, &db, &set, &use, 0);
 249         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 250
 251         resultSetAddTerm(p->zh, p->termset, name[len], db,
 252                          set, use, term_tmp);
 253     }
 254 #endif
 255     (p->isam_p_indx)++;
 256 }
 257
 258 static int grep_handle(char *name, const char *info, void *p)
 259 {
 260     add_isam_p(name, info, (struct grep_info *) p);
 261     return 0;
 262 }
 263
 264 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 265                     const char *ct1, const char *ct2, int first)
 266 {
 267     const char *s1, *s0 = *src;
 268     const char **map;
 269
 270     /* skip white space */
 271     while (*s0)
 272     {
 273         if (ct1 && strchr(ct1, *s0))
 274             break;
 275         if (ct2 && strchr(ct2, *s0))
 276             break;
 277         s1 = s0;
 278         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 279         if (**map != *CHR_SPACE)
 280             break;
 281         s0 = s1;
 282     }
 283     *src = s0;
 284     return *s0;
 285 }
 286
 287
 288 static void esc_str(char *out_buf, int out_size,
 289                     const char *in_buf, int in_size)
 290 {
 291     int k;
 292
 293     assert(out_buf);
 294     assert(in_buf);
 295     assert(out_size > 20);
 296     *out_buf = '\0';
 297     for (k = 0; k<in_size; k++)
 298     {
 299         int c = in_buf[k] & 0xff;
 300         int pc;
 301         if (c < 32 || c > 126)
 302             pc = '?';
 303         else
 304             pc = c;
 305         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 306         if (strlen(out_buf) > out_size-20)
 307         {
 308             strcat(out_buf, "..");
 309             break;
 310         }
 311     }
 312 }
 313
 314 #define REGEX_CHARS " []()|.*+?!"
 315
 316 /* term_100: handle term, where trunc = none(no operators at all) */
 317 static int term_100(ZebraMaps zebra_maps, int reg_type,
 318                     const char **src, char *dst, int space_split,
 319                     char *dst_term)
 320 {
 321     const char *s0;
 322     const char **map;
 323     int i = 0;
 324     int j = 0;
 325
 326     const char *space_start = 0;
 327     const char *space_end = 0;
 328
 329     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 330         return 0;
 331     s0 = *src;
 332     while (*s0)
 333     {
 334         const char *s1 = s0;
 335         int q_map_match = 0;
 336         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 337                                 &q_map_match);
 338         if (space_split)
 339         {
 340             if (**map == *CHR_SPACE)
 341                 break;
 342         }
 343         else  /* complete subfield only. */
 344         {
 345             if (**map == *CHR_SPACE)
 346             {   /* save space mapping for later  .. */
 347                 space_start = s1;
 348                 space_end = s0;
 349                 continue;
 350             }
 351             else if (space_start)
 352             {   /* reload last space */
 353                 while (space_start < space_end)
 354                 {
 355                     if (strchr(REGEX_CHARS, *space_start))
 356                         dst[i++] = '\\';
 357                     dst_term[j++] = *space_start;
 358                     dst[i++] = *space_start++;
 359                 }
 360                 /* and reset */
 361                 space_start = space_end = 0;
 362             }
 363         }
 364         /* add non-space char */
 365         memcpy(dst_term+j, s1, s0 - s1);
 366         j += (s0 - s1);
 367         if (!q_map_match)
 368         {
 369             while (s1 < s0)
 370             {
 371                 if (strchr(REGEX_CHARS, *s1))
 372                     dst[i++] = '\\';
 373                 dst[i++] = *s1++;
 374             }
 375         }
 376         else
 377         {
 378             char tmpbuf[80];
 379             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 380
 381             strcpy(dst + i, map[0]);
 382             i += strlen(map[0]);
 383         }
 384     }
 385     dst[i] = '\0';
 386     dst_term[j] = '\0';
 387     *src = s0;
 388     return i;
 389 }
 390
 391 /* term_101: handle term, where trunc = Process # */
 392 static int term_101(ZebraMaps zebra_maps, int reg_type,
 393                     const char **src, char *dst, int space_split,
 394                     char *dst_term)
 395 {
 396     const char *s0;
 397     const char **map;
 398     int i = 0;
 399     int j = 0;
 400
 401     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 402         return 0;
 403     s0 = *src;
 404     while (*s0)
 405     {
 406         if (*s0 == '#')
 407         {
 408             dst[i++] = '.';
 409             dst[i++] = '*';
 410             dst_term[j++] = *s0++;
 411         }
 412         else
 413         {
 414             const char *s1 = s0;
 415             int q_map_match = 0;
 416             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 417                                     &q_map_match);
 418             if (space_split && **map == *CHR_SPACE)
 419                 break;
 420
 421             /* add non-space char */
 422             memcpy(dst_term+j, s1, s0 - s1);
 423             j += (s0 - s1);
 424             if (!q_map_match)
 425             {
 426                 while (s1 < s0)
 427                 {
 428                     if (strchr(REGEX_CHARS, *s1))
 429                         dst[i++] = '\\';
 430                     dst[i++] = *s1++;
 431                 }
 432             }
 433             else
 434             {
 435                 char tmpbuf[80];
 436                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 437
 438                 strcpy(dst + i, map[0]);
 439                 i += strlen(map[0]);
 440             }
 441         }
 442     }
 443     dst[i] = '\0';
 444     dst_term[j++] = '\0';
 445     *src = s0;
 446     return i;
 447 }
 448
 449 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 450 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 451                     char *dst, int *errors, int space_split,
 452                     char *dst_term)
 453 {
 454     int i = 0;
 455     int j = 0;
 456     const char *s0;
 457     const char **map;
 458
 459     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 460         return 0;
 461     s0 = *src;
 462     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 463         isdigit(((const unsigned char *)s0)[1]))
 464     {
 465         *errors = s0[1] - '0';
 466         s0 += 3;
 467         if (*errors > 3)
 468             *errors = 3;
 469     }
 470     while (*s0)
 471     {
 472         if (strchr("^\\()[].*+?|-", *s0))
 473         {
 474             dst_term[j++] = *s0;
 475             dst[i++] = *s0++;
 476         }
 477         else
 478         {
 479             const char *s1 = s0;
 480             int q_map_match = 0;
 481             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 482                                     &q_map_match);
 483             if (space_split && **map == *CHR_SPACE)
 484                 break;
 485
 486             /* add non-space char */
 487             memcpy(dst_term+j, s1, s0 - s1);
 488             j += (s0 - s1);
 489             if (!q_map_match)
 490             {
 491                 while (s1 < s0)
 492                 {
 493                     if (strchr(REGEX_CHARS, *s1))
 494                         dst[i++] = '\\';
 495                     dst[i++] = *s1++;
 496                 }
 497             }
 498             else
 499             {
 500                 char tmpbuf[80];
 501                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 502
 503                 strcpy(dst + i, map[0]);
 504                 i += strlen(map[0]);
 505             }
 506         }
 507     }
 508     dst[i] = '\0';
 509     dst_term[j] = '\0';
 510     *src = s0;
 511
 512     return i;
 513 }
 514
 515 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 516 static int term_102(ZebraMaps zebra_maps, int reg_type, const char **src,
 517                     char *dst, int space_split, char *dst_term)
 518 {
 519     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 520                     dst_term);
 521 }
 522
 523
 524 /* term_104: handle term, where trunc = Process # and ! */
 525 static int term_104(ZebraMaps zebra_maps, int reg_type,
 526                     const char **src, char *dst, int space_split,
 527                     char *dst_term)
 528 {
 529     const char *s0;
 530     const char **map;
 531     int i = 0;
 532     int j = 0;
 533
 534     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 535         return 0;
 536     s0 = *src;
 537     while (*s0)
 538     {
 539         if (*s0 == '?')
 540         {
 541             dst_term[j++] = *s0++;
 542             if (*s0 >= '0' && *s0 <= '9')
 543             {
 544                 int limit = 0;
 545                 while (*s0 >= '0' && *s0 <= '9')
 546                 {
 547                     limit = limit * 10 + (*s0 - '0');
 548                     dst_term[j++] = *s0++;
 549                 }
 550                 if (limit > 20)
 551                     limit = 20;
 552                 while (--limit >= 0)
 553                 {
 554                     dst[i++] = '.';
 555                     dst[i++] = '?';
 556                 }
 557             }
 558             else
 559             {
 560                 dst[i++] = '.';
 561                 dst[i++] = '*';
 562             }
 563         }
 564         else if (*s0 == '*')
 565         {
 566             dst[i++] = '.';
 567             dst[i++] = '*';
 568             dst_term[j++] = *s0++;
 569         }
 570         else if (*s0 == '#')
 571         {
 572             dst[i++] = '.';
 573             dst_term[j++] = *s0++;
 574         }
 575         else
 576         {
 577             const char *s1 = s0;
 578             int q_map_match = 0;
 579             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 580                                     &q_map_match);
 581             if (space_split && **map == *CHR_SPACE)
 582                 break;
 583
 584             /* add non-space char */
 585             memcpy(dst_term+j, s1, s0 - s1);
 586             j += (s0 - s1);
 587             if (!q_map_match)
 588             {
 589                 while (s1 < s0)
 590                 {
 591                     if (strchr(REGEX_CHARS, *s1))
 592                         dst[i++] = '\\';
 593                     dst[i++] = *s1++;
 594                 }
 595             }
 596             else
 597             {
 598                 char tmpbuf[80];
 599                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 600
 601                 strcpy(dst + i, map[0]);
 602                 i += strlen(map[0]);
 603             }
 604         }
 605     }
 606     dst[i] = '\0';
 607     dst_term[j++] = '\0';
 608     *src = s0;
 609     return i;
 610 }
 611
 612 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 613 static int term_105(ZebraMaps zebra_maps, int reg_type,
 614                     const char **src, char *dst, int space_split,
 615                     char *dst_term, int right_truncate)
 616 {
 617     const char *s0;
 618     const char **map;
 619     int i = 0;
 620     int j = 0;
 621
 622     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 623         return 0;
 624     s0 = *src;
 625     while (*s0)
 626     {
 627         if (*s0 == '*')
 628         {
 629             dst[i++] = '.';
 630             dst[i++] = '*';
 631             dst_term[j++] = *s0++;
 632         }
 633         else if (*s0 == '!')
 634         {
 635             dst[i++] = '.';
 636             dst_term[j++] = *s0++;
 637         }
 638         else
 639         {
 640             const char *s1 = s0;
 641             int q_map_match = 0;
 642             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 643                                     &q_map_match);
 644             if (space_split && **map == *CHR_SPACE)
 645                 break;
 646
 647             /* add non-space char */
 648             memcpy(dst_term+j, s1, s0 - s1);
 649             j += (s0 - s1);
 650             if (!q_map_match)
 651             {
 652                 while (s1 < s0)
 653                 {
 654                     if (strchr(REGEX_CHARS, *s1))
 655                         dst[i++] = '\\';
 656                     dst[i++] = *s1++;
 657                 }
 658             }
 659             else
 660             {
 661                 char tmpbuf[80];
 662                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 663
 664                 strcpy(dst + i, map[0]);
 665                 i += strlen(map[0]);
 666             }
 667         }
 668     }
 669     if (right_truncate)
 670     {
 671         dst[i++] = '.';
 672         dst[i++] = '*';
 673     }
 674     dst[i] = '\0';
 675
 676     dst_term[j++] = '\0';
 677     *src = s0;
 678     return i;
 679 }
 680
 681
 682 /* gen_regular_rel - generate regular expression from relation
 683  *  val:     border value (inclusive)
 684  *  islt:    1 if <=; 0 if >=.
 685  */
 686 static void gen_regular_rel(char *dst, int val, int islt)
 687 {
 688     int dst_p;
 689     int w, d, i;
 690     int pos = 0;
 691     char numstr[20];
 692
 693     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 694     if (val >= 0)
 695     {
 696         if (islt)
 697             strcpy(dst, "(-[0-9]+|(");
 698         else
 699             strcpy(dst, "((");
 700     }
 701     else
 702     {
 703         if (!islt)
 704         {
 705             strcpy(dst, "([0-9]+|-(");
 706             dst_p = strlen(dst);
 707             islt = 1;
 708         }
 709         else
 710         {
 711             strcpy(dst, "(-(");
 712             islt = 0;
 713         }
 714         val = -val;
 715     }
 716     dst_p = strlen(dst);
 717     sprintf(numstr, "%d", val);
 718     for (w = strlen(numstr); --w >= 0; pos++)
 719     {
 720         d = numstr[w];
 721         if (pos > 0)
 722         {
 723             if (islt)
 724             {
 725                 if (d == '0')
 726                     continue;
 727                 d--;
 728             }
 729             else
 730             {
 731                 if (d == '9')
 732                     continue;
 733                 d++;
 734             }
 735         }
 736
 737         strcpy(dst + dst_p, numstr);
 738         dst_p = strlen(dst) - pos - 1;
 739
 740         if (islt)
 741         {
 742             if (d != '0')
 743             {
 744                 dst[dst_p++] = '[';
 745                 dst[dst_p++] = '0';
 746                 dst[dst_p++] = '-';
 747                 dst[dst_p++] = d;
 748                 dst[dst_p++] = ']';
 749             }
 750             else
 751                 dst[dst_p++] = d;
 752         }
 753         else
 754         {
 755             if (d != '9')
 756             {
 757                 dst[dst_p++] = '[';
 758                 dst[dst_p++] = d;
 759                 dst[dst_p++] = '-';
 760                 dst[dst_p++] = '9';
 761                 dst[dst_p++] = ']';
 762             }
 763             else
 764                 dst[dst_p++] = d;
 765         }
 766         for (i = 0; i<pos; i++)
 767         {
 768             dst[dst_p++] = '[';
 769             dst[dst_p++] = '0';
 770             dst[dst_p++] = '-';
 771             dst[dst_p++] = '9';
 772             dst[dst_p++] = ']';
 773         }
 774         dst[dst_p++] = '|';
 775     }
 776     dst[dst_p] = '\0';
 777     if (islt)
 778     {
 779         /* match everything less than 10^(pos-1) */
 780         strcat(dst, "0*");
 781         for (i = 1; i<pos; i++)
 782             strcat(dst, "[0-9]?");
 783     }
 784     else
 785     {
 786         /* match everything greater than 10^pos */
 787         for (i = 0; i <= pos; i++)
 788             strcat(dst, "[0-9]");
 789         strcat(dst, "[0-9]*");
 790     }
 791     strcat(dst, "))");
 792 }
 793
 794 void string_rel_add_char(char **term_p, const char *src, int *indx)
 795 {
 796     if (src[*indx] == '\\')
 797         *(*term_p)++ = src[(*indx)++];
 798     *(*term_p)++ = src[(*indx)++];
 799 }
 800
 801 /*
 802  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 803  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 804  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 805  *              ([^-a].*|a[^-b].*|ab[c-].*)
 806  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 807  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 808  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 809  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 810  */
 811 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 812                            const char **term_sub, char *term_dict,
 813                            oid_value attributeSet,
 814                            int reg_type, int space_split, char *term_dst,
 815                            int *error_code)
 816 {
 817     AttrType relation;
 818     int relation_value;
 819     int i;
 820     char *term_tmp = term_dict + strlen(term_dict);
 821     char term_component[2*IT_MAX_WORD+20];
 822
 823     attr_init(&relation, zapt, 2);
 824     relation_value = attr_find(&relation, NULL);
 825
 826     *error_code = 0;
 827     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 828     switch (relation_value)
 829     {
 830     case 1:
 831         if (!term_100(zh->reg->zebra_maps, reg_type,
 832                       term_sub, term_component,
 833                       space_split, term_dst))
 834             return 0;
 835         yaz_log(log_level_rpn, "Relation <");
 836
 837         *term_tmp++ = '(';
 838         for (i = 0; term_component[i]; )
 839         {
 840             int j = 0;
 841
 842             if (i)
 843                 *term_tmp++ = '|';
 844             while (j < i)
 845                 string_rel_add_char(&term_tmp, term_component, &j);
 846
 847             *term_tmp++ = '[';
 848
 849             *term_tmp++ = '^';
 850             string_rel_add_char(&term_tmp, term_component, &i);
 851             *term_tmp++ = '-';
 852
 853             *term_tmp++ = ']';
 854             *term_tmp++ = '.';
 855             *term_tmp++ = '*';
 856
 857             if ((term_tmp - term_dict) > IT_MAX_WORD)
 858                 break;
 859         }
 860         *term_tmp++ = ')';
 861         *term_tmp = '\0';
 862         break;
 863     case 2:
 864         if (!term_100(zh->reg->zebra_maps, reg_type,
 865                       term_sub, term_component,
 866                       space_split, term_dst))
 867             return 0;
 868         yaz_log(log_level_rpn, "Relation <=");
 869
 870         *term_tmp++ = '(';
 871         for (i = 0; term_component[i]; )
 872         {
 873             int j = 0;
 874
 875             while (j < i)
 876                 string_rel_add_char(&term_tmp, term_component, &j);
 877             *term_tmp++ = '[';
 878
 879             *term_tmp++ = '^';
 880             string_rel_add_char(&term_tmp, term_component, &i);
 881             *term_tmp++ = '-';
 882
 883             *term_tmp++ = ']';
 884             *term_tmp++ = '.';
 885             *term_tmp++ = '*';
 886
 887             *term_tmp++ = '|';
 888
 889             if ((term_tmp - term_dict) > IT_MAX_WORD)
 890                 break;
 891         }
 892         for (i = 0; term_component[i]; )
 893             string_rel_add_char(&term_tmp, term_component, &i);
 894         *term_tmp++ = ')';
 895         *term_tmp = '\0';
 896         break;
 897     case 5:
 898         if (!term_100 (zh->reg->zebra_maps, reg_type,
 899                        term_sub, term_component, space_split, term_dst))
 900             return 0;
 901         yaz_log(log_level_rpn, "Relation >");
 902
 903         *term_tmp++ = '(';
 904         for (i = 0; term_component[i];)
 905         {
 906             int j = 0;
 907
 908             while (j < i)
 909                 string_rel_add_char(&term_tmp, term_component, &j);
 910             *term_tmp++ = '[';
 911
 912             *term_tmp++ = '^';
 913             *term_tmp++ = '-';
 914             string_rel_add_char(&term_tmp, term_component, &i);
 915
 916             *term_tmp++ = ']';
 917             *term_tmp++ = '.';
 918             *term_tmp++ = '*';
 919
 920             *term_tmp++ = '|';
 921
 922             if ((term_tmp - term_dict) > IT_MAX_WORD)
 923                 break;
 924         }
 925         for (i = 0; term_component[i];)
 926             string_rel_add_char(&term_tmp, term_component, &i);
 927         *term_tmp++ = '.';
 928         *term_tmp++ = '+';
 929         *term_tmp++ = ')';
 930         *term_tmp = '\0';
 931         break;
 932     case 4:
 933         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 934                       term_component, space_split, term_dst))
 935             return 0;
 936         yaz_log(log_level_rpn, "Relation >=");
 937
 938         *term_tmp++ = '(';
 939         for (i = 0; term_component[i];)
 940         {
 941             int j = 0;
 942
 943             if (i)
 944                 *term_tmp++ = '|';
 945             while (j < i)
 946                 string_rel_add_char(&term_tmp, term_component, &j);
 947             *term_tmp++ = '[';
 948
 949             if (term_component[i+1])
 950             {
 951                 *term_tmp++ = '^';
 952                 *term_tmp++ = '-';
 953                 string_rel_add_char(&term_tmp, term_component, &i);
 954             }
 955             else
 956             {
 957                 string_rel_add_char(&term_tmp, term_component, &i);
 958                 *term_tmp++ = '-';
 959             }
 960             *term_tmp++ = ']';
 961             *term_tmp++ = '.';
 962             *term_tmp++ = '*';
 963
 964             if ((term_tmp - term_dict) > IT_MAX_WORD)
 965                 break;
 966         }
 967         *term_tmp++ = ')';
 968         *term_tmp = '\0';
 969         break;
 970     case 3:
 971     case 102:
 972     case -1:
 973         yaz_log(log_level_rpn, "Relation =");
 974         if (!term_100(zh->reg->zebra_maps, reg_type, term_sub,
 975                       term_component, space_split, term_dst))
 976             return 0;
 977         strcat(term_tmp, "(");
 978         strcat(term_tmp, term_component);
 979         strcat(term_tmp, ")");
 980         break;
 981     default:
 982         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
 983         return 0;
 984     }
 985     return 1;
 986 }
 987
 988 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 989                              const char **term_sub,
 990                              oid_value attributeSet, NMEM stream,
 991                              struct grep_info *grep_info,
 992                              int reg_type, int complete_flag,
 993                              int num_bases, char **basenames,
 994                              char *term_dst, int xpath_use,
 995                              struct ord_list **ol);
 996
 997 static ZEBRA_RES term_limits_APT(ZebraHandle zh,
 998                                  Z_AttributesPlusTerm *zapt,
 999                                  zint *hits_limit_value,
1000                                  const char **term_ref_id_str,
1001                                  NMEM nmem)
1002 {
1003     AttrType term_ref_id_attr;
1004     AttrType hits_limit_attr;
1005     int term_ref_id_int;
1006
1007     attr_init(&hits_limit_attr, zapt, 9);
1008     *hits_limit_value  = attr_find(&hits_limit_attr, NULL);
1009
1010     attr_init(&term_ref_id_attr, zapt, 10);
1011     term_ref_id_int = attr_find_ex(&term_ref_id_attr, NULL, term_ref_id_str);
1012     if (term_ref_id_int >= 0)
1013     {
1014         char *res = nmem_malloc(nmem, 20);
1015         sprintf(res, "%d", term_ref_id_int);
1016         *term_ref_id_str = res;
1017     }
1018
1019     /* no limit given ? */
1020     if (*hits_limit_value == -1)
1021     {
1022         if (*term_ref_id_str)
1023         {
1024             /* use global if term_ref is present */
1025             *hits_limit_value = zh->approx_limit;
1026         }
1027         else
1028         {
1029             /* no counting if term_ref is not present */
1030             *hits_limit_value = 0;
1031         }
1032     }
1033     else if (*hits_limit_value == 0)
1034     {
1035         /* 0 is the same as global limit */
1036         *hits_limit_value = zh->approx_limit;
1037     }
1038     yaz_log(YLOG_DEBUG, "term_limits_APT ref_id=%s limit=" ZINT_FORMAT,
1039             *term_ref_id_str ? *term_ref_id_str : "none",
1040             *hits_limit_value);
1041     return ZEBRA_OK;
1042 }
1043
1044 static ZEBRA_RES term_trunc(ZebraHandle zh,
1045                             Z_AttributesPlusTerm *zapt,
1046                             const char **term_sub,
1047                             oid_value attributeSet, NMEM stream,
1048                             struct grep_info *grep_info,
1049                             int reg_type, int complete_flag,
1050                             int num_bases, char **basenames,
1051                             char *term_dst,
1052                             const char *rank_type, int xpath_use,
1053                             NMEM rset_nmem,
1054                             RSET *rset,
1055                             struct rset_key_control *kc)
1056 {
1057     ZEBRA_RES res;
1058     struct ord_list *ol;
1059     zint hits_limit_value;
1060     const char *term_ref_id_str = 0;
1061     *rset = 0;
1062
1063     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str,
1064                     stream);
1065     grep_info->isam_p_indx = 0;
1066     res = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
1067                       reg_type, complete_flag, num_bases, basenames,
1068                       term_dst, xpath_use, &ol);
1069     if (res != ZEBRA_OK)
1070         return res;
1071     if (!*term_sub)  /* no more terms ? */
1072         return res;
1073     yaz_log(log_level_rpn, "term: %s", term_dst);
1074     *rset = rset_trunc(zh, grep_info->isam_p_buf,
1075                        grep_info->isam_p_indx, term_dst,
1076                        strlen(term_dst), rank_type, 1 /* preserve pos */,
1077                        zapt->term->which, rset_nmem,
1078                        kc, kc->scope, ol, reg_type, hits_limit_value,
1079                        term_ref_id_str);
1080     if (!*rset)
1081         return ZEBRA_FAIL;
1082     return ZEBRA_OK;
1083 }
1084
1085 static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1086                              const char **term_sub,
1087                              oid_value attributeSet, NMEM stream,
1088                              struct grep_info *grep_info,
1089                              int reg_type, int complete_flag,
1090                              int num_bases, char **basenames,
1091                              char *term_dst, int xpath_use,
1092                              struct ord_list **ol)
1093 {
1094     char term_dict[2*IT_MAX_WORD+4000];
1095     int j, r, base_no;
1096     AttrType truncation;
1097     int truncation_value;
1098     AttrType use;
1099     int use_value;
1100     const char *use_string = 0;
1101     oid_value curAttributeSet = attributeSet;
1102     const char *termp;
1103     struct rpn_char_map_info rcmi;
1104     int space_split = complete_flag ? 0 : 1;
1105
1106     int bases_ok = 0;     /* no of databases with OK attribute */
1107
1108     *ol = ord_list_create(stream);
1109
1110     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1111     attr_init(&use, zapt, 1);
1112     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1113     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1114     attr_init(&truncation, zapt, 5);
1115     truncation_value = attr_find(&truncation, NULL);
1116     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1117
1118     if (use_value == -1)    /* no attribute - assumy "any" */
1119         use_value = 1016;
1120     for (base_no = 0; base_no < num_bases; base_no++)
1121     {
1122         int ord = -1;
1123         int attr_ok = 0;
1124         int regex_range = 0;
1125         int init_pos = 0;
1126         attent attp;
1127         data1_local_attribute id_xpath_attr;
1128         data1_local_attribute *local_attr;
1129         int max_pos, prefix_len = 0;
1130         int relation_error;
1131
1132         termp = *term_sub;
1133
1134         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1135         {
1136             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1137                            basenames[base_no]);
1138             return ZEBRA_FAIL;
1139         }
1140         if (xpath_use > 0 && use_value == -2)
1141         {
1142             /* xpath mode and we have a string attribute */
1143             attp.local_attributes = &id_xpath_attr;
1144             attp.attset_ordinal = VAL_IDXPATH;
1145             id_xpath_attr.next = 0;
1146
1147             use_value = xpath_use;  /* xpath_use as use-attribute now */
1148             id_xpath_attr.local = use_value;
1149         }
1150         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1151         {
1152             /* X-Path attribute, use numeric value directly */
1153             attp.local_attributes = &id_xpath_attr;
1154             attp.attset_ordinal = VAL_IDXPATH;
1155             id_xpath_attr.next = 0;
1156             id_xpath_attr.local = use_value;
1157         }
1158         else if (use_string &&
1159                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1160                                                      reg_type,
1161                                                      use_string)) >= 0)
1162         {
1163             /* we have a match for a raw string attribute */
1164             char ord_buf[32];
1165             int i, ord_len;
1166
1167             if (prefix_len)
1168                 term_dict[prefix_len++] = '|';
1169             else
1170                 term_dict[prefix_len++] = '(';
1171
1172             ord_len = key_SU_encode (ord, ord_buf);
1173             for (i = 0; i<ord_len; i++)
1174             {
1175                 term_dict[prefix_len++] = 1;
1176                 term_dict[prefix_len++] = ord_buf[i];
1177             }
1178             attp.local_attributes = 0;  /* no more attributes */
1179             *ol = ord_list_append(stream, *ol, ord);
1180         }
1181         else
1182         {
1183             /* lookup in the .att files . Allow string as well */
1184             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1185                                       use_string)))
1186             {
1187                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1188                         curAttributeSet, use_value, r);
1189                 if (r == -1)
1190                 {
1191                     /* set was found, but value wasn't defined */
1192                     if (use_string)
1193                         zebra_setError(zh,
1194                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1195                                        use_string);
1196                     else
1197                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1198                                             use_value);
1199                 }
1200                 else
1201                 {
1202                     int oid[OID_SIZE];
1203                     struct oident oident;
1204
1205                     oident.proto = PROTO_Z3950;
1206                     oident.oclass = CLASS_ATTSET;
1207                     oident.value = curAttributeSet;
1208                     oid_ent_to_oid (&oident, oid);
1209
1210                     zebra_setError(zh,
1211                                    YAZ_BIB1_UNSUPP_ATTRIBUTE_SET,
1212                                    oident.desc);
1213
1214                 }
1215                 continue;
1216             }
1217         }
1218         for (local_attr = attp.local_attributes; local_attr;
1219              local_attr = local_attr->next)
1220         {
1221             char ord_buf[32];
1222             int i, ord_len;
1223
1224             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1225                                               reg_type,
1226                                               attp.attset_ordinal,
1227                                               local_attr->local);
1228             if (ord < 0)
1229                 continue;
1230             *ol = ord_list_append(stream, *ol, ord);
1231             if (prefix_len)
1232                 term_dict[prefix_len++] = '|';
1233             else
1234                 term_dict[prefix_len++] = '(';
1235
1236             ord_len = key_SU_encode (ord, ord_buf);
1237             for (i = 0; i<ord_len; i++)
1238             {
1239                 term_dict[prefix_len++] = 1;
1240                 term_dict[prefix_len++] = ord_buf[i];
1241             }
1242             if (ord_len > init_pos)
1243                 init_pos = ord_len;
1244         }
1245         bases_ok++;
1246         if (prefix_len)
1247             attr_ok = 1;
1248
1249         term_dict[prefix_len++] = ')';
1250         term_dict[prefix_len] = '\0';
1251         j = prefix_len;
1252         switch (truncation_value)
1253         {
1254         case -1:         /* not specified */
1255         case 100:        /* do not truncate */
1256             if (!string_relation (zh, zapt, &termp, term_dict,
1257                                   attributeSet,
1258                                   reg_type, space_split, term_dst,
1259                                   &relation_error))
1260             {
1261                 if (relation_error)
1262                 {
1263                     zebra_setError(zh, relation_error, 0);
1264                     return ZEBRA_FAIL;
1265                 }
1266                 *term_sub = 0;
1267                 return ZEBRA_OK;
1268             }
1269             break;
1270         case 1:          /* right truncation */
1271             term_dict[j++] = '(';
1272             if (!term_100(zh->reg->zebra_maps, reg_type,
1273                           &termp, term_dict + j, space_split, term_dst))
1274             {
1275                 *term_sub = 0;
1276                 return ZEBRA_OK;
1277             }
1278             strcat(term_dict, ".*)");
1279             break;
1280         case 2:          /* keft truncation */
1281             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1282             if (!term_100(zh->reg->zebra_maps, reg_type,
1283                           &termp, term_dict + j, space_split, term_dst))
1284             {
1285                 *term_sub = 0;
1286                 return ZEBRA_OK;
1287             }
1288             strcat(term_dict, ")");
1289             break;
1290         case 3:          /* left&right truncation */
1291             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1292             if (!term_100(zh->reg->zebra_maps, reg_type,
1293                           &termp, term_dict + j, space_split, term_dst))
1294             {
1295                 *term_sub = 0;
1296                 return ZEBRA_OK;
1297             }
1298             strcat(term_dict, ".*)");
1299             break;
1300         case 101:        /* process # in term */
1301             term_dict[j++] = '(';
1302             if (!term_101(zh->reg->zebra_maps, reg_type,
1303                           &termp, term_dict + j, space_split, term_dst))
1304             {
1305                 *term_sub = 0;
1306                 return ZEBRA_OK;
1307             }
1308             strcat(term_dict, ")");
1309             break;
1310         case 102:        /* Regexp-1 */
1311             term_dict[j++] = '(';
1312             if (!term_102(zh->reg->zebra_maps, reg_type,
1313                           &termp, term_dict + j, space_split, term_dst))
1314             {
1315                 *term_sub = 0;
1316                 return ZEBRA_OK;
1317             }
1318             strcat(term_dict, ")");
1319             break;
1320         case 103:       /* Regexp-2 */
1321             regex_range = 1;
1322             term_dict[j++] = '(';
1323             if (!term_103(zh->reg->zebra_maps, reg_type,
1324                           &termp, term_dict + j, &regex_range,
1325                           space_split, term_dst))
1326             {
1327                 *term_sub = 0;
1328                 return ZEBRA_OK;
1329             }
1330             strcat(term_dict, ")");
1331             break;
1332         case 104:        /* process # and ! in term */
1333             term_dict[j++] = '(';
1334             if (!term_104(zh->reg->zebra_maps, reg_type,
1335                           &termp, term_dict + j, space_split, term_dst))
1336             {
1337                 *term_sub = 0;
1338                 return ZEBRA_OK;
1339             }
1340             strcat(term_dict, ")");
1341             break;
1342         case 105:        /* process * and ! in term */
1343             term_dict[j++] = '(';
1344             if (!term_105(zh->reg->zebra_maps, reg_type,
1345                           &termp, term_dict + j, space_split, term_dst, 1))
1346             {
1347                 *term_sub = 0;
1348                 return ZEBRA_OK;
1349             }
1350             strcat(term_dict, ")");
1351             break;
1352         case 106:        /* process * and ! in term */
1353             term_dict[j++] = '(';
1354             if (!term_105(zh->reg->zebra_maps, reg_type,
1355                           &termp, term_dict + j, space_split, term_dst, 0))
1356             {
1357                 *term_sub = 0;
1358                 return ZEBRA_OK;
1359             }
1360             strcat(term_dict, ")");
1361             break;
1362         default:
1363             zebra_setError_zint(zh,
1364                                 YAZ_BIB1_UNSUPP_TRUNCATION_ATTRIBUTE,
1365                                 truncation_value);
1366             return ZEBRA_FAIL;
1367         }
1368         if (attr_ok)
1369         {
1370             char buf[80];
1371             const char *input = term_dict + prefix_len;
1372             esc_str(buf, sizeof(buf), input, strlen(input));
1373         }
1374         if (attr_ok)
1375         {
1376             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1377             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1378                                  grep_info, &max_pos, init_pos,
1379                                  grep_handle);
1380             if (r)
1381                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1382         }
1383     }
1384     if (!bases_ok)
1385         return ZEBRA_FAIL;
1386     *term_sub = termp;
1387     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1388     return ZEBRA_OK;
1389 }
1390
1391
1392 /* convert APT search term to UTF8 */
1393 static ZEBRA_RES zapt_term_to_utf8(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1394                                    char *termz)
1395 {
1396     size_t sizez;
1397     Z_Term *term = zapt->term;
1398
1399     switch (term->which)
1400     {
1401     case Z_Term_general:
1402         if (zh->iconv_to_utf8 != 0)
1403         {
1404             char *inbuf = (char *) term->u.general->buf;
1405             size_t inleft = term->u.general->len;
1406             char *outbuf = termz;
1407             size_t outleft = IT_MAX_WORD-1;
1408             size_t ret;
1409
1410             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1411                         &outbuf, &outleft);
1412             if (ret == (size_t)(-1))
1413             {
1414                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1415                 zebra_setError(
1416                     zh,
1417                     YAZ_BIB1_QUERY_TERM_INCLUDES_CHARS_THAT_DO_NOT_TRANSLATE_INTO_,
1418                     0);
1419                 return ZEBRA_FAIL;
1420             }
1421             *outbuf = 0;
1422         }
1423         else
1424         {
1425             sizez = term->u.general->len;
1426             if (sizez > IT_MAX_WORD-1)
1427                 sizez = IT_MAX_WORD-1;
1428             memcpy (termz, term->u.general->buf, sizez);
1429             termz[sizez] = '\0';
1430         }
1431         break;
1432     case Z_Term_characterString:
1433         sizez = strlen(term->u.characterString);
1434         if (sizez > IT_MAX_WORD-1)
1435             sizez = IT_MAX_WORD-1;
1436         memcpy (termz, term->u.characterString, sizez);
1437         termz[sizez] = '\0';
1438         break;
1439     default:
1440         zebra_setError(zh, YAZ_BIB1_UNSUPP_CODED_VALUE_FOR_TERM, 0);
1441         return ZEBRA_FAIL;
1442     }
1443     return ZEBRA_OK;
1444 }
1445
1446 /* convert APT SCAN term to internal cmap */
1447 static ZEBRA_RES trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1448                                  char *termz, int reg_type)
1449 {
1450     char termz0[IT_MAX_WORD];
1451
1452     if (zapt_term_to_utf8(zh, zapt, termz0) == ZEBRA_FAIL)
1453         return ZEBRA_FAIL;    /* error */
1454     else
1455     {
1456         const char **map;
1457         const char *cp = (const char *) termz0;
1458         const char *cp_end = cp + strlen(cp);
1459         const char *src;
1460         int i = 0;
1461         const char *space_map = NULL;
1462         int len;
1463
1464         while ((len = (cp_end - cp)) > 0)
1465         {
1466             map = zebra_maps_input(zh->reg->zebra_maps, reg_type, &cp, len, 0);
1467             if (**map == *CHR_SPACE)
1468                 space_map = *map;
1469             else
1470             {
1471                 if (i && space_map)
1472                     for (src = space_map; *src; src++)
1473                         termz[i++] = *src;
1474                 space_map = NULL;
1475                 for (src = *map; *src; src++)
1476                     termz[i++] = *src;
1477             }
1478         }
1479         termz[i] = '\0';
1480     }
1481     return ZEBRA_OK;
1482 }
1483
1484 static void grep_info_delete(struct grep_info *grep_info)
1485 {
1486 #ifdef TERM_COUNT
1487     xfree(grep_info->term_no);
1488 #endif
1489     xfree(grep_info->isam_p_buf);
1490 }
1491
1492 static ZEBRA_RES grep_info_prepare(ZebraHandle zh,
1493                                    Z_AttributesPlusTerm *zapt,
1494                                    struct grep_info *grep_info,
1495                                    int reg_type)
1496 {
1497     AttrType termset;
1498     int termset_value_numeric;
1499     const char *termset_value_string;
1500
1501 #ifdef TERM_COUNT
1502     grep_info->term_no = 0;
1503 #endif
1504     grep_info->isam_p_size = 0;
1505     grep_info->isam_p_buf = NULL;
1506     grep_info->zh = zh;
1507     grep_info->reg_type = reg_type;
1508     grep_info->termset = 0;
1509
1510     if (!zapt)
1511         return ZEBRA_OK;
1512     attr_init(&termset, zapt, 8);
1513     termset_value_numeric =
1514         attr_find_ex(&termset, NULL, &termset_value_string);
1515     if (termset_value_numeric != -1)
1516     {
1517         char resname[32];
1518         const char *termset_name = 0;
1519         if (termset_value_numeric != -2)
1520         {
1521
1522             sprintf(resname, "%d", termset_value_numeric);
1523             termset_name = resname;
1524         }
1525         else
1526             termset_name = termset_value_string;
1527         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1528         grep_info->termset = resultSetAdd(zh, termset_name, 1);
1529         if (!grep_info->termset)
1530         {
1531             zebra_setError(zh, YAZ_BIB1_ILLEGAL_RESULT_SET_NAME, termset_name);
1532             return ZEBRA_FAIL;
1533         }
1534     }
1535     return ZEBRA_OK;
1536 }
1537
1538 /**
1539   \brief Create result set(s) for list of terms
1540   \param zh Zebra Handle
1541   \param termz term as used in query but converted to UTF-8
1542   \param attributeSet default attribute set
1543   \param stream memory for result
1544   \param reg_type register type ('w', 'p',..)
1545   \param complete_flag whether it's phrases or not
1546   \param rank_type term flags for ranking
1547   \param xpath_use use attribute for X-Path (-1 for no X-path)
1548   \param num_bases number of databases
1549   \param basenames array of databases
1550   \param rset_mem memory for result sets
1551   \param result_sets output result set for each term in list (output)
1552   \param number number of output result sets
1553   \param kc rset key control to be used for created result sets
1554 */
1555 static ZEBRA_RES term_list_trunc(ZebraHandle zh,
1556                                  Z_AttributesPlusTerm *zapt,
1557                                  const char *termz,
1558                                  oid_value attributeSet,
1559                                  NMEM stream,
1560                                  int reg_type, int complete_flag,
1561                                  const char *rank_type, int xpath_use,
1562                                  int num_bases, char **basenames,
1563                                  NMEM rset_nmem,
1564                                  RSET **result_sets, int *num_result_sets,
1565                                  struct rset_key_control *kc)
1566 {
1567     char term_dst[IT_MAX_WORD+1];
1568     struct grep_info grep_info;
1569     const char *termp = termz;
1570     int alloc_sets = 0;
1571
1572     *num_result_sets = 0;
1573     *term_dst = 0;
1574     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1575         return ZEBRA_FAIL;
1576     while(1)
1577     {
1578         ZEBRA_RES res;
1579
1580         if (alloc_sets == *num_result_sets)
1581         {
1582             int add = 10;
1583             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1584                                               sizeof(*rnew));
1585             if (alloc_sets)
1586                 memcpy(rnew, *result_sets, alloc_sets * sizeof(*rnew));
1587             alloc_sets = alloc_sets + add;
1588             *result_sets = rnew;
1589         }
1590         res = term_trunc(zh, zapt, &termp, attributeSet,
1591                          stream, &grep_info,
1592                          reg_type, complete_flag,
1593                          num_bases, basenames,
1594                          term_dst, rank_type,
1595                          xpath_use, rset_nmem,
1596                          &(*result_sets)[*num_result_sets],
1597                          kc);
1598         if (res != ZEBRA_OK)
1599         {
1600             int i;
1601             for (i = 0; i < *num_result_sets; i++)
1602                 rset_delete((*result_sets)[i]);
1603             grep_info_delete (&grep_info);
1604             return res;
1605         }
1606         if ((*result_sets)[*num_result_sets] == 0)
1607             break;
1608         (*num_result_sets)++;
1609     }
1610     grep_info_delete(&grep_info);
1611     return ZEBRA_OK;
1612 }
1613
1614 static ZEBRA_RES rpn_search_APT_phrase(ZebraHandle zh,
1615                                        Z_AttributesPlusTerm *zapt,
1616                                        const char *termz_org,
1617                                        oid_value attributeSet,
1618                                        NMEM stream,
1619                                        int reg_type, int complete_flag,
1620                                        const char *rank_type, int xpath_use,
1621                                        int num_bases, char **basenames,
1622                                        NMEM rset_nmem,
1623                                        RSET *rset,
1624                                        struct rset_key_control *kc)
1625 {
1626     RSET *result_sets = 0;
1627     int num_result_sets = 0;
1628     ZEBRA_RES res =
1629         term_list_trunc(zh, zapt, termz_org, attributeSet,
1630                         stream, reg_type, complete_flag,
1631                         rank_type, xpath_use,
1632                         num_bases, basenames,
1633                         rset_nmem,
1634                         &result_sets, &num_result_sets, kc);
1635     if (res != ZEBRA_OK)
1636         return res;
1637     if (num_result_sets == 0)
1638         *rset = rsnull_create (rset_nmem, kc, 0);
1639     else if (num_result_sets == 1)
1640         *rset = result_sets[0];
1641     else
1642         *rset = rsprox_create(rset_nmem, kc, kc->scope,
1643                               num_result_sets, result_sets,
1644                               1 /* ordered */, 0 /* exclusion */,
1645                               3 /* relation */, 1 /* distance */);
1646     if (!*rset)
1647         return ZEBRA_FAIL;
1648     return ZEBRA_OK;
1649 }
1650
1651 static ZEBRA_RES rpn_search_APT_or_list(ZebraHandle zh,
1652                                         Z_AttributesPlusTerm *zapt,
1653                                         const char *termz_org,
1654                                         oid_value attributeSet,
1655                                         NMEM stream,
1656                                         int reg_type, int complete_flag,
1657                                         const char *rank_type,
1658                                         int xpath_use,
1659                                         int num_bases, char **basenames,
1660                                         NMEM rset_nmem,
1661                                         RSET *rset,
1662                                         struct rset_key_control *kc)
1663 {
1664     RSET *result_sets = 0;
1665     int num_result_sets = 0;
1666     ZEBRA_RES res =
1667         term_list_trunc(zh, zapt, termz_org, attributeSet,
1668                         stream, reg_type, complete_flag,
1669                         rank_type, xpath_use,
1670                         num_bases, basenames,
1671                         rset_nmem,
1672                         &result_sets, &num_result_sets, kc);
1673     if (res != ZEBRA_OK)
1674         return res;
1675     if (num_result_sets == 0)
1676         *rset = rsnull_create (rset_nmem, kc, 0);
1677     else if (num_result_sets == 1)
1678         *rset = result_sets[0];
1679     else
1680         *rset = rsmulti_or_create(rset_nmem, kc, kc->scope, 0 /* termid */,
1681                                   num_result_sets, result_sets);
1682     if (!*rset)
1683         return ZEBRA_FAIL;
1684     return ZEBRA_OK;
1685 }
1686
1687 static ZEBRA_RES rpn_search_APT_and_list(ZebraHandle zh,
1688                                          Z_AttributesPlusTerm *zapt,
1689                                          const char *termz_org,
1690                                          oid_value attributeSet,
1691                                          NMEM stream,
1692                                          int reg_type, int complete_flag,
1693                                          const char *rank_type,
1694                                          int xpath_use,
1695                                          int num_bases, char **basenames,
1696                                          NMEM rset_nmem,
1697                                          RSET *rset,
1698                                          struct rset_key_control *kc)
1699 {
1700     RSET *result_sets = 0;
1701     int num_result_sets = 0;
1702     ZEBRA_RES res =
1703         term_list_trunc(zh, zapt, termz_org, attributeSet,
1704                         stream, reg_type, complete_flag,
1705                         rank_type, xpath_use,
1706                         num_bases, basenames,
1707                         rset_nmem,
1708                         &result_sets, &num_result_sets,
1709                         kc);
1710     if (res != ZEBRA_OK)
1711         return res;
1712     if (num_result_sets == 0)
1713         *rset = rsnull_create (rset_nmem, kc, 0);
1714     else if (num_result_sets == 1)
1715         *rset = result_sets[0];
1716     else
1717         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1718                                    num_result_sets, result_sets);
1719     if (!*rset)
1720         return ZEBRA_FAIL;
1721     return ZEBRA_OK;
1722 }
1723
1724 static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1725                             const char **term_sub,
1726                             char *term_dict,
1727                             oid_value attributeSet,
1728                             struct grep_info *grep_info,
1729                             int *max_pos,
1730                             int reg_type,
1731                             char *term_dst,
1732                             int *error_code)
1733 {
1734     AttrType relation;
1735     int relation_value;
1736     int term_value;
1737     int r;
1738     char *term_tmp = term_dict + strlen(term_dict);
1739
1740     *error_code = 0;
1741     attr_init(&relation, zapt, 2);
1742     relation_value = attr_find(&relation, NULL);
1743
1744     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1745
1746     if (!term_100(zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1747                   term_dst))
1748         return 0;
1749     term_value = atoi (term_tmp);
1750     switch (relation_value)
1751     {
1752     case 1:
1753         yaz_log(log_level_rpn, "Relation <");
1754         gen_regular_rel(term_tmp, term_value-1, 1);
1755         break;
1756     case 2:
1757         yaz_log(log_level_rpn, "Relation <=");
1758         gen_regular_rel(term_tmp, term_value, 1);
1759         break;
1760     case 4:
1761         yaz_log(log_level_rpn, "Relation >=");
1762         gen_regular_rel(term_tmp, term_value, 0);
1763         break;
1764     case 5:
1765         yaz_log(log_level_rpn, "Relation >");
1766         gen_regular_rel(term_tmp, term_value+1, 0);
1767         break;
1768     case -1:
1769     case 3:
1770         yaz_log(log_level_rpn, "Relation =");
1771         sprintf(term_tmp, "(0*%d)", term_value);
1772         break;
1773     default:
1774         *error_code = YAZ_BIB1_UNSUPP_RELATION_ATTRIBUTE;
1775         return 0;
1776     }
1777     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1778     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1779                           0, grep_handle);
1780     if (r)
1781         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1782     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1783     return 1;
1784 }
1785
1786 static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1787                               const char **term_sub,
1788                               oid_value attributeSet,
1789                               struct grep_info *grep_info,
1790                               int reg_type, int complete_flag,
1791                               int num_bases, char **basenames,
1792                               char *term_dst, int xpath_use, NMEM stream)
1793 {
1794     char term_dict[2*IT_MAX_WORD+2];
1795     int r, base_no;
1796     AttrType use;
1797     int use_value;
1798     const char *use_string = 0;
1799     oid_value curAttributeSet = attributeSet;
1800     const char *termp;
1801     struct rpn_char_map_info rcmi;
1802
1803     int bases_ok = 0;     /* no of databases with OK attribute */
1804
1805     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1806     attr_init(&use, zapt, 1);
1807     use_value = attr_find_ex(&use, &curAttributeSet, &use_string);
1808
1809     if (use_value == -1)
1810         use_value = 1016;
1811
1812     for (base_no = 0; base_no < num_bases; base_no++)
1813     {
1814         attent attp;
1815         data1_local_attribute id_xpath_attr;
1816         data1_local_attribute *local_attr;
1817         int max_pos, prefix_len = 0;
1818         int relation_error = 0;
1819
1820         termp = *term_sub;
1821         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1822         {
1823             use_value = xpath_use;
1824             attp.local_attributes = &id_xpath_attr;
1825             attp.attset_ordinal = VAL_IDXPATH;
1826             id_xpath_attr.next = 0;
1827             id_xpath_attr.local = use_value;
1828         }
1829         else if (curAttributeSet == VAL_IDXPATH)
1830         {
1831             attp.local_attributes = &id_xpath_attr;
1832             attp.attset_ordinal = VAL_IDXPATH;
1833             id_xpath_attr.next = 0;
1834             id_xpath_attr.local = use_value;
1835         }
1836         else
1837         {
1838             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1839                                       use_string)))
1840             {
1841                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1842                       curAttributeSet, use_value, r);
1843                 if (r == -1)
1844                 {
1845                     if (use_string)
1846                         zebra_setError(zh,
1847                                        YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1848                                        use_string);
1849                     else
1850                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
1851                                             use_value);
1852                 }
1853                 else
1854                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
1855                 continue;
1856             }
1857         }
1858         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1859         {
1860             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
1861                            basenames[base_no]);
1862             return ZEBRA_FAIL;
1863         }
1864         for (local_attr = attp.local_attributes; local_attr;
1865              local_attr = local_attr->next)
1866         {
1867             int ord;
1868             char ord_buf[32];
1869             int i, ord_len;
1870
1871             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1872                                               reg_type,
1873                                               attp.attset_ordinal,
1874                                               local_attr->local);
1875             if (ord < 0)
1876                 continue;
1877             if (prefix_len)
1878                 term_dict[prefix_len++] = '|';
1879             else
1880                 term_dict[prefix_len++] = '(';
1881
1882             ord_len = key_SU_encode (ord, ord_buf);
1883             for (i = 0; i<ord_len; i++)
1884             {
1885                 term_dict[prefix_len++] = 1;
1886                 term_dict[prefix_len++] = ord_buf[i];
1887             }
1888         }
1889         if (!prefix_len)
1890         {
1891             zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE, use_value);
1892             continue;
1893         }
1894         bases_ok++;
1895         term_dict[prefix_len++] = ')';
1896         term_dict[prefix_len] = '\0';
1897         if (!numeric_relation(zh, zapt, &termp, term_dict,
1898                               attributeSet, grep_info, &max_pos, reg_type,
1899                               term_dst, &relation_error))
1900         {
1901             if (relation_error)
1902             {
1903                 zebra_setError(zh, relation_error, 0);
1904                 return ZEBRA_FAIL;
1905             }
1906             *term_sub = 0;
1907             return ZEBRA_OK;
1908         }
1909     }
1910     if (!bases_ok)
1911         return ZEBRA_FAIL;
1912     *term_sub = termp;
1913     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1914     return ZEBRA_OK;
1915 }
1916
1917
1918 static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh,
1919                                         Z_AttributesPlusTerm *zapt,
1920                                         const char *termz,
1921                                         oid_value attributeSet,
1922                                         NMEM stream,
1923                                         int reg_type, int complete_flag,
1924                                         const char *rank_type, int xpath_use,
1925                                         int num_bases, char **basenames,
1926                                         NMEM rset_nmem,
1927                                         RSET *rset,
1928                                         struct rset_key_control *kc)
1929 {
1930     char term_dst[IT_MAX_WORD+1];
1931     const char *termp = termz;
1932     RSET *result_sets = 0;
1933     int num_result_sets = 0;
1934     ZEBRA_RES res;
1935     struct grep_info grep_info;
1936     int alloc_sets = 0;
1937     zint hits_limit_value;
1938     const char *term_ref_id_str = 0;
1939
1940     term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream);
1941
1942     yaz_log(log_level_rpn, "APT_numeric t='%s'", termz);
1943     if (grep_info_prepare(zh, zapt, &grep_info, reg_type) == ZEBRA_FAIL)
1944         return ZEBRA_FAIL;
1945     while (1)
1946     {
1947         if (alloc_sets == num_result_sets)
1948         {
1949             int add = 10;
1950             RSET *rnew = (RSET *) nmem_malloc(stream, (alloc_sets+add) *
1951                                               sizeof(*rnew));
1952             if (alloc_sets)
1953                 memcpy(rnew, result_sets, alloc_sets * sizeof(*rnew));
1954             alloc_sets = alloc_sets + add;
1955             result_sets = rnew;
1956         }
1957         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1958         grep_info.isam_p_indx = 0;
1959         res = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1960                            reg_type, complete_flag, num_bases, basenames,
1961                            term_dst, xpath_use,
1962                            stream);
1963         if (res == ZEBRA_FAIL || termp == 0)
1964             break;
1965         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1966         result_sets[num_result_sets] =
1967             rset_trunc(zh, grep_info.isam_p_buf,
1968                        grep_info.isam_p_indx, term_dst,
1969                        strlen(term_dst), rank_type,
1970                        0 /* preserve position */,
1971                        zapt->term->which, rset_nmem,
1972                        kc, kc->scope, 0, reg_type,
1973                        hits_limit_value,
1974                        term_ref_id_str);
1975         if (!result_sets[num_result_sets])
1976             break;
1977         num_result_sets++;
1978     }
1979     grep_info_delete(&grep_info);
1980     if (termp)
1981     {
1982         int i;
1983         for (i = 0; i<num_result_sets; i++)
1984             rset_delete(result_sets[i]);
1985         return ZEBRA_FAIL;
1986     }
1987     if (num_result_sets == 0)
1988         *rset = rsnull_create(rset_nmem, kc, 0);
1989     if (num_result_sets == 1)
1990         *rset = result_sets[0];
1991     else
1992         *rset = rsmulti_and_create(rset_nmem, kc, kc->scope,
1993                                    num_result_sets, result_sets);
1994     if (!*rset)
1995         return ZEBRA_FAIL;
1996     return ZEBRA_OK;
1997 }
1998
1999 static ZEBRA_RES rpn_search_APT_local(ZebraHandle zh,
2000                                       Z_AttributesPlusTerm *zapt,
2001                                       const char *termz,
2002                                       oid_value attributeSet,
2003                                       NMEM stream,
2004                                       const char *rank_type, NMEM rset_nmem,
2005                                       RSET *rset,
2006                                       struct rset_key_control *kc)
2007 {
2008     RSFD rsfd;
2009     struct it_key key;
2010     int sys;
2011     *rset = rstemp_create(rset_nmem, kc, kc->scope,
2012                           res_get (zh->res, "setTmpDir"),0 );
2013     rsfd = rset_open(*rset, RSETF_WRITE);
2014
2015     sys = atoi(termz);
2016     if (sys <= 0)
2017         sys = 1;
2018     key.mem[0] = sys;
2019     key.mem[1] = 1;
2020     key.len = 2;
2021     rset_write (rsfd, &key);
2022     rset_close (rsfd);
2023     return ZEBRA_OK;
2024 }
2025
2026 static ZEBRA_RES rpn_sort_spec(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2027                                oid_value attributeSet, NMEM stream,
2028                                Z_SortKeySpecList *sort_sequence,
2029                                const char *rank_type,
2030                                NMEM rset_nmem,
2031                                RSET *rset,
2032                                struct rset_key_control *kc)
2033 {
2034     int i;
2035     int sort_relation_value;
2036     AttrType sort_relation_type;
2037     Z_SortKeySpec *sks;
2038     Z_SortKey *sk;
2039     int oid[OID_SIZE];
2040     oident oe;
2041     char termz[20];
2042
2043     attr_init(&sort_relation_type, zapt, 7);
2044     sort_relation_value = attr_find(&sort_relation_type, &attributeSet);
2045
2046     if (!sort_sequence->specs)
2047     {
2048         sort_sequence->num_specs = 10;
2049         sort_sequence->specs = (Z_SortKeySpec **)
2050             nmem_malloc(stream, sort_sequence->num_specs *
2051                          sizeof(*sort_sequence->specs));
2052         for (i = 0; i<sort_sequence->num_specs; i++)
2053             sort_sequence->specs[i] = 0;
2054     }
2055     if (zapt->term->which != Z_Term_general)
2056         i = 0;
2057     else
2058         i = atoi_n ((char *) zapt->term->u.general->buf,
2059                     zapt->term->u.general->len);
2060     if (i >= sort_sequence->num_specs)
2061         i = 0;
2062     sprintf(termz, "%d", i);
2063
2064     oe.proto = PROTO_Z3950;
2065     oe.oclass = CLASS_ATTSET;
2066     oe.value = attributeSet;
2067     if (!oid_ent_to_oid (&oe, oid))
2068         return ZEBRA_FAIL;
2069
2070     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
2071     sks->sortElement = (Z_SortElement *)
2072         nmem_malloc(stream, sizeof(*sks->sortElement));
2073     sks->sortElement->which = Z_SortElement_generic;
2074     sk = sks->sortElement->u.generic = (Z_SortKey *)
2075         nmem_malloc(stream, sizeof(*sk));
2076     sk->which = Z_SortKey_sortAttributes;
2077     sk->u.sortAttributes = (Z_SortAttributes *)
2078         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
2079
2080     sk->u.sortAttributes->id = oid;
2081     sk->u.sortAttributes->list = zapt->attributes;
2082
2083     sks->sortRelation = (int *)
2084         nmem_malloc(stream, sizeof(*sks->sortRelation));
2085     if (sort_relation_value == 1)
2086         *sks->sortRelation = Z_SortKeySpec_ascending;
2087     else if (sort_relation_value == 2)
2088         *sks->sortRelation = Z_SortKeySpec_descending;
2089     else
2090         *sks->sortRelation = Z_SortKeySpec_ascending;
2091
2092     sks->caseSensitivity = (int *)
2093         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
2094     *sks->caseSensitivity = 0;
2095
2096     sks->which = Z_SortKeySpec_null;
2097     sks->u.null = odr_nullval ();
2098     sort_sequence->specs[i] = sks;
2099     *rset = rsnull_create (rset_nmem, kc, 0);
2100     return ZEBRA_OK;
2101 }
2102
2103
2104 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2105                        oid_value attributeSet,
2106                        struct xpath_location_step *xpath, int max, NMEM mem)
2107 {
2108     oid_value curAttributeSet = attributeSet;
2109     AttrType use;
2110     const char *use_string = 0;
2111
2112     attr_init(&use, zapt, 1);
2113     attr_find_ex(&use, &curAttributeSet, &use_string);
2114
2115     if (!use_string || *use_string != '/')
2116         return -1;
2117
2118     return zebra_parse_xpath_str(use_string, xpath, max, mem);
2119 }
2120
2121
2122
2123 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
2124                         int reg_type, const char *term, int use,
2125                         oid_value curAttributeSet, NMEM rset_nmem,
2126                         struct rset_key_control *kc)
2127 {
2128     RSET rset;
2129     struct grep_info grep_info;
2130     char term_dict[2048];
2131     char ord_buf[32];
2132     int prefix_len = 0;
2133     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_type,
2134                                           curAttributeSet, use);
2135     int ord_len, i, r, max_pos;
2136     int term_type = Z_Term_characterString;
2137     const char *flags = "void";
2138
2139     if (grep_info_prepare(zh, 0 /* zapt */, &grep_info, '0') == ZEBRA_FAIL)
2140         return rsnull_create(rset_nmem, kc, 0);
2141
2142     if (ord < 0)
2143         return rsnull_create(rset_nmem, kc, 0);
2144     if (prefix_len)
2145         term_dict[prefix_len++] = '|';
2146     else
2147         term_dict[prefix_len++] = '(';
2148
2149     ord_len = key_SU_encode (ord, ord_buf);
2150     for (i = 0; i<ord_len; i++)
2151     {
2152         term_dict[prefix_len++] = 1;
2153         term_dict[prefix_len++] = ord_buf[i];
2154     }
2155     term_dict[prefix_len++] = ')';
2156     strcpy(term_dict+prefix_len, term);
2157
2158     grep_info.isam_p_indx = 0;
2159     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
2160                           &grep_info, &max_pos, 0, grep_handle);
2161     yaz_log(YLOG_DEBUG, "%s %d positions", term,
2162              grep_info.isam_p_indx);
2163     rset = rset_trunc(zh, grep_info.isam_p_buf,
2164                       grep_info.isam_p_indx, term, strlen(term),
2165                       flags, 1, term_type,rset_nmem,
2166                       kc, kc->scope, 0, reg_type, 0 /* hits_limit */,
2167                       0 /* term_ref_id_str */);
2168     grep_info_delete(&grep_info);
2169     return rset;
2170 }
2171
2172 static
2173 ZEBRA_RES rpn_search_xpath(ZebraHandle zh,
2174                            oid_value attributeSet,
2175                            int num_bases, char **basenames,
2176                            NMEM stream, const char *rank_type, RSET rset,
2177                            int xpath_len, struct xpath_location_step *xpath,
2178                            NMEM rset_nmem,
2179                            RSET *rset_out,
2180                            struct rset_key_control *kc)
2181 {
2182     oid_value curAttributeSet = attributeSet;
2183     int base_no;
2184     int i;
2185
2186     if (xpath_len < 0)
2187     {
2188         *rset_out = rset;
2189         return ZEBRA_OK;
2190     }
2191
2192     yaz_log(YLOG_DEBUG, "xpath len=%d", xpath_len);
2193     for (i = 0; i<xpath_len; i++)
2194     {
2195         yaz_log(log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2196
2197     }
2198
2199     curAttributeSet = VAL_IDXPATH;
2200
2201     /*
2202       //a    ->    a/.*
2203       //a/b  ->    b/a/.*
2204       /a     ->    a/
2205       /a/b   ->    b/a/
2206
2207       /      ->    none
2208
2209    a[@attr = value]/b[@other = othervalue]
2210
2211  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2212  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2213  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2214  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2215  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2216  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2217
2218     */
2219
2220     dict_grep_cmap (zh->reg->dict, 0, 0);
2221
2222     for (base_no = 0; base_no < num_bases; base_no++)
2223     {
2224         int level = xpath_len;
2225         int first_path = 1;
2226
2227         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2228         {
2229             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2230                            basenames[base_no]);
2231             *rset_out = rset;
2232             return ZEBRA_FAIL;
2233         }
2234         while (--level >= 0)
2235         {
2236             char xpath_rev[128];
2237             int i, len;
2238             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2239
2240             *xpath_rev = 0;
2241             len = 0;
2242             for (i = level; i >= 1; --i)
2243             {
2244                 const char *cp = xpath[i].part;
2245                 if (*cp)
2246                 {
2247                     for (;*cp; cp++)
2248                         if (*cp == '*')
2249                         {
2250                             memcpy (xpath_rev + len, "[^/]*", 5);
2251                             len += 5;
2252                         }
2253                         else if (*cp == ' ')
2254                         {
2255
2256                             xpath_rev[len++] = 1;
2257                             xpath_rev[len++] = ' ';
2258                         }
2259
2260                         else
2261                             xpath_rev[len++] = *cp;
2262                     xpath_rev[len++] = '/';
2263                 }
2264                 else if (i == 1)  /* // case */
2265                 {
2266                     xpath_rev[len++] = '.';
2267                     xpath_rev[len++] = '*';
2268                 }
2269             }
2270             xpath_rev[len] = 0;
2271
2272             if (xpath[level].predicate &&
2273                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2274                 xpath[level].predicate->u.relation.name[0])
2275             {
2276                 WRBUF wbuf = wrbuf_alloc();
2277                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2278                 if (xpath[level].predicate->u.relation.value)
2279                 {
2280                     const char *cp = xpath[level].predicate->u.relation.value;
2281                     wrbuf_putc(wbuf, '=');
2282
2283                     while (*cp)
2284                     {
2285                         if (strchr(REGEX_CHARS, *cp))
2286                             wrbuf_putc(wbuf, '\\');
2287                         wrbuf_putc(wbuf, *cp);
2288                         cp++;
2289                     }
2290                 }
2291                 wrbuf_puts(wbuf, "");
2292                 rset_attr = xpath_trunc(
2293                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2294                     curAttributeSet, rset_nmem, kc);
2295                 wrbuf_free(wbuf, 1);
2296             }
2297             else
2298             {
2299                 if (!first_path)
2300                     continue;
2301             }
2302             yaz_log(log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2303             if (strlen(xpath_rev))
2304             {
2305                 rset_start_tag = xpath_trunc(zh, stream, '0',
2306                         xpath_rev, 1, curAttributeSet, rset_nmem, kc);
2307
2308                 rset_end_tag = xpath_trunc(zh, stream, '0',
2309                         xpath_rev, 2, curAttributeSet, rset_nmem, kc);
2310
2311                 rset = rsbetween_create(rset_nmem, kc, kc->scope,
2312                                         rset_start_tag, rset,
2313                                         rset_end_tag, rset_attr);
2314             }
2315             first_path = 0;
2316         }
2317     }
2318     *rset_out = rset;
2319     return ZEBRA_OK;
2320 }
2321
2322 static ZEBRA_RES rpn_search_APT(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2323                                 oid_value attributeSet, NMEM stream,
2324                                 Z_SortKeySpecList *sort_sequence,
2325                                 int num_bases, char **basenames,
2326                                 NMEM rset_nmem,
2327                                 RSET *rset,
2328                                 struct rset_key_control *kc)
2329 {
2330     ZEBRA_RES res = ZEBRA_OK;
2331     unsigned reg_id;
2332     char *search_type = NULL;
2333     char rank_type[128];
2334     int complete_flag;
2335     int sort_flag;
2336     char termz[IT_MAX_WORD+1];
2337     int xpath_len;
2338     int xpath_use = 0;
2339     struct xpath_location_step xpath[10];
2340
2341     if (!log_level_set)
2342     {
2343         log_level_rpn = yaz_log_module_level("rpn");
2344         log_level_set = 1;
2345     }
2346     zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2347                     rank_type, &complete_flag, &sort_flag);
2348
2349     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2350     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2351     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2352     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2353
2354     if (zapt_term_to_utf8(zh, zapt, termz) == ZEBRA_FAIL)
2355         return ZEBRA_FAIL;
2356
2357     if (sort_flag)
2358         return rpn_sort_spec(zh, zapt, attributeSet, stream, sort_sequence,
2359                              rank_type, rset_nmem, rset, kc);
2360     /* consider if an X-Path query is used */
2361     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2362     if (xpath_len >= 0)
2363     {
2364         xpath_use = 1016;  /* searching for element by default */
2365         if (xpath[xpath_len-1].part[0] == '@')
2366             xpath_use = 1015;  /* last step an attribute .. */
2367     }
2368
2369     /* search using one of the various search type strategies
2370        termz is our UTF-8 search term
2371        attributeSet is top-level default attribute set
2372        stream is ODR for search
2373        reg_id is the register type
2374        complete_flag is 1 for complete subfield, 0 for incomplete
2375        xpath_use is use-attribute to be used for X-Path search, 0 for none
2376     */
2377     if (!strcmp(search_type, "phrase"))
2378     {
2379         res = rpn_search_APT_phrase(zh, zapt, termz, attributeSet, stream,
2380                                     reg_id, complete_flag, rank_type,
2381                                     xpath_use,
2382                                     num_bases, basenames, rset_nmem,
2383                                     rset, kc);
2384     }
2385     else if (!strcmp(search_type, "and-list"))
2386     {
2387         res = rpn_search_APT_and_list(zh, zapt, termz, attributeSet, stream,
2388                                       reg_id, complete_flag, rank_type,
2389                                       xpath_use,
2390                                       num_bases, basenames, rset_nmem,
2391                                       rset, kc);
2392     }
2393     else if (!strcmp(search_type, "or-list"))
2394     {
2395         res = rpn_search_APT_or_list(zh, zapt, termz, attributeSet, stream,
2396                                      reg_id, complete_flag, rank_type,
2397                                      xpath_use,
2398                                      num_bases, basenames, rset_nmem,
2399                                      rset, kc);
2400     }
2401     else if (!strcmp(search_type, "local"))
2402     {
2403         res = rpn_search_APT_local(zh, zapt, termz, attributeSet, stream,
2404                                    rank_type, rset_nmem, rset, kc);
2405     }
2406     else if (!strcmp(search_type, "numeric"))
2407     {
2408         res = rpn_search_APT_numeric(zh, zapt, termz, attributeSet, stream,
2409                                      reg_id, complete_flag, rank_type,
2410                                      xpath_use,
2411                                      num_bases, basenames, rset_nmem,
2412                                      rset, kc);
2413     }
2414     else
2415     {
2416         zebra_setError(zh, YAZ_BIB1_UNSUPP_STRUCTURE_ATTRIBUTE, 0);
2417         res = ZEBRA_FAIL;
2418     }
2419     if (res != ZEBRA_OK)
2420         return res;
2421     if (!*rset)
2422         return ZEBRA_FAIL;
2423     return rpn_search_xpath(zh, attributeSet, num_bases, basenames,
2424                             stream, rank_type, *rset,
2425                             xpath_len, xpath, rset_nmem, rset, kc);
2426 }
2427
2428 static ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2429                                       oid_value attributeSet,
2430                                       NMEM stream, NMEM rset_nmem,
2431                                       Z_SortKeySpecList *sort_sequence,
2432                                       int num_bases, char **basenames,
2433                                       RSET **result_sets, int *num_result_sets,
2434                                       Z_Operator *parent_op,
2435                                       struct rset_key_control *kc);
2436
2437 ZEBRA_RES rpn_search_top(ZebraHandle zh, Z_RPNStructure *zs,
2438                          oid_value attributeSet,
2439                          NMEM stream, NMEM rset_nmem,
2440                          Z_SortKeySpecList *sort_sequence,
2441                          int num_bases, char **basenames,
2442                          RSET *result_set)
2443 {
2444     RSET *result_sets = 0;
2445     int num_result_sets = 0;
2446     ZEBRA_RES res;
2447     struct rset_key_control *kc = zebra_key_control_create(zh);
2448
2449     res = rpn_search_structure(zh, zs, attributeSet,
2450                                stream, rset_nmem,
2451                                sort_sequence,
2452                                num_bases, basenames,
2453                                &result_sets, &num_result_sets,
2454                                0 /* no parent op */,
2455                                kc);
2456     if (res != ZEBRA_OK)
2457     {
2458         int i;
2459         for (i = 0; i<num_result_sets; i++)
2460             rset_delete(result_sets[i]);
2461         *result_set = 0;
2462     }
2463     else
2464     {
2465         assert(num_result_sets == 1);
2466         assert(result_sets);
2467         assert(*result_sets);
2468         *result_set = *result_sets;
2469     }
2470     (*kc->dec)(kc);
2471     return res;
2472 }
2473
2474 ZEBRA_RES rpn_search_structure(ZebraHandle zh, Z_RPNStructure *zs,
2475                                oid_value attributeSet,
2476                                NMEM stream, NMEM rset_nmem,
2477                                Z_SortKeySpecList *sort_sequence,
2478                                int num_bases, char **basenames,
2479                                RSET **result_sets, int *num_result_sets,
2480                                Z_Operator *parent_op,
2481                                struct rset_key_control *kc)
2482 {
2483     *num_result_sets = 0;
2484     if (zs->which == Z_RPNStructure_complex)
2485     {
2486         ZEBRA_RES res;
2487         Z_Operator *zop = zs->u.complex->roperator;
2488         RSET *result_sets_l = 0;
2489         int num_result_sets_l = 0;
2490         RSET *result_sets_r = 0;
2491         int num_result_sets_r = 0;
2492
2493         res = rpn_search_structure(zh, zs->u.complex->s1,
2494                                    attributeSet, stream, rset_nmem,
2495                                    sort_sequence,
2496                                    num_bases, basenames,
2497                                    &result_sets_l, &num_result_sets_l,
2498                                    zop, kc);
2499         if (res != ZEBRA_OK)
2500         {
2501             int i;
2502             for (i = 0; i<num_result_sets_l; i++)
2503                 rset_delete(result_sets_l[i]);
2504             return res;
2505         }
2506         res = rpn_search_structure(zh, zs->u.complex->s2,
2507                                    attributeSet, stream, rset_nmem,
2508                                    sort_sequence,
2509                                    num_bases, basenames,
2510                                    &result_sets_r, &num_result_sets_r,
2511                                    zop, kc);
2512         if (res != ZEBRA_OK)
2513         {
2514             int i;
2515             for (i = 0; i<num_result_sets_l; i++)
2516                 rset_delete(result_sets_l[i]);
2517             for (i = 0; i<num_result_sets_r; i++)
2518                 rset_delete(result_sets_r[i]);
2519             return res;
2520         }
2521
2522         /* make a new list of result for all children */
2523         *num_result_sets = num_result_sets_l + num_result_sets_r;
2524         *result_sets = nmem_malloc(stream, *num_result_sets *
2525                                    sizeof(**result_sets));
2526         memcpy(*result_sets, result_sets_l,
2527                num_result_sets_l * sizeof(**result_sets));
2528         memcpy(*result_sets + num_result_sets_l, result_sets_r,
2529                num_result_sets_r * sizeof(**result_sets));
2530
2531         if (!parent_op || parent_op->which != zop->which
2532             || (zop->which != Z_Operator_and &&
2533                 zop->which != Z_Operator_or))
2534         {
2535             /* parent node different from this one (or non-present) */
2536             /* we must combine result sets now */
2537             RSET rset;
2538             switch (zop->which)
2539             {
2540             case Z_Operator_and:
2541                 rset = rsmulti_and_create(rset_nmem, kc,
2542                                           kc->scope,
2543                                           *num_result_sets, *result_sets);
2544                 break;
2545             case Z_Operator_or:
2546                 rset = rsmulti_or_create(rset_nmem, kc,
2547                                          kc->scope, 0, /* termid */
2548                                          *num_result_sets, *result_sets);
2549                 break;
2550             case Z_Operator_and_not:
2551                 rset = rsbool_create_not(rset_nmem, kc,
2552                                          kc->scope,
2553                                          (*result_sets)[0],
2554                                          (*result_sets)[1]);
2555                 break;
2556             case Z_Operator_prox:
2557                 if (zop->u.prox->which != Z_ProximityOperator_known)
2558                 {
2559                     zebra_setError(zh,
2560                                    YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2561                                    0);
2562                     return ZEBRA_FAIL;
2563                 }
2564                 if (*zop->u.prox->u.known != Z_ProxUnit_word)
2565                 {
2566                     zebra_setError_zint(zh,
2567                                         YAZ_BIB1_UNSUPP_PROX_UNIT_CODE,
2568                                         *zop->u.prox->u.known);
2569                     return ZEBRA_FAIL;
2570                 }
2571                 else
2572                 {
2573                     rset = rsprox_create(rset_nmem, kc,
2574                                          kc->scope,
2575                                          *num_result_sets, *result_sets,
2576                                          *zop->u.prox->ordered,
2577                                          (!zop->u.prox->exclusion ?
2578                                           0 : *zop->u.prox->exclusion),
2579                                          *zop->u.prox->relationType,
2580                                          *zop->u.prox->distance );
2581                 }
2582                 break;
2583             default:
2584                 zebra_setError(zh, YAZ_BIB1_OPERATOR_UNSUPP, 0);
2585                 return ZEBRA_FAIL;
2586             }
2587             *num_result_sets = 1;
2588             *result_sets = nmem_malloc(stream, *num_result_sets *
2589                                        sizeof(**result_sets));
2590             (*result_sets)[0] = rset;
2591         }
2592     }
2593     else if (zs->which == Z_RPNStructure_simple)
2594     {
2595         RSET rset;
2596         ZEBRA_RES res;
2597
2598         if (zs->u.simple->which == Z_Operand_APT)
2599         {
2600             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2601             res = rpn_search_APT(zh, zs->u.simple->u.attributesPlusTerm,
2602                                  attributeSet, stream, sort_sequence,
2603                                  num_bases, basenames, rset_nmem, &rset,
2604                                  kc);
2605             if (res != ZEBRA_OK)
2606                 return res;
2607         }
2608         else if (zs->u.simple->which == Z_Operand_resultSetId)
2609         {
2610             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2611             rset = resultSetRef(zh, zs->u.simple->u.resultSetId);
2612             if (!rset)
2613             {
2614                 zebra_setError(zh,
2615                                YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
2616                                zs->u.simple->u.resultSetId);
2617                 return ZEBRA_FAIL;
2618             }
2619             rset_dup(rset);
2620         }
2621         else
2622         {
2623             zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2624             return ZEBRA_FAIL;
2625         }
2626         *num_result_sets = 1;
2627         *result_sets = nmem_malloc(stream, *num_result_sets *
2628                                    sizeof(**result_sets));
2629         (*result_sets)[0] = rset;
2630     }
2631     else
2632     {
2633         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, 0);
2634         return ZEBRA_FAIL;
2635     }
2636     return ZEBRA_OK;
2637 }
2638
2639 struct scan_info_entry {
2640     char *term;
2641     ISAM_P isam_p;
2642 };
2643
2644 struct scan_info {
2645     struct scan_info_entry *list;
2646     ODR odr;
2647     int before, after;
2648     char prefix[20];
2649 };
2650
2651 static int scan_handle (char *name, const char *info, int pos, void *client)
2652 {
2653     int len_prefix, idx;
2654     struct scan_info *scan_info = (struct scan_info *) client;
2655
2656     len_prefix = strlen(scan_info->prefix);
2657     if (memcmp (name, scan_info->prefix, len_prefix))
2658         return 1;
2659     if (pos > 0)
2660         idx = scan_info->after - pos + scan_info->before;
2661     else
2662         idx = - pos - 1;
2663
2664     if (idx < 0)
2665         return 0;
2666     scan_info->list[idx].term = (char *)
2667         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2668     strcpy(scan_info->list[idx].term, name + len_prefix);
2669     assert (*info == sizeof(ISAM_P));
2670     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAM_P));
2671     return 0;
2672 }
2673
2674 void zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, int reg_type,
2675                               char **dst, const char *src)
2676 {
2677     char term_src[IT_MAX_WORD];
2678     char term_dst[IT_MAX_WORD];
2679
2680     zebra_term_untrans (zh, reg_type, term_src, src);
2681
2682     if (zh->iconv_from_utf8 != 0)
2683     {
2684         int len;
2685         char *inbuf = term_src;
2686         size_t inleft = strlen(term_src);
2687         char *outbuf = term_dst;
2688         size_t outleft = sizeof(term_dst)-1;
2689         size_t ret;
2690
2691         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2692                          &outbuf, &outleft);
2693         if (ret == (size_t)(-1))
2694             len = 0;
2695         else
2696             len = outbuf - term_dst;
2697         *dst = nmem_malloc(stream, len + 1);
2698         if (len > 0)
2699             memcpy (*dst, term_dst, len);
2700         (*dst)[len] = '\0';
2701     }
2702     else
2703         *dst = nmem_strdup(stream, term_src);
2704 }
2705
2706 static void count_set(ZebraHandle zh, RSET rset, zint *count)
2707 {
2708     zint psysno = 0;
2709     struct it_key key;
2710     RSFD rfd;
2711
2712     yaz_log(YLOG_DEBUG, "count_set");
2713
2714     rset->hits_limit = zh->approx_limit;
2715
2716     *count = 0;
2717     rfd = rset_open(rset, RSETF_READ);
2718     while (rset_read(rfd, &key,0 /* never mind terms */))
2719     {
2720         if (key.mem[0] != psysno)
2721         {
2722             psysno = key.mem[0];
2723             if (rfd->counted_items >= rset->hits_limit)
2724                 break;
2725         }
2726     }
2727     rset_close (rfd);
2728     *count = rset->hits_count;
2729 }
2730
2731 ZEBRA_RES rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2732                    oid_value attributeset,
2733                    int num_bases, char **basenames,
2734                    int *position, int *num_entries, ZebraScanEntry **list,
2735                    int *is_partial, RSET limit_set, int return_zero)
2736 {
2737     int i;
2738     int pos = *position;
2739     int num = *num_entries;
2740     int before;
2741     int after;
2742     int base_no;
2743     char termz[IT_MAX_WORD+20];
2744     AttrType use;
2745     int use_value;
2746     const char *use_string = 0;
2747     struct scan_info *scan_info_array;
2748     ZebraScanEntry *glist;
2749     int ords[32], ord_no = 0;
2750     int ptr[32];
2751
2752     int bases_ok = 0;     /* no of databases with OK attribute */
2753     int errCode = 0;      /* err code (if any is not OK) */
2754     char *errString = 0;  /* addinfo */
2755
2756     unsigned reg_id;
2757     char *search_type = NULL;
2758     char rank_type[128];
2759     int complete_flag;
2760     int sort_flag;
2761     NMEM rset_nmem = NULL;
2762     struct rset_key_control *kc = 0;
2763
2764     *list = 0;
2765     *is_partial = 0;
2766
2767     if (attributeset == VAL_NONE)
2768         attributeset = VAL_BIB1;
2769
2770     if (!limit_set)
2771     {
2772         AttrType termset;
2773         int termset_value_numeric;
2774         const char *termset_value_string;
2775         attr_init(&termset, zapt, 8);
2776         termset_value_numeric =
2777             attr_find_ex(&termset, NULL, &termset_value_string);
2778         if (termset_value_numeric != -1)
2779         {
2780             char resname[32];
2781             const char *termset_name = 0;
2782
2783             if (termset_value_numeric != -2)
2784             {
2785
2786                 sprintf(resname, "%d", termset_value_numeric);
2787                 termset_name = resname;
2788             }
2789             else
2790                 termset_name = termset_value_string;
2791
2792             limit_set = resultSetRef (zh, termset_name);
2793         }
2794     }
2795
2796     yaz_log(YLOG_DEBUG, "position = %d, num = %d set=%d",
2797             pos, num, attributeset);
2798
2799     attr_init(&use, zapt, 1);
2800     use_value = attr_find_ex(&use, &attributeset, &use_string);
2801
2802     if (zebra_maps_attr(zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2803                         rank_type, &complete_flag, &sort_flag))
2804     {
2805         *num_entries = 0;
2806         zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_TYPE, 0);
2807         return ZEBRA_FAIL;
2808     }
2809     yaz_log(YLOG_DEBUG, "use_value = %d", use_value);
2810
2811     if (use_value == -1)
2812         use_value = 1016;
2813     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2814     {
2815         data1_local_attribute *local_attr;
2816         attent attp;
2817         int ord;
2818
2819         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2820         {
2821             zebra_setError(zh, YAZ_BIB1_DATABASE_UNAVAILABLE,
2822                            basenames[base_no]);
2823             *num_entries = 0;
2824             return ZEBRA_FAIL;
2825         }
2826
2827         if (use_string &&
2828             (ord = zebraExplain_lookup_attr_str(zh->reg->zei, reg_id,
2829                                                 use_string)) >= 0)
2830         {
2831             /* we have a match for a raw string attribute */
2832             if (ord > 0)
2833                 ords[ord_no++] = ord;
2834             attp.local_attributes = 0;  /* no more attributes */
2835         }
2836         else
2837         {
2838             int r;
2839
2840             if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2841                                       use_string)))
2842             {
2843                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2844                         attributeset, use_value);
2845                 if (r == -1)
2846                 {
2847                     errCode = YAZ_BIB1_UNSUPP_USE_ATTRIBUTE;
2848                     if (use_string)
2849                         zebra_setError(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2850                                        use_string);
2851                     else
2852                         zebra_setError_zint(zh, YAZ_BIB1_UNSUPP_USE_ATTRIBUTE,
2853                                             use_value);
2854                 }
2855                 else
2856                 {
2857                     zebra_setError(zh, YAZ_BIB1_UNSUPP_ATTRIBUTE_SET, 0);
2858                 }
2859                 continue;
2860             }
2861         }
2862         bases_ok++;
2863         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2864              local_attr = local_attr->next)
2865         {
2866             ord = zebraExplain_lookup_attr_su(zh->reg->zei, reg_id,
2867                                               attp.attset_ordinal,
2868                                               local_attr->local);
2869             if (ord > 0)
2870                 ords[ord_no++] = ord;
2871         }
2872     }
2873     if (!bases_ok && errCode)
2874     {
2875         zebra_setError(zh, errCode, errString);
2876         *num_entries = 0;
2877         return ZEBRA_FAIL;
2878     }
2879     if (ord_no == 0)
2880     {
2881         *num_entries = 0;
2882         return ZEBRA_OK;
2883     }
2884     /* prepare dictionary scanning */
2885     if (num < 1)
2886     {
2887         *num_entries = 0;
2888         return ZEBRA_OK;
2889     }
2890     before = pos-1;
2891     if (before < 0)
2892         before = 0;
2893     after = 1+num-pos;
2894     if (after < 0)
2895         after = 0;
2896     yaz_log(YLOG_DEBUG, "rpn_scan pos=%d num=%d before=%d "
2897             "after=%d before+after=%d",
2898             pos, num, before, after, before+after);
2899     scan_info_array = (struct scan_info *)
2900         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2901     for (i = 0; i < ord_no; i++)
2902     {
2903         int j, prefix_len = 0;
2904         int before_tmp = before, after_tmp = after;
2905         struct scan_info *scan_info = scan_info_array + i;
2906         struct rpn_char_map_info rcmi;
2907
2908         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2909
2910         scan_info->before = before;
2911         scan_info->after = after;
2912         scan_info->odr = stream;
2913
2914         scan_info->list = (struct scan_info_entry *)
2915             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2916         for (j = 0; j<before+after; j++)
2917             scan_info->list[j].term = NULL;
2918
2919         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2920         termz[prefix_len] = 0;
2921         strcpy(scan_info->prefix, termz);
2922
2923         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id) == ZEBRA_FAIL)
2924             return ZEBRA_FAIL;
2925
2926         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2927                   scan_info, scan_handle);
2928     }
2929     glist = (ZebraScanEntry *)
2930         odr_malloc(stream, (before+after)*sizeof(*glist));
2931
2932     rset_nmem = nmem_create();
2933     kc = zebra_key_control_create(zh);
2934
2935     /* consider terms after main term */
2936     for (i = 0; i < ord_no; i++)
2937         ptr[i] = before;
2938
2939     *is_partial = 0;
2940     for (i = 0; i<after; i++)
2941     {
2942         int j, j0 = -1;
2943         const char *mterm = NULL;
2944         const char *tst;
2945         RSET rset = 0;
2946         int lo = i + pos-1; /* offset in result list */
2947
2948         /* find: j0 is the first of the minimal values */
2949         for (j = 0; j < ord_no; j++)
2950         {
2951             if (ptr[j] < before+after && ptr[j] >= 0 &&
2952                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2953                 (!mterm || strcmp (tst, mterm) < 0))
2954             {
2955                 j0 = j;
2956                 mterm = tst;
2957             }
2958         }
2959         if (j0 == -1)
2960             break;  /* no value found, stop */
2961
2962         /* get result set for first one , but only if it's within bounds */
2963         if (lo >= 0)
2964         {
2965             /* get result set for first term */
2966             zebra_term_untrans_iconv(zh, stream->mem, reg_id,
2967                                      &glist[lo].term, mterm);
2968             rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2969                               glist[lo].term, strlen(glist[lo].term),
2970                               NULL, 0, zapt->term->which, rset_nmem,
2971                               kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2972                               0 /* term_ref_id_str */);
2973         }
2974         ptr[j0]++; /* move index for this set .. */
2975         /* get result set for remaining scan terms */
2976         for (j = j0+1; j<ord_no; j++)
2977         {
2978             if (ptr[j] < before+after && ptr[j] >= 0 &&
2979                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2980                 !strcmp (tst, mterm))
2981             {
2982                 if (lo >= 0)
2983                 {
2984                     RSET rsets[2];
2985
2986                     rsets[0] = rset;
2987                     rsets[1] =
2988                         rset_trunc(
2989                             zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2990                             glist[lo].term,
2991                             strlen(glist[lo].term), NULL, 0,
2992                             zapt->term->which,rset_nmem,
2993                             kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
2994                             0 /* term_ref_id_str */ );
2995                     rset = rsmulti_or_create(rset_nmem, kc,
2996                                              kc->scope, 0 /* termid */,
2997                                              2, rsets);
2998                 }
2999                 ptr[j]++;
3000             }
3001         }
3002         if (lo >= 0)
3003         {
3004             zint count;
3005             /* merge with limit_set if given */
3006             if (limit_set)
3007             {
3008                 RSET rsets[2];
3009                 rsets[0] = rset;
3010                 rsets[1] = rset_dup(limit_set);
3011
3012                 rset = rsmulti_and_create(rset_nmem, kc,
3013                                           kc->scope,
3014                                           2, rsets);
3015             }
3016             /* count it */
3017             count_set(zh, rset, &count);
3018             glist[lo].occurrences = count;
3019             rset_delete(rset);
3020         }
3021     }
3022     if (i < after)
3023     {
3024         *num_entries -= (after-i);
3025         *is_partial = 1;
3026         if (*num_entries < 0)
3027         {
3028             (*kc->dec)(kc);
3029             nmem_destroy(rset_nmem);
3030             *num_entries = 0;
3031             return ZEBRA_OK;
3032         }
3033     }
3034     /* consider terms before main term */
3035     for (i = 0; i<ord_no; i++)
3036         ptr[i] = 0;
3037
3038     for (i = 0; i<before; i++)
3039     {
3040         int j, j0 = -1;
3041         const char *mterm = NULL;
3042         const char *tst;
3043         RSET rset;
3044         int lo = before-1-i; /* offset in result list */
3045         zint count;
3046
3047         for (j = 0; j <ord_no; j++)
3048         {
3049             if (ptr[j] < before && ptr[j] >= 0 &&
3050                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3051                 (!mterm || strcmp (tst, mterm) > 0))
3052             {
3053                 j0 = j;
3054                     mterm = tst;
3055             }
3056         }
3057         if (j0 == -1)
3058             break;
3059
3060         zebra_term_untrans_iconv(zh, stream->mem, reg_id,
3061                                  &glist[lo].term, mterm);
3062
3063         rset = rset_trunc
3064             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
3065              glist[lo].term, strlen(glist[lo].term),
3066              NULL, 0, zapt->term->which, rset_nmem,
3067              kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3068              0 /* term_ref_id_str */);
3069
3070         ptr[j0]++;
3071
3072         for (j = j0+1; j<ord_no; j++)
3073         {
3074             if (ptr[j] < before && ptr[j] >= 0 &&
3075                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
3076                 !strcmp (tst, mterm))
3077             {
3078                 RSET rsets[2];
3079
3080                 rsets[0] = rset;
3081                 rsets[1] = rset_trunc(
3082                     zh,
3083                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
3084                     glist[lo].term,
3085                     strlen(glist[lo].term), NULL, 0,
3086                     zapt->term->which, rset_nmem,
3087                     kc, kc->scope, 0, reg_id, 0 /* hits_limit */,
3088                     0 /* term_ref_id_str */);
3089                 rset = rsmulti_or_create(rset_nmem, kc,
3090                                          kc->scope, 0 /* termid */, 2, rsets);
3091
3092                 ptr[j]++;
3093             }
3094         }
3095         if (limit_set)
3096         {
3097             RSET rsets[2];
3098             rsets[0] = rset;
3099             rsets[1] = rset_dup(limit_set);
3100
3101             rset = rsmulti_and_create(rset_nmem, kc,
3102                                       kc->scope, 2, rsets);
3103         }
3104         count_set(zh, rset, &count);
3105         glist[lo].occurrences = count;
3106         rset_delete (rset);
3107     }
3108     (*kc->dec)(kc);
3109     nmem_destroy(rset_nmem);
3110     i = before-i;
3111     if (i)
3112     {
3113         *is_partial = 1;
3114         *position -= i;
3115         *num_entries -= i;
3116         if (*num_entries <= 0)
3117         {
3118             *num_entries = 0;
3119             return ZEBRA_OK;
3120         }
3121     }
3122
3123     *list = glist + i;               /* list is set to first 'real' entry */
3124
3125     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
3126             *position, *num_entries);
3127     return ZEBRA_OK;
3128 }
3129