index/zrpn.c

   1 /* $Id: zrpn.c,v 1.171 2005-03-11 17:56:34 adam Exp $
   2    Copyright (C) 1995-2005
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with Zebra; see the file LICENSE.zebra.  If not, write to the
  19 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  20 02111-1307, USA.
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31 #include <ctype.h>
  32
  33 #include "index.h"
  34 #include <zebra_xpath.h>
  35
  36 #include <charmap.h>
  37 #include <rset.h>
  38
  39 /* maximum number of terms in an and/or/phrase item */
  40 #define TERM_LIST_LENGTH_MAX 256
  41
  42 static const struct key_control it_ctrl =
  43 {
  44     sizeof(struct it_key),
  45     2, /* we have sysnos and seqnos in this key, nothing more */
  46     key_compare_it,
  47     key_logdump_txt,   /* FIXME  - clean up these functions */
  48     key_get_seq,
  49 };
  50
  51
  52 const struct key_control *key_it_ctrl = &it_ctrl;
  53
  54 struct rpn_char_map_info
  55 {
  56     ZebraMaps zm;
  57     int reg_type;
  58 };
  59
  60 typedef struct
  61 {
  62     int type;
  63     int major;
  64     int minor;
  65     Z_AttributesPlusTerm *zapt;
  66 } AttrType;
  67
  68
  69 static int log_level_set = 0;
  70 static int log_level_rpn = 0;
  71
  72 static const char **rpn_char_map_handler(void *vp, const char **from, int len)
  73 {
  74     struct rpn_char_map_info *p = (struct rpn_char_map_info *) vp;
  75     const char **out = zebra_maps_input(p->zm, p->reg_type, from, len, 0);
  76 #if 0
  77     if (out && *out)
  78     {
  79         const char *outp = *out;
  80         yaz_log(YLOG_LOG, "---");
  81         while (*outp)
  82         {
  83             yaz_log(YLOG_LOG, "%02X", *outp);
  84             outp++;
  85         }
  86     }
  87 #endif
  88     return out;
  89 }
  90
  91 static void rpn_char_map_prepare(struct zebra_register *reg, int reg_type,
  92                                   struct rpn_char_map_info *map_info)
  93 {
  94     map_info->zm = reg->zebra_maps;
  95     map_info->reg_type = reg_type;
  96     dict_grep_cmap(reg->dict, map_info, rpn_char_map_handler);
  97 }
  98
  99 static int attr_find_ex(AttrType *src, oid_value *attributeSetP,
 100                          const char **string_value)
 101 {
 102     int num_attributes;
 103
 104     num_attributes = src->zapt->attributes->num_attributes;
 105     while (src->major < num_attributes)
 106     {
 107         Z_AttributeElement *element;
 108
 109         element = src->zapt->attributes->attributes[src->major];
 110         if (src->type == *element->attributeType)
 111         {
 112             switch (element->which)
 113             {
 114             case Z_AttributeValue_numeric:
 115                 ++(src->major);
 116                 if (element->attributeSet && attributeSetP)
 117                 {
 118                     oident *attrset;
 119
 120                     attrset = oid_getentbyoid(element->attributeSet);
 121                     *attributeSetP = attrset->value;
 122                 }
 123                 return *element->value.numeric;
 124                 break;
 125             case Z_AttributeValue_complex:
 126                 if (src->minor >= element->value.complex->num_list)
 127                     break;
 128                 if (element->attributeSet && attributeSetP)
 129                 {
 130                     oident *attrset;
 131
 132                     attrset = oid_getentbyoid(element->attributeSet);
 133                     *attributeSetP = attrset->value;
 134                 }
 135                 if (element->value.complex->list[src->minor]->which ==
 136                     Z_StringOrNumeric_numeric)
 137                 {
 138                     ++(src->minor);
 139                     return
 140                         *element->value.complex->list[src->minor-1]->u.numeric;
 141                 }
 142                 else if (element->value.complex->list[src->minor]->which ==
 143                          Z_StringOrNumeric_string)
 144                 {
 145                     if (!string_value)
 146                         break;
 147                     ++(src->minor);
 148                     *string_value =
 149                         element->value.complex->list[src->minor-1]->u.string;
 150                     return -2;
 151                 }
 152                 else
 153                     break;
 154             default:
 155                 assert(0);
 156             }
 157         }
 158         ++(src->major);
 159     }
 160     return -1;
 161 }
 162
 163 static int attr_find(AttrType *src, oid_value *attributeSetP)
 164 {
 165     return attr_find_ex(src, attributeSetP, 0);
 166 }
 167
 168 static void attr_init(AttrType *src, Z_AttributesPlusTerm *zapt,
 169                        int type)
 170 {
 171     src->zapt = zapt;
 172     src->type = type;
 173     src->major = 0;
 174     src->minor = 0;
 175 }
 176
 177 #define TERM_COUNT
 178
 179 struct grep_info {
 180 #ifdef TERM_COUNT
 181     int *term_no;
 182 #endif
 183     ISAMC_P *isam_p_buf;
 184     int isam_p_size;
 185     int isam_p_indx;
 186     ZebraHandle zh;
 187     int reg_type;
 188     ZebraSet termset;
 189 };
 190
 191 static void term_untrans(ZebraHandle zh, int reg_type,
 192                            char *dst, const char *src)
 193 {
 194     int len = 0;
 195     while (*src)
 196     {
 197         const char *cp = zebra_maps_output(zh->reg->zebra_maps,
 198                                            reg_type, &src);
 199         if (!cp && len < IT_MAX_WORD-1)
 200             dst[len++] = *src++;
 201         else
 202             while (*cp && len < IT_MAX_WORD-1)
 203                 dst[len++] = *cp++;
 204     }
 205     dst[len] = '\0';
 206 }
 207
 208 static void add_isam_p(const char *name, const char *info,
 209                        struct grep_info *p)
 210 {
 211     if (!log_level_set)
 212     {
 213         log_level_rpn = yaz_log_module_level("rpn");
 214         log_level_set = 1;
 215     }
 216     if (p->isam_p_indx == p->isam_p_size)
 217     {
 218         ISAMC_P *new_isam_p_buf;
 219 #ifdef TERM_COUNT
 220         int *new_term_no;
 221 #endif
 222         p->isam_p_size = 2*p->isam_p_size + 100;
 223         new_isam_p_buf = (ISAMC_P *) xmalloc(sizeof(*new_isam_p_buf) *
 224                                              p->isam_p_size);
 225         if (p->isam_p_buf)
 226         {
 227             memcpy(new_isam_p_buf, p->isam_p_buf,
 228                     p->isam_p_indx * sizeof(*p->isam_p_buf));
 229             xfree(p->isam_p_buf);
 230         }
 231         p->isam_p_buf = new_isam_p_buf;
 232
 233 #ifdef TERM_COUNT
 234         new_term_no = (int *) xmalloc(sizeof(*new_term_no) * p->isam_p_size);
 235         if (p->term_no)
 236         {
 237             memcpy(new_term_no, p->isam_p_buf,
 238                     p->isam_p_indx * sizeof(*p->term_no));
 239             xfree(p->term_no);
 240         }
 241         p->term_no = new_term_no;
 242 #endif
 243     }
 244     assert(*info == sizeof(*p->isam_p_buf));
 245     memcpy(p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf));
 246
 247 #if 1
 248     if (p->termset)
 249     {
 250         const char *db;
 251         int set, use;
 252         char term_tmp[IT_MAX_WORD];
 253         int su_code = 0;
 254         int len = key_SU_decode (&su_code, name);
 255
 256         term_untrans  (p->zh, p->reg_type, term_tmp, name+len+1);
 257         yaz_log(log_level_rpn, "grep: %d %c %s", su_code, name[len], term_tmp);
 258         zebraExplain_lookup_ord (p->zh->reg->zei,
 259                                  su_code, &db, &set, &use);
 260         yaz_log(log_level_rpn, "grep:  set=%d use=%d db=%s", set, use, db);
 261
 262         resultSetAddTerm(p->zh, p->termset, name[len], db,
 263                          set, use, term_tmp);
 264     }
 265 #endif
 266     (p->isam_p_indx)++;
 267 }
 268
 269 static int grep_handle(char *name, const char *info, void *p)
 270 {
 271     add_isam_p(name, info, (struct grep_info *) p);
 272     return 0;
 273 }
 274
 275 static int term_pre(ZebraMaps zebra_maps, int reg_type, const char **src,
 276                      const char *ct1, const char *ct2, int first)
 277 {
 278     const char *s1, *s0 = *src;
 279     const char **map;
 280
 281     /* skip white space */
 282     while (*s0)
 283     {
 284         if (ct1 && strchr(ct1, *s0))
 285             break;
 286         if (ct2 && strchr(ct2, *s0))
 287             break;
 288         s1 = s0;
 289         map = zebra_maps_input(zebra_maps, reg_type, &s1, strlen(s1), first);
 290         if (**map != *CHR_SPACE)
 291             break;
 292         s0 = s1;
 293     }
 294     *src = s0;
 295     return *s0;
 296 }
 297
 298
 299 static void esc_str(char *out_buf, int out_size,
 300                     const char *in_buf, int in_size)
 301 {
 302     int k;
 303
 304     assert(out_buf);
 305     assert(in_buf);
 306     assert(out_size > 20);
 307     *out_buf = '\0';
 308     for (k = 0; k<in_size; k++)
 309     {
 310         int c = in_buf[k] & 0xff;
 311         int pc;
 312         if (c < 32 || c > 126)
 313             pc = '?';
 314         else
 315             pc = c;
 316         sprintf(out_buf +strlen(out_buf), "%02X:%c  ", c, pc);
 317         if (strlen(out_buf) > out_size-20)
 318         {
 319             strcat(out_buf, "..");
 320             break;
 321         }
 322     }
 323 }
 324
 325 #define REGEX_CHARS " []()|.*+?!"
 326
 327 /* term_100: handle term, where trunc = none(no operators at all) */
 328 static int term_100(ZebraMaps zebra_maps, int reg_type,
 329                      const char **src, char *dst, int space_split,
 330                      char *dst_term)
 331 {
 332     const char *s0;
 333     const char **map;
 334     int i = 0;
 335     int j = 0;
 336
 337     const char *space_start = 0;
 338     const char *space_end = 0;
 339
 340     if (!term_pre(zebra_maps, reg_type, src, NULL, NULL, !space_split))
 341         return 0;
 342     s0 = *src;
 343     while (*s0)
 344     {
 345         const char *s1 = s0;
 346         int q_map_match = 0;
 347         map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 348                                 &q_map_match);
 349         if (space_split)
 350         {
 351             if (**map == *CHR_SPACE)
 352                 break;
 353         }
 354         else  /* complete subfield only. */
 355         {
 356             if (**map == *CHR_SPACE)
 357             {   /* save space mapping for later  .. */
 358                 space_start = s1;
 359                 space_end = s0;
 360                 continue;
 361             }
 362             else if (space_start)
 363             {   /* reload last space */
 364                 while (space_start < space_end)
 365                 {
 366                     if (strchr(REGEX_CHARS, *space_start))
 367                         dst[i++] = '\\';
 368                     dst_term[j++] = *space_start;
 369                     dst[i++] = *space_start++;
 370                 }
 371                 /* and reset */
 372                 space_start = space_end = 0;
 373             }
 374         }
 375         /* add non-space char */
 376         memcpy(dst_term+j, s1, s0 - s1);
 377         j += (s0 - s1);
 378         if (!q_map_match)
 379         {
 380             while (s1 < s0)
 381             {
 382                 if (strchr(REGEX_CHARS, *s1))
 383                     dst[i++] = '\\';
 384                 dst[i++] = *s1++;
 385             }
 386         }
 387         else
 388         {
 389             char tmpbuf[80];
 390             esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 391
 392             strcpy(dst + i, map[0]);
 393             i += strlen(map[0]);
 394         }
 395     }
 396     dst[i] = '\0';
 397     dst_term[j] = '\0';
 398     *src = s0;
 399     return i;
 400 }
 401
 402 /* term_101: handle term, where trunc = Process # */
 403 static int term_101(ZebraMaps zebra_maps, int reg_type,
 404                      const char **src, char *dst, int space_split,
 405                      char *dst_term)
 406 {
 407     const char *s0;
 408     const char **map;
 409     int i = 0;
 410     int j = 0;
 411
 412     if (!term_pre(zebra_maps, reg_type, src, "#", "#", !space_split))
 413         return 0;
 414     s0 = *src;
 415     while (*s0)
 416     {
 417         if (*s0 == '#')
 418         {
 419             dst[i++] = '.';
 420             dst[i++] = '*';
 421             dst_term[j++] = *s0++;
 422         }
 423         else
 424         {
 425             const char *s1 = s0;
 426             int q_map_match = 0;
 427             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 428                                     &q_map_match);
 429             if (space_split && **map == *CHR_SPACE)
 430                 break;
 431
 432             /* add non-space char */
 433             memcpy(dst_term+j, s1, s0 - s1);
 434             j += (s0 - s1);
 435             if (!q_map_match)
 436             {
 437                 while (s1 < s0)
 438                 {
 439                     if (strchr(REGEX_CHARS, *s1))
 440                         dst[i++] = '\\';
 441                     dst[i++] = *s1++;
 442                 }
 443             }
 444             else
 445             {
 446                 char tmpbuf[80];
 447                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 448
 449                 strcpy(dst + i, map[0]);
 450                 i += strlen(map[0]);
 451             }
 452         }
 453     }
 454     dst[i] = '\0';
 455     dst_term[j++] = '\0';
 456     *src = s0;
 457     return i;
 458 }
 459
 460 /* term_103: handle term, where trunc = re-2 (regular expressions) */
 461 static int term_103(ZebraMaps zebra_maps, int reg_type, const char **src,
 462                      char *dst, int *errors, int space_split,
 463                      char *dst_term)
 464 {
 465     int i = 0;
 466     int j = 0;
 467     const char *s0;
 468     const char **map;
 469
 470     if (!term_pre(zebra_maps, reg_type, src, "^\\()[].*+?|", "(", !space_split))
 471         return 0;
 472     s0 = *src;
 473     if (errors && *s0 == '+' && s0[1] && s0[2] == '+' && s0[3] &&
 474         isdigit(((const unsigned char *)s0)[1]))
 475     {
 476         *errors = s0[1] - '0';
 477         s0 += 3;
 478         if (*errors > 3)
 479             *errors = 3;
 480     }
 481     while (*s0)
 482     {
 483         if (strchr("^\\()[].*+?|-", *s0))
 484         {
 485             dst_term[j++] = *s0;
 486             dst[i++] = *s0++;
 487         }
 488         else
 489         {
 490             const char *s1 = s0;
 491             int q_map_match = 0;
 492             map = zebra_maps_search(zebra_maps, reg_type, &s0, strlen(s0),
 493                                     &q_map_match);
 494             if (space_split && **map == *CHR_SPACE)
 495                 break;
 496
 497             /* add non-space char */
 498             memcpy(dst_term+j, s1, s0 - s1);
 499             j += (s0 - s1);
 500             if (!q_map_match)
 501             {
 502                 while (s1 < s0)
 503                 {
 504                     if (strchr(REGEX_CHARS, *s1))
 505                         dst[i++] = '\\';
 506                     dst[i++] = *s1++;
 507                 }
 508             }
 509             else
 510             {
 511                 char tmpbuf[80];
 512                 esc_str(tmpbuf, sizeof(tmpbuf), map[0], strlen(map[0]));
 513
 514                 strcpy(dst + i, map[0]);
 515                 i += strlen(map[0]);
 516             }
 517         }
 518     }
 519     dst[i] = '\0';
 520     dst_term[j] = '\0';
 521     *src = s0;
 522
 523     return i;
 524 }
 525
 526 /* term_103: handle term, where trunc = re-1 (regular expressions) */
 527 static int term_102 (ZebraMaps zebra_maps, int reg_type, const char **src,
 528                      char *dst, int space_split, char *dst_term)
 529 {
 530     return term_103(zebra_maps, reg_type, src, dst, NULL, space_split,
 531                      dst_term);
 532 }
 533
 534
 535 /* term_104: handle term, where trunc = Process # and ! */
 536 static int term_104(ZebraMaps zebra_maps, int reg_type,
 537                      const char **src, char *dst, int space_split,
 538                      char *dst_term)
 539 {
 540     const char *s0, *s1;
 541     const char **map;
 542     int i = 0;
 543     int j = 0;
 544
 545     if (!term_pre(zebra_maps, reg_type, src, "?*#", "?*#", !space_split))
 546         return 0;
 547     s0 = *src;
 548     while (*s0)
 549     {
 550         if (*s0 == '?')
 551         {
 552             dst_term[j++] = *s0++;
 553             if (*s0 >= '0' && *s0 <= '9')
 554             {
 555                 int limit = 0;
 556                 while (*s0 >= '0' && *s0 <= '9')
 557                 {
 558                     limit = limit * 10 + (*s0 - '0');
 559                     dst_term[j++] = *s0++;
 560                 }
 561                 if (limit > 20)
 562                     limit = 20;
 563                 while (--limit >= 0)
 564                 {
 565                     dst[i++] = '.';
 566                     dst[i++] = '?';
 567                 }
 568             }
 569             else
 570             {
 571                 dst[i++] = '.';
 572                 dst[i++] = '*';
 573             }
 574         }
 575         else if (*s0 == '*')
 576         {
 577             dst[i++] = '.';
 578             dst[i++] = '*';
 579             dst_term[j++] = *s0++;
 580         }
 581         else if (*s0 == '#')
 582         {
 583             dst[i++] = '.';
 584             dst_term[j++] = *s0++;
 585         }
 586         else
 587         {
 588             s1 = s0;
 589             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 590             if (space_split && **map == *CHR_SPACE)
 591                 break;
 592             while (s1 < s0)
 593             {
 594                 if (strchr(REGEX_CHARS, *s1))
 595                     dst[i++] = '\\';
 596                 dst_term[j++] = *s1;
 597                 dst[i++] = *s1++;
 598             }
 599         }
 600     }
 601     dst[i] = '\0';
 602     dst_term[j++] = '\0';
 603     *src = s0;
 604     return i;
 605 }
 606
 607 /* term_105/106: handle term, where trunc = Process * and ! and right trunc */
 608 static int term_105 (ZebraMaps zebra_maps, int reg_type,
 609                      const char **src, char *dst, int space_split,
 610                      char *dst_term, int right_truncate)
 611 {
 612     const char *s0, *s1;
 613     const char **map;
 614     int i = 0;
 615     int j = 0;
 616
 617     if (!term_pre(zebra_maps, reg_type, src, "*!", "*!", !space_split))
 618         return 0;
 619     s0 = *src;
 620     while (*s0)
 621     {
 622         if (*s0 == '*')
 623         {
 624             dst[i++] = '.';
 625             dst[i++] = '*';
 626             dst_term[j++] = *s0++;
 627         }
 628         else if (*s0 == '!')
 629         {
 630             dst[i++] = '.';
 631             dst_term[j++] = *s0++;
 632         }
 633         else
 634         {
 635             s1 = s0;
 636             map = zebra_maps_input(zebra_maps, reg_type, &s0, strlen(s0), 0);
 637             if (space_split && **map == *CHR_SPACE)
 638                 break;
 639             while (s1 < s0)
 640             {
 641                 if (strchr(REGEX_CHARS, *s1))
 642                     dst[i++] = '\\';
 643                 dst_term[j++] = *s1;
 644                 dst[i++] = *s1++;
 645             }
 646         }
 647     }
 648     if (right_truncate)
 649     {
 650         dst[i++] = '.';
 651         dst[i++] = '*';
 652     }
 653     dst[i] = '\0';
 654
 655     dst_term[j++] = '\0';
 656     *src = s0;
 657     return i;
 658 }
 659
 660
 661 /* gen_regular_rel - generate regular expression from relation
 662  *  val:     border value (inclusive)
 663  *  islt:    1 if <=; 0 if >=.
 664  */
 665 static void gen_regular_rel(char *dst, int val, int islt)
 666 {
 667     int dst_p;
 668     int w, d, i;
 669     int pos = 0;
 670     char numstr[20];
 671
 672     yaz_log(YLOG_DEBUG, "gen_regular_rel. val=%d, islt=%d", val, islt);
 673     if (val >= 0)
 674     {
 675         if (islt)
 676             strcpy(dst, "(-[0-9]+|(");
 677         else
 678             strcpy(dst, "((");
 679     }
 680     else
 681     {
 682         if (!islt)
 683         {
 684             strcpy(dst, "([0-9]+|-(");
 685             dst_p = strlen(dst);
 686             islt = 1;
 687         }
 688         else
 689         {
 690             strcpy(dst, "(-(");
 691             islt = 0;
 692         }
 693         val = -val;
 694     }
 695     dst_p = strlen(dst);
 696     sprintf(numstr, "%d", val);
 697     for (w = strlen(numstr); --w >= 0; pos++)
 698     {
 699         d = numstr[w];
 700         if (pos > 0)
 701         {
 702             if (islt)
 703             {
 704                 if (d == '0')
 705                     continue;
 706                 d--;
 707             }
 708             else
 709             {
 710                 if (d == '9')
 711                     continue;
 712                 d++;
 713             }
 714         }
 715
 716         strcpy(dst + dst_p, numstr);
 717         dst_p = strlen(dst) - pos - 1;
 718
 719         if (islt)
 720         {
 721             if (d != '0')
 722             {
 723                 dst[dst_p++] = '[';
 724                 dst[dst_p++] = '0';
 725                 dst[dst_p++] = '-';
 726                 dst[dst_p++] = d;
 727                 dst[dst_p++] = ']';
 728             }
 729             else
 730                 dst[dst_p++] = d;
 731         }
 732         else
 733         {
 734             if (d != '9')
 735             {
 736                 dst[dst_p++] = '[';
 737                 dst[dst_p++] = d;
 738                 dst[dst_p++] = '-';
 739                 dst[dst_p++] = '9';
 740                 dst[dst_p++] = ']';
 741             }
 742             else
 743                 dst[dst_p++] = d;
 744         }
 745         for (i = 0; i<pos; i++)
 746         {
 747             dst[dst_p++] = '[';
 748             dst[dst_p++] = '0';
 749             dst[dst_p++] = '-';
 750             dst[dst_p++] = '9';
 751             dst[dst_p++] = ']';
 752         }
 753         dst[dst_p++] = '|';
 754     }
 755     dst[dst_p] = '\0';
 756     if (islt)
 757     {
 758         /* match everything less than 10^(pos-1) */
 759         strcat(dst, "0*");
 760         for (i = 1; i<pos; i++)
 761             strcat(dst, "[0-9]?");
 762     }
 763     else
 764     {
 765         /* match everything greater than 10^pos */
 766         for (i = 0; i <= pos; i++)
 767             strcat(dst, "[0-9]");
 768         strcat(dst, "[0-9]*");
 769     }
 770     strcat(dst, "))");
 771 }
 772
 773 void string_rel_add_char(char **term_p, const char *src, int *indx)
 774 {
 775     if (src[*indx] == '\\')
 776         *(*term_p)++ = src[(*indx)++];
 777     *(*term_p)++ = src[(*indx)++];
 778 }
 779
 780 /*
 781  *   >  abc     ([b-].*|a[c-].*|ab[d-].*|abc.+)
 782  *              ([^-a].*|a[^-b].*ab[^-c].*|abc.+)
 783  *   >= abc     ([b-].*|a[c-].*|ab[c-].*)
 784  *              ([^-a].*|a[^-b].*|ab[c-].*)
 785  *   <  abc     ([-0].*|a[-a].*|ab[-b].*)
 786  *              ([^a-].*|a[^b-].*|ab[^c-].*)
 787  *   <= abc     ([-0].*|a[-a].*|ab[-b].*|abc)
 788  *              ([^a-].*|a[^b-].*|ab[^c-].*|abc)
 789  */
 790 static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 791                             const char **term_sub, char *term_dict,
 792                             oid_value attributeSet,
 793                             int reg_type, int space_split, char *term_dst)
 794 {
 795     AttrType relation;
 796     int relation_value;
 797     int i;
 798     char *term_tmp = term_dict + strlen(term_dict);
 799     char term_component[2*IT_MAX_WORD+20];
 800
 801     attr_init(&relation, zapt, 2);
 802     relation_value = attr_find(&relation, NULL);
 803
 804     yaz_log(YLOG_DEBUG, "string relation value=%d", relation_value);
 805     switch (relation_value)
 806     {
 807     case 1:
 808         if (!term_100 (zh->reg->zebra_maps, reg_type,
 809                        term_sub, term_component,
 810                        space_split, term_dst))
 811             return 0;
 812         yaz_log(log_level_rpn, "Relation <");
 813
 814         *term_tmp++ = '(';
 815         for (i = 0; term_component[i]; )
 816         {
 817             int j = 0;
 818
 819             if (i)
 820                 *term_tmp++ = '|';
 821             while (j < i)
 822                 string_rel_add_char (&term_tmp, term_component, &j);
 823
 824             *term_tmp++ = '[';
 825
 826             *term_tmp++ = '^';
 827             string_rel_add_char (&term_tmp, term_component, &i);
 828             *term_tmp++ = '-';
 829
 830             *term_tmp++ = ']';
 831             *term_tmp++ = '.';
 832             *term_tmp++ = '*';
 833
 834             if ((term_tmp - term_dict) > IT_MAX_WORD)
 835                 break;
 836         }
 837         *term_tmp++ = ')';
 838         *term_tmp = '\0';
 839         break;
 840     case 2:
 841         if (!term_100 (zh->reg->zebra_maps, reg_type,
 842                        term_sub, term_component,
 843                        space_split, term_dst))
 844             return 0;
 845         yaz_log(log_level_rpn, "Relation <=");
 846
 847         *term_tmp++ = '(';
 848         for (i = 0; term_component[i]; )
 849         {
 850             int j = 0;
 851
 852             while (j < i)
 853                 string_rel_add_char (&term_tmp, term_component, &j);
 854             *term_tmp++ = '[';
 855
 856             *term_tmp++ = '^';
 857             string_rel_add_char (&term_tmp, term_component, &i);
 858             *term_tmp++ = '-';
 859
 860             *term_tmp++ = ']';
 861             *term_tmp++ = '.';
 862             *term_tmp++ = '*';
 863
 864             *term_tmp++ = '|';
 865
 866             if ((term_tmp - term_dict) > IT_MAX_WORD)
 867                 break;
 868         }
 869         for (i = 0; term_component[i]; )
 870             string_rel_add_char (&term_tmp, term_component, &i);
 871         *term_tmp++ = ')';
 872         *term_tmp = '\0';
 873         break;
 874     case 5:
 875         if (!term_100 (zh->reg->zebra_maps, reg_type,
 876                        term_sub, term_component, space_split, term_dst))
 877             return 0;
 878         yaz_log(log_level_rpn, "Relation >");
 879
 880         *term_tmp++ = '(';
 881         for (i = 0; term_component[i];)
 882         {
 883             int j = 0;
 884
 885             while (j < i)
 886                 string_rel_add_char (&term_tmp, term_component, &j);
 887             *term_tmp++ = '[';
 888
 889             *term_tmp++ = '^';
 890             *term_tmp++ = '-';
 891             string_rel_add_char (&term_tmp, term_component, &i);
 892
 893             *term_tmp++ = ']';
 894             *term_tmp++ = '.';
 895             *term_tmp++ = '*';
 896
 897             *term_tmp++ = '|';
 898
 899             if ((term_tmp - term_dict) > IT_MAX_WORD)
 900                 break;
 901         }
 902         for (i = 0; term_component[i];)
 903             string_rel_add_char (&term_tmp, term_component, &i);
 904         *term_tmp++ = '.';
 905         *term_tmp++ = '+';
 906         *term_tmp++ = ')';
 907         *term_tmp = '\0';
 908         break;
 909     case 4:
 910         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 911                        term_component, space_split, term_dst))
 912             return 0;
 913         yaz_log(log_level_rpn, "Relation >=");
 914
 915         *term_tmp++ = '(';
 916         for (i = 0; term_component[i];)
 917         {
 918             int j = 0;
 919
 920             if (i)
 921                 *term_tmp++ = '|';
 922             while (j < i)
 923                 string_rel_add_char (&term_tmp, term_component, &j);
 924             *term_tmp++ = '[';
 925
 926             if (term_component[i+1])
 927             {
 928                 *term_tmp++ = '^';
 929                 *term_tmp++ = '-';
 930                 string_rel_add_char (&term_tmp, term_component, &i);
 931             }
 932             else
 933             {
 934                 string_rel_add_char (&term_tmp, term_component, &i);
 935                 *term_tmp++ = '-';
 936             }
 937             *term_tmp++ = ']';
 938             *term_tmp++ = '.';
 939             *term_tmp++ = '*';
 940
 941             if ((term_tmp - term_dict) > IT_MAX_WORD)
 942                 break;
 943         }
 944         *term_tmp++ = ')';
 945         *term_tmp = '\0';
 946         break;
 947     case 3:
 948     default:
 949         yaz_log(log_level_rpn, "Relation =");
 950         if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub,
 951                        term_component, space_split, term_dst))
 952             return 0;
 953         strcat(term_tmp, "(");
 954         strcat(term_tmp, term_component);
 955         strcat(term_tmp, ")");
 956     }
 957     return 1;
 958 }
 959
 960 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 961                         const char **term_sub,
 962                         oid_value attributeSet, NMEM stream,
 963                         struct grep_info *grep_info,
 964                         int reg_type, int complete_flag,
 965                         int num_bases, char **basenames,
 966                         char *term_dst, int xpath_use);
 967
 968 static RSET term_trunc(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
 969                         const char **term_sub,
 970                         oid_value attributeSet, NMEM stream,
 971                         struct grep_info *grep_info,
 972                         int reg_type, int complete_flag,
 973                         int num_bases, char **basenames,
 974                         char *term_dst,
 975                         const char *rank_type, int xpath_use,
 976                         NMEM rset_nmem)
 977 {
 978     int r;
 979     grep_info->isam_p_indx = 0;
 980     r = string_term(zh, zapt, term_sub, attributeSet, stream, grep_info,
 981                      reg_type, complete_flag, num_bases, basenames,
 982                      term_dst, xpath_use);
 983     if (r < 1)
 984         return 0;
 985     yaz_log(log_level_rpn, "term: %s", term_dst);
 986     return rset_trunc(zh, grep_info->isam_p_buf,
 987                        grep_info->isam_p_indx, term_dst,
 988                        strlen(term_dst), rank_type, 1 /* preserve pos */,
 989                        zapt->term->which, rset_nmem,
 990                        key_it_ctrl,key_it_ctrl->scope);
 991 }
 992 static char *nmem_strdup_i(NMEM nmem, int v)
 993 {
 994     char val_str[64];
 995     sprintf (val_str, "%d", v);
 996     return nmem_strdup(nmem, val_str);
 997 }
 998
 999 static int string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1000                        const char **term_sub,
1001                        oid_value attributeSet, NMEM stream,
1002                        struct grep_info *grep_info,
1003                        int reg_type, int complete_flag,
1004                        int num_bases, char **basenames,
1005                        char *term_dst, int xpath_use)
1006 {
1007     char term_dict[2*IT_MAX_WORD+4000];
1008     int j, r, base_no;
1009     AttrType truncation;
1010     int truncation_value;
1011     AttrType use;
1012     int use_value;
1013     const char *use_string = 0;
1014     oid_value curAttributeSet = attributeSet;
1015     const char *termp;
1016     struct rpn_char_map_info rcmi;
1017     int space_split = complete_flag ? 0 : 1;
1018
1019     int bases_ok = 0;     /* no of databases with OK attribute */
1020     int errCode = 0;      /* err code (if any is not OK) */
1021     char *errString = 0;  /* addinfo */
1022
1023     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1024     attr_init (&use, zapt, 1);
1025     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1026     yaz_log(log_level_rpn, "string_term, use value %d", use_value);
1027     attr_init (&truncation, zapt, 5);
1028     truncation_value = attr_find (&truncation, NULL);
1029     yaz_log(log_level_rpn, "truncation value %d", truncation_value);
1030
1031     if (use_value == -1)    /* no attribute - assumy "any" */
1032         use_value = 1016;
1033     for (base_no = 0; base_no < num_bases; base_no++)
1034     {
1035         int ord = -1;
1036         int attr_ok = 0;
1037         int regex_range = 0;
1038         int init_pos = 0;
1039         attent attp;
1040         data1_local_attribute id_xpath_attr;
1041         data1_local_attribute *local_attr;
1042         int max_pos, prefix_len = 0;
1043
1044         termp = *term_sub;
1045
1046         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1047         {
1048             zh->errCode = 109; /* Database unavailable */
1049             zh->errString = basenames[base_no];
1050             return -1;
1051         }
1052         if (xpath_use > 0 && use_value == -2)
1053         {
1054             /* xpath mode and we have a string attribute */
1055             attp.local_attributes = &id_xpath_attr;
1056             attp.attset_ordinal = VAL_IDXPATH;
1057             id_xpath_attr.next = 0;
1058
1059             use_value = xpath_use;  /* xpath_use as use-attribute now */
1060             id_xpath_attr.local = use_value;
1061         }
1062         else if (curAttributeSet == VAL_IDXPATH && use_value >= 0)
1063         {
1064             /* X-Path attribute, use numeric value directly */
1065             attp.local_attributes = &id_xpath_attr;
1066             attp.attset_ordinal = VAL_IDXPATH;
1067             id_xpath_attr.next = 0;
1068             id_xpath_attr.local = use_value;
1069         }
1070         else if (use_string &&
1071                  (ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1072                                                      use_string)) >= 0)
1073         {
1074             /* we have a match for a raw string attribute */
1075             char ord_buf[32];
1076             int i, ord_len;
1077
1078             if (prefix_len)
1079                 term_dict[prefix_len++] = '|';
1080             else
1081                 term_dict[prefix_len++] = '(';
1082
1083             ord_len = key_SU_encode (ord, ord_buf);
1084             for (i = 0; i<ord_len; i++)
1085             {
1086                 term_dict[prefix_len++] = 1;
1087                 term_dict[prefix_len++] = ord_buf[i];
1088             }
1089             attp.local_attributes = 0;  /* no more attributes */
1090         }
1091         else
1092         {
1093             /* lookup in the .att files . Allow string as well */
1094             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1095                                       use_string)))
1096             {
1097                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1098                       curAttributeSet, use_value, r);
1099                 if (r == -1)
1100                 {
1101                     /* set was found, but value wasn't defined */
1102                     errCode = 114;
1103                     if (use_string)
1104                         errString = nmem_strdup(stream, use_string);
1105                     else
1106                         errString = nmem_strdup_i (stream, use_value);
1107                 }
1108                 else
1109                 {
1110                     int oid[OID_SIZE];
1111                     struct oident oident;
1112
1113                     oident.proto = PROTO_Z3950;
1114                     oident.oclass = CLASS_ATTSET;
1115                     oident.value = curAttributeSet;
1116                     oid_ent_to_oid (&oident, oid);
1117
1118                     errCode = 121;
1119                     errString = nmem_strdup (stream, oident.desc);
1120                 }
1121                 continue;
1122             }
1123         }
1124         for (local_attr = attp.local_attributes; local_attr;
1125              local_attr = local_attr->next)
1126         {
1127             char ord_buf[32];
1128             int i, ord_len;
1129
1130             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1131                                               attp.attset_ordinal,
1132                                               local_attr->local);
1133             if (ord < 0)
1134                 continue;
1135             if (prefix_len)
1136                 term_dict[prefix_len++] = '|';
1137             else
1138                 term_dict[prefix_len++] = '(';
1139
1140             ord_len = key_SU_encode (ord, ord_buf);
1141             for (i = 0; i<ord_len; i++)
1142             {
1143                 term_dict[prefix_len++] = 1;
1144                 term_dict[prefix_len++] = ord_buf[i];
1145             }
1146         }
1147         if (!prefix_len)
1148         {
1149 #if 1
1150             bases_ok++;
1151 #else
1152             errCode = 114;
1153             errString = nmem_strdup_i(stream, use_value);
1154             continue;
1155 #endif
1156         }
1157         else
1158         {
1159             bases_ok++; /* this has OK attributes */
1160             attr_ok = 1;
1161         }
1162
1163         term_dict[prefix_len++] = ')';
1164         term_dict[prefix_len++] = 1;
1165         term_dict[prefix_len++] = reg_type;
1166         yaz_log(log_level_rpn, "reg_type = %d", term_dict[prefix_len-1]);
1167         term_dict[prefix_len] = '\0';
1168         j = prefix_len;
1169         switch (truncation_value)
1170         {
1171         case -1:         /* not specified */
1172         case 100:        /* do not truncate */
1173             if (!string_relation (zh, zapt, &termp, term_dict,
1174                                   attributeSet,
1175                                   reg_type, space_split, term_dst))
1176                 return 0;
1177             break;
1178         case 1:          /* right truncation */
1179             term_dict[j++] = '(';
1180             if (!term_100(zh->reg->zebra_maps, reg_type,
1181                           &termp, term_dict + j, space_split, term_dst))
1182                 return 0;
1183             strcat(term_dict, ".*)");
1184             break;
1185         case 2:          /* keft truncation */
1186             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1187             if (!term_100(zh->reg->zebra_maps, reg_type,
1188                           &termp, term_dict + j, space_split, term_dst))
1189                 return 0;
1190             strcat(term_dict, ")");
1191             break;
1192         case 3:          /* left&right truncation */
1193             term_dict[j++] = '('; term_dict[j++] = '.'; term_dict[j++] = '*';
1194             if (!term_100(zh->reg->zebra_maps, reg_type,
1195                           &termp, term_dict + j, space_split, term_dst))
1196                 return 0;
1197             strcat(term_dict, ".*)");
1198             break;
1199         case 101:        /* process # in term */
1200             term_dict[j++] = '(';
1201             if (!term_101(zh->reg->zebra_maps, reg_type,
1202                           &termp, term_dict + j, space_split, term_dst))
1203                 return 0;
1204             strcat(term_dict, ")");
1205             break;
1206         case 102:        /* Regexp-1 */
1207             term_dict[j++] = '(';
1208             if (!term_102(zh->reg->zebra_maps, reg_type,
1209                           &termp, term_dict + j, space_split, term_dst))
1210                 return 0;
1211             strcat(term_dict, ")");
1212             break;
1213         case 103:       /* Regexp-2 */
1214             r = 1;
1215             term_dict[j++] = '(';
1216             init_pos = 2;
1217             if (!term_103 (zh->reg->zebra_maps, reg_type,
1218                            &termp, term_dict + j, &regex_range,
1219                            space_split, term_dst))
1220                 return 0;
1221             strcat(term_dict, ")");
1222         case 104:        /* process # and ! in term */
1223             term_dict[j++] = '(';
1224             if (!term_104 (zh->reg->zebra_maps, reg_type,
1225                            &termp, term_dict + j, space_split, term_dst))
1226                 return 0;
1227             strcat(term_dict, ")");
1228             break;
1229         case 105:        /* process * and ! in term */
1230             term_dict[j++] = '(';
1231             if (!term_105 (zh->reg->zebra_maps, reg_type,
1232                            &termp, term_dict + j, space_split, term_dst, 1))
1233                 return 0;
1234             strcat(term_dict, ")");
1235             break;
1236         case 106:        /* process * and ! in term */
1237             term_dict[j++] = '(';
1238             if (!term_105 (zh->reg->zebra_maps, reg_type,
1239                            &termp, term_dict + j, space_split, term_dst, 0))
1240                 return 0;
1241             strcat(term_dict, ")");
1242             break;
1243         default:
1244             zh->errCode = 120;
1245             zh->errString = nmem_strdup_i(stream, truncation_value);
1246             return -1;
1247         }
1248         if (attr_ok)
1249         {
1250             char buf[80];
1251             const char *input = term_dict + prefix_len;
1252             esc_str(buf, sizeof(buf), input, strlen(input));
1253         }
1254         if (attr_ok)
1255         {
1256             yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_dict+prefix_len);
1257             r = dict_lookup_grep(zh->reg->dict, term_dict, regex_range,
1258                                  grep_info, &max_pos, init_pos,
1259                                  grep_handle);
1260             if (r)
1261                 yaz_log(YLOG_WARN, "dict_lookup_grep fail %d", r);
1262         }
1263     }
1264     if (!bases_ok)
1265     {
1266         zh->errCode = errCode;
1267         zh->errString = errString;
1268         return -1;
1269     }
1270     *term_sub = termp;
1271     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1272     return 1;
1273 }
1274
1275
1276 /* convert APT search term to UTF8 */
1277 static int zapt_term_to_utf8 (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1278                               char *termz)
1279 {
1280     size_t sizez;
1281     Z_Term *term = zapt->term;
1282
1283     switch (term->which)
1284     {
1285     case Z_Term_general:
1286         if (zh->iconv_to_utf8 != 0)
1287         {
1288             char *inbuf = term->u.general->buf;
1289             size_t inleft = term->u.general->len;
1290             char *outbuf = termz;
1291             size_t outleft = IT_MAX_WORD-1;
1292             size_t ret;
1293
1294             ret = yaz_iconv(zh->iconv_to_utf8, &inbuf, &inleft,
1295                         &outbuf, &outleft);
1296             if (ret == (size_t)(-1))
1297             {
1298                 ret = yaz_iconv(zh->iconv_to_utf8, 0, 0, 0, 0);
1299                 zh->errCode = 125;
1300                 return -1;
1301             }
1302             *outbuf = 0;
1303         }
1304         else
1305         {
1306             sizez = term->u.general->len;
1307             if (sizez > IT_MAX_WORD-1)
1308                 sizez = IT_MAX_WORD-1;
1309             memcpy (termz, term->u.general->buf, sizez);
1310             termz[sizez] = '\0';
1311         }
1312         break;
1313     case Z_Term_characterString:
1314         sizez = strlen(term->u.characterString);
1315         if (sizez > IT_MAX_WORD-1)
1316             sizez = IT_MAX_WORD-1;
1317         memcpy (termz, term->u.characterString, sizez);
1318         termz[sizez] = '\0';
1319         break;
1320     default:
1321         zh->errCode = 124;
1322         return -1;
1323     }
1324     return 0;
1325 }
1326
1327 /* convert APT SCAN term to internal cmap */
1328 static int trans_scan_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1329                             char *termz, int reg_type)
1330 {
1331     char termz0[IT_MAX_WORD];
1332
1333     if (zapt_term_to_utf8(zh, zapt, termz0))
1334         return -1;    /* error */
1335     else
1336     {
1337         const char **map;
1338         const char *cp = (const char *) termz0;
1339         const char *cp_end = cp + strlen(cp);
1340         const char *src;
1341         int i = 0;
1342         const char *space_map = NULL;
1343         int len;
1344
1345         while ((len = (cp_end - cp)) > 0)
1346         {
1347             map = zebra_maps_input (zh->reg->zebra_maps, reg_type, &cp, len, 0);
1348             if (**map == *CHR_SPACE)
1349                 space_map = *map;
1350             else
1351             {
1352                 if (i && space_map)
1353                     for (src = space_map; *src; src++)
1354                         termz[i++] = *src;
1355                 space_map = NULL;
1356                 for (src = *map; *src; src++)
1357                     termz[i++] = *src;
1358             }
1359         }
1360         termz[i] = '\0';
1361     }
1362     return 0;
1363 }
1364
1365 char *normalize_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1366                      const char *termz, NMEM stream, unsigned reg_id)
1367 {
1368     WRBUF wrbuf = 0;
1369     AttrType truncation;
1370     int truncation_value;
1371     char *ex_list = 0;
1372
1373     attr_init (&truncation, zapt, 5);
1374     truncation_value = attr_find (&truncation, NULL);
1375
1376     switch (truncation_value)
1377     {
1378     default:
1379         ex_list = "";
1380         break;
1381     case 101:
1382         ex_list = "#";
1383         break;
1384     case 102:
1385     case 103:
1386         ex_list = 0;
1387         break;
1388     case 104:
1389         ex_list = "!#";
1390         break;
1391     case 105:
1392         ex_list = "!*";
1393         break;
1394     }
1395     if (ex_list)
1396         wrbuf = zebra_replace(zh->reg->zebra_maps, reg_id, ex_list,
1397                               termz, strlen(termz));
1398     if (!wrbuf)
1399         return nmem_strdup(stream, termz);
1400     else
1401     {
1402         char *buf = (char*) nmem_malloc(stream, wrbuf_len(wrbuf)+1);
1403         memcpy (buf, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
1404         buf[wrbuf_len(wrbuf)] = '\0';
1405         return buf;
1406     }
1407 }
1408
1409 static void grep_info_delete (struct grep_info *grep_info)
1410 {
1411 #ifdef TERM_COUNT
1412     xfree(grep_info->term_no);
1413 #endif
1414     xfree (grep_info->isam_p_buf);
1415 }
1416
1417 static int grep_info_prepare (ZebraHandle zh,
1418                               Z_AttributesPlusTerm *zapt,
1419                               struct grep_info *grep_info,
1420                               int reg_type,
1421                               NMEM stream)
1422 {
1423     AttrType termset;
1424     int termset_value_numeric;
1425     const char *termset_value_string;
1426
1427 #ifdef TERM_COUNT
1428     grep_info->term_no = 0;
1429 #endif
1430     grep_info->isam_p_size = 0;
1431     grep_info->isam_p_buf = NULL;
1432     grep_info->zh = zh;
1433     grep_info->reg_type = reg_type;
1434     grep_info->termset = 0;
1435
1436     if (!zapt)
1437         return 0;
1438     attr_init (&termset, zapt, 8);
1439     termset_value_numeric =
1440         attr_find_ex (&termset, NULL, &termset_value_string);
1441     if (termset_value_numeric != -1)
1442     {
1443         char resname[32];
1444         const char *termset_name = 0;
1445         if (termset_value_numeric != -2)
1446         {
1447
1448             sprintf (resname, "%d", termset_value_numeric);
1449             termset_name = resname;
1450         }
1451         else
1452             termset_name = termset_value_string;
1453         yaz_log(log_level_rpn, "creating termset set %s", termset_name);
1454         grep_info->termset = resultSetAdd (zh, termset_name, 1);
1455         if (!grep_info->termset)
1456         {
1457             zh->errCode = 128;
1458             zh->errString = nmem_strdup (stream, termset_name);
1459             return -1;
1460         }
1461     }
1462     return 0;
1463 }
1464
1465
1466 static RSET rpn_search_APT_phrase (ZebraHandle zh,
1467                                    Z_AttributesPlusTerm *zapt,
1468                                    const char *termz_org,
1469                                    oid_value attributeSet,
1470                                    NMEM stream,
1471                                    int reg_type, int complete_flag,
1472                                    const char *rank_type, int xpath_use,
1473                                    int num_bases, char **basenames,
1474                                    NMEM rset_nmem)
1475 {
1476     char term_dst[IT_MAX_WORD+1];
1477     RSET rset[TERM_LIST_LENGTH_MAX], result;
1478     size_t rset_no = 0;
1479     struct grep_info grep_info;
1480     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1481     const char *termp = termz;
1482
1483     *term_dst = 0;
1484     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1485         return 0;
1486     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1487     {
1488         yaz_log(log_level_rpn, "APT_phrase termp=%s", termp);
1489         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1490                                     stream, &grep_info,
1491                                     reg_type, complete_flag,
1492                                     num_bases, basenames,
1493                                     term_dst, rank_type,
1494                                     xpath_use,rset_nmem);
1495         if (!rset[rset_no])
1496             break;
1497     }
1498     grep_info_delete (&grep_info);
1499     if (rset_no == 0)
1500         return rsnull_create (rset_nmem,key_it_ctrl);
1501     else if (rset_no == 1)
1502         return (rset[0]);
1503     else
1504         result = rsprox_create( rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1505                        rset_no, rset,
1506                        1 /* ordered */, 0 /* exclusion */,
1507                        3 /* relation */, 1 /* distance */);
1508     return result;
1509 }
1510
1511 static RSET rpn_search_APT_or_list (ZebraHandle zh,
1512                                     Z_AttributesPlusTerm *zapt,
1513                                     const char *termz_org,
1514                                     oid_value attributeSet,
1515                                     NMEM stream,
1516                                     int reg_type, int complete_flag,
1517                                     const char *rank_type,
1518                                     int xpath_use,
1519                                     int num_bases, char **basenames,
1520                                     NMEM rset_nmem)
1521 {
1522     char term_dst[IT_MAX_WORD+1];
1523     RSET rset[TERM_LIST_LENGTH_MAX];
1524     size_t rset_no = 0;
1525     struct grep_info grep_info;
1526     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1527     const char *termp = termz;
1528
1529     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1530         return 0;
1531     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1532     {
1533         yaz_log(log_level_rpn, "APT_or_list termp=%s", termp);
1534         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1535                                     stream, &grep_info,
1536                                     reg_type, complete_flag,
1537                                     num_bases, basenames,
1538                                     term_dst, rank_type,
1539                                     xpath_use,rset_nmem);
1540         if (!rset[rset_no])
1541             break;
1542     }
1543     grep_info_delete (&grep_info);
1544     if (rset_no == 0)
1545         return rsnull_create (rset_nmem,key_it_ctrl);
1546     return rsmulti_or_create(rset_nmem, key_it_ctrl,key_it_ctrl->scope,
1547                              rset_no, rset);
1548 }
1549
1550 static RSET rpn_search_APT_and_list (ZebraHandle zh,
1551                                      Z_AttributesPlusTerm *zapt,
1552                                      const char *termz_org,
1553                                      oid_value attributeSet,
1554                                      NMEM stream,
1555                                      int reg_type, int complete_flag,
1556                                      const char *rank_type,
1557                                      int xpath_use,
1558                                      int num_bases, char **basenames,
1559                                      NMEM rset_nmem)
1560 {
1561     char term_dst[IT_MAX_WORD+1];
1562     RSET rset[TERM_LIST_LENGTH_MAX];
1563     size_t rset_no = 0;
1564     struct grep_info grep_info;
1565     char *termz = normalize_term(zh, zapt, termz_org, stream, reg_type);
1566     const char *termp = termz;
1567
1568     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1569         return 0;
1570     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1571     {
1572         yaz_log(log_level_rpn, "APT_and_list termp=%s", termp);
1573         rset[rset_no] = term_trunc(zh, zapt, &termp, attributeSet,
1574                                     stream, &grep_info,
1575                                     reg_type, complete_flag,
1576                                     num_bases, basenames,
1577                                     term_dst, rank_type,
1578                                     xpath_use, rset_nmem);
1579         if (!rset[rset_no])
1580             break;
1581     }
1582     grep_info_delete (&grep_info);
1583     if (rset_no == 0)
1584         return rsnull_create(rset_nmem,key_it_ctrl);
1585
1586     return rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
1587                               rset_no, rset);
1588 }
1589
1590 static int numeric_relation (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1591                              const char **term_sub,
1592                              char *term_dict,
1593                              oid_value attributeSet,
1594                              struct grep_info *grep_info,
1595                              int *max_pos,
1596                              int reg_type,
1597                              char *term_dst)
1598 {
1599     AttrType relation;
1600     int relation_value;
1601     int term_value;
1602     int r;
1603     char *term_tmp = term_dict + strlen(term_dict);
1604
1605     attr_init (&relation, zapt, 2);
1606     relation_value = attr_find (&relation, NULL);
1607
1608     yaz_log(log_level_rpn, "numeric relation value=%d", relation_value);
1609
1610     if (!term_100 (zh->reg->zebra_maps, reg_type, term_sub, term_tmp, 1,
1611                    term_dst))
1612         return 0;
1613     term_value = atoi (term_tmp);
1614     switch (relation_value)
1615     {
1616     case 1:
1617         yaz_log(log_level_rpn, "Relation <");
1618         gen_regular_rel (term_tmp, term_value-1, 1);
1619         break;
1620     case 2:
1621         yaz_log(log_level_rpn, "Relation <=");
1622         gen_regular_rel (term_tmp, term_value, 1);
1623         break;
1624     case 4:
1625         yaz_log(log_level_rpn, "Relation >=");
1626         gen_regular_rel (term_tmp, term_value, 0);
1627         break;
1628     case 5:
1629         yaz_log(log_level_rpn, "Relation >");
1630         gen_regular_rel (term_tmp, term_value+1, 0);
1631         break;
1632     case 3:
1633     default:
1634         yaz_log(log_level_rpn, "Relation =");
1635         sprintf (term_tmp, "(0*%d)", term_value);
1636     }
1637     yaz_log(log_level_rpn, "dict_lookup_grep: %s", term_tmp);
1638     r = dict_lookup_grep(zh->reg->dict, term_dict, 0, grep_info, max_pos,
1639                           0, grep_handle);
1640     if (r)
1641         yaz_log(YLOG_WARN, "dict_lookup_grep fail, rel = gt: %d", r);
1642     yaz_log(log_level_rpn, "%d positions", grep_info->isam_p_indx);
1643     return 1;
1644 }
1645
1646 static int numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1647                          const char **term_sub,
1648                          oid_value attributeSet, struct grep_info *grep_info,
1649                          int reg_type, int complete_flag,
1650                          int num_bases, char **basenames,
1651                          char *term_dst, int xpath_use, NMEM stream)
1652 {
1653     char term_dict[2*IT_MAX_WORD+2];
1654     int r, base_no;
1655     AttrType use;
1656     int use_value;
1657     const char *use_string = 0;
1658     oid_value curAttributeSet = attributeSet;
1659     const char *termp;
1660     struct rpn_char_map_info rcmi;
1661
1662     int bases_ok = 0;     /* no of databases with OK attribute */
1663     int errCode = 0;      /* err code (if any is not OK) */
1664     char *errString = 0;  /* addinfo */
1665
1666     rpn_char_map_prepare (zh->reg, reg_type, &rcmi);
1667     attr_init (&use, zapt, 1);
1668     use_value = attr_find_ex (&use, &curAttributeSet, &use_string);
1669
1670     if (use_value == -1)
1671         use_value = 1016;
1672
1673     for (base_no = 0; base_no < num_bases; base_no++)
1674     {
1675         attent attp;
1676         data1_local_attribute id_xpath_attr;
1677         data1_local_attribute *local_attr;
1678         int max_pos, prefix_len = 0;
1679
1680         termp = *term_sub;
1681         if (use_value == -2)  /* string attribute (assume IDXPATH/any) */
1682         {
1683             use_value = xpath_use;
1684             attp.local_attributes = &id_xpath_attr;
1685             attp.attset_ordinal = VAL_IDXPATH;
1686             id_xpath_attr.next = 0;
1687             id_xpath_attr.local = use_value;
1688         }
1689         else if (curAttributeSet == VAL_IDXPATH)
1690         {
1691             attp.local_attributes = &id_xpath_attr;
1692             attp.attset_ordinal = VAL_IDXPATH;
1693             id_xpath_attr.next = 0;
1694             id_xpath_attr.local = use_value;
1695         }
1696         else
1697         {
1698             if ((r = att_getentbyatt (zh, &attp, curAttributeSet, use_value,
1699                                             use_string)))
1700             {
1701                 yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d r=%d",
1702                       curAttributeSet, use_value, r);
1703                 if (r == -1)
1704                 {
1705                     errString = nmem_strdup_i(stream, use_value);
1706                     errCode = 114;
1707                 }
1708                 else
1709                     errCode = 121;
1710                 continue;
1711             }
1712         }
1713         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
1714         {
1715             zh->errCode = 109; /* Database unavailable */
1716             zh->errString = basenames[base_no];
1717             return -1;
1718         }
1719         for (local_attr = attp.local_attributes; local_attr;
1720              local_attr = local_attr->next)
1721         {
1722             int ord;
1723             char ord_buf[32];
1724             int i, ord_len;
1725
1726             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
1727                                               attp.attset_ordinal,
1728                                               local_attr->local);
1729             if (ord < 0)
1730                 continue;
1731             if (prefix_len)
1732                 term_dict[prefix_len++] = '|';
1733             else
1734                 term_dict[prefix_len++] = '(';
1735
1736             ord_len = key_SU_encode (ord, ord_buf);
1737             for (i = 0; i<ord_len; i++)
1738             {
1739                 term_dict[prefix_len++] = 1;
1740                 term_dict[prefix_len++] = ord_buf[i];
1741             }
1742         }
1743         if (!prefix_len)
1744         {
1745             errCode = 114;
1746             errString = nmem_strdup_i(stream, use_value);
1747             continue;
1748         }
1749         bases_ok++;
1750         term_dict[prefix_len++] = ')';
1751         term_dict[prefix_len++] = 1;
1752         term_dict[prefix_len++] = reg_type;
1753         yaz_log(YLOG_DEBUG, "reg_type = %d", term_dict[prefix_len-1]);
1754         term_dict[prefix_len] = '\0';
1755         if (!numeric_relation (zh, zapt, &termp, term_dict,
1756                                attributeSet, grep_info, &max_pos, reg_type,
1757                                term_dst))
1758             return 0;
1759     }
1760     if (!bases_ok)
1761     {
1762         zh->errCode = errCode;
1763         zh->errString = errString;
1764         return -1;
1765     }
1766     *term_sub = termp;
1767     yaz_log(YLOG_DEBUG, "%d positions", grep_info->isam_p_indx);
1768     return 1;
1769 }
1770
1771 static RSET rpn_search_APT_numeric (ZebraHandle zh,
1772                                     Z_AttributesPlusTerm *zapt,
1773                                     const char *termz,
1774                                     oid_value attributeSet,
1775                                     NMEM stream,
1776                                     int reg_type, int complete_flag,
1777                                     const char *rank_type, int xpath_use,
1778                                     int num_bases, char **basenames,
1779                                     NMEM rset_nmem)
1780 {
1781     char term_dst[IT_MAX_WORD+1];
1782     const char *termp = termz;
1783     RSET rset[TERM_LIST_LENGTH_MAX];
1784     int  r;
1785     size_t rset_no = 0;
1786     struct grep_info grep_info;
1787
1788     yaz_log(log_level_rpn, "APT_numeric t='%s'",termz);
1789     if (grep_info_prepare (zh, zapt, &grep_info, reg_type, stream))
1790         return 0;
1791     for (; rset_no < sizeof(rset)/sizeof(*rset); rset_no++)
1792     {
1793         yaz_log(YLOG_DEBUG, "APT_numeric termp=%s", termp);
1794         grep_info.isam_p_indx = 0;
1795         r = numeric_term(zh, zapt, &termp, attributeSet, &grep_info,
1796                           reg_type, complete_flag, num_bases, basenames,
1797                           term_dst, xpath_use,
1798                           stream);
1799         if (r < 1)
1800             break;
1801         yaz_log(YLOG_DEBUG, "term: %s", term_dst);
1802         rset[rset_no] = rset_trunc(zh, grep_info.isam_p_buf,
1803                                     grep_info.isam_p_indx, term_dst,
1804                                     strlen(term_dst), rank_type,
1805                                     0 /* preserve position */,
1806                                     zapt->term->which, rset_nmem,
1807                                     key_it_ctrl,key_it_ctrl->scope);
1808         if (!rset[rset_no])
1809             break;
1810     }
1811     grep_info_delete (&grep_info);
1812     if (rset_no == 0)
1813         return rsnull_create(rset_nmem,key_it_ctrl);
1814     if (rset_no == 1)
1815         return rset[0];
1816     return rsmulti_and_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1817                               rset_no, rset);
1818 }
1819
1820 static RSET rpn_search_APT_local (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1821                                   const char *termz,
1822                                   oid_value attributeSet,
1823                                   NMEM stream,
1824                                   const char *rank_type, NMEM rset_nmem)
1825 {
1826     RSET result;
1827     RSFD rsfd;
1828     struct it_key key;
1829     int sys;
1830     result = rstemp_create( rset_nmem,key_it_ctrl,key_it_ctrl->scope,
1831                      res_get (zh->res, "setTmpDir"),0 );
1832     rsfd = rset_open (result, RSETF_WRITE);
1833
1834     sys = atoi(termz);
1835     if (sys <= 0)
1836         sys = 1;
1837     key.mem[0] = sys;
1838     key.mem[1] = 1;
1839     key.len = 2;
1840     rset_write (rsfd, &key);
1841     rset_close (rsfd);
1842     return result;
1843 }
1844
1845 static RSET rpn_sort_spec (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1846                            oid_value attributeSet, NMEM stream,
1847                            Z_SortKeySpecList *sort_sequence,
1848                            const char *rank_type)
1849 {
1850     int i;
1851     int sort_relation_value;
1852     AttrType sort_relation_type;
1853     int use_value;
1854     AttrType use_type;
1855     Z_SortKeySpec *sks;
1856     Z_SortKey *sk;
1857     Z_AttributeElement *ae;
1858     int oid[OID_SIZE];
1859     oident oe;
1860     char termz[20];
1861
1862     attr_init (&sort_relation_type, zapt, 7);
1863     sort_relation_value = attr_find (&sort_relation_type, &attributeSet);
1864
1865     attr_init (&use_type, zapt, 1);
1866     use_value = attr_find (&use_type, &attributeSet);
1867
1868     if (!sort_sequence->specs)
1869     {
1870         sort_sequence->num_specs = 10;
1871         sort_sequence->specs = (Z_SortKeySpec **)
1872             nmem_malloc(stream, sort_sequence->num_specs *
1873                          sizeof(*sort_sequence->specs));
1874         for (i = 0; i<sort_sequence->num_specs; i++)
1875             sort_sequence->specs[i] = 0;
1876     }
1877     if (zapt->term->which != Z_Term_general)
1878         i = 0;
1879     else
1880         i = atoi_n ((char *) zapt->term->u.general->buf,
1881                     zapt->term->u.general->len);
1882     if (i >= sort_sequence->num_specs)
1883         i = 0;
1884     sprintf (termz, "%d", i);
1885
1886     oe.proto = PROTO_Z3950;
1887     oe.oclass = CLASS_ATTSET;
1888     oe.value = attributeSet;
1889     if (!oid_ent_to_oid (&oe, oid))
1890         return 0;
1891
1892     sks = (Z_SortKeySpec *) nmem_malloc(stream, sizeof(*sks));
1893     sks->sortElement = (Z_SortElement *)
1894         nmem_malloc(stream, sizeof(*sks->sortElement));
1895     sks->sortElement->which = Z_SortElement_generic;
1896     sk = sks->sortElement->u.generic = (Z_SortKey *)
1897         nmem_malloc(stream, sizeof(*sk));
1898     sk->which = Z_SortKey_sortAttributes;
1899     sk->u.sortAttributes = (Z_SortAttributes *)
1900         nmem_malloc(stream, sizeof(*sk->u.sortAttributes));
1901
1902     sk->u.sortAttributes->id = oid;
1903     sk->u.sortAttributes->list = (Z_AttributeList *)
1904         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list));
1905     sk->u.sortAttributes->list->num_attributes = 1;
1906     sk->u.sortAttributes->list->attributes = (Z_AttributeElement **)
1907         nmem_malloc(stream, sizeof(*sk->u.sortAttributes->list->attributes));
1908     ae = *sk->u.sortAttributes->list->attributes = (Z_AttributeElement *)
1909         nmem_malloc(stream, sizeof(**sk->u.sortAttributes->list->attributes));
1910     ae->attributeSet = 0;
1911     ae->attributeType = (int *)
1912         nmem_malloc(stream, sizeof(*ae->attributeType));
1913     *ae->attributeType = 1;
1914     ae->which = Z_AttributeValue_numeric;
1915     ae->value.numeric = (int *)
1916         nmem_malloc(stream, sizeof(*ae->value.numeric));
1917     *ae->value.numeric = use_value;
1918
1919     sks->sortRelation = (int *)
1920         nmem_malloc(stream, sizeof(*sks->sortRelation));
1921     if (sort_relation_value == 1)
1922         *sks->sortRelation = Z_SortKeySpec_ascending;
1923     else if (sort_relation_value == 2)
1924         *sks->sortRelation = Z_SortKeySpec_descending;
1925     else
1926         *sks->sortRelation = Z_SortKeySpec_ascending;
1927
1928     sks->caseSensitivity = (int *)
1929         nmem_malloc(stream, sizeof(*sks->caseSensitivity));
1930     *sks->caseSensitivity = 0;
1931
1932     sks->which = Z_SortKeySpec_null;
1933     sks->u.null = odr_nullval ();
1934     sort_sequence->specs[i] = sks;
1935     return rsnull_create (NULL,key_it_ctrl);
1936         /* FIXME - nmem?? */
1937 }
1938
1939
1940 static int parse_xpath(ZebraHandle zh, Z_AttributesPlusTerm *zapt,
1941                        oid_value attributeSet,
1942                        struct xpath_location_step *xpath, int max, NMEM mem)
1943 {
1944     oid_value curAttributeSet = attributeSet;
1945     AttrType use;
1946     const char *use_string = 0;
1947
1948     attr_init (&use, zapt, 1);
1949     attr_find_ex (&use, &curAttributeSet, &use_string);
1950
1951     if (!use_string || *use_string != '/')
1952         return -1;
1953
1954     return zebra_parse_xpath_str(use_string, xpath, max, mem);
1955 }
1956
1957
1958
1959 static RSET xpath_trunc(ZebraHandle zh, NMEM stream,
1960                         int reg_type, const char *term, int use,
1961                         oid_value curAttributeSet, NMEM rset_nmem)
1962 {
1963     RSET rset;
1964     struct grep_info grep_info;
1965     char term_dict[2048];
1966     char ord_buf[32];
1967     int prefix_len = 0;
1968     int ord = zebraExplain_lookup_attr_su(zh->reg->zei, curAttributeSet, use);
1969     int ord_len, i, r, max_pos;
1970     int term_type = Z_Term_characterString;
1971     const char *flags = "void";
1972
1973     if (grep_info_prepare (zh, 0 /* zapt */, &grep_info, '0', stream))
1974         return rsnull_create (rset_nmem,key_it_ctrl);
1975
1976     if (ord < 0)
1977         return rsnull_create (rset_nmem,key_it_ctrl);
1978     if (prefix_len)
1979         term_dict[prefix_len++] = '|';
1980     else
1981         term_dict[prefix_len++] = '(';
1982
1983     ord_len = key_SU_encode (ord, ord_buf);
1984     for (i = 0; i<ord_len; i++)
1985     {
1986         term_dict[prefix_len++] = 1;
1987         term_dict[prefix_len++] = ord_buf[i];
1988     }
1989     term_dict[prefix_len++] = ')';
1990     term_dict[prefix_len++] = 1;
1991     term_dict[prefix_len++] = reg_type;
1992
1993     strcpy(term_dict+prefix_len, term);
1994
1995     grep_info.isam_p_indx = 0;
1996     r = dict_lookup_grep(zh->reg->dict, term_dict, 0,
1997                           &grep_info, &max_pos, 0, grep_handle);
1998     yaz_log (YLOG_DEBUG, "%s %d positions", term,
1999              grep_info.isam_p_indx);
2000     rset = rset_trunc(zh, grep_info.isam_p_buf,
2001                        grep_info.isam_p_indx, term, strlen(term),
2002                        flags, 1, term_type,rset_nmem,
2003                        key_it_ctrl, key_it_ctrl->scope);
2004     grep_info_delete (&grep_info);
2005     return rset;
2006 }
2007
2008 static RSET rpn_search_xpath (ZebraHandle zh,
2009                               oid_value attributeSet,
2010                               int num_bases, char **basenames,
2011                               NMEM stream, const char *rank_type, RSET rset,
2012                               int xpath_len, struct xpath_location_step *xpath,
2013                               NMEM rset_nmem)
2014 {
2015     oid_value curAttributeSet = attributeSet;
2016     int base_no;
2017     int i;
2018
2019     if (xpath_len < 0)
2020         return rset;
2021
2022     yaz_log (YLOG_DEBUG, "xpath len=%d", xpath_len);
2023     for (i = 0; i<xpath_len; i++)
2024     {
2025         yaz_log (log_level_rpn, "XPATH %d %s", i, xpath[i].part);
2026
2027     }
2028
2029     curAttributeSet = VAL_IDXPATH;
2030
2031     /*
2032       //a    ->    a/.*
2033       //a/b  ->    b/a/.*
2034       /a     ->    a/
2035       /a/b   ->    b/a/
2036
2037       /      ->    none
2038
2039    a[@attr = value]/b[@other = othervalue]
2040
2041  /e/@a val      range(e/,range(@a,freetext(w,1015,val),@a),e/)
2042  /a/b val       range(b/a/,freetext(w,1016,val),b/a/)
2043  /a/b/@c val    range(b/a/,range(@c,freetext(w,1016,val),@c),b/a/)
2044  /a/b[@c = y] val range(b/a/,freetext(w,1016,val),b/a/,@c = y)
2045  /a[@c = y]/b val range(a/,range(b/a/,freetext(w,1016,val),b/a/),a/,@c = y)
2046  /a[@c = x]/b[@c = y] range(a/,range(b/a/,freetext(w,1016,val),b/a/,@c = y),a/,@c = x)
2047
2048     */
2049
2050     dict_grep_cmap (zh->reg->dict, 0, 0);
2051
2052     for (base_no = 0; base_no < num_bases; base_no++)
2053     {
2054         int level = xpath_len;
2055         int first_path = 1;
2056
2057         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2058         {
2059             zh->errCode = 109; /* Database unavailable */
2060             zh->errString = basenames[base_no];
2061             return rset;
2062         }
2063         while (--level >= 0)
2064         {
2065             char xpath_rev[128];
2066             int i, len;
2067             RSET rset_start_tag = 0, rset_end_tag = 0, rset_attr = 0;
2068
2069             *xpath_rev = 0;
2070             len = 0;
2071             for (i = level; i >= 1; --i)
2072             {
2073                 const char *cp = xpath[i].part;
2074                 if (*cp)
2075                 {
2076                     for (;*cp; cp++)
2077                         if (*cp == '*')
2078                         {
2079                             memcpy (xpath_rev + len, "[^/]*", 5);
2080                             len += 5;
2081                         }
2082                         else if (*cp == ' ')
2083                         {
2084
2085                             xpath_rev[len++] = 1;
2086                             xpath_rev[len++] = ' ';
2087                         }
2088
2089                         else
2090                             xpath_rev[len++] = *cp;
2091                     xpath_rev[len++] = '/';
2092                 }
2093                 else if (i == 1)  /* // case */
2094                 {
2095                     xpath_rev[len++] = '.';
2096                     xpath_rev[len++] = '*';
2097                 }
2098             }
2099             xpath_rev[len] = 0;
2100
2101             if (xpath[level].predicate &&
2102                 xpath[level].predicate->which == XPATH_PREDICATE_RELATION &&
2103                 xpath[level].predicate->u.relation.name[0])
2104             {
2105                 WRBUF wbuf = wrbuf_alloc();
2106                 wrbuf_puts(wbuf, xpath[level].predicate->u.relation.name+1);
2107                 if (xpath[level].predicate->u.relation.value)
2108                 {
2109                     const char *cp = xpath[level].predicate->u.relation.value;
2110                     wrbuf_putc(wbuf, '=');
2111
2112                     while (*cp)
2113                     {
2114                         if (strchr(REGEX_CHARS, *cp))
2115                             wrbuf_putc(wbuf, '\\');
2116                         wrbuf_putc(wbuf, *cp);
2117                         cp++;
2118                     }
2119                 }
2120                 wrbuf_puts(wbuf, "");
2121                 rset_attr = xpath_trunc(
2122                     zh, stream, '0', wrbuf_buf(wbuf), 3,
2123                     curAttributeSet,rset_nmem);
2124                 wrbuf_free(wbuf, 1);
2125             }
2126             else
2127             {
2128                 if (!first_path)
2129                     continue;
2130             }
2131             yaz_log (log_level_rpn, "xpath_rev (%d) = %s", level, xpath_rev);
2132             if (strlen(xpath_rev))
2133             {
2134                 rset_start_tag = xpath_trunc(zh, stream, '0',
2135                         xpath_rev, 1, curAttributeSet, rset_nmem);
2136
2137                 rset_end_tag = xpath_trunc(zh, stream, '0',
2138                         xpath_rev, 2, curAttributeSet, rset_nmem);
2139
2140                 rset = rsbetween_create(rset_nmem, key_it_ctrl,
2141                                         key_it_ctrl->scope,
2142                                         rset_start_tag, rset,
2143                                         rset_end_tag, rset_attr);
2144             }
2145             first_path = 0;
2146         }
2147     }
2148
2149     return rset;
2150 }
2151
2152
2153
2154 static RSET rpn_search_APT (ZebraHandle zh, Z_AttributesPlusTerm *zapt,
2155                             oid_value attributeSet, NMEM stream,
2156                             Z_SortKeySpecList *sort_sequence,
2157                             int num_bases, char **basenames,
2158                             NMEM rset_nmem)
2159 {
2160     unsigned reg_id;
2161     char *search_type = NULL;
2162     char rank_type[128];
2163     int complete_flag;
2164     int sort_flag;
2165     char termz[IT_MAX_WORD+1];
2166     RSET rset = 0;
2167     int xpath_len;
2168     int xpath_use = 0;
2169     struct xpath_location_step xpath[10];
2170
2171     if (!log_level_set)
2172     {
2173         log_level_rpn = yaz_log_module_level("rpn");
2174         log_level_set = 1;
2175     }
2176     zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2177                      rank_type, &complete_flag, &sort_flag);
2178
2179     yaz_log(YLOG_DEBUG, "reg_id=%c", reg_id);
2180     yaz_log(YLOG_DEBUG, "complete_flag=%d", complete_flag);
2181     yaz_log(YLOG_DEBUG, "search_type=%s", search_type);
2182     yaz_log(YLOG_DEBUG, "rank_type=%s", rank_type);
2183
2184     if (zapt_term_to_utf8(zh, zapt, termz))
2185         return 0;
2186
2187     if (sort_flag)
2188         return rpn_sort_spec (zh, zapt, attributeSet, stream, sort_sequence,
2189                               rank_type);
2190     xpath_len = parse_xpath(zh, zapt, attributeSet, xpath, 10, stream);
2191     if (xpath_len >= 0)
2192     {
2193         xpath_use = 1016;
2194         if (xpath[xpath_len-1].part[0] == '@')
2195             xpath_use = 1015;
2196     }
2197
2198     if (!strcmp (search_type, "phrase"))
2199     {
2200         rset = rpn_search_APT_phrase (zh, zapt, termz, attributeSet, stream,
2201                                       reg_id, complete_flag, rank_type,
2202                                       xpath_use,
2203                                       num_bases, basenames, rset_nmem);
2204     }
2205     else if (!strcmp (search_type, "and-list"))
2206     {
2207         rset = rpn_search_APT_and_list (zh, zapt, termz, attributeSet, stream,
2208                                         reg_id, complete_flag, rank_type,
2209                                         xpath_use,
2210                                         num_bases, basenames, rset_nmem);
2211     }
2212     else if (!strcmp (search_type, "or-list"))
2213     {
2214         rset = rpn_search_APT_or_list (zh, zapt, termz, attributeSet, stream,
2215                                        reg_id, complete_flag, rank_type,
2216                                        xpath_use,
2217                                        num_bases, basenames, rset_nmem);
2218     }
2219     else if (!strcmp (search_type, "local"))
2220     {
2221         rset = rpn_search_APT_local (zh, zapt, termz, attributeSet, stream,
2222                                      rank_type, rset_nmem);
2223     }
2224     else if (!strcmp (search_type, "numeric"))
2225     {
2226         rset = rpn_search_APT_numeric (zh, zapt, termz, attributeSet, stream,
2227                                        reg_id, complete_flag, rank_type,
2228                                        xpath_use,
2229                                        num_bases, basenames, rset_nmem);
2230     }
2231     else if (!strcmp (search_type, "always"))
2232     {
2233         rset = 0;
2234     }
2235     else
2236         zh->errCode = 118;
2237     return rpn_search_xpath (zh, attributeSet, num_bases, basenames,
2238                              stream, rank_type, rset,
2239                              xpath_len, xpath, rset_nmem);
2240 }
2241
2242 static RSET rpn_search_structure (ZebraHandle zh, Z_RPNStructure *zs,
2243                                   oid_value attributeSet,
2244                                   NMEM stream, NMEM rset_nmem,
2245                                   Z_SortKeySpecList *sort_sequence,
2246                                   int num_bases, char **basenames)
2247 {
2248     RSET r = NULL;
2249     if (zs->which == Z_RPNStructure_complex)
2250     {
2251         Z_Operator *zop = zs->u.complex->roperator;
2252         RSET rsets[2]; /* l and r argument */
2253
2254         rsets[0]=rpn_search_structure (zh, zs->u.complex->s1,
2255                                        attributeSet, stream, rset_nmem,
2256                                        sort_sequence,
2257                                        num_bases, basenames);
2258         if (rsets[0] == NULL)
2259             return NULL;
2260         rsets[1]=rpn_search_structure (zh, zs->u.complex->s2,
2261                                        attributeSet, stream, rset_nmem,
2262                                        sort_sequence,
2263                                        num_bases, basenames);
2264         if (rsets[1] == NULL)
2265         {
2266             rset_delete (rsets[0]);
2267             return NULL;
2268         }
2269
2270         switch (zop->which)
2271         {
2272         case Z_Operator_and:
2273             r = rsmulti_and_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2274                                    2, rsets);
2275             break;
2276         case Z_Operator_or:
2277             r = rsmulti_or_create(rset_nmem, key_it_ctrl, key_it_ctrl->scope,
2278                                   2, rsets);
2279             break;
2280         case Z_Operator_and_not:
2281             r = rsbool_create_not(rset_nmem,key_it_ctrl, key_it_ctrl->scope,
2282                     rsets[0],rsets[1]);
2283             break;
2284         case Z_Operator_prox:
2285             if (zop->u.prox->which != Z_ProximityOperator_known)
2286             {
2287                 zh->errCode = 132;
2288                 return NULL;
2289             }
2290             if (*zop->u.prox->u.known != Z_ProxUnit_word)
2291             {
2292                 char *val = (char *) nmem_malloc(stream, 16);
2293                 zh->errCode = 132;
2294                 zh->errString = val;
2295                 sprintf (val, "%d", *zop->u.prox->u.known);
2296                 return NULL;
2297             }
2298             else
2299             {
2300                 /* new / old prox */
2301                 r = rsprox_create(rset_nmem,key_it_ctrl,key_it_ctrl->scope,
2302                          2, rsets,
2303                          *zop->u.prox->ordered,
2304                          (!zop->u.prox->exclusion ?
2305                               0 : *zop->u.prox->exclusion),
2306                          *zop->u.prox->relationType,
2307                          *zop->u.prox->distance );
2308             }
2309             break;
2310         default:
2311             zh->errCode = 110;
2312             return NULL;
2313         }
2314     }
2315     else if (zs->which == Z_RPNStructure_simple)
2316     {
2317         if (zs->u.simple->which == Z_Operand_APT)
2318         {
2319             yaz_log(YLOG_DEBUG, "rpn_search_APT");
2320             r = rpn_search_APT (zh, zs->u.simple->u.attributesPlusTerm,
2321                                 attributeSet, stream, sort_sequence,
2322                                 num_bases, basenames,rset_nmem);
2323         }
2324         else if (zs->u.simple->which == Z_Operand_resultSetId)
2325         {
2326             yaz_log(YLOG_DEBUG, "rpn_search_ref");
2327             r = resultSetRef (zh, zs->u.simple->u.resultSetId);
2328             if (!r)
2329             {
2330                 r = rsnull_create (rset_nmem,key_it_ctrl);
2331                 zh->errCode = 30;
2332                 zh->errString =
2333                     nmem_strdup (stream, zs->u.simple->u.resultSetId);
2334                 return 0;
2335             }
2336             else
2337                 rset_dup(r);
2338         }
2339         else
2340         {
2341             zh->errCode = 3;
2342             return 0;
2343         }
2344     }
2345     else
2346     {
2347         zh->errCode = 3;
2348         return 0;
2349     }
2350     return r;
2351 }
2352
2353
2354 RSET rpn_search(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
2355                 Z_RPNQuery *rpn, int num_bases, char **basenames,
2356                 const char *setname,
2357                 ZebraSet sset)
2358 {
2359     RSET rset;
2360     oident *attrset;
2361     oid_value attributeSet;
2362     Z_SortKeySpecList *sort_sequence;
2363     int sort_status, i;
2364
2365     zh->errCode = 0;
2366     zh->errString = NULL;
2367     zh->hits = 0;
2368
2369     sort_sequence = (Z_SortKeySpecList *)
2370         nmem_malloc(nmem, sizeof(*sort_sequence));
2371     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
2372     sort_sequence->specs = (Z_SortKeySpec **)
2373         nmem_malloc(nmem, sort_sequence->num_specs *
2374                      sizeof(*sort_sequence->specs));
2375     for (i = 0; i<sort_sequence->num_specs; i++)
2376         sort_sequence->specs[i] = 0;
2377
2378     attrset = oid_getentbyoid (rpn->attributeSetId);
2379     attributeSet = attrset->value;
2380     rset = rpn_search_structure (zh, rpn->RPNStructure, attributeSet,
2381                                  nmem, rset_nmem,
2382                                  sort_sequence, num_bases, basenames);
2383     if (!rset)
2384         return 0;
2385
2386     if (zh->errCode)
2387         yaz_log(YLOG_DEBUG, "search error: %d", zh->errCode);
2388
2389     for (i = 0; sort_sequence->specs[i]; i++)
2390         ;
2391     sort_sequence->num_specs = i;
2392     if (!i)
2393         resultSetRank (zh, sset, rset, rset_nmem);
2394     else
2395     {
2396         yaz_log(YLOG_DEBUG, "resultSetSortSingle in rpn_search");
2397         resultSetSortSingle (zh, nmem, sset, rset,
2398                              sort_sequence, &sort_status);
2399         if (zh->errCode)
2400         {
2401             yaz_log(YLOG_DEBUG, "resultSetSortSingle status = %d", zh->errCode);
2402         }
2403     }
2404     return rset;
2405 }
2406
2407 struct scan_info_entry {
2408     char *term;
2409     ISAMC_P isam_p;
2410 };
2411
2412 struct scan_info {
2413     struct scan_info_entry *list;
2414     ODR odr;
2415     int before, after;
2416     char prefix[20];
2417 };
2418
2419 static int scan_handle (char *name, const char *info, int pos, void *client)
2420 {
2421     int len_prefix, idx;
2422     struct scan_info *scan_info = (struct scan_info *) client;
2423
2424     len_prefix = strlen(scan_info->prefix);
2425     if (memcmp (name, scan_info->prefix, len_prefix))
2426         return 1;
2427     if (pos > 0)        idx = scan_info->after - pos + scan_info->before;
2428     else
2429         idx = - pos - 1;
2430     scan_info->list[idx].term = (char *)
2431         odr_malloc(scan_info->odr, strlen(name + len_prefix)+1);
2432     strcpy(scan_info->list[idx].term, name + len_prefix);
2433     assert (*info == sizeof(ISAMC_P));
2434     memcpy (&scan_info->list[idx].isam_p, info+1, sizeof(ISAMC_P));
2435     return 0;
2436 }
2437
2438 static void scan_term_untrans (ZebraHandle zh, NMEM stream, int reg_type,
2439                                char **dst, const char *src)
2440 {
2441     char term_src[IT_MAX_WORD];
2442     char term_dst[IT_MAX_WORD];
2443
2444     term_untrans (zh, reg_type, term_src, src);
2445
2446     if (zh->iconv_from_utf8 != 0)
2447     {
2448         int len;
2449         char *inbuf = term_src;
2450         size_t inleft = strlen(term_src);
2451         char *outbuf = term_dst;
2452         size_t outleft = sizeof(term_dst)-1;
2453         size_t ret;
2454
2455         ret = yaz_iconv (zh->iconv_from_utf8, &inbuf, &inleft,
2456                          &outbuf, &outleft);
2457         if (ret == (size_t)(-1))
2458             len = 0;
2459         else
2460             len = outbuf - term_dst;
2461         *dst = nmem_malloc(stream, len + 1);
2462         if (len > 0)
2463             memcpy (*dst, term_dst, len);
2464         (*dst)[len] = '\0';
2465     }
2466     else
2467         *dst = nmem_strdup(stream, term_src);
2468 }
2469
2470 static void count_set (RSET r, int *count)
2471 {
2472     zint psysno = 0;
2473     int kno = 0;
2474     struct it_key key;
2475     RSFD rfd;
2476
2477     yaz_log(YLOG_DEBUG, "count_set");
2478
2479     *count = 0;
2480     rfd = rset_open (r, RSETF_READ);
2481     while (rset_read (rfd, &key,0 /* never mind terms */))
2482     {
2483         if (key.mem[0] != psysno)
2484         {
2485             psysno = key.mem[0];
2486             (*count)++;
2487         }
2488         kno++;
2489     }
2490     rset_close (rfd);
2491     yaz_log(YLOG_DEBUG, "%d keys, %d records", kno, *count);
2492 }
2493
2494 void rpn_scan(ZebraHandle zh, ODR stream, Z_AttributesPlusTerm *zapt,
2495                oid_value attributeset,
2496                int num_bases, char **basenames,
2497                int *position, int *num_entries, ZebraScanEntry **list,
2498                int *is_partial, RSET limit_set, int return_zero)
2499 {
2500     int i;
2501     int pos = *position;
2502     int num = *num_entries;
2503     int before;
2504     int after;
2505     int base_no;
2506     char termz[IT_MAX_WORD+20];
2507     AttrType use;
2508     int use_value;
2509     const char *use_string = 0;
2510     struct scan_info *scan_info_array;
2511     ZebraScanEntry *glist;
2512     int ords[32], ord_no = 0;
2513     int ptr[32];
2514
2515     int bases_ok = 0;     /* no of databases with OK attribute */
2516     int errCode = 0;      /* err code (if any is not OK) */
2517     char *errString = 0;  /* addinfo */
2518
2519     unsigned reg_id;
2520     char *search_type = NULL;
2521     char rank_type[128];
2522     int complete_flag;
2523     int sort_flag;
2524     NMEM rset_nmem = NULL;
2525
2526     *list = 0;
2527
2528     if (attributeset == VAL_NONE)
2529         attributeset = VAL_BIB1;
2530
2531     if (!limit_set)
2532     {
2533         AttrType termset;
2534         int termset_value_numeric;
2535         const char *termset_value_string;
2536         attr_init (&termset, zapt, 8);
2537         termset_value_numeric =
2538             attr_find_ex (&termset, NULL, &termset_value_string);
2539         if (termset_value_numeric != -1)
2540         {
2541             char resname[32];
2542             const char *termset_name = 0;
2543
2544             if (termset_value_numeric != -2)
2545             {
2546
2547                 sprintf (resname, "%d", termset_value_numeric);
2548                 termset_name = resname;
2549             }
2550             else
2551                 termset_name = termset_value_string;
2552
2553             limit_set = resultSetRef (zh, termset_name);
2554         }
2555     }
2556
2557     yaz_log (YLOG_DEBUG, "position = %d, num = %d set=%d",
2558              pos, num, attributeset);
2559
2560     attr_init (&use, zapt, 1);
2561     use_value = attr_find_ex (&use, &attributeset, &use_string);
2562
2563     if (zebra_maps_attr (zh->reg->zebra_maps, zapt, &reg_id, &search_type,
2564                          rank_type, &complete_flag, &sort_flag))
2565     {
2566         *num_entries = 0;
2567         zh->errCode = 113;
2568         return ;
2569     }
2570     yaz_log (YLOG_DEBUG, "use_value = %d", use_value);
2571
2572     if (use_value == -1)
2573         use_value = 1016;
2574     for (base_no = 0; base_no < num_bases && ord_no < 32; base_no++)
2575     {
2576         int r;
2577         attent attp;
2578         data1_local_attribute *local_attr;
2579
2580         if ((r = att_getentbyatt (zh, &attp, attributeset, use_value,
2581                                 use_string)))
2582         {
2583             yaz_log(YLOG_DEBUG, "att_getentbyatt fail. set=%d use=%d",
2584                   attributeset, use_value);
2585             if (r == -1)
2586             {
2587                 char val_str[32];
2588                 sprintf (val_str, "%d", use_value);
2589                 errCode = 114;
2590                 errString = odr_strdup (stream, val_str);
2591             }
2592             else
2593                 errCode = 121;
2594             continue;
2595         }
2596         if (zebraExplain_curDatabase (zh->reg->zei, basenames[base_no]))
2597         {
2598             zh->errString = basenames[base_no];
2599             zh->errCode = 109; /* Database unavailable */
2600             *num_entries = 0;
2601             return;
2602         }
2603         bases_ok++;
2604         for (local_attr = attp.local_attributes; local_attr && ord_no < 32;
2605              local_attr = local_attr->next)
2606         {
2607             int ord;
2608
2609             ord = zebraExplain_lookup_attr_su(zh->reg->zei,
2610                                               attp.attset_ordinal,
2611                                               local_attr->local);
2612             if (ord > 0)
2613                 ords[ord_no++] = ord;
2614         }
2615     }
2616     if (!bases_ok && errCode)
2617     {
2618         zh->errCode = errCode;
2619         zh->errString = errString;
2620         *num_entries = 0;
2621     }
2622     if (ord_no == 0)
2623     {
2624         *num_entries = 0;
2625         return;
2626     }
2627     /* prepare dictionary scanning */
2628     before = pos-1;
2629     after = 1+num-pos;
2630     scan_info_array = (struct scan_info *)
2631         odr_malloc(stream, ord_no * sizeof(*scan_info_array));
2632     for (i = 0; i < ord_no; i++)
2633     {
2634         int j, prefix_len = 0;
2635         int before_tmp = before, after_tmp = after;
2636         struct scan_info *scan_info = scan_info_array + i;
2637         struct rpn_char_map_info rcmi;
2638
2639         rpn_char_map_prepare (zh->reg, reg_id, &rcmi);
2640
2641         scan_info->before = before;
2642         scan_info->after = after;
2643         scan_info->odr = stream;
2644
2645         scan_info->list = (struct scan_info_entry *)
2646             odr_malloc(stream, (before+after) * sizeof(*scan_info->list));
2647         for (j = 0; j<before+after; j++)
2648             scan_info->list[j].term = NULL;
2649
2650         prefix_len += key_SU_encode (ords[i], termz + prefix_len);
2651         termz[prefix_len++] = reg_id;
2652         termz[prefix_len] = 0;
2653         strcpy(scan_info->prefix, termz);
2654
2655         if (trans_scan_term(zh, zapt, termz+prefix_len, reg_id))
2656             return ;
2657
2658         dict_scan(zh->reg->dict, termz, &before_tmp, &after_tmp,
2659                   scan_info, scan_handle);
2660     }
2661     glist = (ZebraScanEntry *)
2662         odr_malloc(stream, (before+after)*sizeof(*glist));
2663
2664     rset_nmem = nmem_create();
2665
2666     /* consider terms after main term */
2667     for (i = 0; i < ord_no; i++)
2668         ptr[i] = before;
2669
2670     *is_partial = 0;
2671     for (i = 0; i<after; i++)
2672     {
2673         int j, j0 = -1;
2674         const char *mterm = NULL;
2675         const char *tst;
2676         RSET rset;
2677
2678         for (j = 0; j < ord_no; j++)
2679         {
2680             if (ptr[j] < before+after &&
2681                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2682                 (!mterm || strcmp (tst, mterm) < 0))
2683             {
2684                 j0 = j;
2685                 mterm = tst;
2686             }
2687         }
2688         if (j0 == -1)
2689             break;
2690         scan_term_untrans (zh, stream->mem, reg_id,
2691                            &glist[i+before].term, mterm);
2692         rset = rset_trunc(zh, &scan_info_array[j0].list[ptr[j0]].isam_p, 1,
2693                           glist[i+before].term, strlen(glist[i+before].term),
2694                           NULL, 0, zapt->term->which, rset_nmem,
2695                           key_it_ctrl,key_it_ctrl->scope);
2696         ptr[j0]++;
2697         for (j = j0+1; j<ord_no; j++)
2698         {
2699             if (ptr[j] < before+after &&
2700                 (tst = scan_info_array[j].list[ptr[j]].term) &&
2701                 !strcmp (tst, mterm))
2702             {
2703                 RSET rsets[2];
2704
2705                 rsets[0] = rset;
2706                 rsets[1] =
2707                     rset_trunc(zh, &scan_info_array[j].list[ptr[j]].isam_p, 1,
2708                                glist[i+before].term,
2709                                strlen(glist[i+before].term), NULL, 0,
2710                                zapt->term->which,rset_nmem,
2711                                key_it_ctrl, key_it_ctrl->scope);
2712                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2713                                          2, key_it_ctrl->scope, rsets);
2714                 ptr[j]++;
2715             }
2716         }
2717         if (limit_set)
2718         {
2719             RSET rsets[2];
2720             rsets[0] = rset;
2721             rsets[1] = rset_dup(limit_set);
2722
2723             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2724                                       key_it_ctrl->scope, 2, rsets);
2725         }
2726         count_set(rset, &glist[i+before].occurrences);
2727         rset_delete(rset);
2728     }
2729     if (i < after)
2730     {
2731         *num_entries -= (after-i);
2732         *is_partial = 1;
2733     }
2734
2735     /* consider terms before main term */
2736     for (i = 0; i<ord_no; i++)
2737         ptr[i] = 0;
2738
2739     for (i = 0; i<before; i++)
2740     {
2741         int j, j0 = -1;
2742         const char *mterm = NULL;
2743         const char *tst;
2744         RSET rset;
2745
2746         for (j = 0; j <ord_no; j++)
2747         {
2748             if (ptr[j] < before &&
2749                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2750                 (!mterm || strcmp (tst, mterm) > 0))
2751             {
2752                 j0 = j;
2753                     mterm = tst;
2754             }
2755         }
2756         if (j0 == -1)
2757             break;
2758
2759         scan_term_untrans (zh, stream->mem, reg_id,
2760                            &glist[before-1-i].term, mterm);
2761
2762         rset = rset_trunc
2763             (zh, &scan_info_array[j0].list[before-1-ptr[j0]].isam_p, 1,
2764              glist[before-1-i].term, strlen(glist[before-1-i].term),
2765              NULL, 0, zapt->term->which,rset_nmem,
2766              key_it_ctrl,key_it_ctrl->scope);
2767
2768         ptr[j0]++;
2769
2770         for (j = j0+1; j<ord_no; j++)
2771         {
2772             if (ptr[j] < before &&
2773                 (tst = scan_info_array[j].list[before-1-ptr[j]].term) &&
2774                 !strcmp (tst, mterm))
2775             {
2776                 RSET rsets[2];
2777
2778                 rsets[0] = rset;
2779                 rsets[1] = rset_trunc(
2780                     zh,
2781                     &scan_info_array[j].list[before-1-ptr[j]].isam_p, 1,
2782                     glist[before-1-i].term,
2783                     strlen(glist[before-1-i].term), NULL, 0,
2784                     zapt->term->which, rset_nmem,
2785                     key_it_ctrl, key_it_ctrl->scope);
2786                 rset = rsmulti_or_create(rset_nmem, key_it_ctrl,
2787                                          2, key_it_ctrl->scope, rsets);
2788
2789                 ptr[j]++;
2790             }
2791         }
2792         if (limit_set)
2793         {
2794             RSET rsets[2];
2795             rsets[0] = rset;
2796             rsets[1] = rset_dup(limit_set);
2797
2798             rset = rsmulti_and_create(rset_nmem, key_it_ctrl,
2799                                       key_it_ctrl->scope, 2, rsets);
2800         }
2801         count_set (rset, &glist[before-1-i].occurrences);
2802         rset_delete (rset);
2803     }
2804     i = before-i;
2805     if (i)
2806     {
2807         *is_partial = 1;
2808         *position -= i;
2809         *num_entries -= i;
2810     }
2811
2812     nmem_destroy(rset_nmem);
2813     *list = glist + i;               /* list is set to first 'real' entry */
2814
2815     yaz_log(YLOG_DEBUG, "position = %d, num_entries = %d",
2816             *position, *num_entries);
2817     if (zh->errCode)
2818         yaz_log(YLOG_DEBUG, "scan error: %d", zh->errCode);
2819 }
2820