1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2011 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 \brief indexes records and extract tokens for indexing and sorting
43 #include <yaz/snprintf.h>
45 static int log_level_extract = 0;
46 static int log_level_details = 0;
47 static int log_level_initialized = 0;
49 /* 1 if we use eliminitate identical delete/insert keys */
50 /* eventually this the 0-case code will be removed */
54 static void extract_flush_record_keys2(ZebraHandle zh, zint sysno,
55 zebra_rec_keys_t ins_keys,
57 zebra_rec_keys_t del_keys,
60 static void extract_flush_record_keys(ZebraHandle zh, zint sysno,
62 zebra_rec_keys_t reckeys,
66 static void zebra_init_log_level(void)
68 if (!log_level_initialized)
70 log_level_initialized = 1;
72 log_level_extract = yaz_log_module_level("extract");
73 log_level_details = yaz_log_module_level("indexdetails");
77 static WRBUF wrbuf_hex_str(const char *cstr)
80 WRBUF w = wrbuf_alloc();
81 for (i = 0; cstr[i]; i++)
83 if (cstr[i] < ' ' || cstr[i] > 126)
84 wrbuf_printf(w, "\\%02X", cstr[i] & 0xff);
86 wrbuf_putc(w, cstr[i]);
92 static void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
93 int cmd, zebra_rec_keys_t skp);
94 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid);
95 static void extract_token_add(RecWord *p);
97 static void check_log_limit(ZebraHandle zh)
99 if (zh->records_processed + zh->records_skipped == zh->m_file_verbose_limit)
101 yaz_log(YLOG_LOG, "More than %d file log entries. Omitting rest",
102 zh->m_file_verbose_limit);
106 static void logRecord(ZebraHandle zh)
109 ++zh->records_processed;
110 if (!(zh->records_processed % 1000))
112 yaz_log(YLOG_LOG, "Records: "ZINT_FORMAT" i/u/d "
113 ZINT_FORMAT"/"ZINT_FORMAT"/"ZINT_FORMAT,
114 zh->records_processed, zh->records_inserted,
115 zh->records_updated, zh->records_deleted);
119 static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl)
121 ctrl->flagShowRecords = !zh->m_flag_rw;
125 static void extract_add_index_string(RecWord *p,
126 zinfo_index_category_t cat,
127 const char *str, int length);
129 static void extract_set_store_data_prepare(struct recExtractCtrl *p);
131 static void extract_init(struct recExtractCtrl *p, RecWord *w)
134 w->index_name = "any";
142 struct snip_rec_info {
144 zebra_snippets *snippets;
148 static void snippet_add_complete_field(RecWord *p, int ord,
151 struct snip_rec_info *h = p->extractCtrl->handle;
153 const char *b = p->term_buf;
154 char buf[IT_MAX_WORD+1];
155 const char **map = 0;
156 int i = 0, remain = p->term_len;
157 const char *start = b;
158 const char *last = 0;
161 map = zebra_maps_input(zm, &b, remain, 1);
163 while (remain > 0 && i < IT_MAX_WORD)
165 while (map && *map && **map == *CHR_SPACE)
167 remain = p->term_len - (b - p->term_buf);
170 start = b; /* set to first non-ws area */
173 int first = i ? 0 : 1; /* first position */
175 map = zebra_maps_input(zm, &b, remain, first);
183 if (i && i < IT_MAX_WORD)
184 buf[i++] = *CHR_SPACE;
185 while (map && *map && **map != *CHR_SPACE)
187 const char *cp = *map;
189 if (**map == *CHR_CUT)
195 if (i >= IT_MAX_WORD)
197 while (i < IT_MAX_WORD && *cp)
201 remain = p->term_len - (b - p->term_buf);
204 map = zebra_maps_input(zm, &b, remain, 0);
212 if (last && start != last && zebra_maps_is_index(zm))
213 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
214 start, last - start);
218 static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm)
220 struct snip_rec_info *h = p->extractCtrl->handle;
221 const char *b = p->term_buf;
222 int remain = p->term_len;
224 const char **map = 0;
225 const char *start = b;
226 const char *last = b;
229 map = zebra_maps_input(zm, &b, remain, 0);
233 char buf[IT_MAX_WORD+1];
237 while (map && *map && **map == *CHR_SPACE)
239 remain = p->term_len - (b - p->term_buf);
242 map = zebra_maps_input(zm, &b, remain, 0);
248 if (start != last && zebra_maps_is_index(zm))
250 zebra_snippets_appendn(h->snippets, p->seqno, 1, ord,
251 start, last - start);
257 while (map && *map && **map != *CHR_SPACE)
259 const char *cp = *map;
261 while (i < IT_MAX_WORD && *cp)
263 remain = p->term_len - (b - p->term_buf);
266 map = zebra_maps_input(zm, &b, remain, 0);
276 if (zebra_maps_is_first_in_field(zm))
278 /* first in field marker */
282 if (start != last && zebra_maps_is_index(zm))
283 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
284 start, last - start);
291 static void snippet_add_icu(RecWord *p, int ord, zebra_map_t zm)
293 struct snip_rec_info *h = p->extractCtrl->handle;
295 const char *res_buf = 0;
298 const char *display_buf = 0;
299 size_t display_len = 0;
301 zebra_map_tokenize_start(zm, p->term_buf, p->term_len);
302 while (zebra_map_tokenize_next(zm, &res_buf, &res_len,
303 &display_buf, &display_len))
305 if (zebra_maps_is_index(zm))
306 zebra_snippets_appendn(h->snippets, p->seqno, 0, ord,
307 display_buf, display_len);
312 static void snippet_token_add(RecWord *p)
314 struct snip_rec_info *h = p->extractCtrl->handle;
315 ZebraHandle zh = h->zh;
316 zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type);
320 ZebraExplainInfo zei = zh->reg->zei;
321 int ch = zebraExplain_lookup_attr_str(
322 zei, zinfo_index_category_index, p->index_type, p->index_name);
324 if (zebra_maps_is_icu(zm))
325 snippet_add_icu(p, ch, zm);
328 if (zebra_maps_is_complete(zm))
329 snippet_add_complete_field(p, ch, zm);
331 snippet_add_incomplete_field(p, ch, zm);
336 static void snippet_schema_add(
337 struct recExtractCtrl *p, Odr_oid *oid)
342 void extract_snippet(ZebraHandle zh, zebra_snippets *sn,
343 struct ZebraRecStream *stream,
344 RecType rt, void *recTypeClientData)
346 struct recExtractCtrl extractCtrl;
347 struct snip_rec_info info;
350 extractCtrl.stream = stream;
351 extractCtrl.first_record = 1;
352 extractCtrl.init = extract_init;
353 extractCtrl.tokenAdd = snippet_token_add;
354 extractCtrl.schemaAdd = snippet_schema_add;
358 extractCtrl.dh = zh->reg->dh;
362 extractCtrl.handle = &info;
363 extractCtrl.match_criteria[0] = '\0';
364 extractCtrl.staticrank = 0;
365 extractCtrl.action = action_insert;
367 init_extractCtrl(zh, &extractCtrl);
369 extractCtrl.setStoreData = 0;
371 r = (*rt->extract)(recTypeClientData, &extractCtrl);
375 static void searchRecordKey(ZebraHandle zh,
376 zebra_rec_keys_t reckeys,
377 const char *index_name,
378 const char **ws, int ws_length)
382 zinfo_index_category_t cat = zinfo_index_category_index;
384 for (i = 0; i<ws_length; i++)
388 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "0", index_name);
390 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "p", index_name);
392 ch = zebraExplain_lookup_attr_str(zh->reg->zei, cat, "w", index_name);
397 if (zebra_rec_keys_rewind(reckeys))
404 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
406 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
408 seqno = key.mem[key.len-1];
410 if (key.mem[0] == ch)
416 woff = seqno - startSeq;
417 if (woff >= 0 && woff < ws_length)
424 #define FILE_MATCH_BLANK "\t "
426 static char *get_match_from_spec(ZebraHandle zh,
427 zebra_rec_keys_t reckeys,
428 const char *fname, const char *spec)
430 static char dstBuf[2048]; /* static here ??? */
432 const char *s = spec;
436 for (; *s && strchr(FILE_MATCH_BLANK, *s); s++)
443 char attset_str[64], attname_str[64];
447 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
449 for (i = 0; *s && *s != ',' && *s != ')' &&
450 !strchr(FILE_MATCH_BLANK, *s); s++)
451 if (i+1 < sizeof(attset_str))
452 attset_str[i++] = *s;
453 attset_str[i] = '\0';
455 for (; strchr(FILE_MATCH_BLANK, *s); s++)
458 strcpy(attname_str, attset_str);
461 for (s++; strchr(FILE_MATCH_BLANK, *s); s++)
463 for (i = 0; *s && *s != ')' &&
464 !strchr(FILE_MATCH_BLANK, *s); s++)
465 if (i+1 < sizeof(attname_str))
466 attname_str[i++] = *s;
467 attname_str[i] = '\0';
471 yaz_log(YLOG_WARN, "Missing ) in match criteria %s in group %s",
472 spec, zh->m_group ? zh->m_group : "none");
477 searchRecordKey(zh, reckeys, attname_str, ws, 32);
478 if (0) /* for debugging */
480 for (i = 0; i<32; i++)
484 WRBUF w = wrbuf_hex_str(ws[i]);
485 yaz_log(YLOG_LOG, "ws[%d] = %s", i, wrbuf_cstr(w));
491 for (i = 0; i<32; i++)
500 dst += strlen(ws[i]);
504 yaz_log(YLOG_WARN, "Record didn't contain match"
505 " fields in (%s,%s)", attset_str, attname_str);
513 const char *spec_src = NULL;
514 const char *s1 = ++s;
515 while (*s1 && !strchr(FILE_MATCH_BLANK, *s1))
519 if (spec_len > sizeof(special)-1)
520 spec_len = sizeof(special)-1;
521 memcpy(special, s, spec_len);
522 special[spec_len] = '\0';
525 if (!strcmp(special, "group"))
526 spec_src = zh->m_group;
527 else if (!strcmp(special, "database"))
528 spec_src = zh->basenames[0];
529 else if (!strcmp(special, "filename")) {
532 else if (!strcmp(special, "type"))
533 spec_src = zh->m_record_type;
538 strcpy(dst, spec_src);
539 dst += strlen(spec_src);
542 else if (*s == '\"' || *s == '\'')
544 int stopMarker = *s++;
548 while (*s && *s != stopMarker)
550 if (i+1 < sizeof(tmpString))
551 tmpString[i++] = *s++;
556 strcpy(dst, tmpString);
557 dst += strlen(tmpString);
561 yaz_log(YLOG_WARN, "Syntax error in match criteria %s in group %s",
562 spec, zh->m_group ? zh->m_group : "none");
569 yaz_log(YLOG_WARN, "No match criteria for record %s in group %s",
570 fname, zh->m_group ? zh->m_group : "none");
575 if (0) /* for debugging */
577 WRBUF w = wrbuf_hex_str(dstBuf);
578 yaz_log(YLOG_LOG, "get_match_from_spec %s", wrbuf_cstr(w));
585 struct recordLogInfo {
588 struct recordGroup *rGroup;
591 /** \brief add the always-matches index entry and map to real record ID
592 \param ctrl record control
593 \param record_id custom record ID
594 \param sysno system record ID
596 This function serves two purposes.. It adds the always matches
597 entry and makes a pointer from the custom record ID (if defined)
598 back to the system record ID (sysno)
599 See zebra_recid_to_sysno .
601 static void all_matches_add(struct recExtractCtrl *ctrl, zint record_id,
605 extract_init(ctrl, &word);
606 word.record_id = record_id;
607 /* we use the seqno as placeholder for a way to get back to
608 record database from _ALLRECORDS.. This is used if a custom
609 RECORD was defined */
611 word.index_name = "_ALLRECORDS";
612 word.index_type = "w";
614 extract_add_index_string(&word, zinfo_index_category_alwaysmatches,
618 /* forward declaration */
619 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
620 struct ZebraRecStream *stream,
621 enum zebra_recctrl_action_t action,
622 const char *recordType,
624 const char *match_criteria,
627 void *recTypeClientData);
630 ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname,
631 enum zebra_recctrl_action_t action)
633 ZEBRA_RES r = ZEBRA_OK;
638 struct file_read_info *fi = 0;
639 const char *original_record_type = 0;
641 void *recTypeClientData;
642 struct ZebraRecStream stream, *streamp;
644 zebra_init_log_level();
646 if (!zh->m_group || !*zh->m_group)
649 sprintf(gprefix, "%s.", zh->m_group);
651 yaz_log(log_level_extract, "zebra_extract_file %s", fname);
653 /* determine file extension */
655 for (i = strlen(fname); --i >= 0; )
658 else if (fname[i] == '.')
660 strcpy(ext, fname+i+1);
663 /* determine file type - depending on extension */
664 original_record_type = zh->m_record_type;
665 if (!zh->m_record_type)
667 sprintf(ext_res, "%srecordType.%s", gprefix, ext);
668 zh->m_record_type = res_get(zh->res, ext_res);
670 if (!zh->m_record_type)
673 if (zh->records_processed + zh->records_skipped
674 < zh->m_file_verbose_limit)
675 yaz_log(YLOG_LOG, "? %s", fname);
676 zh->records_skipped++;
679 /* determine match criteria */
680 if (!zh->m_record_id)
682 sprintf(ext_res, "%srecordId.%s", gprefix, ext);
683 zh->m_record_id = res_get(zh->res, ext_res);
687 recType_byName(zh->reg->recTypes, zh->res, zh->m_record_type,
688 &recTypeClientData)))
690 yaz_log(YLOG_WARN, "No such record type: %s", zh->m_record_type);
694 switch(recType->version)
699 yaz_log(YLOG_WARN, "Bad filter version: %s", zh->m_record_type);
701 if (sysno && (action == action_delete || action == action_a_delete))
710 if (zh->path_reg && !yaz_is_abspath(fname))
712 strcpy(full_rep, zh->path_reg);
713 strcat(full_rep, "/");
714 strcat(full_rep, fname);
717 strcpy(full_rep, fname);
719 if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1)
721 yaz_log(YLOG_WARN|YLOG_ERRNO, "open %s", full_rep);
722 zh->m_record_type = original_record_type;
726 zebra_create_stream_fd(streamp, fd, 0);
728 r = zebra_extract_records_stream(zh, streamp,
732 0, /*match_criteria */
734 recType, recTypeClientData);
736 stream.destroy(streamp);
737 zh->m_record_type = original_record_type;
742 If sysno is provided, then it's used to identify the reocord.
743 If not, and match_criteria is provided, then sysno is guessed
744 If not, and a record is provided, then sysno is got from there
748 ZEBRA_RES zebra_buffer_extract_record(ZebraHandle zh,
749 const char *buf, size_t buf_size,
750 enum zebra_recctrl_action_t action,
751 const char *recordType,
753 const char *match_criteria,
756 struct ZebraRecStream stream;
761 if (recordType && *recordType)
763 yaz_log(log_level_extract,
764 "Record type explicitly specified: %s", recordType);
765 recType = recType_byName(zh->reg->recTypes, zh->res, recordType,
770 if (!(zh->m_record_type))
772 yaz_log(YLOG_WARN, "No such record type defined");
775 yaz_log(log_level_extract, "Get record type from rgroup: %s",
777 recType = recType_byName(zh->reg->recTypes, zh->res,
778 zh->m_record_type, &clientData);
779 recordType = zh->m_record_type;
784 yaz_log(YLOG_WARN, "No such record type: %s", recordType);
788 zebra_create_stream_mem(&stream, buf, buf_size);
790 res = zebra_extract_records_stream(zh, &stream,
796 recType, clientData);
797 stream.destroy(&stream);
801 static ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh,
802 struct ZebraRecStream *stream,
803 enum zebra_recctrl_action_t action,
804 const char *recordType,
806 const char *match_criteria,
809 void *recTypeClientData,
814 RecordAttr *recordAttr;
815 struct recExtractCtrl extractCtrl;
817 const char *matchStr = 0;
819 off_t start_offset = 0, end_offset = 0;
820 const char *pr_fname = fname; /* filename to print .. */
821 int show_progress = zh->records_processed + zh->records_skipped
822 < zh->m_file_verbose_limit ? 1:0;
824 zebra_init_log_level();
827 pr_fname = "<no file>"; /* make it printable if file is omitted */
829 zebra_rec_keys_reset(zh->reg->keys);
830 zebra_rec_keys_reset(zh->reg->sortKeys);
832 if (zebraExplain_curDatabase(zh->reg->zei, zh->basenames[0]))
834 if (zebraExplain_newDatabase(zh->reg->zei, zh->basenames[0],
835 zh->m_explain_database))
841 off_t null_offset = 0;
842 extractCtrl.stream = stream;
844 start_offset = stream->tellf(stream);
846 extractCtrl.first_record = start_offset ? 0 : 1;
848 stream->endf(stream, &null_offset);;
850 extractCtrl.init = extract_init;
851 extractCtrl.tokenAdd = extract_token_add;
852 extractCtrl.schemaAdd = extract_schema_add;
853 extractCtrl.dh = zh->reg->dh;
854 extractCtrl.handle = zh;
855 extractCtrl.match_criteria[0] = '\0';
856 extractCtrl.staticrank = 0;
857 extractCtrl.action = action;
859 init_extractCtrl(zh, &extractCtrl);
861 extract_set_store_data_prepare(&extractCtrl);
863 r = (*recType->extract)(recTypeClientData, &extractCtrl);
865 if (action == action_update)
867 action = extractCtrl.action;
872 case RECCTRL_EXTRACT_EOF:
874 case RECCTRL_EXTRACT_ERROR_GENERIC:
875 /* error occured during extraction ... */
876 yaz_log(YLOG_WARN, "extract error: generic");
878 case RECCTRL_EXTRACT_ERROR_NO_SUCH_FILTER:
879 /* error occured during extraction ... */
880 yaz_log(YLOG_WARN, "extract error: no such filter");
882 case RECCTRL_EXTRACT_SKIP:
884 yaz_log(YLOG_LOG, "skip %s %s " ZINT_FORMAT,
885 recordType, pr_fname, (zint) start_offset);
888 end_offset = stream->endf(stream, 0);
890 stream->seekf(stream, end_offset);
893 case RECCTRL_EXTRACT_OK:
896 yaz_log(YLOG_WARN, "extract error: unknown error: %d", r);
899 end_offset = stream->endf(stream, 0);
901 stream->seekf(stream, end_offset);
903 end_offset = stream->tellf(stream);
905 if (extractCtrl.match_criteria[0])
906 match_criteria = extractCtrl.match_criteria;
911 if (zh->m_flag_rw == 0)
913 yaz_log(YLOG_LOG, "test %s %s " ZINT_FORMAT, recordType,
914 pr_fname, (zint) start_offset);
915 /* test mode .. Do not perform match */
923 if (match_criteria && *match_criteria)
924 matchStr = match_criteria;
927 if (zh->m_record_id && *zh->m_record_id)
929 matchStr = get_match_from_spec(zh, zh->reg->keys, pr_fname,
933 yaz_log(YLOG_LOG, "error %s %s " ZINT_FORMAT, recordType,
934 pr_fname, (zint) start_offset);
939 WRBUF w = wrbuf_alloc();
941 for (i = 0; i < strlen(matchStr); i++)
943 wrbuf_printf(w, "%02X", matchStr[i] & 0xff);
945 yaz_log(YLOG_LOG, "Got match %s", wrbuf_cstr(w));
952 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
953 char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord,
957 if (log_level_extract)
959 WRBUF w = wrbuf_hex_str(matchStr);
960 yaz_log(log_level_extract, "matchStr: %s", wrbuf_cstr(w));
965 assert(*rinfo == sizeof(*sysno));
966 memcpy(sysno, rinfo+1, sizeof(*sysno));
973 /* new record AKA does not exist already */
974 if (action == action_delete)
976 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
977 pr_fname, (zint) start_offset);
978 yaz_log(YLOG_WARN, "cannot delete record above (seems new)");
981 else if (action == action_a_delete)
984 yaz_log(YLOG_LOG, "adelete %s %s " ZINT_FORMAT, recordType,
985 pr_fname, (zint) start_offset);
988 else if (action == action_replace)
990 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
991 pr_fname, (zint) start_offset);
992 yaz_log(YLOG_WARN, "cannot update record above (seems new)");
996 yaz_log(YLOG_LOG, "add %s %s " ZINT_FORMAT, recordType, pr_fname,
997 (zint) start_offset);
998 rec = rec_new(zh->reg->records);
1000 *sysno = rec->sysno;
1005 all_matches_add(&extractCtrl,
1006 zebra_rec_keys_get_custom_record_id(zh->reg->keys),
1011 recordAttr = rec_init_attr(zh->reg->zei, rec);
1012 if (extractCtrl.staticrank < 0)
1014 yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0");
1015 extractCtrl.staticrank = 0;
1020 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
1021 dict_insert_ord(zh->reg->matchDict, db_ord, matchStr,
1022 sizeof(*sysno), sysno);
1025 extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
1027 extract_flush_record_keys2(zh, *sysno,
1028 zh->reg->keys, extractCtrl.staticrank,
1029 0, recordAttr->staticrank);
1031 extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys,
1032 extractCtrl.staticrank);
1034 recordAttr->staticrank = extractCtrl.staticrank;
1035 zh->records_inserted++;
1039 /* record already exists */
1040 zebra_rec_keys_t delkeys = zebra_rec_keys_open();
1041 zebra_rec_keys_t sortKeys = zebra_rec_keys_open();
1042 if (action == action_insert)
1044 yaz_log(YLOG_LOG, "skipped %s %s " ZINT_FORMAT,
1045 recordType, pr_fname, (zint) start_offset);
1050 rec = rec_get(zh->reg->records, *sysno);
1055 all_matches_add(&extractCtrl,
1056 zebra_rec_keys_get_custom_record_id(zh->reg->keys),
1060 recordAttr = rec_init_attr(zh->reg->zei, rec);
1062 /* decrease total size */
1063 zebraExplain_recordBytesIncrement(zh->reg->zei,
1064 - recordAttr->recordSize);
1066 zebra_rec_keys_set_buf(delkeys,
1067 rec->info[recInfo_delKeys],
1068 rec->size[recInfo_delKeys],
1070 zebra_rec_keys_set_buf(sortKeys,
1071 rec->info[recInfo_sortKeys],
1072 rec->size[recInfo_sortKeys],
1075 extract_flush_sort_keys(zh, *sysno, 0, sortKeys);
1077 extract_flush_record_keys(zh, *sysno, 0, delkeys,
1078 recordAttr->staticrank);
1080 if (action == action_delete || action == action_a_delete)
1082 /* record going to be deleted */
1084 extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank,
1085 delkeys, recordAttr->staticrank);
1087 if (zebra_rec_keys_empty(delkeys))
1089 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
1090 pr_fname, (zint) start_offset);
1091 yaz_log(YLOG_WARN, "cannot delete file above, "
1092 "storeKeys false (3)");
1097 yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType,
1098 pr_fname, (zint) start_offset);
1099 zh->records_deleted++;
1102 int db_ord = zebraExplain_get_database_ord(zh->reg->zei);
1103 dict_delete_ord(zh->reg->matchDict, db_ord, matchStr);
1105 rec_del(zh->reg->records, &rec);
1107 zebra_rec_keys_close(delkeys);
1108 zebra_rec_keys_close(sortKeys);
1114 { /* update or special_update */
1116 yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType,
1117 pr_fname, (zint) start_offset);
1118 extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys);
1121 extract_flush_record_keys2(zh, *sysno,
1122 zh->reg->keys, extractCtrl.staticrank,
1123 delkeys, recordAttr->staticrank);
1125 extract_flush_record_keys(zh, *sysno, 1,
1126 zh->reg->keys, extractCtrl.staticrank);
1128 recordAttr->staticrank = extractCtrl.staticrank;
1129 zh->records_updated++;
1131 zebra_rec_keys_close(delkeys);
1132 zebra_rec_keys_close(sortKeys);
1134 /* update file type */
1135 xfree(rec->info[recInfo_fileType]);
1136 rec->info[recInfo_fileType] =
1137 rec_strdup(recordType, &rec->size[recInfo_fileType]);
1139 /* update filename */
1140 xfree(rec->info[recInfo_filename]);
1141 rec->info[recInfo_filename] =
1142 rec_strdup(fname, &rec->size[recInfo_filename]);
1144 /* update delete keys */
1145 xfree(rec->info[recInfo_delKeys]);
1146 if (!zebra_rec_keys_empty(zh->reg->keys) && zh->m_store_keys == 1)
1148 zebra_rec_keys_get_buf(zh->reg->keys,
1149 &rec->info[recInfo_delKeys],
1150 &rec->size[recInfo_delKeys]);
1154 rec->info[recInfo_delKeys] = NULL;
1155 rec->size[recInfo_delKeys] = 0;
1157 /* update sort keys */
1158 xfree(rec->info[recInfo_sortKeys]);
1160 zebra_rec_keys_get_buf(zh->reg->sortKeys,
1161 &rec->info[recInfo_sortKeys],
1162 &rec->size[recInfo_sortKeys]);
1166 recordAttr->recordSize = end_offset - start_offset;
1167 zebraExplain_recordBytesIncrement(zh->reg->zei,
1168 recordAttr->recordSize);
1171 /* set run-number for this record */
1172 recordAttr->runNumber =
1173 zebraExplain_runNumberIncrement(zh->reg->zei, 0);
1175 /* update store data */
1176 xfree(rec->info[recInfo_storeData]);
1178 /* update store data */
1179 if (zh->store_data_buf)
1181 rec->size[recInfo_storeData] = zh->store_data_size;
1182 rec->info[recInfo_storeData] = zh->store_data_buf;
1183 zh->store_data_buf = 0;
1184 recordAttr->recordSize = zh->store_data_size;
1186 else if (zh->m_store_data)
1188 off_t cur_offset = stream->tellf(stream);
1190 rec->size[recInfo_storeData] = recordAttr->recordSize;
1191 rec->info[recInfo_storeData] = (char *)
1192 xmalloc(recordAttr->recordSize);
1193 stream->seekf(stream, start_offset);
1194 stream->readf(stream, rec->info[recInfo_storeData],
1195 recordAttr->recordSize);
1196 stream->seekf(stream, cur_offset);
1200 rec->info[recInfo_storeData] = NULL;
1201 rec->size[recInfo_storeData] = 0;
1203 /* update database name */
1204 xfree(rec->info[recInfo_databaseName]);
1205 rec->info[recInfo_databaseName] =
1206 rec_strdup(zh->basenames[0], &rec->size[recInfo_databaseName]);
1209 recordAttr->recordOffset = start_offset;
1211 /* commit this record */
1212 rec_put(zh->reg->records, &rec);
1217 /** \brief extracts records from stream
1218 \param zh Zebra Handle
1219 \param stream stream that we read from
1220 \param action (action_insert, action_replace, action_delete, ..)
1221 \param recordType Record filter type "grs.xml", etc.
1222 \param sysno pointer to sysno if already known; NULL otherwise
1223 \param match_criteria (NULL if not already given)
1224 \param fname filename that we read from (for logging purposes only)
1225 \param recType record type
1226 \param recTypeClientData client data for record type
1227 \returns ZEBRA_OK for success; ZEBRA_FAIL for failure
1229 ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh,
1230 struct ZebraRecStream *stream,
1231 enum zebra_recctrl_action_t action,
1232 const char *recordType,
1234 const char *match_criteria,
1237 void *recTypeClientData)
1239 ZEBRA_RES res = ZEBRA_OK;
1243 res = zebra_extract_record_stream(zh, stream,
1249 recType, recTypeClientData, &more);
1255 if (res != ZEBRA_OK)
1263 ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n)
1265 ZebraHandle zh = (ZebraHandle) handle;
1266 struct recExtractCtrl extractCtrl;
1268 if (zebraExplain_curDatabase(zh->reg->zei,
1269 rec->info[recInfo_databaseName]))
1272 if (zebraExplain_newDatabase(zh->reg->zei,
1273 rec->info[recInfo_databaseName], 0))
1277 zebra_rec_keys_reset(zh->reg->keys);
1278 zebra_rec_keys_reset(zh->reg->sortKeys);
1280 extractCtrl.init = extract_init;
1281 extractCtrl.tokenAdd = extract_token_add;
1282 extractCtrl.schemaAdd = extract_schema_add;
1283 extractCtrl.dh = zh->reg->dh;
1285 init_extractCtrl(zh, &extractCtrl);
1287 extractCtrl.flagShowRecords = 0;
1288 extractCtrl.match_criteria[0] = '\0';
1289 extractCtrl.staticrank = 0;
1290 extractCtrl.action = action_update;
1292 extractCtrl.handle = handle;
1293 extractCtrl.first_record = 1;
1295 extract_set_store_data_prepare(&extractCtrl);
1298 grs_extract_tree(&extractCtrl, n);
1300 if (rec->size[recInfo_delKeys])
1302 zebra_rec_keys_t delkeys = zebra_rec_keys_open();
1304 zebra_rec_keys_t sortkeys = zebra_rec_keys_open();
1306 zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys],
1307 rec->size[recInfo_delKeys],
1310 extract_flush_record_keys2(zh, rec->sysno,
1311 zh->reg->keys, 0, delkeys, 0);
1313 extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0);
1314 extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1316 zebra_rec_keys_close(delkeys);
1318 zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys],
1319 rec->size[recInfo_sortKeys],
1322 extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys);
1323 zebra_rec_keys_close(sortkeys);
1328 extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0);
1330 extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0);
1333 extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys);
1335 xfree(rec->info[recInfo_delKeys]);
1336 zebra_rec_keys_get_buf(zh->reg->keys,
1337 &rec->info[recInfo_delKeys],
1338 &rec->size[recInfo_delKeys]);
1340 xfree(rec->info[recInfo_sortKeys]);
1341 zebra_rec_keys_get_buf(zh->reg->sortKeys,
1342 &rec->info[recInfo_sortKeys],
1343 &rec->size[recInfo_sortKeys]);
1347 void zebra_it_key_str_dump(ZebraHandle zh, struct it_key *key,
1348 const char *str, size_t slen, NMEM nmem, int level)
1350 char keystr[200]; /* room for zints to print */
1352 int ord = CAST_ZINT_TO_INT(key->mem[0]);
1353 const char *index_type;
1355 const char *string_index;
1357 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1358 0/* db */, &string_index);
1360 zebra_term_untrans_iconv(zh, nmem, index_type,
1363 for (i = 0; i < key->len; i++)
1365 sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key->mem[i]);
1368 if (*str < CHR_BASE_CHAR)
1371 char dst_buf[200]; /* room for special chars */
1373 strcpy(dst_buf , "?");
1375 if (!strcmp(str, ""))
1376 strcpy(dst_buf, "alwaysmatches");
1377 if (!strcmp(str, FIRST_IN_FIELD_STR))
1378 strcpy(dst_buf, "firstinfield");
1379 else if (!strcmp(str, CHR_UNKNOWN))
1380 strcpy(dst_buf, "unknown");
1381 else if (!strcmp(str, CHR_SPACE))
1382 strcpy(dst_buf, "space");
1384 for (i = 0; i<slen; i++)
1386 sprintf(dst_buf + strlen(dst_buf), " %d", str[i] & 0xff);
1388 yaz_log(level, "%s%s %s %s", keystr, index_type,
1389 string_index, dst_buf);
1393 yaz_log(level, "%s%s %s \"%s\"", keystr, index_type,
1394 string_index, dst_term);
1397 void extract_rec_keys_log(ZebraHandle zh, int is_insert,
1398 zebra_rec_keys_t reckeys,
1401 if (zebra_rec_keys_rewind(reckeys))
1406 NMEM nmem = nmem_create();
1408 while(zebra_rec_keys_read(reckeys, &str, &slen, &key))
1410 zebra_it_key_str_dump(zh, &key, str, slen, nmem, level);
1417 void extract_rec_keys_adjust(ZebraHandle zh, int is_insert,
1418 zebra_rec_keys_t reckeys)
1420 ZebraExplainInfo zei = zh->reg->zei;
1424 struct ord_stat *next;
1427 if (zebra_rec_keys_rewind(reckeys))
1429 struct ord_stat *ord_list = 0;
1433 struct it_key key_in;
1434 while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1436 int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1438 for (p = ord_list; p ; p = p->next)
1446 p = xmalloc(sizeof(*p));
1457 struct ord_stat *p1 = p;
1460 zebraExplain_ord_adjust_occurrences(zei, p->ord, p->no, 1);
1462 zebraExplain_ord_adjust_occurrences(zei, p->ord, - p->no, -1);
1470 static void extract_flush_record_keys2(
1471 ZebraHandle zh, zint sysno,
1472 zebra_rec_keys_t ins_keys, zint ins_rank,
1473 zebra_rec_keys_t del_keys, zint del_rank)
1475 ZebraExplainInfo zei = zh->reg->zei;
1479 if (!zh->reg->key_block)
1481 int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1482 const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", ".");
1483 int use_threads = atoi(res_get_def(zh->res, "threads", "1"));
1484 zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1489 extract_rec_keys_adjust(zh, 1, ins_keys);
1491 zebraExplain_recordCountIncrement(zei, 1);
1492 zebra_rec_keys_rewind(ins_keys);
1496 extract_rec_keys_adjust(zh, 0, del_keys);
1498 zebraExplain_recordCountIncrement(zei, -1);
1499 zebra_rec_keys_rewind(del_keys);
1505 const char *del_str;
1506 struct it_key del_key_in;
1510 const char *ins_str;
1511 struct it_key ins_key_in;
1515 del = zebra_rec_keys_read(del_keys, &del_str, &del_slen,
1518 ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen,
1521 if (del && ins && ins_rank == del_rank
1522 && !key_compare(&del_key_in, &ins_key_in)
1523 && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen))
1533 key_block_write(zh->reg->key_block, sysno,
1534 &del_key_in, 0, del_str, del_slen,
1535 del_rank, zh->m_staticrank);
1537 key_block_write(zh->reg->key_block, sysno,
1538 &ins_key_in, 1, ins_str, ins_slen,
1539 ins_rank, zh->m_staticrank);
1541 yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized);
1544 static void extract_flush_record_keys(
1545 ZebraHandle zh, zint sysno, int cmd,
1546 zebra_rec_keys_t reckeys,
1549 ZebraExplainInfo zei = zh->reg->zei;
1551 extract_rec_keys_adjust(zh, cmd, reckeys);
1553 if (log_level_details)
1555 yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s",
1556 sysno, cmd ? "insert" : "delete");
1557 extract_rec_keys_log(zh, cmd, reckeys, log_level_details);
1560 if (!zh->reg->key_block)
1562 int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8"));
1563 const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", ".");
1564 int use_threads = atoi(res_get_def(zh->res, "threads", "1"));
1565 zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads);
1567 zebraExplain_recordCountIncrement(zei, cmd ? 1 : -1);
1570 yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd);
1571 print_rec_keys(zh, reckeys);
1573 if (zebra_rec_keys_rewind(reckeys))
1577 struct it_key key_in;
1578 while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1580 key_block_write(zh->reg->key_block, sysno,
1581 &key_in, cmd, str, slen,
1582 staticrank, zh->m_staticrank);
1588 ZEBRA_RES zebra_rec_keys_to_snippets1(ZebraHandle zh,
1589 zebra_rec_keys_t reckeys,
1590 zebra_snippets *snippets)
1592 NMEM nmem = nmem_create();
1593 if (zebra_rec_keys_rewind(reckeys))
1598 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1603 const char *index_type;
1605 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1606 seqno = key.mem[key.len-1];
1607 ord = CAST_ZINT_TO_INT(key.mem[0]);
1609 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type,
1610 0/* db */, 0 /* string_index */);
1612 zebra_term_untrans_iconv(zh, nmem, index_type,
1614 zebra_snippets_append(snippets, seqno, 0, ord, dst_term);
1622 void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys)
1624 yaz_log(YLOG_LOG, "print_rec_keys");
1625 if (zebra_rec_keys_rewind(reckeys))
1630 while (zebra_rec_keys_read(reckeys, &str, &slen, &key))
1632 char dst_buf[IT_MAX_WORD];
1634 const char *index_type;
1635 int ord = CAST_ZINT_TO_INT(key.mem[0]);
1637 assert(key.len <= IT_KEY_LEVEL_MAX && key.len > 2);
1639 zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, 0);
1641 seqno = key.mem[key.len-1];
1643 zebra_term_untrans(zh, index_type, dst_buf, str);
1645 yaz_log(YLOG_LOG, "ord=%d seqno=" ZINT_FORMAT
1646 " term=%s", ord, seqno, dst_buf);
1651 static void extract_add_index_string(RecWord *p, zinfo_index_category_t cat,
1652 const char *str, int length)
1655 ZebraHandle zh = p->extractCtrl->handle;
1656 ZebraExplainInfo zei = zh->reg->zei;
1659 ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1661 ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1665 key.mem[i++] = p->record_id;
1666 key.mem[i++] = p->section_id;
1668 if (zh->m_segment_indexing)
1669 key.mem[i++] = p->segment;
1670 key.mem[i++] = p->seqno;
1673 zebra_rec_keys_write(zh->reg->keys, str, length, &key);
1676 static void extract_add_sort_string(RecWord *p, const char *str, int length)
1679 ZebraHandle zh = p->extractCtrl->handle;
1680 ZebraExplainInfo zei = zh->reg->zei;
1682 zinfo_index_category_t cat = zinfo_index_category_sort;
1684 ch = zebraExplain_lookup_attr_str(zei, cat, p->index_type, p->index_name);
1686 ch = zebraExplain_add_attr_str(zei, cat, p->index_type, p->index_name);
1689 key.mem[1] = p->record_id;
1690 key.mem[2] = p->section_id;
1692 zebra_rec_keys_write(zh->reg->sortKeys, str, length, &key);
1695 static void extract_add_staticrank_string(RecWord *p,
1696 const char *str, int length)
1699 struct recExtractCtrl *ctrl = p->extractCtrl;
1701 if (length > sizeof(valz)-1)
1702 length = sizeof(valz)-1;
1704 memcpy(valz, str, length);
1705 valz[length] = '\0';
1706 ctrl->staticrank = atozint(valz);
1709 static void extract_add_string(RecWord *p, zebra_map_t zm,
1710 const char *string, int length)
1716 if (log_level_details)
1719 WRBUF w = wrbuf_alloc();
1721 wrbuf_write_escaped(w, string, length);
1722 yaz_log(log_level_details, "extract_add_string: %s", wrbuf_cstr(w));
1725 if (zebra_maps_is_index(zm))
1727 extract_add_index_string(p, zinfo_index_category_index,
1729 if (zebra_maps_is_alwaysmatches(zm))
1732 memcpy(&word, p, sizeof(word));
1735 extract_add_index_string(
1736 &word, zinfo_index_category_alwaysmatches, "", 0);
1739 else if (zebra_maps_is_sort(zm))
1741 extract_add_sort_string(p, string, length);
1743 else if (zebra_maps_is_staticrank(zm))
1745 extract_add_staticrank_string(p, string, length);
1749 static void extract_add_incomplete_field(RecWord *p, zebra_map_t zm)
1751 const char *b = p->term_buf;
1752 int remain = p->term_len;
1754 const char **map = 0;
1757 map = zebra_maps_input(zm, &b, remain, 0);
1761 char buf[IT_MAX_WORD+1];
1765 while (map && *map && **map == *CHR_SPACE)
1767 remain = p->term_len - (b - p->term_buf);
1769 map = zebra_maps_input(zm, &b, remain, 0);
1776 while (map && *map && **map != *CHR_SPACE)
1778 const char *cp = *map;
1780 while (i < IT_MAX_WORD && *cp)
1782 remain = p->term_len - (b - p->term_buf);
1784 map = zebra_maps_input(zm, &b, remain, 0);
1794 if (zebra_maps_is_first_in_field(zm))
1796 /* first in field marker */
1797 extract_add_string(p, zm, FIRST_IN_FIELD_STR, FIRST_IN_FIELD_LEN);
1801 extract_add_string(p, zm, buf, i);
1806 static void extract_add_complete_field(RecWord *p, zebra_map_t zm)
1808 const char *b = p->term_buf;
1809 char buf[IT_MAX_WORD+1];
1810 const char **map = 0;
1811 int i = 0, remain = p->term_len;
1814 map = zebra_maps_input(zm, &b, remain, 1);
1816 while (remain > 0 && i < IT_MAX_WORD)
1818 while (map && *map && **map == *CHR_SPACE)
1820 remain = p->term_len - (b - p->term_buf);
1824 int first = i ? 0 : 1; /* first position */
1825 map = zebra_maps_input(zm, &b, remain, first);
1833 if (i && i < IT_MAX_WORD)
1834 buf[i++] = *CHR_SPACE;
1835 while (map && *map && **map != *CHR_SPACE)
1837 const char *cp = *map;
1839 if (**map == *CHR_CUT)
1845 if (i >= IT_MAX_WORD)
1847 while (i < IT_MAX_WORD && *cp)
1850 remain = p->term_len - (b - p->term_buf);
1853 map = zebra_maps_input(zm, &b, remain, 0);
1861 extract_add_string(p, zm, buf, i);
1865 static void extract_add_icu(RecWord *p, zebra_map_t zm)
1867 const char *res_buf = 0;
1870 zebra_map_tokenize_start(zm, p->term_buf, p->term_len);
1871 while (zebra_map_tokenize_next(zm, &res_buf, &res_len, 0, 0))
1873 extract_add_string(p, zm, res_buf, res_len);
1879 /** \brief top-level indexing handler for recctrl system
1880 \param p token data to be indexed
1884 extract_add_{in}_complete / extract_add_icu
1887 extract_add_index_string
1889 extract_add_sort_string
1891 extract_add_staticrank_string
1894 static void extract_token_add(RecWord *p)
1896 ZebraHandle zh = p->extractCtrl->handle;
1897 zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type);
1900 if (log_level_details)
1902 yaz_log(log_level_details, "extract_token_add "
1903 "type=%s index=%s seqno=" ZINT_FORMAT " s=%.*s",
1904 p->index_type, p->index_name,
1905 p->seqno, p->term_len, p->term_buf);
1907 if ((wrbuf = zebra_replace(zm, 0, p->term_buf, p->term_len)))
1909 p->term_buf = wrbuf_buf(wrbuf);
1910 p->term_len = wrbuf_len(wrbuf);
1912 if (zebra_maps_is_icu(zm))
1914 extract_add_icu(p, zm);
1918 if (zebra_maps_is_complete(zm))
1919 extract_add_complete_field(p, zm);
1921 extract_add_incomplete_field(p, zm);
1925 static void extract_set_store_data_cb(struct recExtractCtrl *p,
1926 void *buf, size_t sz)
1928 ZebraHandle zh = (ZebraHandle) p->handle;
1930 xfree(zh->store_data_buf);
1931 zh->store_data_buf = 0;
1932 zh->store_data_size = 0;
1935 zh->store_data_buf = xmalloc(sz);
1936 zh->store_data_size = sz;
1937 memcpy(zh->store_data_buf, buf, sz);
1941 static void extract_set_store_data_prepare(struct recExtractCtrl *p)
1943 ZebraHandle zh = (ZebraHandle) p->handle;
1944 xfree(zh->store_data_buf);
1945 zh->store_data_buf = 0;
1946 zh->store_data_size = 0;
1947 p->setStoreData = extract_set_store_data_cb;
1950 static void extract_schema_add(struct recExtractCtrl *p, Odr_oid *oid)
1952 ZebraHandle zh = (ZebraHandle) p->handle;
1953 zebraExplain_addSchema(zh->reg->zei, oid);
1956 void extract_flush_sort_keys(ZebraHandle zh, zint sysno,
1957 int cmd, zebra_rec_keys_t reckeys)
1960 yaz_log(YLOG_LOG, "extract_flush_sort_keys cmd=%d sysno=" ZINT_FORMAT,
1962 extract_rec_keys_log(zh, cmd, reckeys, YLOG_LOG);
1965 if (zebra_rec_keys_rewind(reckeys))
1967 zebra_sort_index_t si = zh->reg->sort_index;
1970 struct it_key key_in;
1972 NMEM nmem = nmem_create();
1973 struct sort_add_ent {
1976 struct sort_add_ent *next;
1981 struct sort_add_ent *sort_ent_list = 0;
1983 while (zebra_rec_keys_read(reckeys, &str, &slen, &key_in))
1985 int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
1986 zint filter_sysno = key_in.mem[1];
1987 zint section_id = key_in.mem[2];
1989 struct sort_add_ent **e = &sort_ent_list;
1990 for (; *e; e = &(*e)->next)
1991 if ((*e)->ord == ord && section_id == (*e)->section_id)
1995 *e = nmem_malloc(nmem, sizeof(**e));
1997 (*e)->wrbuf = wrbuf_alloc();
2000 (*e)->sysno = filter_sysno ? filter_sysno : sysno;
2001 (*e)->section_id = section_id;
2004 wrbuf_write((*e)->wrbuf, str, slen);
2005 wrbuf_putc((*e)->wrbuf, '\0');
2009 zint last_sysno = 0;
2010 struct sort_add_ent *e = sort_ent_list;
2011 for (; e; e = e->next)
2013 if (last_sysno != e->sysno)
2015 zebra_sort_sysno(si, e->sysno);
2016 last_sysno = e->sysno;
2018 zebra_sort_type(si, e->ord);
2020 zebra_sort_add(si, e->section_id, e->wrbuf);
2022 zebra_sort_delete(si, e->section_id);
2023 wrbuf_destroy(e->wrbuf);
2033 * c-file-style: "Stroustrup"
2034 * indent-tabs-mode: nil
2036 * vim: shiftwidth=4 tabstop=8 expandtab