2 * Copyright (C) 1996-2002, Index Data
4 * Sebastian Hammer, Adam Dickmeiss
6 * $Id: charmap.c,v 1.23 2002-07-25 13:06:44 adam Exp $
11 * Support module to handle character-conversions into and out of the
23 static size_t iconv(iconv_t t, char **buf, size_t *inbytesleft,
24 char **outbuf, size_t *outbytesleft)
30 typedef unsigned ucs4_t;
32 #include <yaz/yaz-util.h>
36 #define CHR_MAXSTR 1024
37 #define CHR_MAXEQUIV 32
39 const char *CHR_UNKNOWN = "\001";
40 const char *CHR_SPACE = "\002";
41 const char *CHR_BASE = "\003";
45 chr_t_entry *input; /* mapping table for input data */
46 chr_t_entry *q_input; /* mapping table for queries */
47 unsigned char *output[256]; /* return mapping - for display of registers */
48 int base_uppercase; /* Start of upper-case ordinals */
53 * Character map trie node.
57 chr_t_entry **children; /* array of children */
58 unsigned char **target; /* target for this node, if any */
62 * General argument structure for callback functions (internal use only)
64 typedef struct chrwork
67 char string[CHR_MAXSTR+1];
71 * Add an entry to the character map.
73 static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
74 const char *from, int len, char *to,
81 root = (chr_t_entry *) nmem_malloc(nmem, sizeof(*root));
87 if (!root->target || !root->target[0] || strcmp(root->target[0], to))
90 root->target && root->target[0] && root->target[0][0] &&
91 strcmp (root->target[0], CHR_UNKNOWN))
93 yaz_log (LOG_WARN, "duplicate entry for charmap from '%s'",
96 root->target = (unsigned char **)
97 nmem_malloc(nmem, sizeof(*root->target)*2);
98 root->target[0] = (unsigned char *) nmem_strdup(nmem, to);
108 root->children = (chr_t_entry **)
109 nmem_malloc(nmem, sizeof(chr_t_entry*) * 256);
110 for (i = 0; i < 256; i++)
111 root->children[i] = 0;
113 if (!(root->children[(unsigned char) *from] =
114 set_map_string(root->children[(unsigned char) *from], nmem,
115 from + 1, len - 1, to, from_0)))
121 static chr_t_entry *find_entry(chr_t_entry *t, const char **from, int len)
125 if (len && t->children && t->children[(unsigned char) **from])
127 const char *pos = *from;
130 if ((res = find_entry(t->children[(unsigned char) *pos],
136 /* no children match. use ourselves, if we have a target */
137 return t->target ? t : 0;
140 static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len)
145 { /* switch to next buffer */
151 if (*len > 0 && t->children && t->children[(unsigned char) **from])
153 const char *old_from = *from;
158 if ((res = find_entry_x(t->children[(unsigned char) *old_from],
165 /* no children match. use ourselves, if we have a target */
166 return t->target ? t : 0;
169 const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len)
171 chr_t_entry *t = maptab->input;
174 if (!(res = find_entry_x(t, from, len)))
176 return (const char **) (res->target);
179 const char **chr_map_input(chrmaptab maptab, const char **from, int len)
181 chr_t_entry *t = maptab->input;
187 if (!(res = find_entry_x(t, from, len_tmp)))
189 return (const char **) (res->target);
192 const char *chr_map_output(chrmaptab maptab, const char **from, int len)
194 unsigned char c = ** (unsigned char **) from;
196 return (const char*) maptab->output[c];
199 unsigned char zebra_prim(char **s)
204 yaz_log (LOG_DEBUG, "prim %.3s", *s);
211 case '\\': c = '\\'; (*s)++; break;
212 case 'r': c = '\r'; (*s)++; break;
213 case 'n': c = '\n'; (*s)++; break;
214 case 't': c = '\t'; (*s)++; break;
215 case 's': c = ' '; (*s)++; break;
216 case 'x': sscanf(*s, "x%2x", &i); c = i; *s += 3; break;
227 sscanf(*s, "%3o", &i);
243 ucs4_t zebra_prim_w(ucs4_t **s)
249 yaz_log (LOG_DEBUG, "prim %.3s", (char *) *s);
256 case '\\': c = '\\'; (*s)++; break;
257 case 'r': c = '\r'; (*s)++; break;
258 case 'n': c = '\n'; (*s)++; break;
259 case 't': c = '\t'; (*s)++; break;
260 case 's': c = ' '; (*s)++; break;
266 sscanf(fmtstr, "x%2x", &i);
283 sscanf(fmtstr, "%3o", &i);
296 yaz_log (LOG_DEBUG, "out %d", c);
302 * Add an entry to the value space.
304 static void fun_addentry(const char *s, void *data, int num)
306 chrmaptab tab = (chrmaptab) data;
309 tmp[0] = num; tmp[1] = '\0';
310 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s), tmp, 0);
311 tab->output[num + tab->base_uppercase] =
312 (unsigned char *) nmem_strdup(tab->nmem, s);
317 * Add a space-entry to the value space.
319 static void fun_addspace(const char *s, void *data, int num)
321 chrmaptab tab = (chrmaptab) data;
322 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
323 (char*) CHR_SPACE, 0);
327 * Create a string containing the mapped characters provided.
329 static void fun_mkstring(const char *s, void *data, int num)
331 chrwork *arg = (chrwork *) data;
332 const char **res, *p = s;
334 res = chr_map_input(arg->map, &s, strlen(s));
335 if (*res == (char*) CHR_UNKNOWN)
336 logf(LOG_WARN, "Map: '%s' has no mapping", p);
337 strncat(arg->string, *res, CHR_MAXSTR - strlen(arg->string));
338 arg->string[CHR_MAXSTR] = '\0';
342 * Add a map to the string contained in the argument.
344 static void fun_add_map(const char *s, void *data, int num)
346 chrwork *arg = (chrwork *) data;
348 assert(arg->map->input);
349 logf (LOG_DEBUG, "set map %.*s", (int) strlen(s), s);
350 set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
352 for (s = arg->string; *s; s++)
353 logf (LOG_DEBUG, " %3d", (unsigned char) *s);
357 * Add a query map to the string contained in the argument.
359 static void fun_add_qmap(const char *s, void *data, int num)
361 chrwork *arg = (chrwork *) data;
363 assert(arg->map->q_input);
364 logf (LOG_DEBUG, "set qmap %.*s", (int) strlen(s), s);
365 set_map_string(arg->map->q_input, arg->map->nmem, s,
366 strlen(s), arg->string, 0);
367 for (s = arg->string; *s; s++)
368 logf (LOG_DEBUG, " %3d", (unsigned char) *s);
371 static int scan_to_utf8 (iconv_t t, ucs4_t *from, size_t inlen,
372 char *outbuf, size_t outbytesleft)
374 size_t inbytesleft = inlen * sizeof(ucs4_t);
375 char *inbuf = (char*) from;
378 if (t == (iconv_t)(-1))
379 *outbuf++ = *from; /* ISO-8859-1 is OK here */
383 for (i = 0; i<inlen; i++)
384 yaz_log (LOG_LOG, "%08X", from[i]);
385 ret = iconv (t, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
386 if (ret == (size_t) (-1))
388 yaz_log (LOG_WARN|LOG_ERRNO, "bad unicode sequence");
389 for (i = 0; i<inlen; i++)
390 yaz_log (LOG_LOG, "%08X", from[i]);
398 static int scan_string(char *s_native,
399 iconv_t t_unicode, iconv_t t_utf8,
400 void (*fun)(const char *c, void *data, int num),
401 void *data, int *num)
406 ucs4_t *s0, *s = arg;
407 ucs4_t c, begin, end;
410 if (t_unicode != (iconv_t)(-1))
412 char *outbuf = (char *) arg;
413 char *inbuf = s_native;
414 size_t outbytesleft = sizeof(arg)-4;
415 size_t inbytesleft = strlen(s_native);
417 ret = iconv(t_unicode, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
418 if (ret == (size_t)(-1))
420 i = (outbuf - (char*) arg)/sizeof(ucs4_t);
421 yaz_log (LOG_LOG, "to unicode");
425 for (i = 0; s_native[i]; i++)
426 arg[i] = s_native[i] & 255; /* ISO-8859-1 conversion */
427 yaz_log (LOG_LOG, "to virtual unicode");
429 arg[i] = 0; /* terminate */
430 for (j = 0; j<i; j++)
431 yaz_log (LOG_LOG, " %d %8X %d %c", j, arg[j], arg[j],
432 (arg[j] > 33 && arg[j] < 127) ? arg[j] : '?');
433 if (s[0] == 0xfeff || s[0] == 0xfeff) /* skip byte Order Mark */
441 begin = zebra_prim_w(&s);
444 logf(LOG_FATAL, "Bad range in char-map");
448 end = zebra_prim_w(&s);
451 logf(LOG_FATAL, "Bad range in char-map");
455 for (c = begin; c <= end; c++)
457 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
459 (*fun)(str, data, num ? (*num)++ : 0);
462 case '[': s++; abort(); break;
466 while (*s != ')' || s[-1] == '\\')
469 if (scan_to_utf8 (t_utf8, s0, s - s0, str, sizeof(str)-1))
471 (*fun)(str, data, num ? (*num)++ : 0);
475 c = zebra_prim_w(&s);
476 if (scan_to_utf8 (t_utf8, &c, 1, str, sizeof(str)-1))
478 (*fun)(str, data, num ? (*num)++ : 0);
484 chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
488 char line[512], *argv[50];
492 int argc, num = (int) *CHR_BASE, i;
494 iconv_t t_unicode = (iconv_t)(-1);
495 iconv_t t_utf8 = (iconv_t)(-1);
496 unsigned endian = 31;
497 const char *ucs4_native = "UCS-4";
499 if (*(char*) &endian == 31) /* little endian? */
500 ucs4_native = "UCS-4LE";
503 t_utf8 = iconv_open ("UTF-8", ucs4_native);
505 logf (LOG_DEBUG, "maptab %s open", name);
506 if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
508 logf(LOG_WARN|LOG_ERRNO, "%s", name);
511 nmem = nmem_create ();
512 res = (chrmaptab) nmem_malloc(nmem, sizeof(*res));
514 res->input = (chr_t_entry *) nmem_malloc(res->nmem, sizeof(*res->input));
515 res->input->target = (unsigned char **)
516 nmem_malloc(res->nmem, sizeof(*res->input->target) * 2);
517 res->input->target[0] = (unsigned char*) CHR_UNKNOWN;
518 res->input->target[1] = 0;
519 res->input->children = (chr_t_entry **)
520 nmem_malloc(res->nmem, sizeof(res->input) * 256);
521 for (i = 0; i < 256; i++)
523 res->input->children[i] = (chr_t_entry *)
524 nmem_malloc(res->nmem, sizeof(*res->input));
525 res->input->children[i]->children = 0;
526 res->input->children[i]->target = (unsigned char **)
527 nmem_malloc (res->nmem, 2 * sizeof(unsigned char *));
528 res->input->children[i]->target[1] = 0;
531 res->input->children[i]->target[0] = (unsigned char *)
532 nmem_malloc (res->nmem, 2 * sizeof(unsigned char));
533 res->input->children[i]->target[0][0] = i;
534 res->input->children[i]->target[0][1] = 0;
537 res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
539 res->q_input = (chr_t_entry *)
540 nmem_malloc(res->nmem, sizeof(*res->q_input));
541 res->q_input->target = 0;
542 res->q_input->children = 0;
544 for (i = *CHR_BASE; i < 256; i++)
546 res->output[(int) *CHR_SPACE] = (unsigned char *) " ";
547 res->output[(int) *CHR_UNKNOWN] = (unsigned char*) "@";
548 res->base_uppercase = 0;
550 while (!errors && (argc = readconf_line(f, &lineno, line, 512, argv, 50)))
551 if (!map_only && !yaz_matchstr(argv[0], "lowercase"))
555 logf(LOG_FATAL, "Syntax error in charmap");
558 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
561 logf(LOG_FATAL, "Bad value-set specification");
564 res->base_uppercase = num;
565 res->output[(int) *CHR_SPACE + num] = (unsigned char *) " ";
566 res->output[(int) *CHR_UNKNOWN + num] = (unsigned char*) "@";
567 num = (int) *CHR_BASE;
569 else if (!map_only && !yaz_matchstr(argv[0], "uppercase"))
571 if (!res->base_uppercase)
573 logf(LOG_FATAL, "Uppercase directive with no lowercase set");
578 logf(LOG_FATAL, "Missing arg for uppercase directive");
581 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
584 logf(LOG_FATAL, "Bad value-set specification");
588 else if (!map_only && !yaz_matchstr(argv[0], "space"))
592 logf(LOG_FATAL, "Syntax error in charmap");
595 if (scan_string(argv[1], t_unicode, t_utf8,
596 fun_addspace, res, 0) < 0)
598 logf(LOG_FATAL, "Bad space specification");
602 else if (!yaz_matchstr(argv[0], "map"))
608 logf(LOG_FATAL, "charmap directive map requires 2 args");
612 buf.string[0] = '\0';
613 if (scan_string(argv[2], t_unicode, t_utf8,
614 fun_mkstring, &buf, 0) < 0)
616 logf(LOG_FATAL, "Bad map target");
619 if (scan_string(argv[1], t_unicode, t_utf8,
620 fun_add_map, &buf, 0) < 0)
622 logf(LOG_FATAL, "Bad map source");
626 else if (!yaz_matchstr(argv[0], "qmap"))
632 logf(LOG_FATAL, "charmap directive qmap requires 2 args");
636 buf.string[0] = '\0';
637 if (scan_string(argv[2], t_unicode, t_utf8,
638 fun_mkstring, &buf, 0) < 0)
640 logf(LOG_FATAL, "Bad qmap target");
643 if (scan_string(argv[1], t_unicode, t_utf8,
644 fun_add_qmap, &buf, 0) < 0)
646 logf(LOG_FATAL, "Bad qmap source");
650 else if (!yaz_matchstr(argv[0], "encoding"))
653 if (t_unicode != (iconv_t)(-1))
654 iconv_close (t_unicode);
655 t_unicode = iconv_open (ucs4_native, argv[1]);
657 logf (LOG_WARN, "Encoding ignored. iconv not installed");
662 logf(LOG_WARN, "Syntax error at '%s' in %s", line, name);
668 chrmaptab_destroy(res);
671 logf (LOG_DEBUG, "maptab %s close %d errors", name, errors);
673 if (t_utf8 != (iconv_t)(-1))
675 if (t_unicode != (iconv_t)(-1))
676 iconv_close(t_unicode);
681 void chrmaptab_destroy(chrmaptab tab)
684 nmem_destroy (tab->nmem);