IDZEBRA  2.1.3
retrieve.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <stdio.h>
24 #include <assert.h>
25 
26 #include <fcntl.h>
27 #ifdef WIN32
28 #include <io.h>
29 #include <process.h>
30 #endif
31 #if HAVE_UNISTD_H
32 #include <unistd.h>
33 #endif
34 
35 #include "index.h"
36 #include <yaz/diagbib1.h>
37 #include <yaz/snprintf.h>
38 #include <direntz.h>
39 #include <yaz/oid_db.h>
40 #include <zebra_strmap.h>
41 
42 #define MAX_SYSNOS_PER_RECORD 40
43 
44 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
45 
48  const char *setname;
50  int score;
51  NMEM nmem;
52 };
53 
55  Record *rec,
56  struct ZebraRecStream *stream)
57 {
58  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
59 
60  if ((*rec)->size[recInfo_storeData] > 0
61  || (*rec)->info[recInfo_filename] == 0)
62  zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
63  (*rec)->size[recInfo_storeData]);
64  else
65  {
66  char full_rep[1024];
67  int fd;
68 
69  if (zh->path_reg && !yaz_is_abspath((*rec)->info[recInfo_filename])){
70  strcpy(full_rep, zh->path_reg);
71  strcat(full_rep, "/");
72  strcat(full_rep, (*rec)->info[recInfo_filename]);
73  }
74  else
75  strcpy(full_rep, (*rec)->info[recInfo_filename]);
76 
77  if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
78  yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
79  full_rep);
80  rec_free(rec);
81  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
82  }
83  zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
84  }
85  return 0;
86 }
87 
88 
89 struct index_spec {
90  const char *index_name;
91  const char *index_type;
92  const char *extra;
93  struct index_spec *next;
94 };
95 
96 
97 struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
98  int *error)
99 {
100  struct index_spec *first = 0;
101  struct index_spec **last = &first;
102  const char *cp = elem;
103 
104  *error = 0;
105  if (cp[0] == ':' && cp[1] == ':')
106  {
107 
108  cp++; /* skip first ':' */
109 
110  for (;;)
111  {
112  const char *cp0;
113  struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
114  spec->index_type = 0;
115  spec->next = 0;
116  spec->extra = 0;
117 
118  if (!first)
119  first = spec;
120  *last = spec;
121  last = &spec->next;
122 
123  cp++; /* skip ',' or second ':' */
124  cp0 = cp;
125  while (*cp != ':' && *cp != '\0' && *cp != ',')
126  cp++;
127  spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
128  if (*cp == ':') /* type as well */
129  {
130  cp++;
131  cp0 = cp;
132 
133  while (*cp != '\0' && *cp != ',' && *cp != ':')
134  cp++;
135  spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
136  }
137  if (*cp == ':') /* extra arguments */
138  {
139  cp++;
140  cp0 = cp;
141 
142  while (*cp != '\0' && *cp != ',' && *cp != ':')
143  cp++;
144  spec->extra = nmem_strdupn(nmem, cp0, cp - cp0);
145  }
146  if (*cp != ',')
147  break;
148  }
149  }
150  if (*cp != '\0')
151  *error = 1;
152  return first;
153 }
154 
155 static int sort_fetch(
156  struct special_fetch_s *fi, const char *elemsetname,
157  const Odr_oid *input_format,
158  const Odr_oid **output_format,
159  WRBUF result, WRBUF addinfo)
160 {
161  int ord;
162  ZebraHandle zh = fi->zh;
163  int error;
164  struct index_spec *spec;
165 
166  spec = parse_index_spec(elemsetname, fi->nmem, &error);
167  if (error)
168  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
169 
170  /* for sort fetches.. We allow only one index and type must be given */
171  if (!spec || spec->next || !spec->index_type)
172  return -1;
175  spec->index_type,
176  spec->index_name);
177  if (ord == -1)
178  return -1; /* is not a sort index */
179  else
180  {
181  WRBUF wrbuf_str = wrbuf_alloc();
182  const char *index_type;
183  const char *db = 0;
184  const char *string_index = 0;
185  WRBUF wrbuf_result = result;
186  int off = 0;
187 
188  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
189  &string_index);
190  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
191  {
192  *output_format = yaz_oid_recsyn_xml;
193  wrbuf_printf(wrbuf_result, ZEBRA_XML_HEADER_STR
194  " sysno=\"" ZINT_FORMAT "\""
195  " set=\"zebra::index%s\">\n",
196  fi->sysno, elemsetname);
197  }
198  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
199  {
200  *output_format = yaz_oid_recsyn_sutrs;
201  }
202  else
203  {
204  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
205  elemsetname);
206  *output_format = 0;
207  wrbuf_destroy(wrbuf_str);
208  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
209  }
210  zebra_sort_type(zh->reg->sort_index, ord);
212  zebra_sort_read(zh->reg->sort_index, 0, wrbuf_str);
213 
214  while (off != wrbuf_len(wrbuf_str))
215  {
216  char dst_buf[IT_MAX_WORD];
217  assert(off < wrbuf_len(wrbuf_str));
218  zebra_term_untrans(zh, index_type, dst_buf,
219  wrbuf_buf(wrbuf_str)+off);
220 
221  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
222  {
223  wrbuf_printf(wrbuf_result, " <index name=\"%s\"",
224  string_index);
225  wrbuf_printf(wrbuf_result, " type=\"%s\">", index_type);
226  wrbuf_xmlputs(wrbuf_result, dst_buf);
227  wrbuf_printf(wrbuf_result, "</index>\n");
228  }
229  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
230  {
231  wrbuf_printf(wrbuf_result, "%s %s %s\n", string_index, index_type,
232  dst_buf);
233  }
234  off += strlen(wrbuf_buf(wrbuf_str)+off) + 1;
235  }
236  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
237  {
238  wrbuf_printf(wrbuf_result, "</record>\n");
239  }
240  wrbuf_destroy(wrbuf_str);
241  return 0;
242  }
243 }
244 
246  struct special_fetch_s *fi, const char *elemsetname,
247  const Odr_oid *input_format,
248  const Odr_oid **output_format,
249  WRBUF result, WRBUF addinfo,
250  Record rec)
251 {
252  zebra_rec_keys_t keys;
253  int ret_code = 0;
254  ZebraHandle zh = fi->zh;
255  struct index_spec *spec, *spec_list;
256  int error;
257 
258  /* set output variables before processing possible error states */
259  /* *rec_lenp = 0; */
260 
261  /* only accept XML and SUTRS requests */
262  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
263  && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
264  {
265  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
266  elemsetname);
267  *output_format = 0;
268  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
269  }
270 
271  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
272  if (error)
273  {
274  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
275  }
276 
277  for (spec = spec_list; spec; spec = spec->next)
278  {
281  spec->index_type,
282  spec->index_name) == -1)
283  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
284  }
285 
286  keys = zebra_rec_keys_open();
288  rec->size[recInfo_delKeys], 0);
289 
290  if (!zebra_rec_keys_rewind(keys))
291  {
292  ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
293  }
294  else
295  {
296  size_t slen;
297  const char *str;
298  struct it_key key_in;
299  WRBUF wrbuf = result;
300 
301  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
302  {
303  *output_format = input_format;
304  wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
305  " sysno=\"" ZINT_FORMAT "\""
306  " set=\"zebra::index%s\">\n",
307  fi->sysno, elemsetname);
308  }
309  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
310  *output_format = input_format;
311 
312  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
313  {
314  int i;
315  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
316  const char *index_type;
317  const char *db = 0;
318  const char *string_index = 0;
319  char dst_buf[IT_MAX_WORD];
320  int match = 0;
321 
322  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
323  &string_index);
324  if (!spec_list)
325  match = 1; /* match all if no specs were given */
326  else
327  {
328  for (spec = spec_list; spec; spec = spec->next)
329  {
330  if ((!spec->index_type ||
331  !yaz_matchstr(spec->index_type, index_type))
332  &&
333  !yaz_matchstr(spec->index_name, string_index))
334  match = 1;
335  }
336  }
337  if (match)
338  {
339  if (zebra_term_untrans(zh, index_type, dst_buf, str))
340  *dst_buf = '\0'; /* untrans failed */
341 
342  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
343  {
344  wrbuf_printf(wrbuf, " <index name=\"%s\"",
345  string_index);
346 
347  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
348 
349  wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">",
350  key_in.mem[key_in.len -1]);
351  wrbuf_xmlputs(wrbuf, dst_buf);
352  wrbuf_printf(wrbuf, "</index>\n");
353  }
354  else
355  {
356  wrbuf_printf(wrbuf, "%s ", string_index);
357 
358  wrbuf_printf(wrbuf, "%s", index_type);
359 
360  for (i = 1; i < key_in.len; i++)
361  wrbuf_printf(wrbuf, " " ZINT_FORMAT,
362  key_in.mem[i]);
363 
364  wrbuf_printf(wrbuf, " %s", dst_buf);
365 
366  wrbuf_printf(wrbuf, "\n");
367  }
368  }
369  }
370  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
371  wrbuf_printf(wrbuf, "</record>\n");
372  }
373  zebra_rec_keys_close(keys);
374  return ret_code;
375 }
376 
377 
378 static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
379  const char *value)
380 {
381  if (value)
382  {
383  wrbuf_printf(wrbuf, " %s=\"", name);
384  wrbuf_xmlputs(wrbuf, value);
385  wrbuf_printf(wrbuf, "\"");
386  }
387 }
388 
389 static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
390  const int value)
391 {
392  wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
393 }
394 
395 static void retrieve_puts_str(WRBUF wrbuf, const char *name,
396  const char *value)
397 {
398  if (value)
399  wrbuf_printf(wrbuf, "%s %s\n", name, value);
400 }
401 
402 static void retrieve_puts_int(WRBUF wrbuf, const char *name,
403  const int value)
404 {
405  wrbuf_printf(wrbuf, "%s %i\n", name, value);
406 }
407 
408 
409 static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf,
410  zebra_snippets *doc,
411  const zebra_snippet_word *doc_w,
412  const char *w_index_type)
413 {
414  /* beginning of snippet. See which fields the snippet also
415  occur */
416  const zebra_snippet_word *w;
417  int no = 0;
418  for (w = zebra_snippets_constlist(doc); w; w = w->next)
419  {
420  /* same sequence but other field? */
421  if (w->seqno == doc_w->seqno && w->ord != doc_w->ord)
422  {
423  const char *index_type;
424  const char *db = 0;
425  const char *string_index = 0;
426 
428  &index_type, &db, &string_index);
429  /* only report for same index type */
430  if (!strcmp(w_index_type, index_type))
431  {
432  if (no == 0)
433  wrbuf_printf(wrbuf, " fields=\"%s", string_index);
434  else
435  wrbuf_printf(wrbuf, " %s", string_index);
436  no++;
437  }
438  }
439  }
440  if (no)
441  wrbuf_printf(wrbuf, "\"");
442 }
443 
444 static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
445 {
446  const zebra_snippet_word *doc_w;
447  int mark_state = 0;
448 
449  wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
450  for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
451  {
452  if (doc_w->mark)
453  {
454  const char *index_type;
455  const char *db = 0;
456  const char *string_index = 0;
457 
458  zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord,
459  &index_type, &db, &string_index);
460 
461  if (mark_state == 0)
462  {
463 
464  wrbuf_printf(wrbuf, " <snippet name=\"%s\"", string_index);
465  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
466  snippet_check_fields(zh, wrbuf, doc, doc_w, index_type);
467  wrbuf_printf(wrbuf, ">");
468  }
469  if (doc_w->match)
470  wrbuf_puts(wrbuf, "<s>");
471  /* not printing leading ws */
472  if (mark_state || !doc_w->ws || doc_w->match)
473  wrbuf_xmlputs(wrbuf, doc_w->term);
474  if (doc_w->match)
475  wrbuf_puts(wrbuf, "</s>");
476  }
477  else if (mark_state == 1)
478  {
479  wrbuf_puts(wrbuf, "</snippet>\n");
480  }
481  mark_state = doc_w->mark;
482  }
483  if (mark_state == 1)
484  {
485  wrbuf_puts(wrbuf, "</snippet>\n");
486  }
487  wrbuf_printf(wrbuf, "</record>");
488 }
489 
491  zebra_snippets *snippets)
492 {
493  int return_code = 0;
494  Record rec = rec_get(zh->reg->records, sysno);
495  if (!rec)
496  {
497  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
498  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
499  }
500  else
501  {
502  const char *file_type = rec->info[recInfo_fileType];
503  void *recTypeClientData;
504  RecType rt = recType_byName(zh->reg->recTypes, zh->res,
505  file_type, &recTypeClientData);
506 
507  if (!rt)
508  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
509  else
510  {
511  struct ZebraRecStream stream;
512  return_code = zebra_create_record_stream(zh, &rec, &stream);
513  if (return_code == 0)
514  {
515  extract_snippet(zh, snippets, &stream,
516  rt, recTypeClientData);
517 
518  stream.destroy(&stream);
519  }
520  }
521  rec_free(&rec);
522  }
523  return return_code;
524 }
525 
526 static int snippet_fetch(
527  struct special_fetch_s *fi, const char *elemsetname,
528  const Odr_oid *input_format,
529  const Odr_oid **output_format,
530  WRBUF result, WRBUF addinfo)
531 {
532  ZebraHandle zh = fi->zh;
533  zebra_snippets *rec_snippets = zebra_snippets_create();
534  int return_code = zebra_get_rec_snippets(zh, fi->sysno, rec_snippets);
535 
536  if (!return_code)
537  {
538  WRBUF wrbuf = result;
539  zebra_snippets *hit_snippet = zebra_snippets_create();
540 
541  zebra_snippets_hit_vector(zh, fi->setname, fi->sysno, hit_snippet);
542 
543 #if 0
544  /* for debugging purposes */
545  yaz_log(YLOG_LOG, "---------------------------");
546  yaz_log(YLOG_LOG, "REC SNIPPET:");
547  zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
548  yaz_log(YLOG_LOG, "---------------------------");
549  yaz_log(YLOG_LOG, "HIT SNIPPET:");
550  zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
551 #endif
552 
553  zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
554 
555 #if 0
556  yaz_log(YLOG_LOG, "---------------------------");
557  yaz_log(YLOG_LOG, "RING SNIPPET:");
558  zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
559 #endif
560  snippet_xml_record(zh, wrbuf, rec_snippets);
561 
562  *output_format = yaz_oid_recsyn_xml;
563 
564  zebra_snippets_destroy(hit_snippet);
565  }
566  zebra_snippets_destroy(rec_snippets);
567  return return_code;
568 }
569 
570 struct term_collect {
571  const char *term;
572  int oc;
576 };
577 
578 static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set,
579  zint *first_sysno, zint *first_seq)
580 {
582  char ord_buf[IT_MAX_WORD];
583  int ord_len = key_SU_encode(ord, ord_buf);
584  char *info;
585  zint hits = 0;
586  NMEM nmem = nmem_create();
587 
588  strcpy(ord_buf + ord_len, term);
589 
590  info = dict_lookup(zh->reg->dict, ord_buf);
591  if (info)
592  {
593  ISAM_P isam_p;
594  RSET rsets[2], rset;
595  memcpy(&isam_p, info+1, sizeof(ISAM_P));
596  TERMID termid_key = rset_term_create("", -1,
597  0 /* flags */,
598  Z_Term_general /* type */,
599  nmem,
600  0, /* ord_list */
601  0, /* reg_type */
602  10000, /* hits_limit */
603  0 /* ref_id */);
604 
605  rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p,
606  termid_key);
607  rsets[1] = rset_dup(rset_set);
608 
609  rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
610 
611  zebra_count_set(zh, rset, &hits, zh->approx_limit);
612  RSFD rfd = rset_open(rset, RSETF_READ);
613  TERMID termid;
614  struct it_key key;
615  while (rset_read(rfd, &key, &termid))
616  {
617  if (termid == termid_key)
618  {
619  *first_sysno = key.mem[0];
620  *first_seq = key.mem[key.len - 1];
621  break;
622  }
623  }
624  rset_close(rfd);
625  rset_delete(rsets[0]);
626  rset_delete(rset);
627  }
628  (*kc->dec)(kc);
629  nmem_destroy(nmem);
630  return hits;
631 }
632 
633 static int term_qsort_handle(const void *a, const void *b)
634 {
635  const struct term_collect *l = a;
636  const struct term_collect *r = b;
637  if (l->set_occur < r->set_occur)
638  return 1;
639  else if (l->set_occur > r->set_occur)
640  return -1;
641  else
642  {
643  const char *lterm = l->term ? l->term : "";
644  const char *rterm = r->term ? r->term : "";
645  return strcmp(lterm, rterm);
646  }
647 }
648 
650  struct term_collect *col, int no_terms_collect,
651  int ord, RSET rset, double scale_factor)
652 {
653  int i;
654  for (i = 0; i < no_terms_collect; i++)
655  {
656  if (col[i].term)
657  {
658  if (scale_factor < 0.0)
659  {
660  col[i].set_occur = freq_term(zh, ord, col[i].term, rset,
661  &col[i].first_sysno,
662  &col[i].first_seqno);
663  }
664  else
665  col[i].set_occur = scale_factor * col[i].oc;
666  }
667  }
668  qsort(col, no_terms_collect, sizeof(*col), term_qsort_handle);
669 }
670 
672  int no_terms_collect,
673  NMEM nmem)
674 {
675  const char *term;
676  void *data_buf;
677  size_t data_len;
678  zebra_strmap_it it;
679  struct term_collect *col = nmem_malloc(nmem,
680  sizeof *col *no_terms_collect);
681  int i;
682  for (i = 0; i < no_terms_collect; i++)
683  {
684  col[i].term = 0;
685  col[i].oc = 0;
686  col[i].set_occur = 0;
687  col[i].first_sysno = col[i].first_seqno = 0;
688  }
689  /* iterate over terms and collect the most frequent ones */
690  it = zebra_strmap_it_create(sm);
691  while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
692  {
693  /* invariant:
694  col[0] has lowest oc . col[no_terms_collect-1] has highest oc */
695  int oc = *(int*) data_buf;
696  int j = 0;
697  /* insertion may be slow but terms terms will be "infrequent" and
698  thus number of iterations should be small below
699  */
700  while (j < no_terms_collect && oc > col[j].oc)
701  j++;
702  if (j)
703  { /* oc <= col[j] and oc > col[j-1] */
704  --j;
705  memmove(col, col+1, sizeof(*col) * j);
706  col[j].term = term;
707  col[j].oc = oc;
708  }
709  }
711  return col;
712 }
713 
714 static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array,
715  zebra_strmap_t *map_array,
716  int num_recs, ZebraMetaRecord *poset)
717 {
718  int rec_i;
719  WRBUF w = wrbuf_alloc();
720  int ord_i;
721 
722  for (ord_i = 0; ord_i < no_ord; ord_i++)
723  {
724  for (rec_i = 0; rec_i < num_recs; rec_i++)
725  {
726  if (!poset[rec_i].sysno)
727  continue;
728 
729  zebra_sort_sysno(zh->reg->sort_index, poset[rec_i].sysno);
730  zebra_sort_type(zh->reg->sort_index, ord_array[ord_i]);
731 
732  wrbuf_rewind(w);
733  if (zebra_sort_read(zh->reg->sort_index, 0, w))
734  {
735  zebra_strmap_t sm = map_array[ord_i];
736  int off = 0;
737  while (off != wrbuf_len(w))
738  {
739  const char *str = wrbuf_buf(w) + off;
740  int *freq = zebra_strmap_lookup(sm, str, 0, 0);
741  if (freq)
742  (*freq)++;
743  else
744  {
745  int v = 1;
746  zebra_strmap_add(sm, str, &v, sizeof v);
747  }
748  off += strlen(str)+1;
749  }
750  }
751  }
752  }
753  wrbuf_destroy(w);
754  return 0;
755 }
756 
757 
759  struct special_fetch_s *fi,
760  int no_ord, int *ord_array,
761  zebra_strmap_t *map_array,
762  int num_recs, ZebraMetaRecord *poset,
763  struct index_spec *spec_list)
764 {
765  int max_chunks = 2;
766  int rec_i;
767  res_get_int(zh->res, "facetMaxChunks", &max_chunks);
768 
769  for (rec_i = 0; rec_i < num_recs; rec_i++)
770  {
771  int j;
772  zint sysnos[MAX_SYSNOS_PER_RECORD];
773  int no_sysnos = MAX_SYSNOS_PER_RECORD;
774  if (!poset[rec_i].sysno)
775  continue;
777  poset[rec_i].sysno,
778  sysnos, &no_sysnos);
779  assert(no_sysnos > 0);
780  yaz_log(YLOG_DEBUG, "Analyzing rec=%d ISAM sysno=" ZINT_FORMAT " chunks=%d",
781  rec_i, poset[rec_i].sysno, no_sysnos);
782  for (j = 0; j < no_sysnos && j < max_chunks; j++)
783  {
784  size_t slen;
785  const char *str;
786  struct it_key key_in;
787  Record rec = rec_get(zh->reg->records, sysnos[j]);
789  zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
790  rec->size[recInfo_delKeys], 0);
791 
792  yaz_log(YLOG_DEBUG, "rec %d " ZINT_FORMAT " %s",
793  j, sysnos[j], zebra_rec_keys_empty(keys) ? "empty" : "non-empty");
794  if (zebra_rec_keys_rewind(keys))
795  {
796  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
797  {
798  int ord_i;
799  struct index_spec *spec;
800  for (spec = spec_list, ord_i = 0; ord_i < no_ord;
801  ord_i++, spec = spec->next)
802  {
803  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
804  if (ord == ord_array[ord_i] &&
805  str[0] != FIRST_IN_FIELD_CHAR)
806  {
807  int *freq;
808  zebra_strmap_t sm = map_array[ord_i];
809 
810  freq = zebra_strmap_lookup(sm, str, 0, 0);
811  if (freq)
812  (*freq)++;
813  else
814  {
815  int v = 1;
816  zebra_strmap_add(sm, str, &v, sizeof v);
817  }
818  }
819  }
820  }
821  }
822  zebra_rec_keys_close(keys);
823  rec_free(&rec);
824  }
825  }
826  return 0;
827 }
828 
830  struct special_fetch_s *fi,
831  WRBUF result,
832  int num_recs, ZebraMetaRecord *poset,
833  struct index_spec *spec_list,
834  int no_ord, int *ord_array,
835  int use_xml,
837 {
838  int i;
839  int ret = 0;
840  WRBUF wr = result;
841  struct index_spec *spec;
842  yaz_timing_t timing = yaz_timing_create();
843  zebra_strmap_t *map_array
844  = nmem_malloc(fi->nmem, sizeof *map_array * no_ord);
845  for (i = 0; i < no_ord; i++)
846  map_array[i] = zebra_strmap_create();
847 
848  if (cat == zinfo_index_category_sort)
849  perform_facet_sort(zh, no_ord, ord_array, map_array,
850  num_recs, poset);
851  else
852  perform_facet_index(zh, fi, no_ord, ord_array, map_array,
853  num_recs, poset, spec_list);
854  yaz_timing_stop(timing);
855  yaz_log(YLOG_LOG, "facet first phase real=%4.2f cat=%s",
856  yaz_timing_get_real(timing),
857  (cat == zinfo_index_category_sort) ? "sort" : "index");
858  yaz_timing_start(timing);
859  for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
860  {
861  int j;
862  NMEM nmem = nmem_create();
863  struct term_collect *col;
864  int no_collect_terms = 20;
865 
866  if (spec->extra)
867  no_collect_terms = atoi(spec->extra);
868  if (no_collect_terms < 1)
869  no_collect_terms = 1;
870  col = term_collect_create(map_array[i], no_collect_terms, nmem);
871  term_collect_freq(zh, col, no_collect_terms, ord_array[i],
872  resultSetRef(zh, fi->setname),
873  cat == zinfo_index_category_sort ? 1.0 : -1.0);
874 
875  if (use_xml)
876  wrbuf_printf(wr, " <facet type=\"%s\" index=\"%s\">\n",
877  spec->index_type, spec->index_name);
878  else
879  wrbuf_printf(wr, "facet %s %s\n",
880  spec->index_type, spec->index_name);
881  for (j = 0; j < no_collect_terms; j++)
882  {
883  if (col[j].term)
884  {
885  WRBUF w_buf = wrbuf_alloc();
886  yaz_log(YLOG_LOG, "facet %s %s",
887  spec->index_type, spec->index_name);
888  if (col[j].first_sysno)
889  {
890  zebra_snippets *rec_snippets = zebra_snippets_create();
891  int code = zebra_get_rec_snippets(
892  zh, col[j].first_sysno, rec_snippets);
893  yaz_log(YLOG_LOG, "sysno/seqno "
895  col[j].first_sysno, col[j].first_seqno);
896  if (code == 0)
897  {
898  zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
899  const zebra_snippet_word *sn =
900  zebra_snippets_constlist(rec_snippets);
901  int first = 1; /* ignore leading whitespace */
902  for (; sn; sn = sn->next)
903  {
904  if (sn->ord == ord_array[i] &&
905  sn->seqno == col[j].first_seqno)
906  {
907  if (!sn->ws || !first)
908  {
909  first = 0;
910  wrbuf_puts(w_buf, sn->term);
911  }
912  }
913  }
914  }
915  zebra_snippets_destroy(rec_snippets);
916  }
917  char *dst_buf = 0;
918  if (wrbuf_len(w_buf))
919  zebra_term_untrans_iconv2(zh, nmem, &dst_buf, wrbuf_cstr(w_buf));
920  else
921  zebra_term_untrans_iconv(zh, nmem, spec->index_type, &dst_buf,
922  col[j].term);
923  if (use_xml)
924  {
925  wrbuf_printf(wr, " <term coccur=\"%d\"", col[j].oc);
926  if (col[j].set_occur)
927  wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"",
928  col[j].set_occur);
929  wrbuf_printf(wr, ">");
930  wrbuf_xmlputs(wr, dst_buf);
931  wrbuf_printf(wr, "</term>\n");
932  }
933  else
934  {
935  wrbuf_printf(wr, "term %d", col[j].oc);
936  if (col[j].set_occur)
937  wrbuf_printf(wr, " " ZINT_FORMAT,
938  col[j].set_occur);
939  wrbuf_printf(wr, ": %s\n", dst_buf);
940  }
941  wrbuf_destroy(w_buf);
942  }
943  }
944  if (use_xml)
945  wrbuf_puts(wr, " </facet>\n");
946  nmem_destroy(nmem);
947  }
948  for (i = 0; i < no_ord; i++)
949  zebra_strmap_destroy(map_array[i]);
950  yaz_timing_stop(timing);
951  yaz_log(YLOG_LOG, "facet second phase real=%4.2f",
952  yaz_timing_get_real(timing));
953  yaz_timing_destroy(&timing);
954  return ret;
955 }
956 
957 static int facet_fetch(
958  struct special_fetch_s *fi, const char *elemsetname,
959  const Odr_oid *input_format,
960  const Odr_oid **output_format,
961  WRBUF result, WRBUF addinfo)
962 {
963  zint *pos_array;
964  int i;
965  int num_recs = 10; /* number of records to analyze */
966  ZebraMetaRecord *poset;
967  ZEBRA_RES ret = ZEBRA_OK;
968  int *ord_array;
969  int use_xml = 0;
970  int no_ord = 0;
971  struct index_spec *spec, *spec_list;
972  int error;
973  ZebraHandle zh = fi->zh;
974  /* whether sort or index based */
976 
977  /* see if XML is required for response */
978  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) == 0)
979  use_xml = 1;
980 
981  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
982 
983  if (!spec_list || error)
984  {
985  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
986  }
987 
988  for (spec = spec_list; spec; spec = spec->next)
989  {
990  if (!spec->index_type)
991  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
992  no_ord++;
993  }
994 
995  /* try to see if all specs are sort based.. If not, try the
996  index based ones */
997  ord_array = nmem_malloc(fi->nmem, sizeof(*ord_array) * no_ord);
998 
999  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
1000  {
1001  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1003  spec->index_type,
1004  spec->index_name);
1005  if (ord == -1)
1006  break;
1007  ord_array[i] = ord;
1008  num_recs = 10000;
1009  }
1010  if (spec)
1011  {
1013  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
1014  {
1015  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1017  spec->index_type,
1018  spec->index_name);
1019  if (ord == -1)
1020  break;
1021  ord_array[i] = ord;
1022 
1023  }
1024  }
1025  if (spec)
1026  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1027 
1028  res_get_int(zh->res, "facetNumRecs", &num_recs);
1029 
1030  pos_array = (zint *) nmem_malloc(fi->nmem, num_recs * sizeof(*pos_array));
1031  for (i = 0; i < num_recs; i++)
1032  pos_array[i] = i+1;
1033  poset = zebra_meta_records_create(zh, fi->setname, num_recs, pos_array);
1034  if (!poset)
1035  {
1036  wrbuf_puts(addinfo, fi->setname);
1037  return YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
1038  }
1039  else
1040  {
1041  if (use_xml)
1042  {
1043  wrbuf_printf(result, ZEBRA_XML_HEADER_STR ">\n");
1044  }
1045  ret = perform_facet(zh, fi, result, num_recs, poset,
1046  spec_list, no_ord, ord_array, use_xml,
1047  cat);
1048  if (use_xml)
1049  wrbuf_puts(result, "</record>\n");
1050  }
1051  *output_format = yaz_oid_recsyn_xml;
1052  zebra_meta_records_destroy(zh, poset, num_recs);
1053  return ret;
1054 }
1055 
1056 
1058  void *handle, const char *elemsetname,
1059  const Odr_oid *input_format,
1060  const Odr_oid **output_format,
1061  WRBUF result, WRBUF addinfo)
1062 {
1063  Record rec = 0;
1064  struct special_fetch_s *fi = (struct special_fetch_s *) handle;
1065  ZebraHandle zh = fi->zh;
1066  zint sysno = fi->sysno;
1067 
1068  /* processing zebra::facet */
1069  if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
1070  {
1071  return facet_fetch(fi, elemsetname + 5,
1072  input_format, output_format,
1073  result, addinfo);
1074  }
1075 
1076  if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
1077  {
1078  return snippet_fetch(fi, elemsetname + 7,
1079  input_format, output_format,
1080  result, addinfo);
1081  }
1082 
1083  /* processing zebra::meta::sysno */
1084  if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
1085  {
1086  int ret = 0;
1087  if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1088  {
1089  wrbuf_printf(result, ZINT_FORMAT, fi->sysno);
1090  *output_format = input_format;
1091  }
1092  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1093  {
1094  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1095  " sysno=\"" ZINT_FORMAT "\"/>\n",
1096  fi->sysno);
1097  *output_format = input_format;
1098  }
1099  else
1100  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1101  return ret;
1102  }
1103 
1104  /* processing special elementsetname zebra::index:: for sort elements */
1105  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1106  {
1107  int ret = sort_fetch(
1108  fi, elemsetname + 5,
1109  input_format, output_format,
1110  result, addinfo);
1111  if (ret != -1)
1112  return ret;
1113  /* not a sort index so we continue to get the full record */
1114  }
1115 
1116 
1117  /* fetching binary record up for all other display elementsets */
1118  rec = rec_get(zh->reg->records, sysno);
1119  if (!rec)
1120  {
1121  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1122  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1123  }
1124 
1125  /* processing special elementsetnames zebra::data */
1126  if (elemsetname && 0 == strcmp(elemsetname, "data"))
1127  {
1128  struct ZebraRecStream stream;
1129  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1130  char *b;
1131 
1132  zebra_create_record_stream(zh, &rec, &stream);
1133  *output_format = input_format;
1134 
1135  b = nmem_malloc(fi->nmem, recordAttr->recordSize);
1136  stream.readf(&stream, b, recordAttr->recordSize);
1137  wrbuf_write(result, b, recordAttr->recordSize);
1138 
1139  stream.destroy(&stream);
1140  rec_free(&rec);
1141  return 0;
1142  }
1143 
1144  /* processing special elementsetnames zebra::meta:: */
1145  if (elemsetname && 0 == strcmp(elemsetname, "meta"))
1146  {
1147  int ret = 0;
1148  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1149 
1150  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1151  {
1152  *output_format = input_format;
1153 
1154  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1155  " sysno=\"" ZINT_FORMAT "\"", sysno);
1156  retrieve_puts_attr(result, "base", rec->info[recInfo_databaseName]);
1157  retrieve_puts_attr(result, "file", rec->info[recInfo_filename]);
1158  retrieve_puts_attr(result, "type", rec->info[recInfo_fileType]);
1159  if (fi->score >= 0)
1160  retrieve_puts_attr_int(result, "score", fi->score);
1161 
1162  wrbuf_printf(result,
1163  " rank=\"" ZINT_FORMAT "\""
1164  " size=\"%i\""
1165  " set=\"zebra::%s\"/>\n",
1166  recordAttr->staticrank,
1167  recordAttr->recordSize,
1168  elemsetname);
1169  }
1170  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1171  {
1172  *output_format = input_format;
1173  wrbuf_printf(result, "sysno " ZINT_FORMAT "\n", sysno);
1174  retrieve_puts_str(result, "base", rec->info[recInfo_databaseName]);
1175  retrieve_puts_str(result, "file", rec->info[recInfo_filename]);
1176  retrieve_puts_str(result, "type", rec->info[recInfo_fileType]);
1177  if (fi->score >= 0)
1178  retrieve_puts_int(result, "score", fi->score);
1179 
1180  wrbuf_printf(result,
1181  "rank " ZINT_FORMAT "\n"
1182  "size %i\n"
1183  "set zebra::%s\n",
1184  recordAttr->staticrank,
1185  recordAttr->recordSize,
1186  elemsetname);
1187  }
1188  else
1189  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1190 
1191  rec_free(&rec);
1192  return ret;
1193  }
1194 
1195  /* processing special elementsetnames zebra::index:: */
1196  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1197  {
1198  int ret = special_index_fetch(
1199  fi, elemsetname + 5,
1200  input_format, output_format,
1201  result, addinfo, rec);
1202  rec_free(&rec);
1203  return ret;
1204  }
1205 
1206  if (rec)
1207  rec_free(&rec);
1208  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1209 }
1210 
1212  zint sysno, int score,
1213  ODR odr,
1214  const Odr_oid *input_format, Z_RecordComposition *comp,
1215  const Odr_oid **output_format,
1216  char **rec_bufp, int *rec_lenp, char **basenamep,
1217  WRBUF addinfo_w)
1218 {
1219  Record rec;
1220  char *fname, *file_type, *basename;
1221  const char *elemsetname;
1222  struct ZebraRecStream stream;
1223  RecordAttr *recordAttr;
1224  void *clientData;
1225  int return_code = 0;
1226  zint sysnos[MAX_SYSNOS_PER_RECORD];
1227  int no_sysnos = MAX_SYSNOS_PER_RECORD;
1228  ZEBRA_RES res;
1229  struct special_fetch_s fetch_info;
1230 
1231  res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
1232  if (res != ZEBRA_OK)
1233  return ZEBRA_FAIL;
1234 
1235  sysno = sysnos[0];
1236  *basenamep = 0;
1237  elemsetname = yaz_get_esn(comp);
1238 
1239  fetch_info.zh = zh;
1240  fetch_info.setname = setname;
1241  fetch_info.sysno = sysno;
1242  fetch_info.score = score;
1243  fetch_info.nmem = odr->mem;
1244 
1245  /* processing zebra special elementset names of form 'zebra:: */
1246  if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
1247  {
1248  WRBUF result = wrbuf_alloc();
1249  int r = zebra_special_fetch(&fetch_info, elemsetname + 7,
1250  input_format, output_format,
1251  result, addinfo_w);
1252  if (r == 0)
1253  {
1254  *rec_bufp = odr_strdup(odr, wrbuf_cstr(result));
1255  *rec_lenp = wrbuf_len(result);
1256  }
1257  wrbuf_destroy(result);
1258  return r;
1259  }
1260 
1261  /* processing all other element set names */
1262  rec = rec_get(zh->reg->records, sysno);
1263  if (!rec)
1264  {
1265  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1266  *basenamep = 0;
1267  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1268  }
1269 
1270 
1271  recordAttr = rec_init_attr(zh->reg->zei, rec);
1272 
1273  file_type = rec->info[recInfo_fileType];
1274  fname = rec->info[recInfo_filename];
1275  basename = rec->info[recInfo_databaseName];
1276  *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
1277  strcpy(*basenamep, basename);
1278 
1279  yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
1280  sysno, score);
1281 
1282  return_code = zebra_create_record_stream(zh, &rec, &stream);
1283 
1284  if (rec)
1285  {
1286  RecType rt;
1287  struct recRetrieveCtrl retrieveCtrl;
1288 
1289  retrieveCtrl.stream = &stream;
1290  retrieveCtrl.fname = fname;
1291  retrieveCtrl.localno = sysno;
1292  retrieveCtrl.staticrank = recordAttr->staticrank;
1293  retrieveCtrl.score = score;
1294  retrieveCtrl.recordSize = recordAttr->recordSize;
1295  retrieveCtrl.odr = odr;
1296  retrieveCtrl.input_format = retrieveCtrl.output_format = input_format;
1297  retrieveCtrl.comp = comp;
1298  retrieveCtrl.encoding = zh->record_encoding;
1299  retrieveCtrl.diagnostic = 0;
1300  retrieveCtrl.addinfo = 0;
1301  retrieveCtrl.dh = zh->reg->dh;
1302  retrieveCtrl.res = zh->res;
1303  retrieveCtrl.rec_buf = 0;
1304  retrieveCtrl.rec_len = -1;
1305  retrieveCtrl.handle = &fetch_info;
1306  retrieveCtrl.special_fetch = zebra_special_fetch;
1307 
1308  if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
1309  file_type, &clientData)))
1310  {
1311  wrbuf_printf(addinfo_w, "Could not handle record type %.40s",
1312  file_type);
1313  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1314  }
1315  else
1316  {
1317  (*rt->retrieve)(clientData, &retrieveCtrl);
1318  return_code = retrieveCtrl.diagnostic;
1319 
1320  *output_format = retrieveCtrl.output_format;
1321  *rec_bufp = (char *) retrieveCtrl.rec_buf;
1322  *rec_lenp = retrieveCtrl.rec_len;
1323  if (retrieveCtrl.addinfo)
1324  wrbuf_puts(addinfo_w, retrieveCtrl.addinfo);
1325  }
1326 
1327  stream.destroy(&stream);
1328  rec_free(&rec);
1329  }
1330 
1331  return return_code;
1332 }
1333 
1334 /*
1335  * Local variables:
1336  * c-basic-offset: 4
1337  * c-file-style: "Stroustrup"
1338  * indent-tabs-mode: nil
1339  * End:
1340  * vim: shiftwidth=4 tabstop=8 expandtab
1341  */
1342 
char * addinfo
Definition: recctrl.h:138
char * record_encoding
Definition: index.h:213
void zebra_snippets_ring(zebra_snippets *doc, const zebra_snippets *hit, int before, int after)
Definition: snippet.c:237
Record rec_get(Records p, zint sysno)
gets record - with given system number
Definition: records.c:927
int len
Definition: it_key.h:31
ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh, const char *setname, zint recid, zint *sysnos, int *no_sysnos)
Definition: zsets.c:1481
static void retrieve_puts_str(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:395
static int zebra_special_fetch(void *handle, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:1057
off_t recordOffset
Definition: zinfo.h:107
#define ZEBRA_OK
Definition: util.h:82
RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
Definition: zsets.c:1075
struct rset_key_control * zebra_key_control_create(ZebraHandle zh)
Definition: kcontrol.c:57
zint ISAM_P
Definition: isamc.h:28
int zebra_term_untrans(ZebraHandle zh, const char *index_type, char *dst, const char *src)
Definition: untrans.c:31
int recordSize
Definition: zinfo.h:106
char * path_reg
Definition: index.h:182
struct rset rset
static void retrieve_puts_attr(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:378
ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, zint sysno, zebra_snippets *snippets)
Definition: zsets.c:1340
struct zebra_snippet_word * next
Definition: snippet.h:34
void * zebra_strmap_lookup(zebra_strmap_t st, const char *name, int no, size_t *data_len)
Definition: strmap.c:99
void zebra_create_stream_mem(struct ZebraRecStream *stream, const char *buf, size_t sz)
Definition: stream.c:123
void zebra_snippets_log(const zebra_snippets *l, int log_level, int all)
Definition: snippet.c:104
static int sort_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:155
static int special_index_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo, Record rec)
Definition: retrieve.c:245
RSET rset_dup(RSET rs)
Duplicate an RSET.
Definition: rset.c:255
zint set_occur
Definition: retrieve.c:573
#define MAX_SYSNOS_PER_RECORD
Definition: retrieve.c:42
int zebra_sort_read(zebra_sort_index_t si, zint *section_id, WRBUF w)
reads sort entry
Definition: sortidx.c:469
char * encoding
Definition: recctrl.h:125
void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, zint approx_limit)
Definition: zsets.c:1498
struct zebra_register * reg
Definition: index.h:174
static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc, const zebra_snippet_word *doc_w, const char *w_index_type)
Definition: retrieve.c:409
ZEBRA_RES res_get_int(Res r, const char *name, int *val)
Definition: res.c:432
#define ZEBRA_FAIL
Definition: util.h:81
static int facet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:957
data1_handle dh
Definition: index.h:142
#define RSETF_READ
Definition: rset.h:199
static struct term_collect * term_collect_create(zebra_strmap_t sm, int no_terms_collect, NMEM nmem)
Definition: retrieve.c:671
const zebra_snippet_word * zebra_snippets_constlist(const zebra_snippets *l)
Definition: snippet.c:99
#define rset_open(rs, wflag)
Definition: rset.h:202
static void retrieve_puts_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:402
Records records
Definition: index.h:138
int zebra_rec_keys_rewind(zebra_rec_keys_t keys)
Definition: reckeys.c:240
const char * index_name
Definition: retrieve.c:90
void rec_free(Record *recpp)
frees record (from memory)
Definition: records.c:1043
void zebra_snippets_destroy(zebra_snippets *l)
Definition: snippet.c:45
zint sysno
Definition: api.h:483
static int perform_facet(ZebraHandle zh, struct special_fetch_s *fi, WRBUF result, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list, int no_ord, int *ord_array, int use_xml, zinfo_index_category_t cat)
Definition: retrieve.c:829
void extract_snippet(ZebraHandle zh, zebra_snippets *sn, struct ZebraRecStream *stream, RecType rt, void *recTypeClientData)
Definition: extract.c:334
int(* retrieve)(void *clientData, struct recRetrieveCtrl *ctrl)
Definition: recctrl.h:159
const char * zebra_strmap_it_next(zebra_strmap_it it, void **data_buf, size_t *data_len)
Definition: strmap.c:162
#define IT_MAX_WORD
Definition: it_key.h:27
void error(const char *format,...)
Definition: agrep.c:51
#define ZEBRA_XML_HEADER_STR
Definition: retrieve.c:44
void rset_delete(RSET rs)
Destructor RSETs.
Definition: rset.c:218
static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set, zint *first_sysno, zint *first_seq)
Definition: retrieve.c:578
ZebraMetaRecord * zebra_meta_records_create(ZebraHandle zh, const char *name, int num, zint *positions)
Definition: zsets.c:418
int zebraExplain_lookup_attr_str(ZebraExplainInfo zei, zinfo_index_category_t cat, const char *index_type, const char *str)
lookup ordinal from string index + index type
Definition: zinfo.c:1354
zint staticrank
Definition: zinfo.h:109
#define rset_read(rfd, buf, term)
Definition: rset.h:217
size_t size[REC_NO_INFO]
Definition: recindex.h:35
static int perform_facet_index(ZebraHandle zh, struct special_fetch_s *fi, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list)
Definition: retrieve.c:758
void zebra_strmap_it_destroy(zebra_strmap_it it)
Definition: strmap.c:157
ZebraExplainInfo zei
Definition: index.h:139
void zebra_rec_keys_close(zebra_rec_keys_t p)
Definition: reckeys.c:143
zint first_sysno
Definition: retrieve.c:574
const Odr_oid * output_format
Definition: recctrl.h:134
#define CAST_ZINT_TO_INT(x)
Definition: util.h:96
RSET rset_create_and(NMEM nmem, struct rset_key_control *kcontrol, int scope, int no_rsets, RSET *rsets)
Definition: rsmultiandor.c:280
zebra_strmap_it zebra_strmap_it_create(zebra_strmap_t st)
Definition: strmap.c:148
zebra_rec_keys_t zebra_rec_keys_open(void)
Definition: reckeys.c:88
char * info[REC_NO_INFO]
Definition: recindex.h:34
int zebra_sort_type(zebra_sort_index_t si, int type)
sets type for sort usage
Definition: sortidx.c:234
#define FIRST_IN_FIELD_CHAR
Definition: index.h:419
int zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, const char *index_type, char **dst, const char *src)
Definition: untrans.c:96
int(* readf)(struct ZebraRecStream *s, char *buf, size_t count)
read function
Definition: recctrl.h:75
zint first_seqno
Definition: retrieve.c:575
RSET zebra_create_rset_isam(ZebraHandle zh, NMEM rset_nmem, struct rset_key_control *kctl, int scope, ISAM_P pos, TERMID termid)
Definition: rset_isam.c:32
struct index_spec * parse_index_spec(const char *elem, NMEM nmem, int *error)
Definition: retrieve.c:97
const Odr_oid * input_format
Definition: recctrl.h:123
ZebraHandle zh
Definition: retrieve.c:47
zebra_sort_index_t sort_index
Definition: index.h:134
TERMID rset_term_create(const char *name, int length, const char *flags, int type, NMEM nmem, struct ord_list *ol, int reg_type, zint hits_limit, const char *ref_id)
Creates a TERMID entry.
Definition: rset.c:340
void(* dec)(struct rset_key_control *kc)
Definition: rset.h:138
char * dict_lookup(Dict dict, const char *p)
lookup item in dictionary
Definition: lookup.c:100
void zebra_sort_sysno(zebra_sort_index_t si, zint sysno)
sets sort system number for read / add / delete
Definition: sortidx.c:339
zint mem[IT_KEY_LEVEL_MAX]
Definition: it_key.h:32
struct index_spec * next
Definition: retrieve.c:93
void zebra_term_untrans_iconv2(ZebraHandle zh, NMEM stream, char **dst, const char *src)
Definition: untrans.c:64
static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset)
Definition: retrieve.c:714
const char * extra
Definition: retrieve.c:92
const char * setname
Definition: retrieve.c:48
Z_RecordComposition * comp
Definition: recctrl.h:124
zint staticrank
Definition: recctrl.h:128
void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, int copy_buf)
Definition: reckeys.c:109
data1_handle dh
Definition: recctrl.h:131
Dict dict
Definition: index.h:132
RecTypes recTypes
Definition: index.h:145
zinfo_index_category_t
Definition: zinfo.h:37
int zebraExplain_lookup_ord(ZebraExplainInfo zei, int ord, const char **index_type, const char **db, const char **string_index)
Definition: zinfo.c:1479
struct ZebraRecStream * stream
Definition: recctrl.h:119
int key_SU_encode(int ch, char *out)
Definition: su_codec.c:31
zebra_strmap_t zebra_strmap_create(void)
Definition: strmap.c:45
void zebra_strmap_add(zebra_strmap_t st, const char *name, void *data_buf, size_t data_len)
Definition: strmap.c:80
int zebra_get_rec_snippets(ZebraHandle zh, zint sysno, zebra_snippets *snippets)
Definition: retrieve.c:490
static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
Definition: retrieve.c:444
void * handle
Definition: recctrl.h:141
RecType recType_byName(RecTypes rts, Res res, const char *name, void **clientDataP)
Definition: recctrl.c:264
static int term_qsort_handle(const void *a, const void *b)
Definition: retrieve.c:633
long zint
Zebra integer.
Definition: util.h:66
const char * term
Definition: retrieve.c:571
void(* destroy)(struct ZebraRecStream *s)
close and destroy stream
Definition: recctrl.h:83
int zebra_rec_keys_read(zebra_rec_keys_t keys, const char **str, size_t *slen, struct it_key *key)
Definition: reckeys.c:259
void * rec_buf
Definition: recctrl.h:135
Definition: rset.h:73
Definition: rset.h:150
int zebra_record_fetch(ZebraHandle zh, const char *setname, zint sysno, int score, ODR odr, const Odr_oid *input_format, Z_RecordComposition *comp, const Odr_oid **output_format, char **rec_bufp, int *rec_lenp, char **basenamep, WRBUF addinfo_w)
Definition: retrieve.c:1211
zebra_snippets * zebra_snippets_create(void)
Definition: snippet.c:36
int fd
Definition: tstlockscope.c:38
#define O_BINARY
Definition: agrep.c:46
char * fname
Definition: recctrl.h:130
RecordAttr * rec_init_attr(ZebraExplainInfo zei, Record rec)
Definition: zinfo.c:1595
Definition: it_key.h:30
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
void rset_close(RSFD rfd)
Closes a result set RFD handle.
Definition: rset.c:98
Definition: rset.h:50
const char * index_type
Definition: retrieve.c:91
void zebra_strmap_destroy(zebra_strmap_t st)
Definition: strmap.c:61
static int snippet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:526
int zebra_rec_keys_empty(zebra_rec_keys_t keys)
Definition: reckeys.c:252
zint approx_limit
Definition: index.h:180
void zebra_create_stream_fd(struct ZebraRecStream *stream, int fd, off_t start_offset)
Definition: stream.c:140
static void term_collect_freq(ZebraHandle zh, struct term_collect *col, int no_terms_collect, int ord, RSET rset, double scale_factor)
Definition: retrieve.c:649
static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:389
record reader stream
Definition: recctrl.h:71
int(* special_fetch)(void *handle, const char *esn, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: recctrl.h:142
static int zebra_create_record_stream(ZebraHandle zh, Record *rec, struct ZebraRecStream *stream)
Definition: retrieve.c:54
void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, int num)
Definition: zsets.c:547
#define ZINT_FORMAT
Definition: util.h:72