IDZEBRA  2.1.4
retrieve.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23 #include <stdio.h>
24 #include <assert.h>
25 
26 #include <fcntl.h>
27 #ifdef WIN32
28 #include <io.h>
29 #include <process.h>
30 #endif
31 #if HAVE_UNISTD_H
32 #include <unistd.h>
33 #endif
34 
35 #include "index.h"
36 #include <yaz/diagbib1.h>
37 #include <yaz/snprintf.h>
38 #include <direntz.h>
39 #include <yaz/oid_db.h>
40 #include <zebra_strmap.h>
41 
42 #define MAX_SYSNOS_PER_RECORD 40
43 
44 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
45 
48  const char *setname;
50  int score;
51  NMEM nmem;
52 };
53 
54 static int log_level_mod = 0;
55 static int log_level_set = 0;
56 
58  Record *rec,
59  struct ZebraRecStream *stream)
60 {
61  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
62 
63  if ((*rec)->size[recInfo_storeData] > 0
64  || (*rec)->info[recInfo_filename] == 0)
65  zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
66  (*rec)->size[recInfo_storeData]);
67  else
68  {
69  char full_rep[1024];
70  int fd;
71 
72  if (zh->path_reg && !yaz_is_abspath((*rec)->info[recInfo_filename])){
73  strcpy(full_rep, zh->path_reg);
74  strcat(full_rep, "/");
75  strcat(full_rep, (*rec)->info[recInfo_filename]);
76  }
77  else
78  strcpy(full_rep, (*rec)->info[recInfo_filename]);
79 
80  if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
81  yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
82  full_rep);
83  rec_free(rec);
84  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
85  }
86  zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
87  }
88  return 0;
89 }
90 
91 
92 struct index_spec {
93  const char *index_name;
94  const char *index_type;
95  const char *extra;
96  struct index_spec *next;
97 };
98 
99 
100 struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
101  int *error)
102 {
103  struct index_spec *first = 0;
104  struct index_spec **last = &first;
105  const char *cp = elem;
106 
107  *error = 0;
108  if (cp[0] == ':' && cp[1] == ':')
109  {
110 
111  cp++; /* skip first ':' */
112 
113  for (;;)
114  {
115  const char *cp0;
116  struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
117  spec->index_type = 0;
118  spec->next = 0;
119  spec->extra = 0;
120 
121  if (!first)
122  first = spec;
123  *last = spec;
124  last = &spec->next;
125 
126  cp++; /* skip ',' or second ':' */
127  cp0 = cp;
128  while (*cp != ':' && *cp != '\0' && *cp != ',')
129  cp++;
130  spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
131  if (*cp == ':') /* type as well */
132  {
133  cp++;
134  cp0 = cp;
135 
136  while (*cp != '\0' && *cp != ',' && *cp != ':')
137  cp++;
138  spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
139  }
140  if (*cp == ':') /* extra arguments */
141  {
142  cp++;
143  cp0 = cp;
144 
145  while (*cp != '\0' && *cp != ',' && *cp != ':')
146  cp++;
147  spec->extra = nmem_strdupn(nmem, cp0, cp - cp0);
148  }
149  if (*cp != ',')
150  break;
151  }
152  }
153  if (*cp != '\0')
154  *error = 1;
155  return first;
156 }
157 
158 static int sort_fetch(
159  struct special_fetch_s *fi, const char *elemsetname,
160  const Odr_oid *input_format,
161  const Odr_oid **output_format,
162  WRBUF result, WRBUF addinfo)
163 {
164  int ord;
165  ZebraHandle zh = fi->zh;
166  int error;
167  struct index_spec *spec;
168 
169  spec = parse_index_spec(elemsetname, fi->nmem, &error);
170  if (error)
171  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
172 
173  /* for sort fetches.. We allow only one index and type must be given */
174  if (!spec || spec->next || !spec->index_type)
175  return -1;
178  spec->index_type,
179  spec->index_name);
180  if (ord == -1)
181  return -1; /* is not a sort index */
182  else
183  {
184  WRBUF wrbuf_str = wrbuf_alloc();
185  const char *index_type;
186  const char *db = 0;
187  const char *string_index = 0;
188  WRBUF wrbuf_result = result;
189  int off = 0;
190 
191  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
192  &string_index);
193  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
194  {
195  *output_format = yaz_oid_recsyn_xml;
196  wrbuf_printf(wrbuf_result, ZEBRA_XML_HEADER_STR
197  " sysno=\"" ZINT_FORMAT "\""
198  " set=\"zebra::index%s\">\n",
199  fi->sysno, elemsetname);
200  }
201  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
202  {
203  *output_format = yaz_oid_recsyn_sutrs;
204  }
205  else
206  {
207  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
208  elemsetname);
209  *output_format = 0;
210  wrbuf_destroy(wrbuf_str);
211  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
212  }
213  zebra_sort_type(zh->reg->sort_index, ord);
215  zebra_sort_read(zh->reg->sort_index, 0, wrbuf_str);
216 
217  while (off != wrbuf_len(wrbuf_str))
218  {
219  char dst_buf[IT_MAX_WORD];
220  assert(off < wrbuf_len(wrbuf_str));
221  zebra_term_untrans(zh, index_type, dst_buf,
222  wrbuf_buf(wrbuf_str)+off);
223 
224  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
225  {
226  wrbuf_printf(wrbuf_result, " <index name=\"%s\"",
227  string_index);
228  wrbuf_printf(wrbuf_result, " type=\"%s\">", index_type);
229  wrbuf_xmlputs(wrbuf_result, dst_buf);
230  wrbuf_printf(wrbuf_result, "</index>\n");
231  }
232  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
233  {
234  wrbuf_printf(wrbuf_result, "%s %s %s\n", string_index, index_type,
235  dst_buf);
236  }
237  off += strlen(wrbuf_buf(wrbuf_str)+off) + 1;
238  }
239  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
240  {
241  wrbuf_printf(wrbuf_result, "</record>\n");
242  }
243  wrbuf_destroy(wrbuf_str);
244  return 0;
245  }
246 }
247 
249  struct special_fetch_s *fi, const char *elemsetname,
250  const Odr_oid *input_format,
251  const Odr_oid **output_format,
252  WRBUF result, WRBUF addinfo,
253  Record rec)
254 {
255  zebra_rec_keys_t keys;
256  int ret_code = 0;
257  ZebraHandle zh = fi->zh;
258  struct index_spec *spec, *spec_list;
259  int error;
260 
261  /* set output variables before processing possible error states */
262  /* *rec_lenp = 0; */
263 
264  /* only accept XML and SUTRS requests */
265  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
266  && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
267  {
268  yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
269  elemsetname);
270  *output_format = 0;
271  return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
272  }
273 
274  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
275  if (error)
276  {
277  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
278  }
279 
280  for (spec = spec_list; spec; spec = spec->next)
281  {
284  spec->index_type,
285  spec->index_name) == -1)
286  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
287  }
288 
289  keys = zebra_rec_keys_open();
291  rec->size[recInfo_delKeys], 0);
292 
293  if (!zebra_rec_keys_rewind(keys))
294  {
295  ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
296  }
297  else
298  {
299  size_t slen;
300  const char *str;
301  struct it_key key_in;
302  WRBUF wrbuf = result;
303 
304  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
305  {
306  *output_format = input_format;
307  wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
308  " sysno=\"" ZINT_FORMAT "\""
309  " set=\"zebra::index%s\">\n",
310  fi->sysno, elemsetname);
311  }
312  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
313  *output_format = input_format;
314 
315  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
316  {
317  int i;
318  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
319  const char *index_type;
320  const char *db = 0;
321  const char *string_index = 0;
322  char dst_buf[IT_MAX_WORD];
323  int match = 0;
324 
325  zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
326  &string_index);
327  if (!spec_list)
328  match = 1; /* match all if no specs were given */
329  else
330  {
331  for (spec = spec_list; spec; spec = spec->next)
332  {
333  if ((!spec->index_type ||
334  !yaz_matchstr(spec->index_type, index_type))
335  &&
336  !yaz_matchstr(spec->index_name, string_index))
337  match = 1;
338  }
339  }
340  if (match)
341  {
342  if (zebra_term_untrans(zh, index_type, dst_buf, str))
343  *dst_buf = '\0'; /* untrans failed */
344 
345  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
346  {
347  wrbuf_printf(wrbuf, " <index name=\"%s\"",
348  string_index);
349 
350  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
351 
352  wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">",
353  key_in.mem[key_in.len -1]);
354  wrbuf_xmlputs(wrbuf, dst_buf);
355  wrbuf_printf(wrbuf, "</index>\n");
356  }
357  else
358  {
359  wrbuf_printf(wrbuf, "%s ", string_index);
360 
361  wrbuf_printf(wrbuf, "%s", index_type);
362 
363  for (i = 1; i < key_in.len; i++)
364  wrbuf_printf(wrbuf, " " ZINT_FORMAT,
365  key_in.mem[i]);
366 
367  wrbuf_printf(wrbuf, " %s", dst_buf);
368 
369  wrbuf_printf(wrbuf, "\n");
370  }
371  }
372  }
373  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
374  wrbuf_printf(wrbuf, "</record>\n");
375  }
376  zebra_rec_keys_close(keys);
377  return ret_code;
378 }
379 
380 
381 static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
382  const char *value)
383 {
384  if (value)
385  {
386  wrbuf_printf(wrbuf, " %s=\"", name);
387  wrbuf_xmlputs(wrbuf, value);
388  wrbuf_printf(wrbuf, "\"");
389  }
390 }
391 
392 static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
393  const int value)
394 {
395  wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
396 }
397 
398 static void retrieve_puts_str(WRBUF wrbuf, const char *name,
399  const char *value)
400 {
401  if (value)
402  wrbuf_printf(wrbuf, "%s %s\n", name, value);
403 }
404 
405 static void retrieve_puts_int(WRBUF wrbuf, const char *name,
406  const int value)
407 {
408  wrbuf_printf(wrbuf, "%s %i\n", name, value);
409 }
410 
411 
412 static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf,
413  zebra_snippets *doc,
414  const zebra_snippet_word *doc_w,
415  const char *w_index_type)
416 {
417  /* beginning of snippet. See which fields the snippet also
418  occur */
419  const zebra_snippet_word *w;
420  int no = 0;
421  for (w = zebra_snippets_constlist(doc); w; w = w->next)
422  {
423  /* same sequence but other field? */
424  if (w->seqno == doc_w->seqno && w->ord != doc_w->ord)
425  {
426  const char *index_type;
427  const char *db = 0;
428  const char *string_index = 0;
429 
431  &index_type, &db, &string_index);
432  /* only report for same index type */
433  if (!strcmp(w_index_type, index_type))
434  {
435  if (no == 0)
436  wrbuf_printf(wrbuf, " fields=\"%s", string_index);
437  else
438  wrbuf_printf(wrbuf, " %s", string_index);
439  no++;
440  }
441  }
442  }
443  if (no)
444  wrbuf_printf(wrbuf, "\"");
445 }
446 
447 static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
448 {
449  const zebra_snippet_word *doc_w;
450  int mark_state = 0;
451 
452  wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
453  for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
454  {
455  if (doc_w->mark)
456  {
457  const char *index_type;
458  const char *db = 0;
459  const char *string_index = 0;
460 
461  zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord,
462  &index_type, &db, &string_index);
463 
464  if (mark_state == 0)
465  {
466 
467  wrbuf_printf(wrbuf, " <snippet name=\"%s\"", string_index);
468  wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
469  snippet_check_fields(zh, wrbuf, doc, doc_w, index_type);
470  wrbuf_printf(wrbuf, ">");
471  }
472  if (doc_w->match)
473  wrbuf_puts(wrbuf, "<s>");
474  /* not printing leading ws */
475  if (mark_state || !doc_w->ws || doc_w->match)
476  wrbuf_xmlputs(wrbuf, doc_w->term);
477  if (doc_w->match)
478  wrbuf_puts(wrbuf, "</s>");
479  }
480  else if (mark_state == 1)
481  {
482  wrbuf_puts(wrbuf, "</snippet>\n");
483  }
484  mark_state = doc_w->mark;
485  }
486  if (mark_state == 1)
487  {
488  wrbuf_puts(wrbuf, "</snippet>\n");
489  }
490  wrbuf_printf(wrbuf, "</record>");
491 }
492 
494  zebra_snippets *snippets)
495 {
496  int return_code = 0;
497  Record rec;
498 
499  if (!log_level_set)
500  {
501  log_level_mod = yaz_log_module_level("retrieve");
502  log_level_set = 1;
503  }
504  rec = rec_get(zh->reg->records, sysno);
505  if (!rec)
506  {
507  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
508  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
509  }
510  else
511  {
512  const char *file_type = rec->info[recInfo_fileType];
513  void *recTypeClientData;
514  RecType rt = recType_byName(zh->reg->recTypes, zh->res,
515  file_type, &recTypeClientData);
516 
517  if (!rt)
518  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
519  else
520  {
521  struct ZebraRecStream stream;
522  return_code = zebra_create_record_stream(zh, &rec, &stream);
523  if (return_code == 0)
524  {
525  extract_snippet(zh, snippets, &stream,
526  rt, recTypeClientData);
527 
528  stream.destroy(&stream);
529  }
530  }
531  rec_free(&rec);
532  }
533  return return_code;
534 }
535 
536 static int snippet_fetch(
537  struct special_fetch_s *fi, const char *elemsetname,
538  const Odr_oid *input_format,
539  const Odr_oid **output_format,
540  WRBUF result, WRBUF addinfo)
541 {
542  ZebraHandle zh = fi->zh;
543  zebra_snippets *rec_snippets = zebra_snippets_create();
544  int return_code = zebra_get_rec_snippets(zh, fi->sysno, rec_snippets);
545 
546  if (!return_code)
547  {
548  WRBUF wrbuf = result;
549  zebra_snippets *hit_snippet = zebra_snippets_create();
550 
551  zebra_snippets_hit_vector(zh, fi->setname, fi->sysno, hit_snippet);
552 
553  if (log_level_mod)
554  {
555  /* for debugging purposes */
556  yaz_log(log_level_mod, "---------------------------");
557  yaz_log(log_level_mod, "REC SNIPPET:");
558  zebra_snippets_log(rec_snippets, log_level_mod, 1);
559  yaz_log(log_level_mod, "---------------------------");
560  yaz_log(log_level_mod, "HIT SNIPPET:");
561  zebra_snippets_log(hit_snippet, log_level_mod, 1);
562  }
563 
564  zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
565 
566  if (log_level_mod)
567  {
568  yaz_log(log_level_mod, "---------------------------");
569  yaz_log(log_level_mod, "RING SNIPPET:");
570  zebra_snippets_log(rec_snippets, log_level_mod, 1);
571  }
572  snippet_xml_record(zh, wrbuf, rec_snippets);
573 
574  *output_format = yaz_oid_recsyn_xml;
575 
576  zebra_snippets_destroy(hit_snippet);
577  }
578  zebra_snippets_destroy(rec_snippets);
579  return return_code;
580 }
581 
582 struct term_collect {
583  const char *term;
584  int oc;
588 };
589 
590 static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set,
591  zint *first_sysno, zint *first_seq)
592 {
594  char ord_buf[IT_MAX_WORD];
595  int ord_len = key_SU_encode(ord, ord_buf);
596  char *info;
597  zint hits = 0;
598  NMEM nmem = nmem_create();
599 
600  strcpy(ord_buf + ord_len, term);
601 
602  info = dict_lookup(zh->reg->dict, ord_buf);
603  if (info)
604  {
605  ISAM_P isam_p;
606  RSET rsets[2], rset;
607  memcpy(&isam_p, info+1, sizeof(ISAM_P));
608  TERMID termid_key = rset_term_create("", -1,
609  0 /* flags */,
610  Z_Term_general /* type */,
611  nmem,
612  0, /* ord_list */
613  0, /* reg_type */
614  10000, /* hits_limit */
615  0 /* ref_id */);
616 
617  rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p,
618  termid_key);
619  rsets[1] = rset_dup(rset_set);
620 
621  rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
622 
623  zebra_count_set(zh, rset, &hits, zh->approx_limit);
624  RSFD rfd = rset_open(rset, RSETF_READ);
625  TERMID termid;
626  struct it_key key;
627  while (rset_read(rfd, &key, &termid))
628  {
629  if (termid == termid_key)
630  {
631  *first_sysno = key.mem[0];
632  *first_seq = key.mem[key.len - 1];
633  break;
634  }
635  }
636  rset_close(rfd);
637  rset_delete(rsets[0]);
638  rset_delete(rset);
639  }
640  (*kc->dec)(kc);
641  nmem_destroy(nmem);
642  return hits;
643 }
644 
645 static int term_qsort_handle(const void *a, const void *b)
646 {
647  const struct term_collect *l = a;
648  const struct term_collect *r = b;
649  if (l->set_occur < r->set_occur)
650  return 1;
651  else if (l->set_occur > r->set_occur)
652  return -1;
653  else
654  {
655  const char *lterm = l->term ? l->term : "";
656  const char *rterm = r->term ? r->term : "";
657  return strcmp(lterm, rterm);
658  }
659 }
660 
662  struct term_collect *col, int no_terms_collect,
663  int ord, RSET rset, double scale_factor)
664 {
665  int i;
666  for (i = 0; i < no_terms_collect; i++)
667  {
668  if (col[i].term)
669  {
670  if (scale_factor < 0.0)
671  {
672  col[i].set_occur = freq_term(zh, ord, col[i].term, rset,
673  &col[i].first_sysno,
674  &col[i].first_seqno);
675  }
676  else
677  col[i].set_occur = scale_factor * col[i].oc;
678  }
679  }
680  qsort(col, no_terms_collect, sizeof(*col), term_qsort_handle);
681 }
682 
684  int no_terms_collect,
685  NMEM nmem)
686 {
687  const char *term;
688  void *data_buf;
689  size_t data_len;
690  zebra_strmap_it it;
691  struct term_collect *col = nmem_malloc(nmem,
692  sizeof *col *no_terms_collect);
693  int i;
694  for (i = 0; i < no_terms_collect; i++)
695  {
696  col[i].term = 0;
697  col[i].oc = 0;
698  col[i].set_occur = 0;
699  col[i].first_sysno = col[i].first_seqno = 0;
700  }
701  /* iterate over terms and collect the most frequent ones */
702  it = zebra_strmap_it_create(sm);
703  while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
704  {
705  /* invariant:
706  col[0] has lowest oc . col[no_terms_collect-1] has highest oc */
707  int oc = *(int*) data_buf;
708  int j = 0;
709  /* insertion may be slow but terms terms will be "infrequent" and
710  thus number of iterations should be small below
711  */
712  while (j < no_terms_collect && oc > col[j].oc)
713  j++;
714  if (j)
715  { /* oc <= col[j] and oc > col[j-1] */
716  --j;
717  memmove(col, col+1, sizeof(*col) * j);
718  col[j].term = term;
719  col[j].oc = oc;
720  }
721  }
723  return col;
724 }
725 
726 static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array,
727  zebra_strmap_t *map_array,
728  int num_recs, ZebraMetaRecord *poset)
729 {
730  int rec_i;
731  WRBUF w = wrbuf_alloc();
732  int ord_i;
733 
734  for (ord_i = 0; ord_i < no_ord; ord_i++)
735  {
736  for (rec_i = 0; rec_i < num_recs; rec_i++)
737  {
738  if (!poset[rec_i].sysno)
739  continue;
740 
741  zebra_sort_sysno(zh->reg->sort_index, poset[rec_i].sysno);
742  zebra_sort_type(zh->reg->sort_index, ord_array[ord_i]);
743 
744  wrbuf_rewind(w);
745  if (zebra_sort_read(zh->reg->sort_index, 0, w))
746  {
747  zebra_strmap_t sm = map_array[ord_i];
748  int off = 0;
749  while (off != wrbuf_len(w))
750  {
751  const char *str = wrbuf_buf(w) + off;
752  int *freq = zebra_strmap_lookup(sm, str, 0, 0);
753  if (freq)
754  (*freq)++;
755  else
756  {
757  int v = 1;
758  zebra_strmap_add(sm, str, &v, sizeof v);
759  }
760  off += strlen(str)+1;
761  }
762  }
763  }
764  }
765  wrbuf_destroy(w);
766  return 0;
767 }
768 
769 
771  struct special_fetch_s *fi,
772  int no_ord, int *ord_array,
773  zebra_strmap_t *map_array,
774  int num_recs, ZebraMetaRecord *poset,
775  struct index_spec *spec_list)
776 {
777  int max_chunks = 2;
778  int rec_i;
779  res_get_int(zh->res, "facetMaxChunks", &max_chunks);
780 
781  for (rec_i = 0; rec_i < num_recs; rec_i++)
782  {
783  int j;
784  zint sysnos[MAX_SYSNOS_PER_RECORD];
785  int no_sysnos = MAX_SYSNOS_PER_RECORD;
786  if (!poset[rec_i].sysno)
787  continue;
789  poset[rec_i].sysno,
790  sysnos, &no_sysnos);
791  assert(no_sysnos > 0);
792  yaz_log(YLOG_DEBUG, "Analyzing rec=%d ISAM sysno=" ZINT_FORMAT " chunks=%d",
793  rec_i, poset[rec_i].sysno, no_sysnos);
794  for (j = 0; j < no_sysnos && j < max_chunks; j++)
795  {
796  size_t slen;
797  const char *str;
798  struct it_key key_in;
799  Record rec = rec_get(zh->reg->records, sysnos[j]);
801  zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
802  rec->size[recInfo_delKeys], 0);
803 
804  yaz_log(YLOG_DEBUG, "rec %d " ZINT_FORMAT " %s",
805  j, sysnos[j], zebra_rec_keys_empty(keys) ? "empty" : "non-empty");
806  if (zebra_rec_keys_rewind(keys))
807  {
808  while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
809  {
810  int ord_i;
811  struct index_spec *spec;
812  for (spec = spec_list, ord_i = 0; ord_i < no_ord;
813  ord_i++, spec = spec->next)
814  {
815  int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
816  if (ord == ord_array[ord_i] &&
817  str[0] != FIRST_IN_FIELD_CHAR)
818  {
819  int *freq;
820  zebra_strmap_t sm = map_array[ord_i];
821 
822  freq = zebra_strmap_lookup(sm, str, 0, 0);
823  if (freq)
824  (*freq)++;
825  else
826  {
827  int v = 1;
828  zebra_strmap_add(sm, str, &v, sizeof v);
829  }
830  }
831  }
832  }
833  }
834  zebra_rec_keys_close(keys);
835  rec_free(&rec);
836  }
837  }
838  return 0;
839 }
840 
842  struct special_fetch_s *fi,
843  WRBUF result,
844  int num_recs, ZebraMetaRecord *poset,
845  struct index_spec *spec_list,
846  int no_ord, int *ord_array,
847  int use_xml,
849 {
850  int i;
851  int ret = 0;
852  WRBUF wr = result;
853  struct index_spec *spec;
854  yaz_timing_t timing = yaz_timing_create();
855  zebra_strmap_t *map_array
856  = nmem_malloc(fi->nmem, sizeof *map_array * no_ord);
857  for (i = 0; i < no_ord; i++)
858  map_array[i] = zebra_strmap_create();
859 
860  if (cat == zinfo_index_category_sort)
861  perform_facet_sort(zh, no_ord, ord_array, map_array,
862  num_recs, poset);
863  else
864  perform_facet_index(zh, fi, no_ord, ord_array, map_array,
865  num_recs, poset, spec_list);
866  yaz_timing_stop(timing);
867  yaz_log(YLOG_LOG, "facet first phase real=%4.2f cat=%s",
868  yaz_timing_get_real(timing),
869  (cat == zinfo_index_category_sort) ? "sort" : "index");
870  yaz_timing_start(timing);
871  for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
872  {
873  int j;
874  NMEM nmem = nmem_create();
875  struct term_collect *col;
876  int no_collect_terms = 20;
877 
878  if (spec->extra)
879  no_collect_terms = atoi(spec->extra);
880  if (no_collect_terms < 1)
881  no_collect_terms = 1;
882  col = term_collect_create(map_array[i], no_collect_terms, nmem);
883  term_collect_freq(zh, col, no_collect_terms, ord_array[i],
884  resultSetRef(zh, fi->setname),
885  cat == zinfo_index_category_sort ? 1.0 : -1.0);
886 
887  if (use_xml)
888  wrbuf_printf(wr, " <facet type=\"%s\" index=\"%s\">\n",
889  spec->index_type, spec->index_name);
890  else
891  wrbuf_printf(wr, "facet %s %s\n",
892  spec->index_type, spec->index_name);
893  for (j = 0; j < no_collect_terms; j++)
894  {
895  if (col[j].term)
896  {
897  WRBUF w_buf = wrbuf_alloc();
898  yaz_log(log_level_mod, "facet %s %s",
899  spec->index_type, spec->index_name);
900  if (col[j].first_sysno)
901  {
902  zebra_snippets *rec_snippets = zebra_snippets_create();
903  int code = zebra_get_rec_snippets(
904  zh, col[j].first_sysno, rec_snippets);
905  yaz_log(log_level_mod, "sysno/seqno "
907  col[j].first_sysno, col[j].first_seqno);
908  if (code == 0)
909  {
910  if (log_level_mod)
911  zebra_snippets_log(rec_snippets,
912  log_level_mod, 1);
913  const zebra_snippet_word *sn =
914  zebra_snippets_constlist(rec_snippets);
915  int first = 1; /* ignore leading whitespace */
916  for (; sn; sn = sn->next)
917  {
918  if (sn->ord == ord_array[i] &&
919  sn->seqno == col[j].first_seqno)
920  {
921  if (!sn->ws || !first)
922  {
923  first = 0;
924  wrbuf_puts(w_buf, sn->term);
925  }
926  }
927  }
928  }
929  zebra_snippets_destroy(rec_snippets);
930  }
931  char *dst_buf = 0;
932  if (wrbuf_len(w_buf))
933  zebra_term_untrans_iconv2(zh, nmem, &dst_buf, wrbuf_cstr(w_buf));
934  else
935  zebra_term_untrans_iconv(zh, nmem, spec->index_type, &dst_buf,
936  col[j].term);
937  if (use_xml)
938  {
939  wrbuf_printf(wr, " <term coccur=\"%d\"", col[j].oc);
940  if (col[j].set_occur)
941  wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"",
942  col[j].set_occur);
943  wrbuf_printf(wr, ">");
944  wrbuf_xmlputs(wr, dst_buf);
945  wrbuf_printf(wr, "</term>\n");
946  }
947  else
948  {
949  wrbuf_printf(wr, "term %d", col[j].oc);
950  if (col[j].set_occur)
951  wrbuf_printf(wr, " " ZINT_FORMAT,
952  col[j].set_occur);
953  wrbuf_printf(wr, ": %s\n", dst_buf);
954  }
955  wrbuf_destroy(w_buf);
956  }
957  }
958  if (use_xml)
959  wrbuf_puts(wr, " </facet>\n");
960  nmem_destroy(nmem);
961  }
962  for (i = 0; i < no_ord; i++)
963  zebra_strmap_destroy(map_array[i]);
964  yaz_timing_stop(timing);
965  yaz_log(YLOG_LOG, "facet second phase real=%4.2f",
966  yaz_timing_get_real(timing));
967  yaz_timing_destroy(&timing);
968  return ret;
969 }
970 
971 static int facet_fetch(
972  struct special_fetch_s *fi, const char *elemsetname,
973  const Odr_oid *input_format,
974  const Odr_oid **output_format,
975  WRBUF result, WRBUF addinfo)
976 {
977  zint *pos_array;
978  int i;
979  int num_recs = 10; /* number of records to analyze */
980  ZebraMetaRecord *poset;
981  ZEBRA_RES ret = ZEBRA_OK;
982  int *ord_array;
983  int use_xml = 0;
984  int no_ord = 0;
985  struct index_spec *spec, *spec_list;
986  int error;
987  ZebraHandle zh = fi->zh;
988  /* whether sort or index based */
990 
991  /* see if XML is required for response */
992  if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) == 0)
993  use_xml = 1;
994 
995  spec_list = parse_index_spec(elemsetname, fi->nmem, &error);
996 
997  if (!spec_list || error)
998  {
999  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1000  }
1001 
1002  for (spec = spec_list; spec; spec = spec->next)
1003  {
1004  if (!spec->index_type)
1005  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1006  no_ord++;
1007  }
1008 
1009  /* try to see if all specs are sort based.. If not, try the
1010  index based ones */
1011  ord_array = nmem_malloc(fi->nmem, sizeof(*ord_array) * no_ord);
1012 
1013  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
1014  {
1015  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1017  spec->index_type,
1018  spec->index_name);
1019  if (ord == -1)
1020  break;
1021  ord_array[i] = ord;
1022  num_recs = 10000;
1023  }
1024  if (spec)
1025  {
1027  for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
1028  {
1029  int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
1031  spec->index_type,
1032  spec->index_name);
1033  if (ord == -1)
1034  break;
1035  ord_array[i] = ord;
1036 
1037  }
1038  }
1039  if (spec)
1040  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1041 
1042  res_get_int(zh->res, "facetNumRecs", &num_recs);
1043 
1044  pos_array = (zint *) nmem_malloc(fi->nmem, num_recs * sizeof(*pos_array));
1045  for (i = 0; i < num_recs; i++)
1046  pos_array[i] = i+1;
1047  poset = zebra_meta_records_create(zh, fi->setname, num_recs, pos_array);
1048  if (!poset)
1049  {
1050  wrbuf_puts(addinfo, fi->setname);
1051  return YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST;
1052  }
1053  else
1054  {
1055  if (use_xml)
1056  {
1057  wrbuf_printf(result, ZEBRA_XML_HEADER_STR ">\n");
1058  }
1059  ret = perform_facet(zh, fi, result, num_recs, poset,
1060  spec_list, no_ord, ord_array, use_xml,
1061  cat);
1062  if (use_xml)
1063  wrbuf_puts(result, "</record>\n");
1064  }
1065  *output_format = yaz_oid_recsyn_xml;
1066  zebra_meta_records_destroy(zh, poset, num_recs);
1067  return ret;
1068 }
1069 
1070 
1072  void *handle, const char *elemsetname,
1073  const Odr_oid *input_format,
1074  const Odr_oid **output_format,
1075  WRBUF result, WRBUF addinfo)
1076 {
1077  Record rec = 0;
1078  struct special_fetch_s *fi = (struct special_fetch_s *) handle;
1079  ZebraHandle zh = fi->zh;
1080  zint sysno = fi->sysno;
1081 
1082  /* processing zebra::facet */
1083  if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
1084  {
1085  return facet_fetch(fi, elemsetname + 5,
1086  input_format, output_format,
1087  result, addinfo);
1088  }
1089 
1090  if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
1091  {
1092  return snippet_fetch(fi, elemsetname + 7,
1093  input_format, output_format,
1094  result, addinfo);
1095  }
1096 
1097  /* processing zebra::meta::sysno */
1098  if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
1099  {
1100  int ret = 0;
1101  if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1102  {
1103  wrbuf_printf(result, ZINT_FORMAT, fi->sysno);
1104  *output_format = input_format;
1105  }
1106  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1107  {
1108  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1109  " sysno=\"" ZINT_FORMAT "\"/>\n",
1110  fi->sysno);
1111  *output_format = input_format;
1112  }
1113  else
1114  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1115  return ret;
1116  }
1117 
1118  /* processing special elementsetname zebra::index:: for sort elements */
1119  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1120  {
1121  int ret = sort_fetch(
1122  fi, elemsetname + 5,
1123  input_format, output_format,
1124  result, addinfo);
1125  if (ret != -1)
1126  return ret;
1127  /* not a sort index so we continue to get the full record */
1128  }
1129 
1130 
1131  /* fetching binary record up for all other display elementsets */
1132  rec = rec_get(zh->reg->records, sysno);
1133  if (!rec)
1134  {
1135  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1136  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1137  }
1138 
1139  /* processing special elementsetnames zebra::data */
1140  if (elemsetname && 0 == strcmp(elemsetname, "data"))
1141  {
1142  struct ZebraRecStream stream;
1143  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1144  char *b;
1145 
1146  zebra_create_record_stream(zh, &rec, &stream);
1147  *output_format = input_format;
1148 
1149  b = nmem_malloc(fi->nmem, recordAttr->recordSize);
1150  stream.readf(&stream, b, recordAttr->recordSize);
1151  wrbuf_write(result, b, recordAttr->recordSize);
1152 
1153  stream.destroy(&stream);
1154  rec_free(&rec);
1155  return 0;
1156  }
1157 
1158  /* processing special elementsetnames zebra::meta:: */
1159  if (elemsetname && 0 == strcmp(elemsetname, "meta"))
1160  {
1161  int ret = 0;
1162  RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1163 
1164  if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1165  {
1166  *output_format = input_format;
1167 
1168  wrbuf_printf(result, ZEBRA_XML_HEADER_STR
1169  " sysno=\"" ZINT_FORMAT "\"", sysno);
1170  retrieve_puts_attr(result, "base", rec->info[recInfo_databaseName]);
1171  retrieve_puts_attr(result, "file", rec->info[recInfo_filename]);
1172  retrieve_puts_attr(result, "type", rec->info[recInfo_fileType]);
1173  if (fi->score >= 0)
1174  retrieve_puts_attr_int(result, "score", fi->score);
1175 
1176  wrbuf_printf(result,
1177  " rank=\"" ZINT_FORMAT "\""
1178  " size=\"%i\""
1179  " set=\"zebra::%s\"/>\n",
1180  recordAttr->staticrank,
1181  recordAttr->recordSize,
1182  elemsetname);
1183  }
1184  else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1185  {
1186  *output_format = input_format;
1187  wrbuf_printf(result, "sysno " ZINT_FORMAT "\n", sysno);
1188  retrieve_puts_str(result, "base", rec->info[recInfo_databaseName]);
1189  retrieve_puts_str(result, "file", rec->info[recInfo_filename]);
1190  retrieve_puts_str(result, "type", rec->info[recInfo_fileType]);
1191  if (fi->score >= 0)
1192  retrieve_puts_int(result, "score", fi->score);
1193 
1194  wrbuf_printf(result,
1195  "rank " ZINT_FORMAT "\n"
1196  "size %i\n"
1197  "set zebra::%s\n",
1198  recordAttr->staticrank,
1199  recordAttr->recordSize,
1200  elemsetname);
1201  }
1202  else
1203  ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1204 
1205  rec_free(&rec);
1206  return ret;
1207  }
1208 
1209  /* processing special elementsetnames zebra::index:: */
1210  if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1211  {
1212  int ret = special_index_fetch(
1213  fi, elemsetname + 5,
1214  input_format, output_format,
1215  result, addinfo, rec);
1216  rec_free(&rec);
1217  return ret;
1218  }
1219 
1220  if (rec)
1221  rec_free(&rec);
1222  return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1223 }
1224 
1226  zint sysno, int score,
1227  ODR odr,
1228  const Odr_oid *input_format, Z_RecordComposition *comp,
1229  const Odr_oid **output_format,
1230  char **rec_bufp, int *rec_lenp, char **basenamep,
1231  WRBUF addinfo_w)
1232 {
1233  Record rec;
1234  char *fname, *file_type, *basename;
1235  const char *elemsetname;
1236  struct ZebraRecStream stream;
1237  RecordAttr *recordAttr;
1238  void *clientData;
1239  int return_code = 0;
1240  zint sysnos[MAX_SYSNOS_PER_RECORD];
1241  int no_sysnos = MAX_SYSNOS_PER_RECORD;
1242  ZEBRA_RES res;
1243  struct special_fetch_s fetch_info;
1244 
1245  if (!log_level_set)
1246  {
1247  log_level_mod = yaz_log_module_level("retrieve");
1248  log_level_set = 1;
1249  }
1250  res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
1251  if (res != ZEBRA_OK)
1252  return ZEBRA_FAIL;
1253 
1254  sysno = sysnos[0];
1255  *basenamep = 0;
1256  elemsetname = yaz_get_esn(comp);
1257 
1258  fetch_info.zh = zh;
1259  fetch_info.setname = setname;
1260  fetch_info.sysno = sysno;
1261  fetch_info.score = score;
1262  fetch_info.nmem = odr->mem;
1263 
1264  /* processing zebra special elementset names of form 'zebra:: */
1265  if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
1266  {
1267  WRBUF result = wrbuf_alloc();
1268  int r = zebra_special_fetch(&fetch_info, elemsetname + 7,
1269  input_format, output_format,
1270  result, addinfo_w);
1271  if (r == 0)
1272  {
1273  *rec_bufp = odr_strdup(odr, wrbuf_cstr(result));
1274  *rec_lenp = wrbuf_len(result);
1275  }
1276  wrbuf_destroy(result);
1277  return r;
1278  }
1279 
1280  /* processing all other element set names */
1281  rec = rec_get(zh->reg->records, sysno);
1282  if (!rec)
1283  {
1284  yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1285  *basenamep = 0;
1286  return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1287  }
1288 
1289 
1290  recordAttr = rec_init_attr(zh->reg->zei, rec);
1291 
1292  file_type = rec->info[recInfo_fileType];
1293  fname = rec->info[recInfo_filename];
1294  basename = rec->info[recInfo_databaseName];
1295  *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
1296  strcpy(*basenamep, basename);
1297 
1298  yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
1299  sysno, score);
1300 
1301  return_code = zebra_create_record_stream(zh, &rec, &stream);
1302 
1303  if (rec)
1304  {
1305  RecType rt;
1306  struct recRetrieveCtrl retrieveCtrl;
1307 
1308  retrieveCtrl.stream = &stream;
1309  retrieveCtrl.fname = fname;
1310  retrieveCtrl.localno = sysno;
1311  retrieveCtrl.staticrank = recordAttr->staticrank;
1312  retrieveCtrl.score = score;
1313  retrieveCtrl.recordSize = recordAttr->recordSize;
1314  retrieveCtrl.odr = odr;
1315  retrieveCtrl.input_format = retrieveCtrl.output_format = input_format;
1316  retrieveCtrl.comp = comp;
1317  retrieveCtrl.encoding = zh->record_encoding;
1318  retrieveCtrl.diagnostic = 0;
1319  retrieveCtrl.addinfo = 0;
1320  retrieveCtrl.dh = zh->reg->dh;
1321  retrieveCtrl.res = zh->res;
1322  retrieveCtrl.rec_buf = 0;
1323  retrieveCtrl.rec_len = -1;
1324  retrieveCtrl.handle = &fetch_info;
1325  retrieveCtrl.special_fetch = zebra_special_fetch;
1326 
1327  if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
1328  file_type, &clientData)))
1329  {
1330  wrbuf_printf(addinfo_w, "Could not handle record type %.40s",
1331  file_type);
1332  return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1333  }
1334  else
1335  {
1336  (*rt->retrieve)(clientData, &retrieveCtrl);
1337  return_code = retrieveCtrl.diagnostic;
1338 
1339  *output_format = retrieveCtrl.output_format;
1340  *rec_bufp = (char *) retrieveCtrl.rec_buf;
1341  *rec_lenp = retrieveCtrl.rec_len;
1342  if (retrieveCtrl.addinfo)
1343  wrbuf_puts(addinfo_w, retrieveCtrl.addinfo);
1344  }
1345 
1346  stream.destroy(&stream);
1347  rec_free(&rec);
1348  }
1349 
1350  return return_code;
1351 }
1352 
1353 /*
1354  * Local variables:
1355  * c-basic-offset: 4
1356  * c-file-style: "Stroustrup"
1357  * indent-tabs-mode: nil
1358  * End:
1359  * vim: shiftwidth=4 tabstop=8 expandtab
1360  */
1361 
char * addinfo
Definition: recctrl.h:138
char * record_encoding
Definition: index.h:213
void zebra_snippets_ring(zebra_snippets *doc, const zebra_snippets *hit, int before, int after)
Definition: snippet.c:237
Record rec_get(Records p, zint sysno)
gets record - with given system number
Definition: records.c:927
int len
Definition: it_key.h:31
ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh, const char *setname, zint recid, zint *sysnos, int *no_sysnos)
Definition: zsets.c:1481
static void retrieve_puts_str(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:398
static int zebra_special_fetch(void *handle, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:1071
off_t recordOffset
Definition: zinfo.h:107
#define ZEBRA_OK
Definition: util.h:82
RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
Definition: zsets.c:1075
struct rset_key_control * zebra_key_control_create(ZebraHandle zh)
Definition: kcontrol.c:57
zint ISAM_P
Definition: isamc.h:28
int zebra_term_untrans(ZebraHandle zh, const char *index_type, char *dst, const char *src)
Definition: untrans.c:31
int recordSize
Definition: zinfo.h:106
char * path_reg
Definition: index.h:182
struct rset rset
static void retrieve_puts_attr(WRBUF wrbuf, const char *name, const char *value)
Definition: retrieve.c:381
ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, zint sysno, zebra_snippets *snippets)
Definition: zsets.c:1340
struct zebra_snippet_word * next
Definition: snippet.h:34
void * zebra_strmap_lookup(zebra_strmap_t st, const char *name, int no, size_t *data_len)
Definition: strmap.c:99
void zebra_create_stream_mem(struct ZebraRecStream *stream, const char *buf, size_t sz)
Definition: stream.c:123
void zebra_snippets_log(const zebra_snippets *l, int log_level, int all)
Definition: snippet.c:104
static int sort_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:158
static int special_index_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo, Record rec)
Definition: retrieve.c:248
RSET rset_dup(RSET rs)
Duplicate an RSET.
Definition: rset.c:255
zint set_occur
Definition: retrieve.c:585
#define MAX_SYSNOS_PER_RECORD
Definition: retrieve.c:42
int zebra_sort_read(zebra_sort_index_t si, zint *section_id, WRBUF w)
reads sort entry
Definition: sortidx.c:469
char * encoding
Definition: recctrl.h:125
void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, zint approx_limit)
Definition: zsets.c:1498
struct zebra_register * reg
Definition: index.h:174
static void snippet_check_fields(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc, const zebra_snippet_word *doc_w, const char *w_index_type)
Definition: retrieve.c:412
ZEBRA_RES res_get_int(Res r, const char *name, int *val)
Definition: res.c:432
#define ZEBRA_FAIL
Definition: util.h:81
static int facet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:971
data1_handle dh
Definition: index.h:142
#define RSETF_READ
Definition: rset.h:199
static struct term_collect * term_collect_create(zebra_strmap_t sm, int no_terms_collect, NMEM nmem)
Definition: retrieve.c:683
const zebra_snippet_word * zebra_snippets_constlist(const zebra_snippets *l)
Definition: snippet.c:99
#define rset_open(rs, wflag)
Definition: rset.h:202
static void retrieve_puts_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:405
Records records
Definition: index.h:138
int zebra_rec_keys_rewind(zebra_rec_keys_t keys)
Definition: reckeys.c:240
const char * index_name
Definition: retrieve.c:93
void rec_free(Record *recpp)
frees record (from memory)
Definition: records.c:1043
void zebra_snippets_destroy(zebra_snippets *l)
Definition: snippet.c:45
zint sysno
Definition: api.h:483
static int perform_facet(ZebraHandle zh, struct special_fetch_s *fi, WRBUF result, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list, int no_ord, int *ord_array, int use_xml, zinfo_index_category_t cat)
Definition: retrieve.c:841
void extract_snippet(ZebraHandle zh, zebra_snippets *sn, struct ZebraRecStream *stream, RecType rt, void *recTypeClientData)
Definition: extract.c:334
int(* retrieve)(void *clientData, struct recRetrieveCtrl *ctrl)
Definition: recctrl.h:159
const char * zebra_strmap_it_next(zebra_strmap_it it, void **data_buf, size_t *data_len)
Definition: strmap.c:162
#define IT_MAX_WORD
Definition: it_key.h:27
void error(const char *format,...)
Definition: agrep.c:51
#define ZEBRA_XML_HEADER_STR
Definition: retrieve.c:44
void rset_delete(RSET rs)
Destructor RSETs.
Definition: rset.c:218
static zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set, zint *first_sysno, zint *first_seq)
Definition: retrieve.c:590
ZebraMetaRecord * zebra_meta_records_create(ZebraHandle zh, const char *name, int num, zint *positions)
Definition: zsets.c:418
int zebraExplain_lookup_attr_str(ZebraExplainInfo zei, zinfo_index_category_t cat, const char *index_type, const char *str)
lookup ordinal from string index + index type
Definition: zinfo.c:1354
zint staticrank
Definition: zinfo.h:109
#define rset_read(rfd, buf, term)
Definition: rset.h:217
size_t size[REC_NO_INFO]
Definition: recindex.h:35
static int perform_facet_index(ZebraHandle zh, struct special_fetch_s *fi, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset, struct index_spec *spec_list)
Definition: retrieve.c:770
void zebra_strmap_it_destroy(zebra_strmap_it it)
Definition: strmap.c:157
ZebraExplainInfo zei
Definition: index.h:139
void zebra_rec_keys_close(zebra_rec_keys_t p)
Definition: reckeys.c:143
static int log_level_set
Definition: retrieve.c:55
zint first_sysno
Definition: retrieve.c:586
const Odr_oid * output_format
Definition: recctrl.h:134
#define CAST_ZINT_TO_INT(x)
Definition: util.h:96
RSET rset_create_and(NMEM nmem, struct rset_key_control *kcontrol, int scope, int no_rsets, RSET *rsets)
Definition: rsmultiandor.c:280
static int log_level_mod
Definition: retrieve.c:54
zebra_strmap_it zebra_strmap_it_create(zebra_strmap_t st)
Definition: strmap.c:148
zebra_rec_keys_t zebra_rec_keys_open(void)
Definition: reckeys.c:88
char * info[REC_NO_INFO]
Definition: recindex.h:34
int zebra_sort_type(zebra_sort_index_t si, int type)
sets type for sort usage
Definition: sortidx.c:234
#define FIRST_IN_FIELD_CHAR
Definition: index.h:419
int zebra_term_untrans_iconv(ZebraHandle zh, NMEM stream, const char *index_type, char **dst, const char *src)
Definition: untrans.c:96
int(* readf)(struct ZebraRecStream *s, char *buf, size_t count)
read function
Definition: recctrl.h:75
zint first_seqno
Definition: retrieve.c:587
RSET zebra_create_rset_isam(ZebraHandle zh, NMEM rset_nmem, struct rset_key_control *kctl, int scope, ISAM_P pos, TERMID termid)
Definition: rset_isam.c:32
struct index_spec * parse_index_spec(const char *elem, NMEM nmem, int *error)
Definition: retrieve.c:100
const Odr_oid * input_format
Definition: recctrl.h:123
ZebraHandle zh
Definition: retrieve.c:47
zebra_sort_index_t sort_index
Definition: index.h:134
TERMID rset_term_create(const char *name, int length, const char *flags, int type, NMEM nmem, struct ord_list *ol, int reg_type, zint hits_limit, const char *ref_id)
Creates a TERMID entry.
Definition: rset.c:340
void(* dec)(struct rset_key_control *kc)
Definition: rset.h:138
char * dict_lookup(Dict dict, const char *p)
lookup item in dictionary
Definition: lookup.c:100
void zebra_sort_sysno(zebra_sort_index_t si, zint sysno)
sets sort system number for read / add / delete
Definition: sortidx.c:339
zint mem[IT_KEY_LEVEL_MAX]
Definition: it_key.h:32
struct index_spec * next
Definition: retrieve.c:96
void zebra_term_untrans_iconv2(ZebraHandle zh, NMEM stream, char **dst, const char *src)
Definition: untrans.c:64
static int perform_facet_sort(ZebraHandle zh, int no_ord, int *ord_array, zebra_strmap_t *map_array, int num_recs, ZebraMetaRecord *poset)
Definition: retrieve.c:726
const char * extra
Definition: retrieve.c:95
const char * setname
Definition: retrieve.c:48
Z_RecordComposition * comp
Definition: recctrl.h:124
zint staticrank
Definition: recctrl.h:128
void zebra_rec_keys_set_buf(zebra_rec_keys_t p, char *buf, size_t sz, int copy_buf)
Definition: reckeys.c:109
data1_handle dh
Definition: recctrl.h:131
Dict dict
Definition: index.h:132
RecTypes recTypes
Definition: index.h:145
zinfo_index_category_t
Definition: zinfo.h:37
int zebraExplain_lookup_ord(ZebraExplainInfo zei, int ord, const char **index_type, const char **db, const char **string_index)
Definition: zinfo.c:1479
struct ZebraRecStream * stream
Definition: recctrl.h:119
int key_SU_encode(int ch, char *out)
Definition: su_codec.c:31
zebra_strmap_t zebra_strmap_create(void)
Definition: strmap.c:45
void zebra_strmap_add(zebra_strmap_t st, const char *name, void *data_buf, size_t data_len)
Definition: strmap.c:80
int zebra_get_rec_snippets(ZebraHandle zh, zint sysno, zebra_snippets *snippets)
Definition: retrieve.c:493
static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
Definition: retrieve.c:447
void * handle
Definition: recctrl.h:141
RecType recType_byName(RecTypes rts, Res res, const char *name, void **clientDataP)
Definition: recctrl.c:264
static int term_qsort_handle(const void *a, const void *b)
Definition: retrieve.c:645
long zint
Zebra integer.
Definition: util.h:66
const char * term
Definition: retrieve.c:583
void(* destroy)(struct ZebraRecStream *s)
close and destroy stream
Definition: recctrl.h:83
int zebra_rec_keys_read(zebra_rec_keys_t keys, const char **str, size_t *slen, struct it_key *key)
Definition: reckeys.c:259
void * rec_buf
Definition: recctrl.h:135
Definition: rset.h:73
Definition: rset.h:150
int zebra_record_fetch(ZebraHandle zh, const char *setname, zint sysno, int score, ODR odr, const Odr_oid *input_format, Z_RecordComposition *comp, const Odr_oid **output_format, char **rec_bufp, int *rec_lenp, char **basenamep, WRBUF addinfo_w)
Definition: retrieve.c:1225
zebra_snippets * zebra_snippets_create(void)
Definition: snippet.c:36
int fd
Definition: tstlockscope.c:38
#define O_BINARY
Definition: agrep.c:46
char * fname
Definition: recctrl.h:130
RecordAttr * rec_init_attr(ZebraExplainInfo zei, Record rec)
Definition: zinfo.c:1595
Definition: it_key.h:30
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
void rset_close(RSFD rfd)
Closes a result set RFD handle.
Definition: rset.c:98
Definition: rset.h:50
const char * index_type
Definition: retrieve.c:94
void zebra_strmap_destroy(zebra_strmap_t st)
Definition: strmap.c:61
static int snippet_fetch(struct special_fetch_s *fi, const char *elemsetname, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: retrieve.c:536
int zebra_rec_keys_empty(zebra_rec_keys_t keys)
Definition: reckeys.c:252
zint approx_limit
Definition: index.h:180
void zebra_create_stream_fd(struct ZebraRecStream *stream, int fd, off_t start_offset)
Definition: stream.c:140
static void term_collect_freq(ZebraHandle zh, struct term_collect *col, int no_terms_collect, int ord, RSET rset, double scale_factor)
Definition: retrieve.c:661
static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name, const int value)
Definition: retrieve.c:392
record reader stream
Definition: recctrl.h:71
int(* special_fetch)(void *handle, const char *esn, const Odr_oid *input_format, const Odr_oid **output_format, WRBUF result, WRBUF addinfo)
Definition: recctrl.h:142
static int zebra_create_record_stream(ZebraHandle zh, Record *rec, struct ZebraRecStream *stream)
Definition: retrieve.c:57
void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, int num)
Definition: zsets.c:547
#define ZINT_FORMAT
Definition: util.h:72