pazpar2  1.14.1
session.c
Go to the documentation of this file.
1 /* This file is part of Pazpar2.
2  Copyright (C) Index Data
3 
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
24 #if HAVE_CONFIG_H
25 #include <config.h>
26 #endif
27 
28 #include <time.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <string.h>
32 #if HAVE_SYS_TIME_H
33 #include <sys/time.h>
34 #endif
35 #if HAVE_UNISTD_H
36 #include <unistd.h>
37 #endif
38 #ifdef WIN32
39 #include <windows.h>
40 #endif
41 #include <signal.h>
42 #include <ctype.h>
43 #include <assert.h>
44 #include <math.h>
45 
46 #include <yaz/marcdisp.h>
47 #include <yaz/comstack.h>
48 #include <yaz/tcpip.h>
49 #include <yaz/proto.h>
50 #include <yaz/readconf.h>
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/yaz-util.h>
54 #include <yaz/nmem.h>
55 #include <yaz/query-charset.h>
56 #include <yaz/querytowrbuf.h>
57 #include <yaz/oid_db.h>
58 #include <yaz/snprintf.h>
59 #include <yaz/xml_get.h>
60 
61 #define USE_TIMING 0
62 #if USE_TIMING
63 #include <yaz/timing.h>
64 #endif
65 
66 #include "ppmutex.h"
67 #include "parameters.h"
68 #include "session.h"
69 #include "eventl.h"
70 #include "http.h"
71 #include "termlists.h"
72 #include "reclists.h"
73 #include "relevance.h"
74 #include "database.h"
75 #include "client.h"
76 #include "settings.h"
77 #include "normalize7bit.h"
78 
79 #include <libxml/tree.h>
80 
81 #define MAX_CHUNK 15
82 
83 #define MAX(a,b) ((a)>(b)?(a):(b))
84 
85 // Note: Some things in this structure will eventually move to configuration
87 {
88  0, // dump_records
89  0, // debug_mode
90  0, // predictable sessions
91 };
92 
93 struct client_list {
94  struct client *client;
95  struct client_list *next;
96 };
97 
98 /* session counting (1) , disable client counting (0) */
99 static YAZ_MUTEX g_session_mutex = 0;
100 static int no_sessions = 0;
101 
102 static int session_use(int delta)
103 {
104  int sessions;
105  if (!g_session_mutex)
106  yaz_mutex_create(&g_session_mutex);
107  yaz_mutex_enter(g_session_mutex);
108  no_sessions += delta;
109  sessions = no_sessions;
110  yaz_mutex_leave(g_session_mutex);
111  yaz_log(YLOG_DEBUG, "%s sessions=%d", delta == 0 ? "" :
112  (delta > 0 ? "INC" : "DEC"), no_sessions);
113  return sessions;
114 }
115 
117 {
118  return session_use(0);
119 }
120 
121 static void log_xml_doc(xmlDoc *doc)
122 {
123  FILE *lf = yaz_log_file();
124  xmlChar *result = 0;
125  int len = 0;
126 #if LIBXML_VERSION >= 20600
127  xmlDocDumpFormatMemory(doc, &result, &len, 1);
128 #else
129  xmlDocDumpMemory(doc, &result, &len);
130 #endif
131  if (lf && len)
132  {
133  (void) fwrite(result, 1, len, lf);
134  fprintf(lf, "\n");
135  }
136  xmlFree(result);
137 }
138 
139 static void session_enter(struct session *s, const char *caller)
140 {
141  if (caller)
142  session_log(s, YLOG_DEBUG, "Session lock by %s", caller);
143  yaz_mutex_enter(s->session_mutex);
144 }
145 
146 static void session_leave(struct session *s, const char *caller)
147 {
148  yaz_mutex_leave(s->session_mutex);
149  if (caller)
150  session_log(s, YLOG_DEBUG, "Session unlock by %s", caller);
151 }
152 
153 static int run_icu(struct session *s, const char *icu_chain_id,
154  const char *value,
155  WRBUF norm_wr, WRBUF disp_wr)
156 {
157  const char *facet_component;
158  struct conf_service *service = s->service;
159  pp2_charset_token_t prt =
160  pp2_charset_token_create(service->charsets, icu_chain_id);
161  if (!prt)
162  {
163  session_log(s, YLOG_FATAL,
164  "Unknown ICU chain '%s'", icu_chain_id);
165  return 0;
166  }
167  pp2_charset_token_first(prt, value, 0);
168  while ((facet_component = pp2_charset_token_next(prt)))
169  {
170  const char *display_component;
171  if (*facet_component)
172  {
173  if (wrbuf_len(norm_wr))
174  wrbuf_puts(norm_wr, " ");
175  wrbuf_puts(norm_wr, facet_component);
176  }
177  display_component = pp2_get_display(prt);
178  if (display_component)
179  {
180  if (wrbuf_len(disp_wr))
181  wrbuf_puts(disp_wr, " ");
182  wrbuf_puts(disp_wr, display_component);
183  }
184  }
186  return 1;
187 }
188 
189 static void session_normalize_facet(struct session *s,
190  const char *type, const char *value,
191  WRBUF display_wrbuf, WRBUF facet_wrbuf)
192 {
193  struct conf_service *service = s->service;
194  int i;
195  const char *icu_chain_id = 0;
196 
197  for (i = 0; i < service->num_metadata; i++)
198  if (!strcmp((service->metadata + i)->name, type))
199  icu_chain_id = (service->metadata + i)->facetrule;
200  if (!icu_chain_id)
201  icu_chain_id = "facet";
202 
203  run_icu(s, icu_chain_id, value, facet_wrbuf, display_wrbuf);
204 }
205 
206 struct facet_id {
207  char *client_id;
208  char *type;
209  char *id;
210  char *term;
211  struct facet_id *next;
212 };
213 
214 static void session_add_id_facet(struct session *s, struct client *cl,
215  const char *type,
216  const char *id,
217  size_t id_len,
218  const char *term)
219 {
220  struct facet_id *t = nmem_malloc(s->session_nmem, sizeof(*t));
221 
222  t->client_id = nmem_strdup(s->session_nmem, client_get_id(cl));
223  t->type = nmem_strdup(s->session_nmem, type);
224  t->id = nmem_strdupn(s->session_nmem, id, id_len);
225  t->term = nmem_strdup(s->session_nmem, term);
226  t->next = s->facet_id_list;
227  s->facet_id_list = t;
228 }
229 
230 
231 // Look up a facet term, and return matching id
232 // If facet type not found, returns 0
233 // If facet type found, but no matching term, returns ""
234 const char *session_lookup_id_facet(struct session *s, struct client *cl,
235  const char *type,
236  const char *term)
237 {
238  char *retval = 0;
239  struct facet_id *t = s->facet_id_list;
240  for (; t; t = t->next)
241  {
242  if (!strcmp(client_get_id(cl), t->client_id) && !strcmp(t->type, type) )
243  {
244  retval = "";
245  if ( !strcmp(t->term, term))
246  {
247  return t->id;
248  }
249  }
250  }
251  return retval;
252 }
253 
254 void add_facet(struct session *s, const char *type, const char *value, int count, struct client *cl)
255 {
256  WRBUF facet_wrbuf = wrbuf_alloc();
257  WRBUF display_wrbuf = wrbuf_alloc();
258  const char *id = 0;
259  size_t id_len = 0;
260 
261  /* inspect pz:facetmap:split:name ?? */
262  if (!strncmp(type, "split:", 6))
263  {
264  const char *cp = strchr(value, ':');
265  if (cp)
266  {
267  id = value;
268  id_len = cp - value;
269  value = cp + 1;
270  }
271  type += 6;
272  }
273 
274  session_normalize_facet(s, type, value, display_wrbuf, facet_wrbuf);
275  if (wrbuf_len(facet_wrbuf))
276  {
277  struct named_termlist **tp = &s->termlists;
278  for (; (*tp); tp = &(*tp)->next)
279  if (!strcmp((*tp)->name, type))
280  break;
281  if (!*tp)
282  {
283  *tp = nmem_malloc(s->nmem, sizeof(**tp));
284  (*tp)->name = nmem_strdup(s->nmem, type);
285  (*tp)->termlist = termlist_create(s->nmem);
286  (*tp)->next = 0;
287  }
288  termlist_insert((*tp)->termlist, wrbuf_cstr(display_wrbuf),
289  wrbuf_cstr(facet_wrbuf), id, id_len, count);
290  if (id)
291  session_add_id_facet(s, cl, type, id, id_len,
292  wrbuf_cstr(display_wrbuf));
293  }
294  wrbuf_destroy(facet_wrbuf);
295  wrbuf_destroy(display_wrbuf);
296 }
297 
298 static xmlDoc *record_to_xml(struct session *se,
299  struct session_database *sdb, const char *rec)
300 {
301  struct database *db = sdb->database;
302  xmlDoc *rdoc = 0;
303 
304  rdoc = xmlParseMemory(rec, strlen(rec));
305 
306  if (!rdoc)
307  {
308  session_log(se, YLOG_WARN, "Non-wellformed XML");
309  return 0;
310  }
311 
313  {
314  session_log(se, YLOG_LOG, "Un-normalized record from %s", db->id);
315  log_xml_doc(rdoc);
316  }
317 
318  return rdoc;
319 }
320 
321 #define MAX_XSLT_ARGS 16
322 
323 // Add static values from session database settings if applicable
325  struct conf_service *service,
326  char **parms,
327  NMEM nmem)
328 {
329  int i;
330  int nparms = 0;
331  int offset = 0;
332 
333  for (i = 0; i < service->num_metadata; i++)
334  {
335  struct conf_metadata *md = &service->metadata[i];
336  int setting;
337 
338  if (md->setting == Metadata_setting_parameter &&
339  (setting = settings_lookup_offset(service, md->name)) >= 0)
340  {
341  const char *val = session_setting_oneval(sdb, setting);
342  if (val && nparms < MAX_XSLT_ARGS)
343  {
344  char *buf;
345  int len = strlen(val);
346  buf = nmem_malloc(nmem, len + 3);
347  buf[0] = '\'';
348  strcpy(buf + 1, val);
349  buf[len+1] = '\'';
350  buf[len+2] = '\0';
351  parms[offset++] = md->name;
352  parms[offset++] = buf;
353  nparms++;
354  }
355  }
356  }
357  parms[offset] = 0;
358 }
359 
360 // Add static values from session database settings if applicable
361 static void insert_settings_values(struct session_database *sdb, xmlDoc *doc,
362  xmlNode *root,
363  struct conf_service *service)
364 {
365  int i;
366 
367  for (i = 0; i < service->num_metadata; i++)
368  {
369  struct conf_metadata *md = &service->metadata[i];
370  int offset;
371 
372  if (md->setting == Metadata_setting_postproc &&
373  (offset = settings_lookup_offset(service, md->name)) >= 0)
374  {
375  const char *val = session_setting_oneval(sdb, offset);
376  if (val)
377  {
378  xmlNode *n = xmlNewTextChild(root, 0, (xmlChar *) "metadata",
379  (xmlChar *) val);
380  xmlSetProp(n, (xmlChar *) "type", (xmlChar *) md->name);
381  }
382  }
383  }
384 }
385 
386 static xmlDoc *normalize_record(struct session *se,
387  struct session_database *sdb,
388  struct conf_service *service,
389  const char *rec, NMEM nmem)
390 {
391  xmlDoc *rdoc = record_to_xml(se, sdb, rec);
392 
393  if (rdoc)
394  {
395  char *parms[MAX_XSLT_ARGS*2+1];
396 
397  insert_settings_parameters(sdb, service, parms, nmem);
398 
399  if (normalize_record_transform(sdb->map, &rdoc, (const char **)parms))
400  {
401  session_log(se, YLOG_WARN, "Normalize failed");
402  }
403  }
404  return rdoc;
405 }
406 
408  struct session_database *db,
409  WRBUF w)
410 {
411  if (db->settings)
412  {
413  int i, num = db->num_settings;
414  for (i = 0; i < num; i++)
415  {
416  struct setting *s = db->settings[i];
417  for (;s ; s = s->next)
418  {
419  wrbuf_puts(w, "<set name=\"");
420  wrbuf_xmlputs(w, s->name);
421  wrbuf_puts(w, "\" value=\"");
422  wrbuf_xmlputs(w, s->value);
423  wrbuf_puts(w, "\"/>");
424  }
425  if (db->settings[i])
426  wrbuf_puts(w, "\n");
427  }
428  }
429 }
430 
431 // Retrieve first defined value for 'name' for given database.
432 // Will be extended to take into account user associated with session
433 const char *session_setting_oneval(struct session_database *db, int offset)
434 {
435  if (offset >= db->num_settings || !db->settings[offset])
436  return "";
437  return db->settings[offset]->value;
438 }
439 
440 // Prepare XSLT stylesheets for record normalization
441 // Structures are allocated on the session_wide nmem to avoid having
442 // to recompute this for every search. This would lead
443 // to leaking if a single session was to repeatedly change the PZ_XSLT
444 // setting. However, this is not a realistic use scenario.
445 static int prepare_map(struct session *se, struct session_database *sdb)
446 {
447  if (sdb->settings && !sdb->map)
448  {
449  const char *s;
450 
451  if (sdb->settings[PZ_XSLT] &&
452  (s = session_setting_oneval(sdb, PZ_XSLT)))
453  {
454  char auto_stylesheet[256];
455 
456  if (!strcmp(s, "auto"))
457  {
458  const char *request_syntax = session_setting_oneval(
459  sdb, PZ_REQUESTSYNTAX);
460  if (request_syntax)
461  {
462  char *cp;
463  yaz_snprintf(auto_stylesheet, sizeof(auto_stylesheet),
464  "%s.xsl", request_syntax);
465  for (cp = auto_stylesheet; *cp; cp++)
466  {
467  /* deliberately only consider ASCII */
468  if (*cp > 32 && *cp < 127)
469  *cp = tolower(*cp);
470  }
471  s = auto_stylesheet;
472  }
473  else
474  {
475  session_log(se, YLOG_WARN,
476  "No pz:requestsyntax for auto stylesheet");
477  }
478  }
480  se->service, s);
481  if (!sdb->map)
482  return -1;
483  }
484  }
485  return 0;
486 }
487 
488 // called if watch should be removed because http_channel is to be destroyed
489 static void session_watch_cancel(void *data, struct http_channel *c,
490  void *data2)
491 {
492  struct session_watchentry *ent = data;
493 
494  ent->fun = 0;
495  ent->data = 0;
496  ent->obs = 0;
497 }
498 
499 // set watch. Returns 0=OK, -1 if watch is already set
500 int session_set_watch(struct session *s, int what,
501  session_watchfun fun, void *data,
502  struct http_channel *chan)
503 {
504  int ret;
505  session_enter(s, "session_set_watch");
506  if (s->watchlist[what].fun)
507  ret = -1;
508  else
509  {
510 
511  s->watchlist[what].fun = fun;
512  s->watchlist[what].data = data;
513  s->watchlist[what].obs = http_add_observer(chan, &s->watchlist[what],
515  ret = 0;
516  }
517  session_leave(s, "session_set_watch");
518  return ret;
519 }
520 
521 void session_alert_watch(struct session *s, int what)
522 {
523  assert(s);
524  session_enter(s, "session_alert_watch");
525  if (s->watchlist[what].fun)
526  {
527  /* our watch is no longer associated with http_channel */
528  void *data;
530 
532  fun = s->watchlist[what].fun;
533  data = s->watchlist[what].data;
534 
535  /* reset watch before fun is invoked - in case fun wants to set
536  it again */
537  s->watchlist[what].fun = 0;
538  s->watchlist[what].data = 0;
539  s->watchlist[what].obs = 0;
540 
541  session_leave(s, "session_alert_watch");
542  session_log(s, YLOG_DEBUG,
543  "Alert Watch: %d calling function: %p", what, fun);
544  fun(data);
545  }
546  else
547  session_leave(s,"session_alert_watch");
548 }
549 
550 //callback for grep_databases
551 static void select_targets_callback(struct session *se,
552  struct session_database *db)
553 {
554  struct client *cl;
555  struct client_list *l;
556 
557  for (l = se->clients_cached; l; l = l->next)
558  if (client_get_database(l->client) == db)
559  break;
560 
561  if (l)
562  cl = l->client;
563  else
564  {
565  cl = client_create(db->database->id);
566  client_set_database(cl, db);
567 
568  l = xmalloc(sizeof(*l));
569  l->client = cl;
570  l->next = se->clients_cached;
571  se->clients_cached = l;
572  }
573  client_set_session(cl, se);
574 
575  l = xmalloc(sizeof(*l));
576  l->client = cl;
577  l->next = se->clients_active;
578  se->clients_active = l;
579 }
580 
581 static void session_reset_active_clients(struct session *se,
582  struct client_list *new_list)
583 {
584  struct client_list *l;
585 
586  session_enter(se, "session_reset_active_clients");
587  l = se->clients_active;
588  se->clients_active = new_list;
589  session_leave(se, "session_reset_active_clients");
590 
591  while (l)
592  {
593  struct client_list *l_next = l->next;
594 
595  client_lock(l->client);
596  client_set_session(l->client, 0); /* mark client inactive */
597  client_unlock(l->client);
598 
599  xfree(l);
600  l = l_next;
601  }
602 }
603 
604 static void session_remove_cached_clients(struct session *se)
605 {
606  struct client_list *l;
607 
609 
610  session_enter(se, "session_remove_cached_clients");
611  l = se->clients_cached;
612  se->clients_cached = 0;
613  session_leave(se, "session_remove_cached_clients");
614 
615  while (l)
616  {
617  struct client_list *l_next = l->next;
618  client_lock(l->client);
619  client_set_session(l->client, 0);
622  client_unlock(l->client);
624  xfree(l);
625  l = l_next;
626  }
627 }
628 
629 // Associates a set of clients with a session;
630 // Note: Session-databases represent databases with per-session
631 // setting overrides
632 static int select_targets(struct session *se, const char *filter)
633 {
635 }
636 
638 {
639  struct client_list *l;
640  int res = 0;
641 
642  for (l = s->clients_active; l; l = l->next)
643  if (client_is_active(l->client))
644  res++;
645 
646  return res;
647 }
648 
650 {
651  struct client_list *l;
652  int res = 0;
653 
654  for (l = s->clients_active; l; l = l->next)
656  res++;
657  session_log(s, YLOG_DEBUG, "Has %d active preferred clients.", res);
658  return res == 0;
659 }
660 
661 static void session_clear_set(struct session *se, struct reclist_sortparms *sp)
662 {
664  if (nmem_total(se->nmem))
665  session_log(se, YLOG_DEBUG, "NMEN operation usage %zd",
666  nmem_total(se->nmem));
667  nmem_reset(se->nmem);
668  se->total_records = se->total_merged = 0;
669  se->termlists = 0;
671 
672  /* reset list of sorted results and clear to relevance search */
673  se->sorted_results = nmem_malloc(se->nmem, sizeof(*se->sorted_results));
674  se->sorted_results->name = nmem_strdup(se->nmem, sp->name);
676  se->sorted_results->type = sp->type;
677  se->sorted_results->next = 0;
678 
679  session_log(se, YLOG_DEBUG, "clear_set session_sort: field=%s increasing=%d type=%d configured",
680  sp->name, sp->increasing, sp->type);
681 
682  se->reclist = reclist_create(se->nmem);
683 }
684 
685 void session_sort(struct session *se, struct reclist_sortparms *sp,
686  const char *mergekey, const char *rank)
687 {
688  struct client_list *l;
689  const char *field = sp->name;
690  int increasing = sp->increasing;
691  int type = sp->type;
692  int clients_research = 0;
693 
694  session_enter(se, "session_sort");
695  session_log(se, YLOG_DEBUG, "session_sort field=%s increasing=%d type=%d",
696  field, increasing, type);
697 
698  if (rank && (!se->rank || strcmp(se->rank, rank)))
699  {
700  /* new rank must research/reingest anyway */
701  assert(rank);
702  xfree(se->rank);
703  se->rank = *rank ? xstrdup(rank) : 0;
704  clients_research = 1;
705  session_log(se, YLOG_DEBUG, "session_sort: new rank = %s",
706  rank);
707  }
708  if (mergekey && (!se->mergekey || strcmp(se->mergekey, mergekey)))
709  {
710  /* new mergekey must research/reingest anyway */
711  assert(mergekey);
712  xfree(se->mergekey);
713  se->mergekey = *mergekey ? xstrdup(mergekey) : 0;
714  clients_research = 1;
715  session_log(se, YLOG_DEBUG, "session_sort: new mergekey = %s",
716  mergekey);
717  }
718  if (clients_research == 0)
719  {
720  struct reclist_sortparms *sr;
721  for (sr = se->sorted_results; sr; sr = sr->next)
722  if (!reclist_sortparms_cmp(sr, sp))
723  break;
724  if (sr)
725  {
726  session_log(se, YLOG_LOG, "session_sort: field=%s increasing=%d type=%d already fetched",
727  field, increasing, type);
728  session_leave(se, "session_sort");
729  return;
730  }
731  }
732  session_log(se, YLOG_DEBUG, "session_sort: field=%s increasing=%d type=%d must fetch",
733  field, increasing, type);
734 
735  // We need to reset reclist on every sort that changes the records, not just for position
736  // So if just one client requires new searching, we need to clear set.
737  // Ask each of the client if sorting requires re-search due to native sort
738  // If it does it will require us to
739  for (l = se->clients_active; l; l = l->next)
740  {
741  struct client *cl = l->client;
742  // Assume no re-search is required.
743  client_parse_init(cl, 1);
744  clients_research += client_parse_sort(cl, sp, 0);
745  }
746  if (!clients_research || se->clients_starting)
747  {
748  // A new sorting based on same record set
749  struct reclist_sortparms *sr = nmem_malloc(se->nmem, sizeof(*sr));
750  sr->name = nmem_strdup(se->nmem, field);
751  sr->increasing = increasing;
752  sr->type = type;
753  sr->next = se->sorted_results;
754  se->sorted_results = sr;
755  session_log(se, YLOG_DEBUG, "session_sort: no research/ingesting done");
756  session_leave(se, "session_sort");
757  }
758  else
759  {
760  se->clients_starting = 1;
761  session_log(se, YLOG_DEBUG,
762  "session_sort: reset results due to %d clients researching",
763  clients_research);
764  session_clear_set(se, sp);
765  session_log(se, YLOG_DEBUG, "Re- search/ingesting for clients due to change in sort order");
766 
767  session_leave(se, "session_sort");
768  for (l = se->clients_active; l; l = l->next)
769  {
770  struct client *cl = l->client;
771  if (client_get_state(cl) == Client_Connecting ||
772  client_get_state(cl) == Client_Idle ||
775  }
776  else
777  {
778  session_log(se, YLOG_DEBUG,
779  "session_sort: %s: No re-start/ingest in show. "
780  "Wrong client state: %d",
782  }
783  }
784  session_enter(se, "session_sort");
785  se->clients_starting = 0;
786  se->force_position = 0;
787  session_leave(se, "session_sort");
788  }
789 }
790 
791 void session_stop(struct session *se)
792 {
793  struct client_list *l;
794  session_enter(se, "session_stop1");
795  if (se->clients_starting)
796  {
797  session_leave(se, "session_stop1");
798  return;
799  }
800  se->clients_starting = 1;
801  session_leave(se, "session_stop1");
802 
807 
808  for (l = se->clients_active; l; l = l->next)
809  {
810  struct client *cl = l->client;
811  client_stop(cl);
812  }
813  session_enter(se, "session_stop2");
814  se->clients_starting = 0;
815  session_leave(se, "session_stop2");
816 }
817 
819  const char *query,
820  const char *startrecs,
821  const char *maxrecs,
822  const char *filter,
823  const char *limit,
824  const char **addinfo,
825  const char **addinfo2,
826  struct reclist_sortparms *sp,
827  const char *mergekey,
828  const char *rank)
829 {
830  int live_channels = 0;
831  int no_working = 0;
832  int no_failed_query = 0;
833  int no_failed_limit = 0;
834  int no_sortmap = 0;
835  struct client_list *l;
836 
837  session_log(se, YLOG_LOG, "search query %s", query);
838  if (filter)
839  session_log(se, YLOG_LOG, "search filter %s", filter);
840  if (limit)
841  session_log(se, YLOG_LOG, "search limit %s", limit);
842 
843  *addinfo = 0;
844 
845  session_enter(se, "session_search0");
846  if (se->clients_starting)
847  {
848  session_leave(se, "session_search0");
849  return PAZPAR2_NO_ERROR;
850  }
851  se->clients_starting = 1;
852  se->force_position = 0;
853  session_leave(se, "session_search0");
854 
855  if (se->settings_modified) {
857  }
858  else
860 
861  session_enter(se, "session_search");
862  se->settings_modified = 0;
863 
864  if (mergekey)
865  {
866  xfree(se->mergekey);
867  se->mergekey = *mergekey ? xstrdup(mergekey) : 0;
868  }
869  if (rank)
870  {
871  xfree(se->rank);
872  se->rank = *rank ? xstrdup(rank) : 0;
873  }
874 
875  session_clear_set(se, sp);
877 
878  live_channels = select_targets(se, filter);
879  if (!live_channels)
880  {
881  session_leave(se, "session_search");
882  se->clients_starting = 0;
883  return PAZPAR2_NO_TARGETS;
884  }
885 
887  se->facet_limits = facet_limits_create(limit);
888  if (!se->facet_limits)
889  {
890  *addinfo = "limit";
891  session_leave(se, "session_search");
892  se->clients_starting = 0;
895  }
896 
897  session_leave(se, "session_search");
898 
903 
904  for (l = se->clients_active; l; l = l->next)
905  {
906  int parse_ret;
907  struct client *cl = l->client;
908  client_parse_init(cl, 1);
909  if (prepare_map(se, client_get_database(cl)) < 0)
910  continue;
911 
912  parse_ret = client_parse_query(cl, query, se->facet_limits, addinfo2);
913  if (parse_ret == -1)
914  no_failed_query++;
915  else if (parse_ret == -2)
916  no_failed_limit++;
917  else if (parse_ret < 0)
918  no_working++; /* other error, such as bad CCL map */
919  else
920  {
922  client_parse_sort(cl, sp, &no_sortmap);
924  no_working++;
925  }
926  }
927  session_log(se, YLOG_LOG, "search "
928  "working %d sortmap %d failed-query %d failed-limit %d",
929  no_working, no_sortmap, no_failed_query, no_failed_limit);
930  session_enter(se, "session_search2");
931  if (no_working == 1 && no_sortmap == 1)
932  {
933  se->force_position = 1;
934  yaz_log(YLOG_LOG, "force_position=1");
935  }
936  se->clients_starting = 0;
937  session_leave(se, "session_search2");
938  if (no_working == 0)
939  {
940  if (no_failed_query > 0)
941  {
942  *addinfo = "query";
944  }
945  else if (no_failed_limit > 0)
946  {
947  *addinfo = "limit";
949  }
950  else
951  return PAZPAR2_NO_TARGETS;
952  }
953  return PAZPAR2_NO_ERROR;
954 }
955 
956 // Creates a new session_database object for a database
957 static void session_init_databases_fun(void *context, struct database *db)
958 {
959  struct session *se = (struct session *) context;
960  struct session_database *new = nmem_malloc(se->session_nmem, sizeof(*new));
961  int i;
962 
963  new->database = db;
964 
965  new->map = 0;
966  assert(db->settings);
967  new->settings = nmem_malloc(se->session_nmem,
968  sizeof(struct settings *) * db->num_settings);
969  new->num_settings = db->num_settings;
970  for (i = 0; i < db->num_settings; i++)
971  {
972  struct setting *setting = db->settings[i];
973  new->settings[i] = setting;
974  }
975  new->next = se->databases;
976  se->databases = new;
977 }
978 
979 // Doesn't free memory associated with sdb -- nmem takes care of that
981 {
982  sdb->map = 0;
983 }
984 
985 // Initialize session_database list -- this represents this session's view
986 // of the database list -- subject to modification by the settings ws command
988 {
989  se->databases = 0;
991 }
992 
993 // Probably session_init_databases_fun should be refactored instead of
994 // called here.
996  const char *id)
997 {
999  session_init_databases_fun((void*) se, db);
1000 
1001  // New sdb is head of se->databases list
1002  return se->databases;
1003 }
1004 
1005 // Find an existing session database. If not found, load it
1007  const char *id)
1008 {
1009  struct session_database *sdb;
1010 
1011  for (sdb = se->databases; sdb; sdb = sdb->next)
1012  if (!strcmp(sdb->database->id, id))
1013  return sdb;
1014  return load_session_database(se, id);
1015 }
1016 
1017 // Apply a session override to a database
1018 void session_apply_setting(struct session *se, const char *dbname,
1019  const char *name, const char *value)
1020 {
1021  session_enter(se, "session_apply_setting");
1022  {
1023  struct session_database *sdb = find_session_database(se, dbname);
1024  struct conf_service *service = se->service;
1025  struct setting *s;
1026  int offset = settings_create_offset(service, name);
1027 
1028  expand_settings_array(&sdb->settings, &sdb->num_settings, offset,
1029  se->session_nmem);
1030  // Force later recompute of settings-driven data structures
1031  // (happens when a search starts and client connections are prepared)
1032  if (offset == PZ_XSLT)
1033  sdb->map = 0;
1034  se->settings_modified = 1;
1035  for (s = sdb->settings[offset]; s; s = s->next)
1036  if (!strcmp(s->name, name) &&
1037  dbname && s->target && !strcmp(dbname, s->target))
1038  break;
1039  if (!s)
1040  {
1041  s = nmem_malloc(se->session_nmem, sizeof(*s));
1042  s->precedence = 0;
1043  s->target = nmem_strdup(se->session_nmem, dbname);
1044  s->name = nmem_strdup(se->session_nmem, name);
1045  s->next = sdb->settings[offset];
1046  sdb->settings[offset] = s;
1047  }
1048  s->value = nmem_strdup(se->session_nmem, value);
1049  }
1050  session_leave(se, "session_apply_setting");
1051 }
1052 
1053 void session_destroy(struct session *se)
1054 {
1055  struct session_database *sdb;
1056  struct facet_id *t;
1057  int sessions_total = session_use(-1);
1058  int no_facet_ids = 0;
1059 
1060  for (t = se->facet_id_list; t; t = t->next)
1061  no_facet_ids++;
1062  session_log(se, YLOG_LOG, "destroy "
1063  "session-total %d nmem-op %zd nmem-ses %zd facets-ids %d",
1064  sessions_total,
1065  nmem_total(se->nmem), nmem_total(se->session_nmem),
1066  no_facet_ids);
1068 
1069  for (sdb = se->databases; sdb; sdb = sdb->next)
1073  reclist_destroy(se->reclist);
1074  xfree(se->mergekey);
1075  xfree(se->rank);
1077  nmem_destroy(se->nmem);
1078  service_destroy(se->service);
1079  yaz_mutex_destroy(&se->session_mutex);
1080 }
1081 
1083  size_t session_nmem;
1084  if (session == 0)
1085  return 0;
1086  session_enter(session, "session_get_memory_status");
1087  session_nmem = nmem_total(session->nmem);
1088  session_leave(session, "session_get_memory_status");
1089  return session_nmem;
1090 }
1091 
1092 
1094  unsigned session_id)
1095 {
1096  int i;
1097  struct session *session = nmem_malloc(nmem, sizeof(*session));
1098 
1099  char tmp_str[50];
1100 
1101  sprintf(tmp_str, "session#%u", session_id);
1102 
1104  session_log(session, YLOG_DEBUG, "New");
1105  session->service = service;
1106  session->relevance = 0;
1107  session->total_records = 0;
1110  session->termlists = 0;
1112  session->clients_active = 0;
1113  session->clients_cached = 0;
1116  session->facet_id_list = 0;
1117  session->nmem = nmem_create();
1118  session->databases = 0;
1119  session->sorted_results = 0;
1120  session->facet_limits = 0;
1121  session->mergekey = 0;
1122  session->rank = 0;
1124  session->force_position = 0;
1125 
1126  for (i = 0; i <= SESSION_WATCH_MAX; i++)
1127  {
1128  session->watchlist[i].data = 0;
1129  session->watchlist[i].fun = 0;
1130  }
1132  session->session_mutex = 0;
1134 
1135  i = session_use(1);
1136  session_log(session, YLOG_LOG, "create session-total %d", i);
1137  return session;
1138 }
1139 
1140 static struct hitsbytarget *hitsbytarget_nb(struct session *se,
1141  int *count, NMEM nmem)
1142 {
1143  struct hitsbytarget *res = 0;
1144  struct client_list *l;
1145  size_t sz = 0;
1146 
1147  for (l = se->clients_active; l; l = l->next)
1148  sz++;
1149 
1150  res = nmem_malloc(nmem, sizeof(*res) * sz);
1151  *count = 0;
1152  for (l = se->clients_active; l; l = l->next)
1153  {
1154  struct client *cl = l->client;
1155  WRBUF w = wrbuf_alloc();
1157  PZ_NAME);
1158  res[*count].id = client_get_id(cl);
1159  res[*count].name = *name ? name : "Unknown";
1160  res[*count].hits = client_get_hits(cl);
1161  res[*count].approximation = client_get_approximation(cl);
1162  res[*count].records = client_get_num_records(cl,
1163  &res[*count].filtered,
1164  0, 0);
1165  res[*count].diagnostic =
1166  client_get_diagnostic(cl, &res[*count].message,
1167  &res[*count].addinfo);
1168  res[*count].state = client_get_state_str(cl);
1169  res[*count].connected = client_get_connection(cl) ? 1 : 0;
1171  res[*count].settings_xml = nmem_strdup(nmem, wrbuf_cstr(w));
1172  wrbuf_rewind(w);
1173  res[*count].suggestions_xml =
1174  nmem_strdup(nmem, client_get_suggestions_xml(cl, w));
1175 
1176  res[*count].query_data =
1177  client_get_query(cl, &res[*count].query_type, nmem);
1178  wrbuf_destroy(w);
1179  (*count)++;
1180  }
1181  return res;
1182 }
1183 
1184 struct hitsbytarget *get_hitsbytarget(struct session *se, int *count, NMEM nmem)
1185 {
1186  struct hitsbytarget *p;
1187  session_enter(se, "get_hitsbytarget");
1188  p = hitsbytarget_nb(se, count, nmem);
1189  session_leave(se, "get_hitsbytarget");
1190  return p;
1191 }
1192 
1193 // Compares two hitsbytarget nodes by hitcount
1194 static int cmp_ht(const void *p1, const void *p2)
1195 {
1196  const struct hitsbytarget *h1 = p1;
1197  const struct hitsbytarget *h2 = p2;
1198  return h2->hits - h1->hits;
1199 }
1200 
1201 // Compares two hitsbytarget nodes by hitcount
1202 static int cmp_ht_approx(const void *p1, const void *p2)
1203 {
1204  const struct hitsbytarget *h1 = p1;
1205  const struct hitsbytarget *h2 = p2;
1206  return h2->approximation - h1->approximation;
1207 }
1208 
1209 static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num,
1210  NMEM nmem, int version)
1211 {
1212  struct hitsbytarget *ht;
1213  int count, i;
1214 
1215  ht = hitsbytarget_nb(se, &count, nmem);
1216  if (version >= 2)
1217  qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht_approx);
1218  else
1219  qsort(ht, count, sizeof(struct hitsbytarget), cmp_ht);
1220  for (i = 0; i < count && i < num && ht[i].hits > 0; i++)
1221  {
1222 
1223  // do only print terms which have display names
1224 
1225  wrbuf_puts(wrbuf, "<term>\n");
1226 
1227  wrbuf_puts(wrbuf, "<id>");
1228  wrbuf_xmlputs(wrbuf, ht[i].id);
1229  wrbuf_puts(wrbuf, "</id>\n");
1230 
1231  wrbuf_puts(wrbuf, "<name>");
1232  if (!ht[i].name || !ht[i].name[0])
1233  wrbuf_xmlputs(wrbuf, "NO TARGET NAME");
1234  else
1235  wrbuf_xmlputs(wrbuf, ht[i].name);
1236  wrbuf_puts(wrbuf, "</name>\n");
1237 
1238  wrbuf_printf(wrbuf, "<frequency>" ODR_INT_PRINTF "</frequency>\n",
1239  ht[i].hits);
1240 
1241  if (version >= 2) {
1242  // Should not print if we know it isn't a approximation.
1243  wrbuf_printf(wrbuf, "<approximation>" ODR_INT_PRINTF "</approximation>\n", ht[i].approximation);
1244  wrbuf_printf(wrbuf, "<records>%d</records>\n", ht[i].records - ht[i].filtered);
1245  wrbuf_printf(wrbuf, "<filtered>%d</filtered>\n", ht[i].filtered);
1246  }
1247 
1248  wrbuf_puts(wrbuf, "<state>");
1249  wrbuf_xmlputs(wrbuf, ht[i].state);
1250  wrbuf_puts(wrbuf, "</state>\n");
1251 
1252  wrbuf_printf(wrbuf, "<diagnostic>%d</diagnostic>\n",
1253  ht[i].diagnostic);
1254  wrbuf_puts(wrbuf, "</term>\n");
1255  }
1256  return count;
1257 }
1258 
1259 void perform_termlist(struct http_channel *c, struct session *se,
1260  const char *name, int num, int version)
1261 {
1262  int j;
1263  NMEM nmem_tmp = nmem_create();
1264  char **names;
1265  int num_names = 0;
1266 
1267  if (!name)
1268  name = "*";
1269 
1270  nmem_strsplit(nmem_tmp, ",", name, &names, &num_names);
1271 
1272  session_enter(se, "perform_termlist");
1273 
1274  for (j = 0; j < num_names; j++)
1275  {
1276  const char *tname;
1277  int must_generate_empty = 1; /* bug 5350 */
1278 
1279  struct named_termlist *t = se->termlists;
1280  for (; t; t = t->next)
1281  {
1282  tname = t->name;
1283  if (!strcmp(names[j], tname) || !strcmp(names[j], "*"))
1284  {
1285  struct termlist_score **p = 0;
1286  int len;
1287 
1288  wrbuf_puts(c->wrbuf, "<list name=\"");
1289  wrbuf_xmlputs(c->wrbuf, tname);
1290  wrbuf_puts(c->wrbuf, "\">\n");
1291  must_generate_empty = 0;
1292 
1293  p = termlist_highscore(t->termlist, &len, nmem_tmp);
1294  if (p)
1295  {
1296  int i;
1297  for (i = 0; i < len && i < num; i++)
1298  {
1299  // prevent sending empty term elements
1300  if (!p[i]->display_term || !p[i]->display_term[0])
1301  continue;
1302 
1303  wrbuf_puts(c->wrbuf, "<term>");
1304  wrbuf_puts(c->wrbuf, "<name>");
1305  wrbuf_xmlputs(c->wrbuf, p[i]->display_term);
1306  wrbuf_puts(c->wrbuf, "</name>");
1307  wrbuf_printf(c->wrbuf,
1308  "<frequency>%d</frequency>",
1309  p[i]->frequency);
1310  wrbuf_puts(c->wrbuf, "</term>\n");
1311  }
1312  }
1313  wrbuf_puts(c->wrbuf, "</list>\n");
1314  }
1315  }
1316  tname = "xtargets";
1317  if (!strcmp(names[j], tname) || !strcmp(names[j], "*"))
1318  {
1319  wrbuf_puts(c->wrbuf, "<list name=\"");
1320  wrbuf_xmlputs(c->wrbuf, tname);
1321  wrbuf_puts(c->wrbuf, "\">\n");
1322 
1323  targets_termlist_nb(c->wrbuf, se, num, c->nmem, version);
1324  wrbuf_puts(c->wrbuf, "</list>\n");
1325  must_generate_empty = 0;
1326  }
1327  if (must_generate_empty)
1328  {
1329  wrbuf_puts(c->wrbuf, "<list name=\"");
1330  wrbuf_xmlputs(c->wrbuf, names[j]);
1331  wrbuf_puts(c->wrbuf, "\"/>\n");
1332  }
1333  }
1334  session_leave(se, "perform_termlist");
1335  nmem_destroy(nmem_tmp);
1336 }
1337 
1338 #ifdef MISSING_HEADERS
1339 void report_nmem_stats(void)
1340 {
1341  size_t in_use, is_free;
1342 
1343  nmem_get_memory_in_use(&in_use);
1344  nmem_get_memory_free(&is_free);
1345 
1346  yaz_log(YLOG_LOG, "nmem stat: use=%ld free=%ld",
1347  (long) in_use, (long) is_free);
1348 }
1349 #endif
1350 
1351 struct record_cluster *show_single_start(struct session *se, const char *id,
1352  struct record_cluster **prev_r,
1353  struct record_cluster **next_r)
1354 {
1355  struct record_cluster *r = 0;
1356 
1357  session_enter(se, "show_single_start");
1358  *prev_r = 0;
1359  *next_r = 0;
1360  reclist_limit(se->reclist, se, 1);
1361 
1362  reclist_enter(se->reclist);
1363  while ((r = reclist_read_record(se->reclist)))
1364  {
1365  if (!strcmp(r->recid, id))
1366  {
1367  *next_r = reclist_read_record(se->reclist);
1368  break;
1369  }
1370  *prev_r = r;
1371  }
1372  reclist_leave(se->reclist);
1373  if (!r)
1374  session_leave(se, "show_single_start");
1375  return r;
1376 }
1377 
1378 void show_single_stop(struct session *se, struct record_cluster *rec)
1379 {
1380  session_leave(se, "show_single_stop");
1381 }
1382 
1383 
1385 {
1386  struct client_list *l;
1387  int ret = 0;
1388 
1389  for (l = se->clients_active; l; l = l->next)
1390  {
1391  struct client *cl = l->client;
1392  if (client_get_state(cl) == Client_Idle)
1393  {
1394  if (client_fetch_more(cl))
1395  {
1396  session_log(se, YLOG_LOG, "%s: more to fetch",
1397  client_get_id(cl));
1398  ret = 1;
1399  }
1400  else
1401  {
1402  int filtered;
1403  int ingest_failures;
1404  int record_failures;
1405  int num = client_get_num_records(
1407 
1408  session_log(se, YLOG_LOG, "%s: hits=" ODR_INT_PRINTF
1409  " fetched=%d filtered=%d",
1410  client_get_id(cl),
1411  client_get_hits(cl),
1412  num, filtered);
1414  {
1415  session_log(se, YLOG_WARN, "%s:"
1416  " ingest failures=%d record failures=%d",
1417  client_get_id(cl),
1419  }
1420  }
1421  }
1422  else
1423  {
1424  session_log(se, YLOG_LOG, "%s: no fetch due to state=%s",
1426  }
1427 
1428  }
1429  return ret;
1430 }
1431 
1433  struct reclist_sortparms *sp,
1434  int start, int *num, int *total,
1435  Odr_int *sumhits, Odr_int *approx_hits,
1436  void (*show_records_ready)(void *data),
1437  struct http_channel *chan)
1438 {
1439  struct record_cluster **recs = 0;
1440  struct reclist_sortparms *spp;
1441  struct client_list *l;
1442  int i;
1443  NMEM nmem_tmp = 0;
1444 #if USE_TIMING
1445  yaz_timing_t t = yaz_timing_create();
1446 #endif
1447  session_enter(se, "show_range_start");
1448  *sumhits = 0;
1449  *approx_hits = 0;
1450  *total = 0;
1451  reclist_limit(se->reclist, se, 0);
1452  if (se->relevance)
1453  {
1454  for (spp = sp; spp; spp = spp->next)
1455  if (spp->type == Metadata_type_relevance)
1456  {
1458  break;
1459  }
1460  for (l = se->clients_active; l; l = l->next) {
1461  *sumhits += client_get_hits(l->client);
1462  *approx_hits += client_get_approximation(l->client);
1463  }
1464  }
1465  if (se->force_position)
1466  {
1467  nmem_tmp = nmem_create();
1468  sp = reclist_parse_sortparms(nmem_tmp, "position:1", 0);
1469  assert(sp);
1470  }
1471  reclist_sort(se->reclist, sp);
1472  if (nmem_tmp)
1473  nmem_destroy(nmem_tmp);
1474 
1475  reclist_enter(se->reclist);
1476  *total = reclist_get_num_records(se->reclist);
1477 
1478  for (l = se->clients_active; l; l = l->next)
1480 
1481  for (i = 0; i < start; i++)
1482  {
1483  struct record_cluster *r = reclist_read_record(se->reclist);
1484  if (!r)
1485  {
1486  *num = 0;
1487  break;
1488  }
1489  else
1490  {
1491  struct record *rec = r->records;
1492  for (;rec; rec = rec->next)
1494  }
1495  }
1496  recs = nmem_malloc(se->nmem, (*num > 0 ? *num : 1) * sizeof(*recs));
1497  for (i = 0; i < *num; i++)
1498  {
1499  struct record_cluster *r = reclist_read_record(se->reclist);
1500  if (!r)
1501  {
1502  *num = i;
1503  break;
1504  }
1505  else
1506  {
1507  struct record *rec = r->records;
1508  for (;rec; rec = rec->next)
1510  recs[i] = r;
1511  }
1512  }
1513  reclist_leave(se->reclist);
1514 #if USE_TIMING
1515  yaz_timing_stop(t);
1516  session_log(se, YLOG_LOG, "show %6.5f %3.2f %3.2f",
1517  yaz_timing_get_real(t), yaz_timing_get_user(t),
1518  yaz_timing_get_sys(t));
1519  yaz_timing_destroy(&t);
1520 #endif
1521 
1522  if (!session_fetch_more(se))
1523  session_log(se, YLOG_LOG, "can not fetch more");
1524  else
1525  {
1526  show_range_stop(se, recs);
1527  session_log(se, YLOG_LOG, "fetching more in progress");
1529  show_records_ready, chan, chan))
1530  {
1531  session_log(se, YLOG_WARN, "Ignoring show block");
1532  session_enter(se, "show_range_start");
1533  }
1534  else
1535  {
1536  session_log(se, YLOG_LOG, "session watch OK");
1537  return 0;
1538  }
1539  }
1540  return recs;
1541 }
1542 
1543 void show_range_stop(struct session *se, struct record_cluster **recs)
1544 {
1545  session_leave(se, "show_range_stop");
1546 }
1547 
1548 void statistics(struct session *se, struct statistics *stat)
1549 {
1550  struct client_list *l;
1551  int count = 0;
1552 
1553  memset(stat, 0, sizeof(*stat));
1554  stat->num_hits = 0;
1555  for (l = se->clients_active; l; l = l->next)
1556  {
1557  struct client *cl = l->client;
1558  if (!client_get_connection(cl))
1559  stat->num_no_connection++;
1560  stat->num_hits += client_get_hits(cl);
1561  switch (client_get_state(cl))
1562  {
1563  case Client_Connecting: stat->num_connecting++; break;
1564  case Client_Working: stat->num_working++; break;
1565  case Client_Idle: stat->num_idle++; break;
1566  case Client_Failed: stat->num_failed++; break;
1567  case Client_Error: stat->num_error++; break;
1568  default: break;
1569  }
1570  count++;
1571  }
1572  stat->num_records = se->total_records;
1573 
1574  stat->num_clients = count;
1575 }
1576 
1578  NMEM nmem, const char *value, const char *norm,
1579  enum conf_metadata_type type,
1580  struct _xmlAttr *attr)
1581 {
1582  struct record_metadata *rec_md = record_metadata_create(nmem);
1583  struct record_metadata_attr **attrp = &rec_md->attributes;
1584 
1585  for (; attr; attr = attr->next)
1586  {
1587  if (attr->children && attr->children->content)
1588  {
1589  if (strcmp((const char *) attr->name, "type")
1590  && strcmp((const char *) attr->name, "empty"))
1591  { /* skip the "type" + "empty" attribute..
1592  The "Type" is already part of the element in output
1593  (md-%s) and so repeating it here is redundant */
1594  *attrp = nmem_malloc(nmem, sizeof(**attrp));
1595  (*attrp)->name =
1596  nmem_strdup(nmem, (const char *) attr->name);
1597  (*attrp)->value =
1598  nmem_strdup(nmem, (const char *) attr->children->content);
1599  attrp = &(*attrp)->next;
1600  }
1601  }
1602  }
1603  *attrp = 0;
1604 
1605  switch (type)
1606  {
1607  case Metadata_type_generic:
1609  if (norm)
1610  {
1611  rec_md->data.text.disp = nmem_strdup(nmem, value);
1612  rec_md->data.text.norm = nmem_strdup(nmem, norm);
1613  }
1614  else
1615  {
1616  if (strstr(value, "://")) /* looks like a URL */
1617  rec_md->data.text.disp = nmem_strdup(nmem, value);
1618  else
1619  rec_md->data.text.disp =
1620  normalize7bit_generic(nmem_strdup(nmem, value), " ,/.:([");
1621  rec_md->data.text.norm = rec_md->data.text.disp;
1622  }
1623  rec_md->data.text.sort = 0;
1624  rec_md->data.text.snippet = 0;
1625  break;
1626  case Metadata_type_year:
1627  case Metadata_type_date:
1628  {
1629  int first, last;
1630  int longdate = 0;
1631 
1632  if (type == Metadata_type_date)
1633  longdate = 1;
1634  if (extract7bit_dates((char *) value, &first, &last, longdate) < 0)
1635  return 0;
1636 
1637  rec_md->data.number.min = first;
1638  rec_md->data.number.max = last;
1639  }
1640  break;
1641  case Metadata_type_float:
1642  rec_md->data.fnumber = atof(value);
1643  break;
1647  return 0;
1648  }
1649  return rec_md;
1650 }
1651 
1653  WRBUF norm_wr, const char *value)
1654 {
1655  const char *norm_str;
1656  pp2_charset_token_t prt =
1657  pp2_charset_token_create(charsets, "mergekey");
1658 
1659  pp2_charset_token_first(prt, value, 0);
1660  while ((norm_str = pp2_charset_token_next(prt)))
1661  {
1662  if (*norm_str)
1663  {
1664  if (wrbuf_len(norm_wr))
1665  wrbuf_puts(norm_wr, " ");
1666  wrbuf_puts(norm_wr, norm_str);
1667  }
1668  }
1670 }
1671 
1672 static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name,
1673  struct conf_service *service, WRBUF norm_wr)
1674 {
1675  xmlNode *n;
1676  int no_found = 0;
1677  for (n = root->children; n; n = n->next)
1678  {
1679  if (n->type != XML_ELEMENT_NODE)
1680  continue;
1681  if (!strcmp((const char *) n->name, "metadata"))
1682  {
1683  const char *type = yaz_xml_get_prop(n, "type");
1684  if (type == NULL) {
1685  yaz_log(YLOG_FATAL, "Missing type attribute on metadata element. Skipping!");
1686  }
1687  else if (!strcmp(name, (const char *) type))
1688  {
1689  xmlChar *value = xmlNodeListGetString(doc, n->children, 1);
1690  if (value && *value)
1691  {
1692  if (wrbuf_len(norm_wr) > 0)
1693  wrbuf_puts(norm_wr, " ");
1694  wrbuf_puts(norm_wr, name);
1695  mergekey_norm_wr(service->charsets, norm_wr,
1696  (const char *) value);
1697  no_found++;
1698  }
1699  if (value)
1700  xmlFree(value);
1701  }
1702  }
1703  }
1704  return no_found;
1705 }
1706 
1707 static const char *get_mergekey(xmlDoc *doc, xmlNode *root,
1708  struct client *cl, int record_no,
1709  struct conf_service *service, NMEM nmem,
1710  const char *session_mergekey)
1711 {
1712  char *mergekey_norm = 0;
1713  WRBUF norm_wr = wrbuf_alloc();
1714  const char *mergekey;
1715 
1716  if (session_mergekey)
1717  {
1718  int i, num = 0;
1719  char **values = 0;
1720  nmem_strsplit_escape2(nmem, ",", session_mergekey, &values,
1721  &num, 1, '\\', 1);
1722 
1723  for (i = 0; i < num; i++)
1724  get_mergekey_from_doc(doc, root, values[i], service, norm_wr);
1725  }
1726  else if ((mergekey = yaz_xml_get_prop(root, "mergekey")))
1727  {
1728  mergekey_norm_wr(service->charsets, norm_wr, mergekey);
1729  }
1730  else
1731  {
1732  /* no mergekey defined in XSL. Look for mergekey metadata instead */
1733  int field_id;
1734  for (field_id = 0; field_id < service->num_metadata; field_id++)
1735  {
1736  struct conf_metadata *ser_md = &service->metadata[field_id];
1737  if (ser_md->mergekey != Metadata_mergekey_no)
1738  {
1739  int r = get_mergekey_from_doc(doc, root, ser_md->name,
1740  service, norm_wr);
1741  if (r == 0 && ser_md->mergekey == Metadata_mergekey_required)
1742  {
1743  /* no mergekey on this one and it is required..
1744  Generate unique key instead */
1745  wrbuf_rewind(norm_wr);
1746  break;
1747  }
1748  }
1749  }
1750  }
1751 
1752  /* generate unique key if none is not generated already or is empty */
1753  if (wrbuf_len(norm_wr) == 0)
1754  {
1755  wrbuf_printf(norm_wr, "position: %s-%06d",
1756  client_get_id(cl), record_no);
1757  }
1758  else
1759  {
1760  const char *lead = "content: ";
1761  wrbuf_insert(norm_wr, 0, lead, strlen(lead));
1762  }
1763  if (wrbuf_len(norm_wr) > 0)
1764  mergekey_norm = nmem_strdup(nmem, wrbuf_cstr(norm_wr));
1765  wrbuf_destroy(norm_wr);
1766  return mergekey_norm;
1767 }
1768 
1779 static int check_record_filter(xmlNode *root, struct session_database *sdb)
1780 {
1781  int match = 0;
1782  xmlNode *n;
1783  const char *s;
1785 
1786  if (!s || !*s)
1787  return 1;
1788 
1789  for (n = root->children; n; n = n->next)
1790  {
1791  if (n->type != XML_ELEMENT_NODE)
1792  continue;
1793  if (!strcmp((const char *) n->name, "metadata"))
1794  {
1795  const char *type = yaz_xml_get_prop(n, "type");
1796  if (type)
1797  {
1798  size_t len;
1799  int substring;
1800  const char *eq;
1801 
1802  if ((eq = strchr(s, '=')))
1803  substring = 0;
1804  else if ((eq = strchr(s, '~')))
1805  substring = 1;
1806  if (eq)
1807  len = eq - s;
1808  else
1809  len = strlen(s);
1810  if (len == strlen((const char *)type) &&
1811  !memcmp((const char *) type, s, len))
1812  {
1813  xmlChar *value = xmlNodeGetContent(n);
1814  if (value && *value)
1815  {
1816  if (!eq ||
1817  (substring && strstr((const char *) value, eq+1)) ||
1818  (!substring && !strcmp((const char *) value, eq + 1)))
1819  match = 1;
1820  }
1821  xmlFree(value);
1822  }
1823  }
1824  }
1825  }
1826  return match;
1827 }
1828 
1829 static int ingest_to_cluster(struct client *cl,
1830  WRBUF wrbuf_disp,
1831  WRBUF wrbuf_norm,
1832  xmlDoc *xdoc,
1833  xmlNode *root,
1834  int record_no,
1835  struct record_metadata_attr *mergekey);
1836 
1837 static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root,
1838  int record_no, NMEM nmem,
1839  struct session_database *sdb,
1840  struct record_metadata_attr *mergekeys)
1841 {
1842  int ret = 0;
1843  struct session *se = client_get_session(cl);
1844  WRBUF wrbuf_disp, wrbuf_norm;
1845 
1846  if (!check_record_filter(root, sdb))
1847  {
1848  session_log(se, YLOG_LOG,
1849  "Filtered out record no %d from %s",
1850  record_no, sdb->database->id);
1851  return 0;
1852  }
1853  wrbuf_disp = wrbuf_alloc();
1854  wrbuf_norm = wrbuf_alloc();
1855  session_enter(se, "ingest_sub_record");
1856  if (client_get_session(cl) == se && se->relevance)
1857  ret = ingest_to_cluster(cl, wrbuf_disp, wrbuf_norm,
1858  xdoc, root, record_no, mergekeys);
1859  session_leave(se, "ingest_sub_record");
1860  wrbuf_destroy(wrbuf_norm);
1861  wrbuf_destroy(wrbuf_disp);
1862  return ret;
1863 }
1864 
1874 int ingest_record(struct client *cl, const char *rec,
1875  int record_no, NMEM nmem)
1876 {
1877  struct session *se = client_get_session(cl);
1878  struct session_database *sdb = client_get_database(cl);
1879  struct conf_service *service = se->service;
1880  xmlDoc *xdoc = normalize_record(se, sdb, service, rec, nmem);
1881  int r = ingest_xml_record(cl, xdoc, record_no, nmem, 0);
1882  client_store_xdoc(cl, record_no, xdoc);
1883  return r;
1884 }
1885 
1886 int ingest_xml_record(struct client *cl, xmlDoc *xdoc,
1887  int record_no, NMEM nmem, int cached_copy)
1888 {
1889  struct session *se = client_get_session(cl);
1890  struct session_database *sdb = client_get_database(cl);
1891  struct conf_service *service = se->service;
1892  xmlNode *root;
1893  int r = 0;
1894  if (!xdoc)
1895  return -1;
1896 
1898  {
1899  session_log(se, YLOG_LOG, "Normalized record from %s",
1900  sdb->database->id);
1901  log_xml_doc(xdoc);
1902  }
1903 
1904  root = xmlDocGetRootElement(xdoc);
1905 
1906  if (!strcmp((const char *) root->name, "cluster"))
1907  {
1908  int no_merge_keys = 0;
1909  int no_merge_dups = 0;
1910  xmlNode *sroot;
1911  struct record_metadata_attr *mk = 0;
1912 
1913  for (sroot = root->children; sroot; sroot = sroot->next)
1914  if (sroot->type == XML_ELEMENT_NODE &&
1915  !strcmp((const char *) sroot->name, "record"))
1916  {
1917  struct record_metadata_attr **mkp;
1918  const char *mergekey_norm =
1919  get_mergekey(xdoc, sroot, cl, record_no, service, nmem,
1920  se->mergekey);
1921  if (!mergekey_norm)
1922  {
1923  r = -1;
1924  break;
1925  }
1926  for (mkp = &mk; *mkp; mkp = &(*mkp)->next)
1927  if (!strcmp((*mkp)->value, mergekey_norm))
1928  break;
1929  if (!*mkp)
1930  {
1931  *mkp = (struct record_metadata_attr*)
1932  nmem_malloc(nmem, sizeof(**mkp));
1933  (*mkp)->name = 0;
1934  (*mkp)->value = nmem_strdup(nmem, mergekey_norm);
1935  (*mkp)->next = 0;
1936  no_merge_keys++;
1937  }
1938  else
1939  no_merge_dups++;
1940  }
1941  if (no_merge_keys > 1 || no_merge_dups > 0)
1942  {
1943  yaz_log(YLOG_LOG, "Got %d mergekeys, %d dups for position %d",
1944  no_merge_keys, no_merge_dups, record_no);
1945  }
1946  for (sroot = root->children; !r && sroot; sroot = sroot->next)
1947  if (sroot->type == XML_ELEMENT_NODE &&
1948  !strcmp((const char *) sroot->name, "record"))
1949  {
1950  if (!cached_copy)
1951  insert_settings_values(sdb, xdoc, root, service);
1952  r = ingest_sub_record(cl, xdoc, sroot, record_no, nmem, sdb,
1953  mk);
1954  }
1955  }
1956  else if (!strcmp((const char *) root->name, "record"))
1957  {
1958  const char *mergekey_norm =
1959  get_mergekey(xdoc, root, cl, record_no, service, nmem,
1960  se->mergekey);
1961  if (mergekey_norm)
1962  {
1963  struct record_metadata_attr *mk = (struct record_metadata_attr*)
1964  nmem_malloc(nmem, sizeof(*mk));
1965  mk->name = 0;
1966  mk->value = nmem_strdup(nmem, mergekey_norm);
1967  mk->next = 0;
1968 
1969  if (!cached_copy)
1970  insert_settings_values(sdb, xdoc, root, service);
1971  r = ingest_sub_record(cl, xdoc, root, record_no, nmem, sdb, mk);
1972  }
1973  }
1974  else
1975  {
1976  session_log(se, YLOG_WARN, "Bad pz root element: %s",
1977  (const char *) root->name);
1978  r = -1;
1979  }
1980  return r;
1981 }
1982 
1983 
1984 // struct conf_metadata *ser_md = &service->metadata[md_field_id];
1985 // struct record_metadata *rec_md = record->metadata[md_field_id];
1986 static int match_metadata_local(struct conf_service *service,
1987  struct conf_metadata *ser_md,
1988  struct record_metadata *rec_md0,
1989  char **values, int num_v)
1990 {
1991  int i;
1992  struct record_metadata *rec_md = rec_md0;
1993  WRBUF val_wr = 0;
1994  WRBUF text_wr = wrbuf_alloc();
1995  for (i = 0; i < num_v; )
1996  {
1997  if (rec_md)
1998  {
1999  if (ser_md->type == Metadata_type_year
2000  || ser_md->type == Metadata_type_date)
2001  {
2002  int y = atoi(values[i]);
2003  if (y >= rec_md->data.number.min
2004  && y <= rec_md->data.number.max)
2005  break;
2006  }
2007  else
2008  {
2009  if (!val_wr)
2010  {
2011  val_wr = wrbuf_alloc();
2012  mergekey_norm_wr(service->charsets, val_wr, values[i]);
2013  }
2014  wrbuf_rewind(text_wr);
2015  mergekey_norm_wr(service->charsets, text_wr,
2016  rec_md->data.text.disp);
2017  if (!strcmp(wrbuf_cstr(val_wr), wrbuf_cstr(text_wr)))
2018  break;
2019  }
2020  rec_md = rec_md->next;
2021  }
2022  else
2023  {
2024  rec_md = rec_md0;
2025  wrbuf_destroy(val_wr);
2026  val_wr = 0;
2027  i++;
2028  }
2029  }
2030  wrbuf_destroy(val_wr);
2031  wrbuf_destroy(text_wr);
2032  return i < num_v ? 1 : 0;
2033 }
2034 
2036 {
2037  int i;
2038  struct conf_service *service = se->service;
2039  int ret = 1;
2040  const char *name;
2041  const char *value;
2042  NMEM nmem_tmp = nmem_create();
2043 
2044  for (i = 0; (name = facet_limits_get(se->facet_limits, i, &value)); i++)
2045  {
2046  int j;
2047  for (j = 0; j < service->num_metadata; j++)
2048  {
2049  struct conf_metadata *md = service->metadata + j;
2050  if (!strcmp(md->name, name) && md->limitcluster)
2051  {
2052  char **values = 0;
2053  int num = 0;
2054  int md_field_id =
2056  md->limitcluster);
2057 
2058  if (md_field_id < 0)
2059  {
2060  ret = 0;
2061  break;
2062  }
2063 
2064  nmem_strsplit_escape2(nmem_tmp, "|", value, &values,
2065  &num, 1, '\\', 1);
2066 
2067  if (!match_metadata_local(service,
2068  &service->metadata[md_field_id],
2069  rec->metadata[md_field_id],
2070  values, num))
2071  {
2072  ret = 0;
2073  break;
2074  }
2075  }
2076  }
2077  }
2078  nmem_destroy(nmem_tmp);
2079  return ret;
2080 }
2081 
2082 // Skip record on non-zero
2083 static int check_limit_local(struct client *cl,
2084  struct record *record,
2085  int record_no)
2086 {
2087  int skip_record = 0;
2088  struct session *se = client_get_session(cl);
2089  struct conf_service *service = se->service;
2090  NMEM nmem_tmp = nmem_create();
2091  struct session_database *sdb = client_get_database(cl);
2092  int l = 0;
2093  while (!skip_record)
2094  {
2095  int md_field_id;
2096  char **values = 0;
2097  int num_v = 0;
2098  const char *name =
2099  client_get_facet_limit_local(cl, sdb, &l, nmem_tmp,
2100  &num_v, &values);
2101  if (!name)
2102  break;
2103 
2104  if (!strcmp(name, "*"))
2105  {
2106  for (md_field_id = 0; md_field_id < service->num_metadata;
2107  md_field_id++)
2108  {
2110  service,
2111  &service->metadata[md_field_id],
2112  record->metadata[md_field_id],
2113  values, num_v))
2114  break;
2115  }
2116  if (md_field_id == service->num_metadata)
2117  skip_record = 1;
2118  }
2119  else
2120  {
2121  md_field_id = conf_service_metadata_field_id(service, name);
2122  if (md_field_id < 0)
2123  {
2124  skip_record = 1;
2125  break;
2126  }
2127  if (!match_metadata_local(
2128  service,
2129  &service->metadata[md_field_id],
2130  record->metadata[md_field_id],
2131  values, num_v))
2132  {
2133  skip_record = 1;
2134  }
2135  }
2136  }
2137  nmem_destroy(nmem_tmp);
2138  return skip_record;
2139 }
2140 
2141 static int ingest_to_cluster(struct client *cl,
2142  WRBUF wrbuf_disp,
2143  WRBUF wrbuf_norm,
2144  xmlDoc *xdoc,
2145  xmlNode *root,
2146  int record_no,
2147  struct record_metadata_attr *merge_keys)
2148 {
2149  xmlNode *n;
2150  struct session *se = client_get_session(cl);
2151  struct conf_service *service = se->service;
2152  int term_factor = 1;
2153  struct record_cluster *cluster;
2154  struct record_metadata **metadata0;
2155  struct session_database *sdb = client_get_database(cl);
2156  NMEM ingest_nmem = 0;
2157  char **rank_values = 0;
2158  int rank_num = 0;
2159  struct record *record = record_create(se->nmem,
2160  service->num_metadata,
2161  service->num_sortkeys, cl,
2162  record_no);
2163 
2164  for (n = root->children; n; n = n->next)
2165  {
2166  if (n->type != XML_ELEMENT_NODE)
2167  continue;
2168  if (!strcmp((const char *) n->name, "metadata"))
2169  {
2170  struct conf_metadata *ser_md = 0;
2171  struct record_metadata **wheretoput = 0;
2172  struct record_metadata *rec_md = 0;
2173  int md_field_id = -1;
2174  xmlChar *value0;
2175  const char *type = yaz_xml_get_prop(n, "type");
2176 
2177  if (!type)
2178  continue;
2179 
2180  md_field_id
2181  = conf_service_metadata_field_id(service, (const char *) type);
2182  if (md_field_id < 0)
2183  {
2185  {
2186  session_log(se, YLOG_WARN,
2187  "Ignoring unknown metadata element: %s", type);
2188  }
2190  continue;
2191  }
2192 
2193  wrbuf_rewind(wrbuf_disp);
2194  value0 = xmlNodeListGetString(xdoc, n->children, 1);
2195  if (!value0 || !*value0)
2196  {
2197  const char *empty = yaz_xml_get_prop(n, "empty");
2198  if (value0)
2199  xmlFree(value0);
2200  if (!empty)
2201  continue;
2202  wrbuf_puts(wrbuf_disp, (const char *) empty);
2203  }
2204  else
2205  {
2206  wrbuf_puts(wrbuf_disp, (const char *) value0);
2207  xmlFree(value0);
2208  }
2209  ser_md = &service->metadata[md_field_id];
2210 
2211  // non-merged metadata
2212  rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp), 0,
2213  ser_md->type, n->properties);
2214  if (!rec_md)
2215  {
2216  session_log(se, YLOG_WARN, "bad metadata data '%s' "
2217  "for element '%s'", wrbuf_cstr(wrbuf_disp), type);
2218  continue;
2219  }
2220 
2221  if (ser_md->type == Metadata_type_generic)
2222  {
2223  WRBUF w = wrbuf_alloc();
2224  if (relevance_snippet(se->relevance,
2225  wrbuf_cstr(wrbuf_disp), ser_md->name, w))
2226  rec_md->data.text.snippet = nmem_strdup(se->nmem,
2227  wrbuf_cstr(w));
2228  wrbuf_destroy(w);
2229  }
2230 
2231 
2232  wheretoput = &record->metadata[md_field_id];
2233  while (*wheretoput)
2234  wheretoput = &(*wheretoput)->next;
2235  *wheretoput = rec_md;
2236  }
2237  }
2238 
2239  if (check_limit_local(cl, record, record_no))
2240  {
2241  return -2;
2242  }
2243  cluster = reclist_insert(se->reclist, se->relevance, service, record,
2244  merge_keys, &se->total_merged);
2245  if (!cluster)
2246  {
2247  return 0; // complete match with existing record
2248  }
2249 
2250  {
2251  const char *use_term_factor_str =
2253  if (use_term_factor_str && use_term_factor_str[0] == '1')
2254  {
2255  int maxrecs = client_get_maxrecs(cl);
2256  int hits = (int) client_get_hits(cl);
2257  term_factor = MAX(hits, maxrecs) / MAX(1, maxrecs);
2258  assert(term_factor >= 1);
2259  session_log(se, YLOG_DEBUG, "Using term factor: %d (%d / %d)",
2260  term_factor, MAX(hits, maxrecs), MAX(1, maxrecs));
2261  }
2262  }
2263 
2265  session_log(se, YLOG_LOG, "Cluster id %s from %s (#%d)", cluster->recid,
2266  sdb->database->id, record_no);
2267 
2268  // original metadata, to check if first existence of a field
2269  metadata0 = xmalloc(sizeof(*metadata0) * service->num_metadata);
2270  memcpy(metadata0, cluster->metadata,
2271  sizeof(*metadata0) * service->num_metadata);
2272 
2273  ingest_nmem = nmem_create();
2274  if (se->rank)
2275  {
2276  yaz_log(YLOG_LOG, "local in sort : %s", se->rank);
2277  nmem_strsplit_escape2(ingest_nmem, ",", se->rank, &rank_values,
2278  &rank_num, 1, '\\', 1);
2279  }
2280 
2281  // now parsing XML record and adding data to cluster or record metadata
2282  for (n = root->children; n; n = n->next)
2283  {
2284  if (n->type != XML_ELEMENT_NODE)
2285  continue;
2286  if (!strcmp((const char *) n->name, "metadata"))
2287  {
2288  struct conf_metadata *ser_md = 0;
2289  struct conf_sortkey *ser_sk = 0;
2290  struct record_metadata **wheretoput = 0;
2291  struct record_metadata *rec_md = 0;
2292  int md_field_id = -1;
2293  int sk_field_id = -1;
2294  const char *rank = 0;
2295  const char *xml_rank = 0;
2296  const char *type = 0;
2297  xmlChar *value0;
2298 
2299  type = yaz_xml_get_prop(n, "type");
2300  if (!type)
2301  continue;
2302 
2303  md_field_id
2304  = conf_service_metadata_field_id(service, (const char *) type);
2305  if (md_field_id < 0)
2306  continue;
2307 
2308  ser_md = &service->metadata[md_field_id];
2309 
2310  if (ser_md->sortkey_offset >= 0)
2311  {
2312  sk_field_id = ser_md->sortkey_offset;
2313  ser_sk = &service->sortkeys[sk_field_id];
2314  }
2315 
2316  wrbuf_rewind(wrbuf_disp);
2317  wrbuf_rewind(wrbuf_norm);
2318 
2319  value0 = xmlNodeListGetString(xdoc, n->children, 1);
2320  if (!value0 || !*value0)
2321  {
2322  if (value0)
2323  xmlFree(value0);
2324  continue;
2325  }
2326 
2327  if (ser_md->icurule)
2328  {
2329  run_icu(se, ser_md->icurule, (const char *) value0,
2330  wrbuf_norm, wrbuf_disp);
2331  yaz_log(YLOG_LOG, "run_icu input=%s norm=%s disp=%s",
2332  (const char *) value0,
2333  wrbuf_cstr(wrbuf_norm), wrbuf_cstr(wrbuf_disp));
2334  rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp),
2335  wrbuf_cstr(wrbuf_norm),
2336  ser_md->type, 0);
2337  }
2338  else
2339  {
2340  wrbuf_puts(wrbuf_disp, (const char *) value0);
2341  rec_md = record_metadata_init(se->nmem, wrbuf_cstr(wrbuf_disp),
2342  0,
2343  ser_md->type, 0);
2344  }
2345 
2346  xmlFree(value0);
2347 
2348  // see if the field was not in cluster already (from beginning)
2349  if (!rec_md)
2350  continue;
2351 
2352  if (rank_num)
2353  {
2354  int i;
2355  for (i = 0; i < rank_num; i++)
2356  {
2357  const char *val = rank_values[i];
2358  const char *cp = strchr(val, '=');
2359  if (!cp)
2360  continue;
2361  if ((cp - val) == strlen((const char *) type)
2362  && !memcmp(val, type, cp - val))
2363  {
2364  rank = cp + 1;
2365  break;
2366  }
2367  }
2368  }
2369  else
2370  {
2371  xml_rank = yaz_xml_get_prop(n, "rank");
2372  rank = xml_rank ? (const char *) xml_rank : ser_md->rank;
2373  }
2374 
2375  wheretoput = &cluster->metadata[md_field_id];
2376 
2377  if (ser_md->merge == Metadata_merge_first)
2378  {
2379  if (!metadata0[md_field_id])
2380  {
2381  while (*wheretoput)
2382  wheretoput = &(*wheretoput)->next;
2383  *wheretoput = rec_md;
2384  }
2385  }
2386  else if (ser_md->merge == Metadata_merge_unique)
2387  {
2388  while (*wheretoput)
2389  {
2390  if (!strcmp((const char *) (*wheretoput)->data.text.norm,
2391  rec_md->data.text.norm))
2392  break;
2393  wheretoput = &(*wheretoput)->next;
2394  }
2395  if (!*wheretoput)
2396  *wheretoput = rec_md;
2397  }
2398  else if (ser_md->merge == Metadata_merge_longest)
2399  {
2400  if (!*wheretoput
2401  || strlen(rec_md->data.text.norm)
2402  > strlen((*wheretoput)->data.text.norm))
2403  {
2404  *wheretoput = rec_md;
2405  if (ser_sk)
2406  {
2407  pp2_charset_token_t prt;
2408  const char *sort_str = 0;
2409  int skip_article =
2410  ser_sk->type == Metadata_type_skiparticle;
2411 
2412  if (!cluster->sortkeys[sk_field_id])
2413  cluster->sortkeys[sk_field_id] =
2414  nmem_malloc(se->nmem,
2415  sizeof(union data_types));
2416 
2417  prt =
2418  pp2_charset_token_create(service->charsets, "sort");
2419 
2420  pp2_charset_token_first(prt, rec_md->data.text.disp,
2421  skip_article);
2422 
2424 
2425  sort_str = pp2_get_sort(prt);
2426 
2427  cluster->sortkeys[sk_field_id]->text.disp =
2428  rec_md->data.text.disp;
2429  if (!sort_str)
2430  {
2431  sort_str = rec_md->data.text.disp;
2432  session_log(se, YLOG_WARN,
2433  "Could not make sortkey. Bug #1858");
2434  }
2435  cluster->sortkeys[sk_field_id]->text.sort =
2436  nmem_strdup(se->nmem, sort_str);
2438  }
2439  }
2440  }
2441  else if (ser_md->merge == Metadata_merge_all)
2442  {
2443  while (*wheretoput)
2444  wheretoput = &(*wheretoput)->next;
2445  *wheretoput = rec_md;
2446  }
2447  else if (ser_md->merge == Metadata_merge_range)
2448  {
2449  if (!*wheretoput)
2450  {
2451  *wheretoput = rec_md;
2452  if (ser_sk)
2453  cluster->sortkeys[sk_field_id]
2454  = &rec_md->data;
2455  }
2456  else
2457  {
2458  int this_min = rec_md->data.number.min;
2459  int this_max = rec_md->data.number.max;
2460  if (this_min < (*wheretoput)->data.number.min)
2461  (*wheretoput)->data.number.min = this_min;
2462  if (this_max > (*wheretoput)->data.number.max)
2463  (*wheretoput)->data.number.max = this_max;
2464  }
2465  }
2466 
2467  // ranking of _all_ fields enabled ...
2468  if (rank)
2469  {
2470  relevance_countwords(se->relevance, cluster,
2471  wrbuf_cstr(wrbuf_disp),
2472  rank, ser_md->name);
2473  }
2474  // construct facets ... unless the client already has reported them
2475  if (ser_md->termlist && !client_has_facet(cl, (char *) type))
2476  {
2477  if (ser_md->type == Metadata_type_year)
2478  {
2479  char year[64];
2480  sprintf(year, "%d", rec_md->data.number.max);
2481 
2482  add_facet(se, (char *) type, year, term_factor, cl);
2483  if (rec_md->data.number.max != rec_md->data.number.min)
2484  {
2485  sprintf(year, "%d", rec_md->data.number.min);
2486  add_facet(se, (char *) type, year, term_factor, cl);
2487  }
2488  }
2489  else
2490  add_facet(se, type, wrbuf_cstr(wrbuf_disp), term_factor, cl);
2491  }
2492  }
2493  else
2494  {
2496  session_log(se, YLOG_WARN,
2497  "Unexpected element in internal record: %s", n->name);
2499  }
2500  }
2501  nmem_destroy(ingest_nmem);
2502  xfree(metadata0);
2503  relevance_donerecord(se->relevance, cluster);
2504  se->total_records++;
2505 
2506  return 0;
2507 }
2508 
2509 void session_log(struct session *s, int level, const char *fmt, ...)
2510 {
2511  char buf[1024];
2512  va_list ap;
2513  va_start(ap, fmt);
2514 
2515  yaz_vsnprintf(buf, sizeof(buf)-30, fmt, ap);
2516  yaz_log(level, "Session %u: %s", s ? s->session_id : 0, buf);
2517 
2518  va_end(ap);
2519 }
2520 
2521 /*
2522  * Local variables:
2523  * c-basic-offset: 4
2524  * c-file-style: "Stroustrup"
2525  * indent-tabs-mode: nil
2526  * End:
2527  * vim: shiftwidth=4 tabstop=8 expandtab
2528  */
2529 
const char * pp2_charset_token_next(pp2_charset_token_t prt)
Definition: charsets.c:360
pp2_charset_token_t pp2_charset_token_create(pp2_charset_fact_t pft, const char *id)
Definition: charsets.c:282
const char * pp2_get_display(pp2_charset_token_t prt)
Definition: charsets.c:371
void pp2_charset_token_first(pp2_charset_token_t prt, const char *buf, int skip_article)
Definition: charsets.c:314
void pp2_charset_token_destroy(pp2_charset_token_t prt)
Definition: charsets.c:346
const char * pp2_get_sort(pp2_charset_token_t prt)
Definition: charsets.c:366
int client_get_diagnostic(struct client *cl, const char **message, const char **addinfo)
Definition: client.c:1789
void client_unlock(struct client *c)
Definition: client.c:1096
void client_store_xdoc(struct client *cl, int record_no, xmlDoc *xdoc)
Definition: client.c:228
int client_parse_query(struct client *cl, const char *query, facet_limits_t facet_limits, const char **error_msg)
Definition: client.c:1502
void client_set_database(struct client *cl, struct session_database *db)
Definition: client.c:1828
void client_update_show_stat(struct client *cl, int cmd)
Definition: client.c:814
void client_stop(struct client *cl)
Definition: client.c:1176
int client_parse_sort(struct client *cl, struct reclist_sortparms *sp, int *has_sortmap)
Definition: client.c:1654
Odr_int client_get_hits(struct client *cl)
Definition: client.c:1746
int client_fetch_more(struct client *cl)
Definition: client.c:822
const char * client_get_state_str(struct client *cl)
Definition: client.c:161
int client_is_active_preferred(struct client *cl)
Definition: client.c:1734
void client_lock(struct client *c)
Definition: client.c:1091
const char * client_get_id(struct client *cl)
Definition: client.c:1833
const char * client_get_facet_limit_local(struct client *cl, struct session_database *sdb, int *l, NMEM nmem, int *num, char ***values)
Definition: client.c:1289
Odr_int client_get_approximation(struct client *cl)
Definition: client.c:1751
struct connection * client_get_connection(struct client *cl)
Definition: client.c:246
void client_set_session(struct client *cl, struct session *se)
Definition: client.c:1721
struct client * client_create(const char *id)
Definition: client.c:1055
int client_start_search(struct client *cl)
Definition: client.c:910
struct session * client_get_session(struct client *cl)
Definition: client.c:256
int client_get_maxrecs(struct client *cl)
Definition: client.c:1838
int client_has_facet(struct client *cl, const char *name)
Definition: client.c:773
int client_parse_range(struct client *cl, const char *startrecs, const char *maxrecs)
Definition: client.c:876
void client_mark_dead(struct client *cl)
Definition: client.c:1170
enum client_state client_get_state(struct client *cl)
Definition: client.c:166
int client_is_active(struct client *cl)
Definition: client.c:1726
int client_parse_init(struct client *cl, int same_search)
Definition: client.c:867
const char * client_get_suggestions_xml(struct client *cl, WRBUF wrbuf)
Definition: client.c:1799
int client_destroy(struct client *c)
Definition: client.c:1108
const char * client_get_query(struct client *cl, const char **type, NMEM nmem)
Definition: client.c:894
struct session_database * client_get_database(struct client *cl)
Definition: client.c:251
int client_get_num_records(struct client *cl, int *filtered, int *ingest, int *failed)
Definition: client.c:1765
Z39.50 client.
@ Client_Working
Definition: client.h:37
@ Client_Idle
Definition: client.h:36
@ Client_Failed
Definition: client.h:39
@ Client_Error
Definition: client.h:38
@ Client_Connecting
Definition: client.h:35
int session_grep_databases(struct session *se, const char *filter, void(*fun)(struct session *se, struct session_database *db))
Definition: database.c:266
int predef_grep_databases(void *context, struct conf_service *service, void(*fun)(void *context, struct database *db))
Definition: database.c:291
struct database * new_database_inherit_settings(const char *id, NMEM nmem, struct settings_array *service_settings)
Definition: database.c:57
const char * facet_limits_get(facet_limits_t fl, int idx, const char **value)
Definition: facet_limit.c:94
void facet_limits_destroy(facet_limits_t fl)
Definition: facet_limit.c:102
facet_limits_t facet_limits_create(const char *param)
Definition: facet_limit.c:64
void http_remove_observer(http_channel_observer_t obs)
Definition: http.c:1443
http_channel_observer_t http_add_observer(struct http_channel *c, void *data, http_channel_destroy_t des)
Definition: http.c:1430
void(* fun)(struct http_channel *c)
char * name
static void show_records_ready(void *data)
char * normalize7bit_generic(char *str, const char *rm_chars)
removes leading whitespace.. Removes suffix cahrs in rm_chars
Definition: normalize7bit.c:36
int extract7bit_dates(const char *buf, int *first, int *last, int longdate)
Definition: normalize7bit.c:73
normalize_record_t normalize_cache_get(normalize_cache_t nc, struct conf_service *service, const char *spec)
normalize_cache_t normalize_cache_create(void)
void normalize_cache_destroy(normalize_cache_t nc)
int normalize_record_transform(normalize_record_t nt, xmlDoc **doc, const char **parms)
void service_destroy(struct conf_service *service)
int conf_service_metadata_field_id(struct conf_service *service, const char *name)
conf_metadata_type
@ Metadata_type_float
@ Metadata_type_relevance
@ Metadata_type_position
@ Metadata_type_retrieval
@ Metadata_type_generic
@ Metadata_type_date
@ Metadata_type_year
@ Metadata_type_skiparticle
@ Metadata_merge_all
@ Metadata_merge_longest
@ Metadata_merge_range
@ Metadata_merge_unique
@ Metadata_merge_first
@ Metadata_mergekey_required
@ Metadata_mergekey_no
@ Metadata_setting_parameter
@ Metadata_setting_postproc
void pazpar2_mutex_create(YAZ_MUTEX *p, const char *name)
Definition: ppmutex.c:39
control MUTEX debugging
struct record_cluster * reclist_insert(struct reclist *l, struct relevance *r, struct conf_service *service, struct record *record, struct record_metadata_attr *merge_keys, int *total)
Definition: reclists.c:482
void reclist_sort(struct reclist *l, struct reclist_sortparms *parms)
Definition: reclists.c:298
struct reclist_sortparms * reclist_parse_sortparms(NMEM nmem, const char *parms, struct conf_service *service)
Definition: reclists.c:71
void reclist_enter(struct reclist *l)
Definition: reclists.c:343
void reclist_destroy(struct reclist *l)
Definition: reclists.c:380
void reclist_leave(struct reclist *l)
Definition: reclists.c:350
struct record_cluster * reclist_read_record(struct reclist *l)
Definition: reclists.c:331
int reclist_get_num_records(struct reclist *l)
Definition: reclists.c:400
void reclist_limit(struct reclist *l, struct session *se, int lazy)
Definition: reclists.c:259
int reclist_sortparms_cmp(struct reclist_sortparms *sort1, struct reclist_sortparms *sort2)
Definition: reclists.c:564
struct reclist * reclist_create(NMEM nmem)
Definition: reclists.c:362
struct record * record_create(NMEM nmem, int num_metadata, int num_sortkeys, struct client *client, int position)
Definition: record.c:54
struct record_metadata * record_metadata_create(NMEM nmem)
Definition: record.c:91
void relevance_clear(struct relevance *r)
Definition: relevance.c:530
void relevance_donerecord(struct relevance *r, struct record_cluster *cluster)
Definition: relevance.c:616
void relevance_destroy(struct relevance **rp)
Definition: relevance.c:575
int relevance_snippet(struct relevance *r, const char *words, const char *name, WRBUF w_snippet)
Definition: relevance.c:349
void relevance_countwords(struct relevance *r, struct record_cluster *cluster, const char *words, const char *rank, const char *name)
Definition: relevance.c:398
void relevance_prepare_read(struct relevance *rel, struct reclist *reclist)
Definition: relevance.c:630
static int check_record_filter(xmlNode *root, struct session_database *sdb)
see if metadata for pz:recordfilter exists
Definition: session.c:1779
const char * session_setting_oneval(struct session_database *db, int offset)
Definition: session.c:433
static int cmp_ht(const void *p1, const void *p2)
Definition: session.c:1194
static void session_init_databases_fun(void *context, struct database *db)
Definition: session.c:957
enum pazpar2_error_code session_search(struct session *se, const char *query, const char *startrecs, const char *maxrecs, const char *filter, const char *limit, const char **addinfo, const char **addinfo2, struct reclist_sortparms *sp, const char *mergekey, const char *rank)
Definition: session.c:818
static const char * get_mergekey(xmlDoc *doc, xmlNode *root, struct client *cl, int record_no, struct conf_service *service, NMEM nmem, const char *session_mergekey)
Definition: session.c:1707
void session_alert_watch(struct session *s, int what)
Definition: session.c:521
void session_stop(struct session *se)
Definition: session.c:791
static int targets_termlist_nb(WRBUF wrbuf, struct session *se, int num, NMEM nmem, int version)
Definition: session.c:1209
void session_apply_setting(struct session *se, const char *dbname, const char *name, const char *value)
Definition: session.c:1018
static void mergekey_norm_wr(pp2_charset_fact_t charsets, WRBUF norm_wr, const char *value)
Definition: session.c:1652
static int session_use(int delta)
Definition: session.c:102
static struct session_database * find_session_database(struct session *se, const char *id)
Definition: session.c:1006
static YAZ_MUTEX g_session_mutex
Definition: session.c:99
static int no_sessions
Definition: session.c:100
void session_sort(struct session *se, struct reclist_sortparms *sp, const char *mergekey, const char *rank)
Definition: session.c:685
static void session_remove_cached_clients(struct session *se)
Definition: session.c:604
struct session * session_create(NMEM nmem, struct conf_service *service, unsigned session_id)
Definition: session.c:1093
int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem)
ingest XML record
Definition: session.c:1874
static struct session_database * load_session_database(struct session *se, const char *id)
Definition: session.c:995
static void insert_settings_parameters(struct session_database *sdb, struct conf_service *service, char **parms, NMEM nmem)
Definition: session.c:324
static void insert_settings_values(struct session_database *sdb, xmlDoc *doc, xmlNode *root, struct conf_service *service)
Definition: session.c:361
static xmlDoc * record_to_xml(struct session *se, struct session_database *sdb, const char *rec)
Definition: session.c:298
static void session_watch_cancel(void *data, struct http_channel *c, void *data2)
Definition: session.c:489
static void session_enter(struct session *s, const char *caller)
Definition: session.c:139
static int ingest_sub_record(struct client *cl, xmlDoc *xdoc, xmlNode *root, int record_no, NMEM nmem, struct session_database *sdb, struct record_metadata_attr *mergekeys)
Definition: session.c:1837
static struct hitsbytarget * hitsbytarget_nb(struct session *se, int *count, NMEM nmem)
Definition: session.c:1140
void session_init_databases(struct session *se)
Definition: session.c:987
void show_range_stop(struct session *se, struct record_cluster **recs)
Definition: session.c:1543
int session_check_cluster_limit(struct session *se, struct record_cluster *rec)
Definition: session.c:2035
static void select_targets_callback(struct session *se, struct session_database *db)
Definition: session.c:551
static int check_limit_local(struct client *cl, struct record *record, int record_no)
Definition: session.c:2083
int session_set_watch(struct session *s, int what, session_watchfun fun, void *data, struct http_channel *chan)
Definition: session.c:500
void statistics(struct session *se, struct statistics *stat)
Definition: session.c:1548
struct hitsbytarget * get_hitsbytarget(struct session *se, int *count, NMEM nmem)
Definition: session.c:1184
int ingest_xml_record(struct client *cl, xmlDoc *xdoc, int record_no, NMEM nmem, int cached_copy)
Definition: session.c:1886
static int ingest_to_cluster(struct client *cl, WRBUF wrbuf_disp, WRBUF wrbuf_norm, xmlDoc *xdoc, xmlNode *root, int record_no, struct record_metadata_attr *mergekey)
Definition: session.c:2141
static int select_targets(struct session *se, const char *filter)
Definition: session.c:632
size_t session_get_memory_status(struct session *session)
Definition: session.c:1082
struct parameters global_parameters
Definition: session.c:86
void perform_termlist(struct http_channel *c, struct session *se, const char *name, int num, int version)
Definition: session.c:1259
static void session_leave(struct session *s, const char *caller)
Definition: session.c:146
int sessions_get_count(void)
Definition: session.c:116
static void session_add_id_facet(struct session *s, struct client *cl, const char *type, const char *id, size_t id_len, const char *term)
Definition: session.c:214
struct record_cluster ** show_range_start(struct session *se, struct reclist_sortparms *sp, int start, int *num, int *total, Odr_int *sumhits, Odr_int *approx_hits, void(*show_records_ready)(void *data), struct http_channel *chan)
Definition: session.c:1432
static xmlDoc * normalize_record(struct session *se, struct session_database *sdb, struct conf_service *service, const char *rec, NMEM nmem)
Definition: session.c:386
void session_destroy(struct session *se)
Definition: session.c:1053
void add_facet(struct session *s, const char *type, const char *value, int count, struct client *cl)
Definition: session.c:254
static void log_xml_doc(xmlDoc *doc)
Definition: session.c:121
static struct record_metadata * record_metadata_init(NMEM nmem, const char *value, const char *norm, enum conf_metadata_type type, struct _xmlAttr *attr)
Definition: session.c:1577
static int cmp_ht_approx(const void *p1, const void *p2)
Definition: session.c:1202
int session_fetch_more(struct session *se)
Definition: session.c:1384
static void session_clear_set(struct session *se, struct reclist_sortparms *sp)
Definition: session.c:661
static void session_reset_active_clients(struct session *se, struct client_list *new_list)
Definition: session.c:581
static int match_metadata_local(struct conf_service *service, struct conf_metadata *ser_md, struct record_metadata *rec_md0, char **values, int num_v)
Definition: session.c:1986
static void session_normalize_facet(struct session *s, const char *type, const char *value, WRBUF display_wrbuf, WRBUF facet_wrbuf)
Definition: session.c:189
void show_single_stop(struct session *se, struct record_cluster *rec)
Definition: session.c:1378
#define MAX_XSLT_ARGS
Definition: session.c:321
void session_log(struct session *s, int level, const char *fmt,...)
Definition: session.c:2509
int session_active_clients(struct session *s)
Definition: session.c:637
const char * session_lookup_id_facet(struct session *s, struct client *cl, const char *type, const char *term)
Definition: session.c:234
int session_is_preferred_clients_ready(struct session *s)
Definition: session.c:649
static int get_mergekey_from_doc(xmlDoc *doc, xmlNode *root, const char *name, struct conf_service *service, WRBUF norm_wr)
Definition: session.c:1672
struct record_cluster * show_single_start(struct session *se, const char *id, struct record_cluster **prev_r, struct record_cluster **next_r)
Definition: session.c:1351
static void session_database_destroy(struct session_database *sdb)
Definition: session.c:980
#define MAX(a, b)
Definition: session.c:83
static int run_icu(struct session *s, const char *icu_chain_id, const char *value, WRBUF norm_wr, WRBUF disp_wr)
Definition: session.c:153
void session_settings_dump(struct session *se, struct session_database *db, WRBUF w)
Definition: session.c:407
static int prepare_map(struct session *se, struct session_database *sdb)
Definition: session.c:445
#define SESSION_WATCH_BYTARGET
Definition: session.h:71
pazpar2_error_code
Definition: session.h:36
@ PAZPAR2_MALFORMED_PARAMETER_VALUE
Definition: session.h:41
@ PAZPAR2_NO_TARGETS
Definition: session.h:46
@ PAZPAR2_NO_ERROR
Definition: session.h:37
#define SESSION_WATCH_TERMLIST
Definition: session.h:70
#define SESSION_WATCH_MAX
Definition: session.h:72
#define SESSION_WATCH_SHOW
Definition: session.h:67
#define SESSION_WATCH_SHOW_PREF
Definition: session.h:69
void(* session_watchfun)(void *data)
Definition: session.h:74
void expand_settings_array(struct setting ***set_ar, int *num, int offset, NMEM nmem)
Definition: settings.c:336
int settings_create_offset(struct conf_service *service, const char *name)
Definition: settings.c:145
int settings_lookup_offset(struct conf_service *service, const char *name)
Definition: settings.c:150
#define PZ_XSLT
Definition: settings.h:27
#define PZ_REQUESTSYNTAX
Definition: settings.h:25
#define PZ_RECORDFILTER
Definition: settings.h:41
#define PZ_TERMLIST_TERM_FACTOR
Definition: settings.h:46
#define PZ_NAME
Definition: settings.h:33
struct client_list * next
Definition: session.c:95
struct client * client
Definition: session.c:94
Represents client state for a connection to one search target.
Definition: client.c:99
int filtered
Definition: client.c:109
int maxrecs
Definition: client.c:112
int ingest_failures
Definition: client.c:110
char * message
Definition: client.c:115
int startrecs
Definition: client.c:113
char * addinfo
Definition: client.c:105
int record_failures
Definition: client.c:111
enum conf_metadata_merge merge
const char * rank
char * limitcluster
enum conf_metadata_type type
enum conf_setting_type setting
enum conf_metadata_mergekey mergekey
pp2_charset_fact_t charsets
struct conf_sortkey * sortkeys
struct conf_metadata * metadata
struct settings_array * settings
enum conf_metadata_type type
char * id
Definition: settings.h:76
int num_settings
Definition: settings.h:77
struct setting ** settings
Definition: settings.h:78
char * id
Definition: session.c:209
char * term
Definition: session.c:210
char * type
Definition: session.c:208
char * client_id
Definition: session.c:207
struct facet_id * next
Definition: session.c:211
const char * id
Definition: session.h:133
const char * state
Definition: session.h:142
Odr_int approximation
Definition: session.h:136
const char * name
Definition: session.h:134
Odr_int hits
Definition: session.h:135
int filtered
Definition: session.h:141
const char * query_data
Definition: session.h:147
int connected
Definition: session.h:143
int diagnostic
Definition: session.h:137
char * suggestions_xml
Definition: session.h:145
int records
Definition: session.h:140
char * settings_xml
Definition: session.h:144
NMEM nmem
Definition: http.h:47
WRBUF wrbuf
Definition: http.h:48
char * name
Definition: session.h:78
struct termlist * termlist
Definition: session.h:79
struct named_termlist * next
Definition: session.h:80
global parameters
Definition: parameters.h:26
int dump_records
Definition: parameters.h:27
struct reclist_sortparms * next
Definition: reclists.h:36
enum conf_metadata_type type
Definition: reclists.h:33
struct record_metadata ** metadata
Definition: record.h:87
union data_types ** sortkeys
Definition: record.h:88
struct record * records
Definition: record.h:100
char * recid
Definition: record.h:96
struct record_metadata_attr * next
Definition: record.h:45
union data_types data
Definition: record.h:49
struct record_metadata * next
Definition: record.h:51
struct record_metadata_attr * attributes
Definition: record.h:52
Definition: record.h:60
struct record * next
Definition: record.h:67
struct client * client
Definition: record.h:61
struct record_metadata ** metadata
Definition: record.h:63
struct database * database
Definition: session.h:60
normalize_record_t map
Definition: session.h:63
struct session_database * next
Definition: session.h:64
struct setting ** settings
Definition: session.h:62
int num_settings
Definition: session.h:61
http_channel_observer_t obs
Definition: session.h:85
session_watchfun fun
Definition: session.h:86
int number_of_warnings_unknown_elements
Definition: session.h:107
int force_position
Definition: session.h:115
YAZ_MUTEX session_mutex
Definition: session.h:110
NMEM nmem
Definition: session.h:98
struct client_list * clients_cached
Definition: session.h:96
facet_limits_t facet_limits
Definition: session.h:113
struct reclist * reclist
Definition: session.h:101
struct client_list * clients_active
Definition: session.h:95
int number_of_warnings_unknown_metadata
Definition: session.h:108
struct session_watchentry watchlist[SESSION_WATCH_MAX+1]
Definition: session.h:104
struct named_termlist * termlists
Definition: session.h:99
struct session_database * databases
Definition: session.h:94
struct facet_id * facet_id_list
Definition: session.h:117
int total_records
Definition: session.h:105
int settings_modified
Definition: session.h:112
struct conf_service * service
Definition: session.h:93
struct relevance * relevance
Definition: session.h:100
int clients_starting
Definition: session.h:114
char * rank
Definition: session.h:103
unsigned session_id
Definition: session.h:111
NMEM session_nmem
Definition: session.h:97
int total_merged
Definition: session.h:106
char * mergekey
Definition: session.h:102
struct reclist_sortparms * sorted_results
Definition: session.h:116
normalize_cache_t normalize_cache
Definition: session.h:109
const char * name
Definition: settings.h:69
const char * value
Definition: settings.h:70
const char * target
Definition: settings.h:68
struct setting * next
Definition: settings.h:71
int precedence
Definition: settings.h:67
int num_connecting
Definition: session.h:123
int num_working
Definition: session.h:124
int num_failed
Definition: session.h:126
int num_clients
Definition: session.h:121
int num_no_connection
Definition: session.h:122
int num_idle
Definition: session.h:125
int num_error
Definition: session.h:127
Odr_int num_hits
Definition: session.h:128
int num_records
Definition: session.h:129
char * display_term
Definition: termlists.h:28
struct termlist_score ** termlist_highscore(struct termlist *tl, int *len, NMEM nmem)
Definition: termlists.c:109
void termlist_insert(struct termlist *tl, const char *display_term, const char *norm_term, const char *id, size_t id_len, int freq)
Definition: termlists.c:64
struct termlist * termlist_create(NMEM nmem)
Definition: termlists.c:52
const char * norm
Definition: record.h:30
double fnumber
Definition: record.h:38
const char * snippet
Definition: record.h:32
const char * sort
Definition: record.h:31
struct data_types::@3 text
int max
Definition: record.h:36
struct data_types::@4 number
const char * disp
Definition: record.h:29
int min
Definition: record.h:35