IDZEBRA  2.2.7
records.c
Go to the documentation of this file.
1 /* This file is part of the Zebra server.
2  Copyright (C) Index Data
3 
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8 
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 for more details.
13 
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 
18 */
19 
20 /*
21  * Format of first block (assumes a 512 block size)
22  * next (8 bytes)
23  * ref_count (2 bytes)
24  * block (500 bytes)
25  *
26  * Format of subsequent blocks
27  * next (8 bytes)
28  * block (502 bytes)
29  *
30  * Format of each record
31  * sysno
32  * (length, data) - pairs
33  * length = 0 if same as previous
34  */
35 #if HAVE_CONFIG_H
36 #include <config.h>
37 #endif
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <assert.h>
41 #include <string.h>
42 
43 #include <yaz/yaz-util.h>
44 #include <yaz/snprintf.h>
45 #include <idzebra/bfile.h>
46 #include "recindex.h"
47 
48 #if HAVE_BZLIB_H
49 #include <bzlib.h>
50 #endif
51 #if HAVE_ZLIB_H
52 #include <zlib.h>
53 #endif
54 
55 #define REC_BLOCK_TYPES 2
56 #define REC_HEAD_MAGIC "recindex"
57 #define REC_VERSION 5
58 
59 struct records_info {
60  int rw;
62 
64 
67 
68  char *tmp_buf;
69  int tmp_size;
70 
73  int cache_cur;
74  int cache_max;
75 
77 
79 
80  struct records_head {
81  char magic[8];
82  char version[4];
88 
93 
94  } head;
95 };
96 
99 
102  enum recordCacheFlag flag;
103 };
104 
105 struct record_index_entry {
106  zint next; /* first block of record info / next free entry */
107  int size; /* size of record or 0 if free entry */
108 };
109 
110 Record rec_cp(Record rec);
111 
112 /* Modify argument to if below: 1=normal, 0=sysno testing */
113 #if 1
114 /* If this is used sysno are not converted (no testing) */
115 #define FAKE_OFFSET 0
116 #define USUAL_RANGE 6000000000LL
117 
118 #else
119 /* Use a fake > 2^32 offset so we can test for proper 64-bit handling */
120 #define FAKE_OFFSET 6000000000LL
121 #define USUAL_RANGE 2000000000LL
122 #endif
123 
125 {
126  assert(sysno >= 0 && sysno <= USUAL_RANGE);
127  return sysno + FAKE_OFFSET;
128 }
129 
131 {
132  assert(sysno >= FAKE_OFFSET && sysno <= FAKE_OFFSET + USUAL_RANGE);
133  return sysno - FAKE_OFFSET;
134 }
135 
136 static void rec_tmp_expand(Records p, int size)
137 {
138  if (p->tmp_size < size + 2048 ||
140  {
141  xfree(p->tmp_buf);
142  p->tmp_size = size + (int)
143  (p->head.block_size[REC_BLOCK_TYPES-1])*2 + 2048;
144  p->tmp_buf = (char *) xmalloc(p->tmp_size);
145  }
146 }
147 
149 {
150  struct record_index_entry entry;
151  zint freeblock;
152  char block_and_ref[sizeof(zint) + sizeof(short)];
153  int dst_type;
154  int first = 1;
155 
156  if (recindex_read_indx(p->recindex, sysno, &entry, sizeof(entry), 1) != 1)
157  return ZEBRA_FAIL;
158 
159  freeblock = entry.next;
160  assert(freeblock > 0);
161  dst_type = CAST_ZINT_TO_INT(freeblock & 7);
162  assert(dst_type < REC_BLOCK_TYPES);
163  freeblock = freeblock / 8;
164  while (freeblock)
165  {
166  if (bf_read(p->data_BFile[dst_type], freeblock, 0,
167  first ? sizeof(block_and_ref) : sizeof(zint),
168  block_and_ref) != 1)
169  {
170  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in rec_del_single");
171  return ZEBRA_FAIL;
172  }
173  if (first)
174  {
175  short ref;
176  memcpy(&ref, block_and_ref + sizeof(freeblock), sizeof(ref));
177  --ref;
178  memcpy(block_and_ref + sizeof(freeblock), &ref, sizeof(ref));
179  if (ref)
180  {
181  /* there is still a reference to this block.. */
182  if (bf_write(p->data_BFile[dst_type], freeblock, 0,
183  sizeof(block_and_ref), block_and_ref))
184  {
185  yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
186  return ZEBRA_FAIL;
187  }
188  return ZEBRA_OK;
189  }
190  /* the list of blocks can all be removed (ref == 0) */
191  first = 0;
192  }
193 
194  if (bf_write(p->data_BFile[dst_type], freeblock, 0, sizeof(freeblock),
195  &p->head.block_free[dst_type]))
196  {
197  yaz_log(YLOG_FATAL|YLOG_ERRNO, "write in rec_del_single");
198  return ZEBRA_FAIL;
199  }
200  p->head.block_free[dst_type] = freeblock;
201  memcpy(&freeblock, block_and_ref, sizeof(freeblock));
202 
203  p->head.block_used[dst_type]--;
204  }
205  p->head.total_bytes -= entry.size;
206  return ZEBRA_OK;
207 }
208 
210 {
211  struct record_index_entry entry;
212 
213  /* all data in entry must be reset, since it's written verbatim */
214  memset(&entry, '\0', sizeof(entry));
216  return ZEBRA_FAIL;
217 
218  entry.next = p->head.index_free;
219  entry.size = 0;
221  recindex_write_indx(p->recindex, rec_sysno_to_int(rec->sysno), &entry, sizeof(entry));
222  return ZEBRA_OK;
223 }
224 
225 static ZEBRA_RES rec_write_tmp_buf(Records p, int size, zint *sysnos)
226 {
227  struct record_index_entry entry;
228  int no_written = 0;
229  char *cptr = p->tmp_buf;
230  zint block_prev = -1, block_free;
231  int dst_type = 0;
232  int i;
233 
234  /* all data in entry must be reset, since it's written verbatim */
235  memset(&entry, '\0', sizeof(entry));
236 
237  for (i = 1; i<REC_BLOCK_TYPES; i++)
238  if (size >= p->head.block_move[i])
239  dst_type = i;
240  while (no_written < size)
241  {
242  block_free = p->head.block_free[dst_type];
243  if (block_free)
244  {
245  if (bf_read(p->data_BFile[dst_type],
246  block_free, 0, sizeof(*p->head.block_free),
247  &p->head.block_free[dst_type]) != 1)
248  {
249  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
250  ZINT_FORMAT,
251  p->data_fname[dst_type], block_free);
252  return ZEBRA_FAIL;
253  }
254  }
255  else
256  block_free = p->head.block_last[dst_type]++;
257  if (block_prev == -1)
258  {
259  entry.next = block_free*8 + dst_type;
260  entry.size = size;
261  p->head.total_bytes += size;
262  while (*sysnos > 0)
263  {
264  recindex_write_indx(p->recindex, *sysnos, &entry, sizeof(entry));
265  sysnos++;
266  }
267  }
268  else
269  {
270  memcpy(cptr, &block_free, sizeof(block_free));
271  bf_write(p->data_BFile[dst_type], block_prev, 0, 0, cptr);
272  cptr = p->tmp_buf + no_written;
273  }
274  block_prev = block_free;
275  no_written += CAST_ZINT_TO_INT(p->head.block_size[dst_type])
276  - sizeof(zint);
277  p->head.block_used[dst_type]++;
278  }
279  assert(block_prev != -1);
280  block_free = 0;
281  memcpy(cptr, &block_free, sizeof(block_free));
282  bf_write(p->data_BFile[dst_type], block_prev, 0,
283  sizeof(block_free) + (p->tmp_buf+size) - cptr, cptr);
284  return ZEBRA_OK;
285 }
286 
287 int rec_check_compression_method(int compression_method)
288 {
289  switch(compression_method)
290  {
291  case REC_COMPRESS_ZLIB:
292 #if HAVE_ZLIB_H
293  return 1;
294 #else
295  return 0;
296 #endif
297  case REC_COMPRESS_BZIP2:
298 #if HAVE_BZLIB_H
299  return 1;
300 #else
301  return 0;
302 #endif
303  case REC_COMPRESS_NONE:
304  return 1;
305  }
306  return 0;
307 }
308 
309 Records rec_open(BFiles bfs, int rw, int compression_method)
310 {
311  Records p;
312  int i, r;
313  int version;
314  ZEBRA_RES ret = ZEBRA_OK;
315 
316  p = (Records) xmalloc(sizeof(*p));
317  memset(&p->head, '\0', sizeof(p->head));
318  p->compression_method = compression_method;
319  p->rw = rw;
320  p->tmp_size = 4096;
321  p->tmp_buf = (char *) xmalloc(p->tmp_size);
322  p->compression_chunk_size = 0;
323  if (compression_method == REC_COMPRESS_BZIP2)
324  p->compression_chunk_size = 90000;
325  p->recindex = recindex_open(bfs, rw, 0 /* 1=isamb for recindex */);
326  r = recindex_read_head(p->recindex, p->tmp_buf);
327  switch (r)
328  {
329  case 0:
330  memcpy(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic));
331  yaz_snprintf(p->head.version, sizeof(p->head.version),
332  "%3d", REC_VERSION);
333  p->head.index_free = 0;
334  p->head.index_last = 1;
335  p->head.no_records = 0;
336  p->head.total_bytes = 0;
337  for (i = 0; i<REC_BLOCK_TYPES; i++)
338  {
339  p->head.block_free[i] = 0;
340  p->head.block_last[i] = 1;
341  p->head.block_used[i] = 0;
342  }
343  p->head.block_size[0] = 256;
344  p->head.block_move[0] = 0;
345  for (i = 1; i<REC_BLOCK_TYPES; i++)
346  {
347  p->head.block_size[i] = p->head.block_size[i-1] * 8;
348  p->head.block_move[i] = p->head.block_size[i] * 2;
349  }
350  if (rw)
351  {
353  &p->head, sizeof(p->head)) != ZEBRA_OK)
354  ret = ZEBRA_FAIL;
355  }
356  break;
357  case 1:
358  memcpy(&p->head, p->tmp_buf, sizeof(p->head));
359  if (memcmp(p->head.magic, REC_HEAD_MAGIC, sizeof(p->head.magic)))
360  {
361  yaz_log(YLOG_FATAL, "file %s has bad format",
363  ret = ZEBRA_FAIL;
364  }
365  version = atoi(p->head.version);
366  if (version != REC_VERSION)
367  {
368  yaz_log(YLOG_FATAL, "file %s is version %d, but version"
369  " %d is required",
370  recindex_get_fname(p->recindex), version, REC_VERSION);
371  ret = ZEBRA_FAIL;
372  }
373  break;
374  }
375  for (i = 0; i<REC_BLOCK_TYPES; i++)
376  {
377  char str[80];
378  yaz_snprintf(str, sizeof(str), "recd%c", i + 'A');
379  p->data_fname[i] = (char *) xstrdup(str);
380  p->data_BFile[i] = NULL;
381  }
382  for (i = 0; i<REC_BLOCK_TYPES; i++)
383  {
384  if (!(p->data_BFile[i] =
385  bf_open(bfs, p->data_fname[i],
386  CAST_ZINT_TO_INT(p->head.block_size[i]), rw)))
387  {
388  yaz_log(YLOG_FATAL|YLOG_ERRNO, "bf_open %s", p->data_fname[i]);
389  ret = ZEBRA_FAIL;
390  break;
391  }
392  }
393  p->cache_max = 400;
394  p->cache_cur = 0;
395  p->record_cache = (struct record_cache_entry *)
396  xmalloc(sizeof(*p->record_cache)*p->cache_max);
397  zebra_mutex_init(&p->mutex);
398  if (ret == ZEBRA_FAIL)
399  rec_close(&p);
400  return p;
401 }
402 
403 static void rec_encode_unsigned(unsigned n, unsigned char *buf, int *len)
404 {
405  (*len) = 0;
406  while (n > 127)
407  {
408  buf[*len] = 128 + (n & 127);
409  n = n >> 7;
410  (*len)++;
411  }
412  buf[*len] = n;
413  (*len)++;
414 }
415 
416 static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
417 {
418  unsigned n = 0;
419  unsigned w = 1;
420  (*len) = 0;
421 
422  while (buf[*len] > 127)
423  {
424  n += w*(buf[*len] & 127);
425  w = w << 7;
426  (*len)++;
427  }
428  n += w * buf[*len];
429  (*len)++;
430  *np = n;
431 }
432 
433 static void rec_encode_zint(zint n, unsigned char *buf, int *len)
434 {
435  (*len) = 0;
436  while (n > 127)
437  {
438  buf[*len] = (unsigned) (128 + (n & 127));
439  n = n >> 7;
440  (*len)++;
441  }
442  buf[*len] = (unsigned) n;
443  (*len)++;
444 }
445 
446 static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
447 {
448  zint n = 0;
449  zint w = 1;
450  (*len) = 0;
451 
452  while (buf[*len] > 127)
453  {
454  n += w*(buf[*len] & 127);
455  w = w << 7;
456  (*len)++;
457  }
458  n += w * buf[*len];
459  (*len)++;
460  *np = n;
461 }
462 
463 static void rec_cache_flush_block1(Records p, Record rec, Record last_rec,
464  char **out_buf, int *out_size,
465  int *out_offset)
466 {
467  int i;
468  int len;
469 
470  for (i = 0; i<REC_NO_INFO; i++)
471  {
472  if (*out_offset + CAST_ZINT_TO_INT(rec->size[i]) + 20 > *out_size)
473  {
474  int new_size = *out_offset + rec->size[i] + 65536;
475  char *np = (char *) xmalloc(new_size);
476  if (*out_offset)
477  memcpy(np, *out_buf, *out_offset);
478  xfree(*out_buf);
479  *out_size = new_size;
480  *out_buf = np;
481  }
482  if (i == 0)
483  {
485  (unsigned char *) *out_buf + *out_offset, &len);
486  (*out_offset) += len;
487  }
488  if (rec->size[i] == 0)
489  {
490  rec_encode_unsigned(1, (unsigned char *) *out_buf + *out_offset,
491  &len);
492  (*out_offset) += len;
493  }
494  else if (last_rec && rec->size[i] == last_rec->size[i] &&
495  !memcmp(rec->info[i], last_rec->info[i], rec->size[i]))
496  {
497  rec_encode_unsigned(0, (unsigned char *) *out_buf + *out_offset,
498  &len);
499  (*out_offset) += len;
500  }
501  else
502  {
504  (unsigned char *) *out_buf + *out_offset,
505  &len);
506  (*out_offset) += len;
507  memcpy(*out_buf + *out_offset, rec->info[i], rec->size[i]);
508  (*out_offset) += rec->size[i];
509  }
510  }
511 }
512 
513 static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos,
514  char *out_buf, int out_offset)
515 {
516  ZEBRA_RES ret = ZEBRA_OK;
517  if (ref_count)
518  {
519 #if HAVE_BZLIB_H
520  int i;
521 #endif
522  unsigned int csize = 0; /* indicate compression "not performed yet" */
523  char compression_method = p->compression_method;
524  switch (compression_method)
525  {
526  case REC_COMPRESS_ZLIB:
527 #if HAVE_ZLIB_H
528  csize = out_offset + (out_offset >> 6) + 620;
529  while (1)
530  {
531  int r;
532  uLongf destLen = csize;
533  rec_tmp_expand(p, csize);
534  r = compress((Bytef *) p->tmp_buf+sizeof(zint)+sizeof(short)+
535  sizeof(char),
536  &destLen, (const Bytef *) out_buf, out_offset);
537  csize = destLen;
538  if (r == Z_OK)
539  {
540  break;
541  }
542  if (r != Z_MEM_ERROR)
543  {
544  yaz_log(YLOG_WARN, "compress error: %d", r);
545  csize = 0;
546  break;
547  }
548  csize = csize * 2;
549  }
550 #endif
551  break;
552  case REC_COMPRESS_BZIP2:
553 #if HAVE_BZLIB_H
554  csize = out_offset + (out_offset >> 6) + 620;
555  rec_tmp_expand(p, csize);
556 #ifdef BZ_CONFIG_ERROR
557  i = BZ2_bzBuffToBuffCompress
558 #else
559  i = bzBuffToBuffCompress
560 #endif
561  (p->tmp_buf+sizeof(zint)+sizeof(short)+
562  sizeof(char),
563  &csize, out_buf, out_offset, 1, 0, 30);
564  if (i != BZ_OK)
565  {
566  yaz_log(YLOG_WARN, "bzBuffToBuffCompress error code=%d", i);
567  csize = 0;
568  }
569 #endif
570  break;
571  case REC_COMPRESS_NONE:
572  break;
573  }
574  if (!csize)
575  {
576  /* either no compression or compression not supported ... */
577  csize = out_offset;
578  rec_tmp_expand(p, csize);
579  memcpy(p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char),
580  out_buf, out_offset);
581  csize = out_offset;
582  compression_method = REC_COMPRESS_NONE;
583  }
584  memcpy(p->tmp_buf + sizeof(zint), &ref_count, sizeof(ref_count));
585  memcpy(p->tmp_buf + sizeof(zint)+sizeof(short),
586  &compression_method, sizeof(compression_method));
587 
588  /* -------- compression */
589  if (rec_write_tmp_buf(p, csize + sizeof(short) + sizeof(char), sysnos)
590  != ZEBRA_OK)
591  ret = ZEBRA_FAIL;
592  }
593  return ret;
594 }
595 
596 static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
597 {
598  int i;
599  short ref_count = 0;
600  Record last_rec = 0;
601  int out_size = 1000;
602  int out_offset = 0;
603  char *out_buf = (char *) xmalloc(out_size);
604  zint *sysnos = (zint *) xmalloc(sizeof(*sysnos) * (p->cache_cur + 1));
605  zint *sysnop = sysnos;
606  ZEBRA_RES ret = ZEBRA_OK;
607 
608  for (i = 0; i<p->cache_cur - saveCount; i++)
609  {
610  struct record_cache_entry *e = p->record_cache + i;
611  switch (e->flag)
612  {
613  case recordFlagNew:
614  rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
615  &out_size, &out_offset);
616  *sysnop++ = rec_sysno_to_int(e->rec->sysno);
617  ref_count++;
618  e->flag = recordFlagNop;
619  last_rec = e->rec;
620  break;
621  case recordFlagWrite:
623  != ZEBRA_OK)
624  ret = ZEBRA_FAIL;
625 
626  rec_cache_flush_block1(p, e->rec, last_rec, &out_buf,
627  &out_size, &out_offset);
628  *sysnop++ = rec_sysno_to_int(e->rec->sysno);
629  ref_count++;
630  e->flag = recordFlagNop;
631  last_rec = e->rec;
632  break;
633  case recordFlagDelete:
634  if (rec_delete_single(p, e->rec) != ZEBRA_OK)
635  ret = ZEBRA_FAIL;
636 
637  e->flag = recordFlagNop;
638  break;
639  case recordFlagNop:
640  break;
641  default:
642  break;
643  }
644  }
645 
646  *sysnop = -1;
647  rec_flush_shared(p, ref_count, sysnos, out_buf, out_offset);
648  xfree(out_buf);
649  xfree(sysnos);
650  return ret;
651 }
652 
653 static ZEBRA_RES rec_cache_flush(Records p, int saveCount)
654 {
655  int i, j;
656  ZEBRA_RES ret;
657 
658  if (saveCount >= p->cache_cur)
659  saveCount = 0;
660 
661  ret = rec_write_multiple(p, saveCount);
662 
663  for (i = 0; i<p->cache_cur - saveCount; i++)
664  {
665  struct record_cache_entry *e = p->record_cache + i;
666  rec_free(&e->rec);
667  }
668  /* i still being used ... */
669  for (j = 0; j<saveCount; j++, i++)
670  memcpy(p->record_cache+j, p->record_cache+i,
671  sizeof(*p->record_cache));
672  p->cache_cur = saveCount;
673  return ret;
674 }
675 
677  enum recordCacheFlag flag)
678 {
679  int i;
680  for (i = 0; i<p->cache_cur; i++)
681  {
682  struct record_cache_entry *e = p->record_cache + i;
683  if (e->rec->sysno == sysno)
684  {
685  if (flag != recordFlagNop && e->flag == recordFlagNop)
686  e->flag = flag;
687  return &e->rec;
688  }
689  }
690  return NULL;
691 }
692 
694 {
695  struct record_cache_entry *e;
696  ZEBRA_RES ret = ZEBRA_OK;
697 
698  if (p->cache_cur == p->cache_max)
699  ret = rec_cache_flush(p, 1);
700  else if (p->cache_cur > 0)
701  {
702  int i, j;
703  int used = 0;
704  for (i = 0; i<p->cache_cur; i++)
705  {
706  Record r = (p->record_cache + i)->rec;
707  for (j = 0; j<REC_NO_INFO; j++)
708  used += r->size[j];
709  }
710  if (used > p->compression_chunk_size)
711  ret = rec_cache_flush(p, 1);
712  }
713  assert(p->cache_cur < p->cache_max);
714 
715  e = p->record_cache + (p->cache_cur)++;
716  e->flag = flag;
717  e->rec = rec_cp(rec);
718  return ret;
719 }
720 
722 {
723  Records p = *pp;
724  int i;
725  ZEBRA_RES ret = ZEBRA_OK;
726 
727  if (!p)
728  return ret;
729 
731  if (rec_cache_flush(p, 0) != ZEBRA_OK)
732  ret = ZEBRA_FAIL;
733 
734  xfree(p->record_cache);
735 
736  if (p->rw)
737  {
738  if (recindex_write_head(p->recindex, &p->head, sizeof(p->head)) != ZEBRA_OK)
739  ret = ZEBRA_FAIL;
740  }
741 
743 
744  for (i = 0; i<REC_BLOCK_TYPES; i++)
745  {
746  if (p->data_BFile[i])
747  bf_close(p->data_BFile[i]);
748  xfree(p->data_fname[i]);
749  }
750  xfree(p->tmp_buf);
751  xfree(p);
752  *pp = NULL;
753  return ret;
754 }
755 
756 static Record rec_get_int(Records p, zint sysno)
757 {
758  int i, in_size, r;
759  Record rec, *recp;
760  struct record_index_entry entry;
761  zint freeblock;
762  int dst_type;
763  char *nptr, *cptr;
764  char *in_buf = 0;
765  char *bz_buf = 0;
766  char compression_method;
767 
768  assert(sysno > 0);
769  assert(p);
770 
771  if ((recp = rec_cache_lookup(p, sysno, recordFlagNop)))
772  return rec_cp(*recp);
773 
774  if (recindex_read_indx(p->recindex, rec_sysno_to_int(sysno), &entry, sizeof(entry), 1) < 1)
775  return NULL; /* record is not there! */
776 
777  if (!entry.size)
778  return NULL; /* record is deleted */
779 
780  dst_type = (int) (entry.next & 7);
781  assert(dst_type < REC_BLOCK_TYPES);
782  freeblock = entry.next / 8;
783 
784  assert(freeblock > 0);
785 
786  rec_tmp_expand(p, entry.size);
787 
788  cptr = p->tmp_buf;
789  r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
790  if (r < 0)
791  return 0;
792  memcpy(&freeblock, cptr, sizeof(freeblock));
793 
794  while (freeblock)
795  {
796  zint tmp;
797 
798  cptr += p->head.block_size[dst_type] - sizeof(freeblock);
799 
800  memcpy(&tmp, cptr, sizeof(tmp));
801  r = bf_read(p->data_BFile[dst_type], freeblock, 0, 0, cptr);
802  if (r < 0)
803  return 0;
804  memcpy(&freeblock, cptr, sizeof(freeblock));
805  memcpy(cptr, &tmp, sizeof(tmp));
806  }
807 
808  rec = (Record) xmalloc(sizeof(*rec));
809  rec->sysno = sysno;
810  memcpy(&compression_method, p->tmp_buf + sizeof(zint) + sizeof(short),
811  sizeof(compression_method));
812  in_buf = p->tmp_buf + sizeof(zint) + sizeof(short) + sizeof(char);
813  in_size = entry.size - sizeof(short) - sizeof(char);
814  switch (compression_method)
815  {
816  case REC_COMPRESS_ZLIB:
817 #if HAVE_ZLIB_H
818  if (1)
819  {
820  unsigned int bz_size = entry.size * 20 + 100;
821  while (1)
822  {
823  uLongf destLen = bz_size;
824  bz_buf = (char *) xmalloc(bz_size);
825  i = uncompress((Bytef *) bz_buf, &destLen,
826  (const Bytef *) in_buf, in_size);
827  if (i == Z_OK)
828  {
829  bz_size = destLen;
830  break;
831  }
832  yaz_log(YLOG_LOG, "failed");
833  xfree(bz_buf);
834  bz_size *= 2;
835  }
836  in_buf = bz_buf;
837  in_size = bz_size;
838  }
839 #else
840  yaz_log(YLOG_FATAL, "cannot decompress record(s) in ZLIB format");
841  return 0;
842 #endif
843  break;
844  case REC_COMPRESS_BZIP2:
845 #if HAVE_BZLIB_H
846  if (1)
847  {
848  unsigned int bz_size = entry.size * 20 + 100;
849  while (1)
850  {
851  bz_buf = (char *) xmalloc(bz_size);
852 #ifdef BZ_CONFIG_ERROR
853  i = BZ2_bzBuffToBuffDecompress
854 #else
855  i = bzBuffToBuffDecompress
856 #endif
857  (bz_buf, &bz_size, in_buf, in_size, 0, 0);
858  if (i == BZ_OK)
859  break;
860  yaz_log(YLOG_LOG, "failed");
861  xfree(bz_buf);
862  bz_size *= 2;
863  }
864  in_buf = bz_buf;
865  in_size = bz_size;
866  }
867 #else
868  yaz_log(YLOG_FATAL, "cannot decompress record(s) in BZIP2 format");
869  return 0;
870 #endif
871  break;
872  case REC_COMPRESS_NONE:
873  break;
874  }
875  for (i = 0; i<REC_NO_INFO; i++)
876  rec->info[i] = 0;
877 
878  nptr = in_buf; /* skip ref count */
879  while (nptr < in_buf + in_size)
880  {
881  zint this_sysno;
882  int len;
883  rec_decode_zint(&this_sysno, (unsigned char *) nptr, &len);
884  nptr += len;
885 
886  for (i = 0; i < REC_NO_INFO; i++)
887  {
888  unsigned int this_size;
889  rec_decode_unsigned(&this_size, (unsigned char *) nptr, &len);
890  nptr += len;
891 
892  if (this_size == 0)
893  continue;
894  rec->size[i] = this_size-1;
895 
896  if (rec->size[i])
897  {
898  rec->info[i] = nptr;
899  nptr += rec->size[i];
900  }
901  else
902  rec->info[i] = NULL;
903  }
904  if (this_sysno == rec_sysno_to_int(sysno))
905  break;
906  }
907  for (i = 0; i<REC_NO_INFO; i++)
908  {
909  if (rec->info[i] && rec->size[i])
910  {
911  char *np = xmalloc(rec->size[i]+1);
912  memcpy(np, rec->info[i], rec->size[i]);
913  np[rec->size[i]] = '\0';
914  rec->info[i] = np;
915  }
916  else
917  {
918  assert(rec->info[i] == 0);
919  assert(rec->size[i] == 0);
920  }
921  }
922  xfree(bz_buf);
923  if (rec_cache_insert(p, rec, recordFlagNop) != ZEBRA_OK)
924  return 0;
925  return rec;
926 }
927 
929 {
930  Record rec;
931  zebra_mutex_lock(&p->mutex);
932 
933  rec = rec_get_int(p, sysno);
935  return rec;
936 }
937 
939 {
940  return rec_get(p, rec_sysno_to_ext(1));
941 }
942 
944 {
945  Record next = 0;
946  zint next_sysno_int = rec_sysno_to_int(rec->sysno);
947 
948  while (!next)
949  {
950  ++next_sysno_int;
951  if (next_sysno_int == p->head.index_last)
952  break;
953  next = rec_get(p, rec_sysno_to_ext(next_sysno_int));
954  }
955  return next;
956 }
957 
959 {
960  int i;
961  zint sysno;
962  Record rec;
963 
964  assert(p);
965  rec = (Record) xmalloc(sizeof(*rec));
966  if (1 || p->head.index_free == 0)
967  sysno = (p->head.index_last)++;
968  else
969  {
970  struct record_index_entry entry;
971 
972  if (recindex_read_indx(p->recindex, p->head.index_free, &entry, sizeof(entry), 0) < 1)
973  {
974  xfree(rec);
975  return 0;
976  }
977  sysno = p->head.index_free;
978  p->head.index_free = entry.next;
979  }
980  (p->head.no_records)++;
981  rec->sysno = rec_sysno_to_ext(sysno);
982  for (i = 0; i < REC_NO_INFO; i++)
983  {
984  rec->info[i] = NULL;
985  rec->size[i] = 0;
986  }
988  return rec;
989 }
990 
992 {
993  Record rec;
994  zebra_mutex_lock(&p->mutex);
995 
996  rec = rec_new_int(p);
998  return rec;
999 }
1000 
1002 {
1003  Record *recp;
1004  ZEBRA_RES ret = ZEBRA_OK;
1005 
1006  zebra_mutex_lock(&p->mutex);
1007  (p->head.no_records)--;
1008  if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagDelete)))
1009  {
1010  rec_free(recp);
1011  *recp = *recpp;
1012  }
1013  else
1014  {
1015  ret = rec_cache_insert(p, *recpp, recordFlagDelete);
1016  rec_free(recpp);
1017  }
1019  *recpp = NULL;
1020  return ret;
1021 }
1022 
1024 {
1025  Record *recp;
1026  ZEBRA_RES ret = ZEBRA_OK;
1027 
1028  zebra_mutex_lock(&p->mutex);
1029  if ((recp = rec_cache_lookup(p, (*recpp)->sysno, recordFlagWrite)))
1030  {
1031  rec_free(recp);
1032  *recp = *recpp;
1033  }
1034  else
1035  {
1036  ret = rec_cache_insert(p, *recpp, recordFlagWrite);
1037  rec_free(recpp);
1038  }
1040  *recpp = NULL;
1041  return ret;
1042 }
1043 
1044 void rec_free(Record *recpp)
1045 {
1046  int i;
1047 
1048  if (!*recpp)
1049  return ;
1050  for (i = 0; i < REC_NO_INFO; i++)
1051  xfree((*recpp)->info[i]);
1052  xfree(*recpp);
1053  *recpp = NULL;
1054 }
1055 
1057 {
1058  Record n;
1059  int i;
1060 
1061  n = (Record) xmalloc(sizeof(*n));
1062  n->sysno = rec->sysno;
1063  for (i = 0; i < REC_NO_INFO; i++)
1064  if (!rec->info[i])
1065  {
1066  n->info[i] = NULL;
1067  n->size[i] = 0;
1068  }
1069  else
1070  {
1071  n->size[i] = rec->size[i];
1072  n->info[i] = (char *) xmalloc(rec->size[i]+1);
1073  memcpy(n->info[i], rec->info[i], rec->size[i]);
1074  n->info[i][rec->size[i]] = '\0';
1075  }
1076  return n;
1077 }
1078 
1079 
1080 char *rec_strdup(const char *s, size_t *len)
1081 {
1082  char *p;
1083 
1084  if (!s)
1085  {
1086  *len = 0;
1087  return NULL;
1088  }
1089  *len = strlen(s)+1;
1090  p = (char *) xmalloc(*len);
1091  strcpy(p, s);
1092  return p;
1093 }
1094 
1095 void rec_prstat(Records records, int verbose)
1096 {
1097  int i;
1098  zint total_bytes = 0;
1099 
1100  yaz_log (YLOG_LOG,
1101  "Total records %8" ZINT_FORMAT0,
1102  records->head.no_records);
1103 
1104  for (i = 0; i< REC_BLOCK_TYPES; i++)
1105  {
1106  yaz_log (YLOG_LOG, "Record blocks of size "ZINT_FORMAT,
1107  records->head.block_size[i]);
1108  yaz_log (YLOG_LOG,
1109  " Used/Total/Bytes used "
1111  records->head.block_used[i], records->head.block_last[i]-1,
1112  records->head.block_used[i] * records->head.block_size[i]);
1113  total_bytes +=
1114  records->head.block_used[i] * records->head.block_size[i];
1115 
1116  yaz_log(YLOG_LOG, " Block Last " ZINT_FORMAT, records->head.block_last[i]);
1117  if (verbose)
1118  { /* analyse free lists */
1119  zint no_free = 0;
1120  zint block_free = records->head.block_free[i];
1121  WRBUF w = wrbuf_alloc();
1122  while (block_free)
1123  {
1124  zint nblock;
1125  no_free++;
1126  wrbuf_printf(w, " " ZINT_FORMAT, block_free);
1127  if (bf_read(records->data_BFile[i],
1128  block_free, 0, sizeof(nblock), &nblock) != 1)
1129  {
1130  yaz_log(YLOG_FATAL|YLOG_ERRNO, "read in %s at free block "
1131  ZINT_FORMAT,
1132  records->data_fname[i], block_free);
1133  break;
1134  }
1135  block_free = nblock;
1136  }
1137  yaz_log (YLOG_LOG,
1138  " Number in free list %8" ZINT_FORMAT0, no_free);
1139  if (no_free)
1140  yaz_log(YLOG_LOG, "%s", wrbuf_cstr(w));
1141  wrbuf_destroy(w);
1142  }
1143  }
1144  yaz_log (YLOG_LOG,
1145  "Total size of record index in bytes %8" ZINT_FORMAT0,
1146  records->head.total_bytes);
1147  yaz_log (YLOG_LOG,
1148  "Total size with overhead %8" ZINT_FORMAT0,
1149  total_bytes);
1150 }
1151 
1152 /*
1153  * Local variables:
1154  * c-basic-offset: 4
1155  * c-file-style: "Stroustrup"
1156  * indent-tabs-mode: nil
1157  * End:
1158  * vim: shiftwidth=4 tabstop=8 expandtab
1159  */
1160 
Zebra Block File Layer.
int bf_read(BFile bf, zint no, int offset, int nbytes, void *buf)
read from block file (may call exit)
Definition: bfile.c:205
void bf_close(BFile bf)
closes a Block file (may call exit)
Definition: bfile.c:139
BFile bf_open(BFiles bfs, const char *name, int block_size, int wflag)
opens and returns a Block file handle
Definition: bfile.c:150
int bf_write(BFile bf, zint no, int offset, int nbytes, const void *buf)
writes block of bytes to file (may call exit)
Definition: bfile.c:232
int recindex_read_indx(recindex_t p, zint sysno, void *buf, int itemsize, int ignoreError)
Definition: recindex.c:225
void recindex_write_indx(recindex_t p, zint sysno, void *buf, int itemsize)
Definition: recindex.c:312
recindex_t recindex_open(BFiles bfs, int rw, int use_isamb)
opens record index handle
Definition: recindex.c:134
void recindex_close(recindex_t p)
closes record index handle
Definition: recindex.c:182
const char * recindex_get_fname(recindex_t p)
Definition: recindex.c:203
ZEBRA_RES recindex_write_head(recindex_t p, const void *buf, size_t len)
Definition: recindex.c:208
int recindex_read_head(recindex_t p, void *buf)
Definition: recindex.c:198
struct records_info * Records
Definition: recindex.h:41
struct record_info * Record
#define REC_COMPRESS_BZIP2
BZIP2 compression (slow and requires big chunks)
Definition: recindex.h:113
#define REC_COMPRESS_ZLIB
zlib compression (faster and works off small chunks)
Definition: recindex.h:115
#define REC_COMPRESS_NONE
No compression ("none")
Definition: recindex.h:111
#define REC_NO_INFO
Definition: recindex.h:29
static ZEBRA_RES rec_cache_flush(Records p, int saveCount)
Definition: records.c:653
#define FAKE_OFFSET
Definition: records.c:115
static void rec_tmp_expand(Records p, int size)
Definition: records.c:136
static void rec_encode_unsigned(unsigned n, unsigned char *buf, int *len)
Definition: records.c:403
static void rec_cache_flush_block1(Records p, Record rec, Record last_rec, char **out_buf, int *out_size, int *out_offset)
Definition: records.c:463
recordCacheFlag
Definition: records.c:97
@ recordFlagNew
Definition: records.c:97
@ recordFlagDelete
Definition: records.c:98
@ recordFlagNop
Definition: records.c:97
@ recordFlagWrite
Definition: records.c:97
static void rec_decode_zint(zint *np, unsigned char *buf, int *len)
Definition: records.c:446
static ZEBRA_RES rec_write_tmp_buf(Records p, int size, zint *sysnos)
Definition: records.c:225
zint rec_sysno_to_int(zint sysno)
Definition: records.c:130
Record rec_new(Records p)
creates new record (to be written to file storage)
Definition: records.c:991
#define USUAL_RANGE
Definition: records.c:116
static zint rec_sysno_to_ext(zint sysno)
Definition: records.c:124
static ZEBRA_RES rec_write_multiple(Records p, int saveCount)
Definition: records.c:596
Records rec_open(BFiles bfs, int rw, int compression_method)
opens records system
Definition: records.c:309
Record rec_get(Records p, zint sysno)
gets record - with given system number
Definition: records.c:928
static Record rec_get_int(Records p, zint sysno)
Definition: records.c:756
ZEBRA_RES rec_put(Records p, Record *recpp)
puts record (writes into file storage)
Definition: records.c:1023
static Record rec_new_int(Records p)
Definition: records.c:958
static Record * rec_cache_lookup(Records p, zint sysno, enum recordCacheFlag flag)
Definition: records.c:676
static void rec_decode_unsigned(unsigned *np, unsigned char *buf, int *len)
Definition: records.c:416
static ZEBRA_RES rec_cache_insert(Records p, Record rec, enum recordCacheFlag flag)
Definition: records.c:693
static ZEBRA_RES rec_release_blocks(Records p, zint sysno)
Definition: records.c:148
#define REC_HEAD_MAGIC
Definition: records.c:56
ZEBRA_RES rec_close(Records *pp)
Definition: records.c:721
char * rec_strdup(const char *s, size_t *len)
Definition: records.c:1080
void rec_free(Record *recpp)
frees record (from memory)
Definition: records.c:1044
Record rec_get_next(Records p, Record rec)
gets next record - with given records
Definition: records.c:943
#define REC_BLOCK_TYPES
Definition: records.c:55
static ZEBRA_RES rec_delete_single(Records p, Record rec)
Definition: records.c:209
ZEBRA_RES rec_del(Records p, Record *recpp)
marks record for deletion (on file storage)
Definition: records.c:1001
void rec_prstat(Records records, int verbose)
Definition: records.c:1095
#define REC_VERSION
Definition: records.c:57
Record rec_cp(Record rec)
Definition: records.c:1056
int rec_check_compression_method(int compression_method)
check whether a compression method is supported
Definition: records.c:287
static ZEBRA_RES rec_flush_shared(Records p, short ref_count, zint *sysnos, char *out_buf, int out_offset)
Definition: records.c:513
Record rec_get_root(Records p)
gets root record
Definition: records.c:938
static void rec_encode_zint(zint n, unsigned char *buf, int *len)
Definition: records.c:433
Definition: records.c:100
enum recordCacheFlag flag
Definition: records.c:102
Record rec
Definition: records.c:101
Definition: recindex.c:42
int size
Definition: recindex.c:44
zint next
Definition: recindex.c:43
char * info[REC_NO_INFO]
Definition: recindex.h:34
size_t size[REC_NO_INFO]
Definition: recindex.h:35
zint sysno
Definition: recindex.h:32
zint block_free[REC_BLOCK_TYPES]
Definition: records.c:84
zint block_size[REC_BLOCK_TYPES]
Definition: records.c:83
zint block_move[REC_BLOCK_TYPES]
Definition: records.c:87
zint block_used[REC_BLOCK_TYPES]
Definition: records.c:86
zint block_last[REC_BLOCK_TYPES]
Definition: records.c:85
struct record_cache_entry * record_cache
Definition: records.c:71
int compression_method
Definition: records.c:61
int cache_size
Definition: records.c:72
Zebra_mutex mutex
Definition: records.c:78
int compression_chunk_size
Definition: records.c:76
int tmp_size
Definition: records.c:69
char * data_fname[REC_BLOCK_TYPES]
Definition: records.c:65
struct records_info::records_head head
int cache_cur
Definition: records.c:73
char * tmp_buf
Definition: records.c:68
BFile data_BFile[REC_BLOCK_TYPES]
Definition: records.c:66
recindex_t recindex
Definition: records.c:63
int cache_max
Definition: records.c:74
long zint
Zebra integer.
Definition: util.h:66
#define ZEBRA_FAIL
Definition: util.h:81
#define ZINT_FORMAT
Definition: util.h:72
#define CAST_ZINT_TO_INT(x)
Definition: util.h:96
#define ZEBRA_OK
Definition: util.h:82
#define ZINT_FORMAT0
Definition: util.h:67
short ZEBRA_RES
Common return type for Zebra API.
Definition: util.h:80
int zebra_mutex_unlock(Zebra_mutex *p)
Definition: zebra-lock.c:74
int zebra_mutex_init(Zebra_mutex *p)
Definition: zebra-lock.c:31
int zebra_mutex_lock(Zebra_mutex *p)
Definition: zebra-lock.c:59
int zebra_mutex_destroy(Zebra_mutex *p)
Definition: zebra-lock.c:43