BitMagic-C++
bmsparsevec_serial.h
Go to the documentation of this file.
1 #ifndef BMSPARSEVEC_SERIAL__H__INCLUDED__
2 #define BMSPARSEVEC_SERIAL__H__INCLUDED__
3 /*
4 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5 
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9 
10  http://www.apache.org/licenses/LICENSE-2.0
11 
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 
18 For more information please visit: http://bitmagic.io
19 */
20 
21 /*! \file bmsparsevec_serial.h
22  \brief Serialization for sparse_vector<>
23 */
24 
25 
26 #ifndef BM__H__INCLUDED__
27 // BitMagic utility headers do not include main "bm.h" declaration
28 // #include "bm.h" or "bm64.h" explicitly
29 # error missing include (bm.h or bm64.h)
30 #endif
31 
32 #include "bmsparsevec.h"
33 #include "bmserial.h"
34 #include "bmdef.h"
35 
36 namespace bm
37 {
38 
39 /** \defgroup svserial Sparse vector serialization
40  Sparse vector serialization
41  \ingroup svector
42  */
43 
44 
45 /*!
46  \brief layout class for serialization buffer structure
47 
48  Class keeps a memory block sized for the target sparse vector BLOB.
49  This class also provides acess to bit-plane memory, so it becomes possible
50  to use parallel storage methods to save bit-plains into
51  different storage shards.
52 
53  \ingroup svserial
54 
55 */
56 template<class SV>
58 {
59  typedef typename SV::value_type value_type;
60  typedef typename SV::bvector_type bvector_type;
62 
64 
66 
67  /*!
68  \brief resize capacity
69  \param capacity - new capacity
70  \return new buffer or 0 if failed
71  */
72  unsigned char* reserve(size_t capacity)
73  {
74  if (capacity == 0)
75  {
76  freemem();
77  return 0;
78  }
79  buf_.reinit(capacity);
80  return buf_.data();
81  }
82 
83  /// return current serialized size
84  size_t size() const { return buf_.size(); }
85 
86  /// Set new serialized size
87  void resize(size_t ssize) { buf_.resize(ssize); }
88 
89  /// return serialization buffer capacity
90  size_t capacity() const { return buf_.capacity(); }
91 
92  /// free memory
93  void freemem() { buf_.release(); }
94 
95  /// Set plain output pointer and size
96  void set_plain(unsigned i, unsigned char* ptr, size_t buf_size)
97  {
98  plain_ptrs_[i] = ptr;
99  plane_size_[i] = buf_size;
100  }
101 
102  /// Get plain pointer
103  const unsigned char* get_plain(unsigned i) const { return plain_ptrs_[i]; }
104 
105  /// Return serialization buffer pointer
106  const unsigned char* buf() const { return buf_.buf(); }
107 
108 private:
110  void operator=(const sparse_vector_serial_layout&);
111 protected:
112  buffer_type buf_; ///< serialization buffer
113  unsigned char* plain_ptrs_[SV::sv_plains]; ///< pointers on serialized bit-plains
114  size_t plane_size_[SV::sv_plains]; ///< serialized plain size
115 };
116 
117 // -------------------------------------------------------------------------
118 
119 /*!
120  \brief Serialize sparse vector into a memory buffer(s) structure
121 
122  Serialization format:
123 
124  | HEADER | BIT-VECTORS ... | REMAP_MATRIX
125 
126  Header structure:
127  -----------------
128  BYTE+BYTE: Magic-signature 'BM' or 'BC' (c-compressed)
129  BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
130  {
131  BYTE : Number of Bit-vector plains (total) (non-zero when < 255 plains)
132  |
133  BYTE: zero - flag of large plain matrix
134  INT64: Nnmber of bit-vector plains
135  }
136  INT64: Vector size
137  INT64: Offset of plain 0 from the header start (value 0 means plain is empty)
138  INT64: Offset of plain 1 from
139  ...
140  INT32: reserved
141 
142 Bit-vectors:
143 ------------
144  Based on current bit-vector serialization
145 
146 Remap Matrix:
147  SubHeader | Matrix BLOB
148 
149  sub-header:
150  BYTE: 'R' (remapping) or 'N' (no remapping)
151  N - means no other info is saved on the stream
152  INT64: remap matrix size
153 
154  \ingroup svserial
155 */
156 template<typename SV>
158 {
159 public:
160  typedef typename SV::bvector_type bvector_type;
163  typedef typename SV::value_type value_type;
164  typedef typename SV::size_type size_type;
166 public:
168 
169  /*!
170  \brief Serialize sparse vector into a memory buffer(s) structure
171 
172  \param sv - sparse vector to serialize
173  \param sv_layout - buffer structure to keep the result
174  \param temp_block - temporary buffer
175  (allocate with BM_DECLARE_TEMP_BLOCK(x) for speed)
176  \param bv_serialization_flags - bit-vector serialization flags
177  as defined in bm::serialization_flags
178  */
179  void serialize(const SV& sv,
181 private:
183  sparse_vector_serializer& operator=(const sparse_vector_serializer&) = delete;
184 protected:
186 };
187 
188 /**
189  sparse vector de-serializer
190 */
191 template<typename SV>
193 {
194 public:
195  typedef typename SV::bvector_type bvector_type;
198  typedef typename SV::value_type value_type;
199  typedef typename SV::size_type size_type;
201 public:
203 
204  void deserialize(SV& sv, const unsigned char* buf);
205 
206 private:
208  sparse_vector_deserializer& operator=(const sparse_vector_deserializer&) = delete;
209 protected:
211 
212 };
213 
214 
215 
216 /*!
217  \brief Serialize sparse vector into a memory buffer(s) structure
218 
219  \param sv - sparse vector to serialize
220  \param sv_layout - buffer structure to keep the result
221  \param temp_block - temporary buffer
222  (allocate with BM_DECLARE_TEMP_BLOCK(x) for speed)
223  \param bv_serialization_flags - bit-vector serialization flags
224  as defined in bm::serialization_flags
225 
226  \ingroup svserial
227 
228  @sa serialization_flags
229 */
230 template<class SV>
232  const SV& sv,
234  bm::word_t* temp_block = 0)
235 {
236  (void)temp_block;
237  bm::sparse_vector_serializer<SV> sv_serializer;
238  sv_serializer.serialize(sv, sv_layout);
239 }
240 
241 // -------------------------------------------------------------------------
242 
243 /*!
244  \brief Deserialize sparse vector
245  \param sv - target sparse vector
246  \param buf - source memory buffer
247  \param temp_block - temporary block buffer to avoid re-allocations
248 
249  \return 0 (error processing via std::logic_error)
250 
251  \ingroup svector
252 */
253 template<class SV>
255  const unsigned char* buf,
256  bm::word_t* temp_block=0)
257 {
258  (void)temp_block;
259  bm::sparse_vector_deserializer<SV> sv_deserializer;
260  sv_deserializer.deserialize(sv, buf);
261  return 0;
262 }
263 
264 // -------------------------------------------------------------------------
265 
266 /**
267  Seriaizer for compressed collections
268 */
269 template<class CBC>
271 {
272 public:
274  typedef typename CBC::bvector_type bvector_type;
275  typedef typename CBC::buffer_type buffer_type;
276  typedef typename CBC::statistics statistics_type;
277  typedef typename CBC::address_resolver_type address_resolver_type;
278 
279 public:
280  void serialize(const CBC& buffer_coll,
281  buffer_type& buf,
282  bm::word_t* temp_block = 0);
283 };
284 
285 /**
286  Deseriaizer for compressed collections
287 */
288 template<class CBC>
290 {
291 public:
293  typedef typename CBC::bvector_type bvector_type;
294  typedef typename CBC::buffer_type buffer_type;
295  typedef typename CBC::statistics statistics_type;
296  typedef typename CBC::address_resolver_type address_resolver_type;
297  typedef typename CBC::container_type container_type;
298 
299 public:
300  int deserialize(CBC& buffer_coll,
301  const unsigned char* buf,
302  bm::word_t* temp_block=0);
303 };
304 
305 
306 // -------------------------------------------------------------------------
307 
308 /**
309  \brief Serialize compressed collection into memory buffer
310 
311 Serialization format:
312 
313 
314 <pre>
315  | MAGIC_HEADER | ADDRESS_BITVECTROR | LIST_OF_BUFFER_SIZES | BUFFER(s)
316 
317  MAGIC_HEADER:
318  BYTE+BYTE: Magic-signature 'BM' or 'BC'
319  BYTE : Byte order ( 0 - Big Endian, 1 - Little Endian)
320 
321  ADDRESS_BITVECTROR:
322  INT64: address bit-vector size
323  <memblock>: serialized address bit-vector
324 
325  LIST_OF_BUFFER_SIZES:
326  INT64 - buffer sizes count
327  INT32 - buffer size 0
328  INT32 - buffer size 1
329  ...
330 
331  BUFFERS:
332  <memblock>: block0
333  <memblock>: block1
334  ...
335 
336 </pre>
337 */
338 
339 template<class CBC>
341  buffer_type& buf,
342  bm::word_t* temp_block)
343 {
344  statistics_type st;
345  buffer_coll.calc_stat(&st);
346 
347  buf.resize(st.max_serialize_mem);
348 
349  // ptr where bit-plains start
350  unsigned char* buf_ptr = buf.data();
351 
352  bm::encoder enc(buf.data(), buf.capacity());
354  enc.put_8('B');
355  enc.put_8('C');
356  enc.put_8((unsigned char)bo);
357 
358  unsigned char* mbuf1 = enc.get_pos(); // bookmark position
359  enc.put_64(0); // address vector size (reservation)
360 
361  buf_ptr = enc.get_pos();
362 
363  const address_resolver_type& addr_res = buffer_coll.resolver();
364  const bvector_type& bv = addr_res.get_bvector();
365  {
366  bm::serializer<bvector_type > bvs(temp_block);
367  bvs.gap_length_serialization(false);
368 
369  size_t addr_bv_size = bvs.serialize(bv, buf_ptr, buf.size());
370  buf_ptr += addr_bv_size;
371 
372  enc.set_pos(mbuf1); // rewind to bookmark
373  enc.put_64(addr_bv_size); // save the address vector size
374  }
375  enc.set_pos(buf_ptr); // restore stream position
376  size_t coll_size = buffer_coll.size();
377 
378  enc.put_64(coll_size);
379 
380  // pass 1 (save buffer sizes)
381  {
382  for (unsigned i = 0; i < buffer_coll.size(); ++i)
383  {
384  const buffer_type& cbuf = buffer_coll.get(i);
385  size_t sz = cbuf.size();
386  enc.put_64(sz);
387  } // for i
388  }
389  // pass 2 (save buffers)
390  {
391  for (unsigned i = 0; i < buffer_coll.size(); ++i)
392  {
393  const buffer_type& cbuf = buffer_coll.get(i);
394  size_t sz = cbuf.size();
395  enc.memcpy(cbuf.buf(), sz);
396  } // for i
397  }
398  buf.resize(enc.size());
399 }
400 
401 // -------------------------------------------------------------------------
402 template<class CBC>
404  CBC& buffer_coll,
405  const unsigned char* buf,
406  bm::word_t* temp_block)
407 {
408  // TODO: implement correct processing of byte-order corect deserialization
409  // ByteOrder bo_current = globals<true>::byte_order();
410 
411  bm::decoder dec(buf);
412  unsigned char h1 = dec.get_8();
413  unsigned char h2 = dec.get_8();
414 
415  BM_ASSERT(h1 == 'B' && h2 == 'C');
416  if (h1 != 'B' && h2 != 'C') // no magic header? issue...
417  {
418  return -1;
419  }
420  //unsigned char bv_bo =
421  dec.get_8();
422 
423  // -----------------------------------------
424  // restore address resolver
425  //
426  bm::id64_t addr_bv_size = dec.get_64();
427 
428  const unsigned char* bv_buf_ptr = dec.get_pos();
429 
430  address_resolver_type& addr_res = buffer_coll.resolver();
431  bvector_type& bv = addr_res.get_bvector();
432  bv.clear();
433 
434  bm::deserialize(bv, bv_buf_ptr, temp_block);
435  addr_res.sync();
436 
437  typename bvector_type::size_type addr_cnt = bv.count();
438  dec.seek((int)addr_bv_size);
439 
440  // -----------------------------------------
441  // read buffer sizes
442  //
443  bm::id64_t coll_size = dec.get_64();
444  if (coll_size != addr_cnt)
445  {
446  return -2; // buffer size collection does not match address vector
447  }
448 
449  typedef std::vector<unsigned>::size_type vect_size_type;
450  std::vector<bm::id64_t> buf_size_vec;
451  buf_size_vec.resize(vect_size_type(coll_size));
452  {
453  for (unsigned i = 0; i < coll_size; ++i)
454  {
455  bm::id64_t sz = dec.get_64();
456  buf_size_vec[i] = sz;
457  } // for i
458  }
459 
460  {
461  container_type& buf_vect = buffer_coll.container();
462  buf_vect.resize(vect_size_type(coll_size));
463  for (unsigned i = 0; i < coll_size; ++i)
464  {
465  bm::id64_t sz = buf_size_vec[i];
466  buffer_type& b = buf_vect.at(i);
467  b.resize(sz);
468  dec.memcpy(b.data(), sz);
469  } // for i
470  }
471  buffer_coll.sync();
472  return 0;
473 }
474 
475 // -------------------------------------------------------------------------
476 //
477 // -------------------------------------------------------------------------
478 
479 template<typename SV>
481 {
482  bvs_.gap_length_serialization(false);
483 }
484 
485 // -------------------------------------------------------------------------
486 
487 template<typename SV>
490 {
491  typename SV::statistics sv_stat;
492  sv.calc_stat(&sv_stat);
493  unsigned char* buf = sv_layout.reserve(sv_stat.max_serialize_mem);
494 
495  bm::encoder enc(buf, (unsigned)sv_layout.capacity());
496  unsigned plains = sv.stored_plains();
497 
498  // header size in bytes
499  unsigned h_size = 1 + 1 + // "BM" or "BC" (magic header)
500  1 + // byte-order
501  1 + // number of bit-plains (for vector)
502  8 + // size (internal 64-bit)
503  (8 * plains) + // offsets of all plains
504  4; // reserve
505  // for large plain matrixes
506  {
507  h_size += 1 + // version number
508  8; // number of plains (64-bit)
509  }
510 
511  // -----------------------------------------------------
512  // Serialize all bvector plains
513  //
514 
515  unsigned char* buf_ptr = buf + h_size; // ptr where plains start (start+hdr)
516 
517  unsigned i;
518  for (i = 0; i < plains; ++i)
519  {
520  typename SV::bvector_type_const_ptr bv = sv.get_plain(i);
521  if (!bv) // empty plain
522  {
523  sv_layout.set_plain(i, 0, 0);
524  continue;
525  }
526 
527  size_t buf_size =
528  bvs_.serialize(*bv, buf_ptr, sv_stat.max_serialize_mem);
529 
530  sv_layout.set_plain(i, buf_ptr, buf_size);
531  buf_ptr += buf_size;
532  if (sv_stat.max_serialize_mem > buf_size)
533  {
534  sv_stat.max_serialize_mem -= buf_size;
535  }
536  else
537  {
538  BM_ASSERT(0); // TODO: throw an exception here
539  }
540  } // for i
541 
542  // -----------------------------------------------------
543  // serialize the re-map matrix
544  //
545  if (bm::conditional<SV::is_remap_support::value>::test()) // test remapping trait
546  {
547  bm::encoder enc_m(buf_ptr, sv_stat.max_serialize_mem);
548  if (sv.is_remap())
549  {
550  bm::id64_t remap_size = sv.remap_size();
551  const unsigned char* matrix_buf = sv.get_remap_buffer();
552  BM_ASSERT(matrix_buf);
553  BM_ASSERT(remap_size);
554 
555  enc_m.put_8('R');
556  enc_m.put_64(remap_size);
557  enc_m.memcpy(matrix_buf, remap_size);
558  enc_m.put_8('E'); // end of matrix (integrity check token)
559  }
560  else
561  {
562  enc_m.put_8('N');
563  }
564  buf_ptr += enc_m.size(); // add mattrix encoded data size
565  }
566 
567  sv_layout.resize(size_t(buf_ptr - buf)); // set the true occupied size
568 
569  // -----------------------------------------------------
570  // save the header
571  //
573 
574  enc.put_8('B'); // magic header 'BM' - bit matrix 'BC' - bit compressed
575  if (sv.is_compressed())
576  enc.put_8('C');
577  else
578  enc.put_8('M');
579 
580  enc.put_8((unsigned char)bo); // byte order
581 
582  unsigned char matr_s_ser = 1;
583 #ifdef BM64ADDR
584  matr_s_ser = 2;
585 #endif
586 
587  enc.put_8(0); // number of plains == 0 (legacy magic number)
588  enc.put_8(matr_s_ser); // matrix serialization version
589  enc.put_64(plains); // number of rows in the bit-matrix
590  enc.put_64(sv.size_internal());
591 
592  // save the offset table (part of the header)
593  //
594  for (i = 0; i < plains; ++i)
595  {
596  const unsigned char* p = sv_layout.get_plain(i);
597  if (!p)
598  {
599  enc.put_64(0);
600  continue;
601  }
602  size_t offset = size_t(p - buf);
603  enc.put_64(offset);
604  } // for
605 }
606 
607 // -------------------------------------------------------------------------
608 //
609 // -------------------------------------------------------------------------
610 
611 template<typename SV>
613 {
614 }
615 
616 // -------------------------------------------------------------------------
617 
618 template<typename SV>
620  const unsigned char* buf)
621 {
622  // TODO: implement correct processing of byte-order corect deserialization
623  // ByteOrder bo_current = globals<true>::byte_order();
624 
625  bm::decoder dec(buf);
626  unsigned char h1 = dec.get_8();
627  unsigned char h2 = dec.get_8();
628 
629  BM_ASSERT(h1 == 'B' && (h2 == 'M' || h2 == 'C'));
630 
631  if (h1 != 'B' && (h2 != 'M' || h2 != 'C')) // no magic header?
632  {
633  #ifndef BM_NO_STL
634  throw std::logic_error("Invalid serialization signature header");
635  #else
636  BM_THROW(BM_ERR_SERIALFORMAT);
637  #endif
638  }
639  unsigned char matr_s_ser = 0;
640  //unsigned char bv_bo =
641  dec.get_8();
642  unsigned plains = dec.get_8();
643  if (plains == 0) // bit-matrix
644  {
645  matr_s_ser = dec.get_8(); // matrix serialization version
646  plains = (unsigned) dec.get_64(); // number of rows in the bit-matrix
647  }
648  #ifdef BM64ADDR
649  #else
650  if (matr_s_ser == 2) // 64-bit matrix
651  {
652  #ifndef BM_NO_STL
653  throw std::logic_error("Invalid serialization target (64-bit BLOB)");
654  #else
655  BM_THROW(BM_ERR_SERIALFORMAT);
656  #endif
657  }
658  #endif
659 
660 
661  unsigned sv_plains = sv.stored_plains();
662  if (!plains || plains > sv_plains)
663  {
664  #ifndef BM_NO_STL
665  throw std::logic_error("Invalid serialization target (bit depth)");
666  #else
667  BM_THROW(BM_ERR_SERIALFORMAT);
668  #endif
669  }
670 
671  sv.clear();
672 
673  bm::id64_t sv_size = dec.get_64();
674  if (sv_size == 0)
675  return; // empty vector
676 
677  sv.resize_internal(size_type(sv_size));
678  bm::word_t* temp_block = 0;
679 
680  const unsigned char* remap_buf_ptr = 0;
681 
682  unsigned i = 0;
683  for (i = 0; i < plains; ++i)
684  {
685  size_t offset = (size_t) dec.get_64();
686  if (offset == 0) // null vector
687  {
688  continue;
689  }
690  const unsigned char* bv_buf_ptr = buf + offset; // seek to position
691  bvector_type* bv = sv.get_plain(i);
692  BM_ASSERT(bv);
693  if (!temp_block)
694  {
695  typename bvector_type::blocks_manager_type& bv_bm =
696  bv->get_blocks_manager();
697  temp_block = bv_bm.check_allocate_tempblock();
698  }
699  size_t read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block);
700  remap_buf_ptr = bv_buf_ptr + read_bytes;
701  } // for i
702 
703  // ----------------------------
704  // load the remap matrix
705  //
706  if (bm::conditional<SV::is_remap_support::value>::test()) // test remapping trait
707  {
708  if (matr_s_ser > 0)
709  {
710  if (remap_buf_ptr)
711  {
712  bm::decoder dec_m(remap_buf_ptr);
713  unsigned char rh = dec_m.get_8();
714  if (rh == 'N') // no remap matrix here
715  {
716  }
717  else
718  if (rh == 'R')
719  {
720  size_t remap_size = (size_t) dec_m.get_64();
721  unsigned char* remap_buf = sv.init_remap_buffer();
722  BM_ASSERT(remap_buf);
723  size_t target_remap_size = sv.remap_size();
724  if (!remap_size || !remap_buf || remap_size != target_remap_size)
725  {
726  #ifndef BM_NO_STL
727  throw std::logic_error("Invalid serialization format (remap size)");
728  #else
729  BM_THROW(BM_ERR_SERIALFORMAT);
730  #endif
731  }
732  dec_m.memcpy(remap_buf, remap_size);
733  unsigned char end_tok = dec_m.get_8();
734  if (end_tok != 'E')
735  {
736  #ifndef BM_NO_STL
737  throw std::logic_error("Invalid serialization format");
738  #else
739  BM_THROW(BM_ERR_SERIALFORMAT);
740  #endif
741  }
742  sv.set_remap();
743  }
744  else // unknown serialization token
745  {
746  #ifndef BM_NO_STL
747  throw std::logic_error("Invalid serialization format (remap error)");
748  #else
749  BM_THROW(BM_ERR_SERIALFORMAT);
750  #endif
751  }
752 
753  }
754  }
755  } // if remap traits
756 
757  sv.sync(true); // force sync, recalculate RS index, remap tables, etc
758 }
759 
760 // -------------------------------------------------------------------------
761 
762 
763 } // namespace bm
764 
765 #include "bmundef.h"
766 
767 #endif
bm::compressed_collection_serializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:275
bm::compressed_collection_serializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:277
bm::sparse_vector_deserializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:198
bm::sparse_vector_serial_layout::plane_size_
size_t plane_size_[SV::sv_plains]
serialized plain size
Definition: bmsparsevec_serial.h:114
bm::sparse_vector_serializer::bvs_
bm::serializer< bvector_type > bvs_
Definition: bmsparsevec_serial.h:185
bm::sparse_vector_deserializer
sparse vector de-serializer
Definition: bmsparsevec_serial.h:192
bm::sparse_vector_deserializer::deserial_
bm::deserializer< typename SV::bvector_type, bm::decoder > deserial_
Definition: bmsparsevec_serial.h:210
bm::sparse_vector_deserialize
int sparse_vector_deserialize(SV &sv, const unsigned char *buf, bm::word_t *temp_block=0)
Deserialize sparse vector.
Definition: bmsparsevec_serial.h:254
bm::sparse_vector_serial_layout::buf_
buffer_type buf_
serialization buffer
Definition: bmsparsevec_serial.h:112
bm::sparse_vector_serializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:160
bm::decoder_base::get_pos
const unsigned char * get_pos() const
Return current buffer pointer.
Definition: encoding.h:99
bm::deserialize
size_t deserialize(BV &bv, const unsigned char *buf, bm::word_t *temp_block=0)
Bitvector deserialization from memory.
Definition: bmserial.h:1900
bm::sparse_vector_serial_layout::buf
const unsigned char * buf() const
Return serialization buffer pointer.
Definition: bmsparsevec_serial.h:106
bm::encoder::get_pos
unsigned char * get_pos() const
Get current memory stream position.
Definition: encoding.h:480
bm::sparse_vector_serializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:165
bm::id64_t
unsigned long long int id64_t
Definition: bmconst.h:34
bmsparsevec.h
Sparse constainer sparse_vector<> for integer types using bit-transposition transform.
bm::serializer::serialize
size_type serialize(const BV &bv, unsigned char *buf, size_t buf_size)
Bitvector serialization into memory block.
Definition: bmserial.h:1608
bm::sparse_vector_serial_layout::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:60
bm::sparse_vector_serializer::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:163
bm::compressed_collection_deserializer::container_type
CBC::container_type container_type
Definition: bmsparsevec_serial.h:297
bm::compressed_collection_deserializer::buffer_type
CBC::buffer_type buffer_type
Definition: bmsparsevec_serial.h:294
bm::decoder_base::memcpy
void memcpy(unsigned char *dst, size_t count)
read bytes from the decode buffer
Definition: encoding.h:573
bm::encoder::size
size_t size() const
Returns size of the current encoding stream.
Definition: encoding.h:472
bm::sparse_vector_deserializer::sparse_vector_deserializer
sparse_vector_deserializer()
Definition: bmsparsevec_serial.h:612
bm::compressed_collection_deserializer::deserialize
int deserialize(CBC &buffer_coll, const unsigned char *buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:403
bm::sparse_vector_deserializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:197
bm::sparse_vector_serialize
void sparse_vector_serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout, bm::word_t *temp_block=0)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:231
bm::compressed_collection_deserializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:293
bm::sparse_vector_serial_layout
layout class for serialization buffer structure
Definition: bmsparsevec_serial.h:57
bmundef.h
pre-processor un-defines to avoid global space pollution (internal)
bm::globals::byte_order
static ByteOrder byte_order()
Definition: bmconst.h:465
bm::sparse_vector_serial_layout::sparse_vector_serial_layout
sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:63
bm::compressed_collection_serializer
Seriaizer for compressed collections.
Definition: bmsparsevec_serial.h:270
bm::sparse_vector_serial_layout::~sparse_vector_serial_layout
~sparse_vector_serial_layout()
Definition: bmsparsevec_serial.h:65
bm::sparse_vector_serial_layout::capacity
size_t capacity() const
return serialization buffer capacity
Definition: bmsparsevec_serial.h:90
bm::serializer::gap_length_serialization
void gap_length_serialization(bool value)
Set GAP length serialization (serializes GAP levels of the original vector)
Definition: bmserial.h:797
bm::compressed_collection_serializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:273
bm::sparse_vector_serial_layout::reserve
unsigned char * reserve(size_t capacity)
resize capacity
Definition: bmsparsevec_serial.h:72
bm::bvector::allocator_pool_type
allocator_type::allocator_pool_type allocator_pool_type
Definition: bm.h:111
bm::encoder::put_64
void put_64(bm::id64_t w)
Puts 64 bits word into encoding buffer.
Definition: encoding.h:517
bm::compressed_collection_serializer::bvector_type
CBC::bvector_type bvector_type
Definition: bmsparsevec_serial.h:274
bm::sparse_vector_deserializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:199
bm::sparse_vector_serializer::serialize
void serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)
Serialize sparse vector into a memory buffer(s) structure.
Definition: bmsparsevec_serial.h:488
bm::sparse_vector_deserializer::allocator_pool_type
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
Definition: bmsparsevec_serial.h:200
bm::sparse_vector_serial_layout::buffer_type
serializer< bvector_type >::buffer buffer_type
Definition: bmsparsevec_serial.h:61
bm::conditional
ad-hoc conditional expressions
Definition: bmfunc.h:139
BM_ASSERT
#define BM_ASSERT
Definition: bmdef.h:117
bm::encoder::set_pos
void set_pos(unsigned char *buf_pos)
Set current memory stream position.
Definition: encoding.h:488
bm::sparse_vector_serial_layout::resize
void resize(size_t ssize)
Set new serialized size.
Definition: bmsparsevec_serial.h:87
bvector_type
bm::bvector bvector_type
Definition: strsvsample01.cpp:39
bm::deserializer< typename SV::bvector_type, bm::decoder >
bm::sparse_vector_deserializer::deserialize
void deserialize(SV &sv, const unsigned char *buf)
Definition: bmsparsevec_serial.h:619
bmdef.h
Definitions(internal)
bm::compressed_collection_serializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:276
bm::sparse_vector_serializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:161
bm::serializer
Bit-vector serialization class.
Definition: bmserial.h:77
bm::compressed_collection_deserializer
Deseriaizer for compressed collections.
Definition: bmsparsevec_serial.h:289
bm::compressed_collection_serializer::serialize
void serialize(const CBC &buffer_coll, buffer_type &buf, bm::word_t *temp_block=0)
Definition: bmsparsevec_serial.h:340
bm::encoder::put_8
void put_8(unsigned char c)
Puts one character into the encoding buffer.
Definition: encoding.h:407
bm::sparse_vector_deserializer::bvector_type_const_ptr
const typedef bvector_type * bvector_type_const_ptr
Definition: bmsparsevec_serial.h:196
bm::sparse_vector_serial_layout::set_plain
void set_plain(unsigned i, unsigned char *ptr, size_t buf_size)
Set plain output pointer and size.
Definition: bmsparsevec_serial.h:96
bm::compressed_collection_deserializer::address_resolver_type
CBC::address_resolver_type address_resolver_type
Definition: bmsparsevec_serial.h:296
bm::ByteOrder
ByteOrder
Byte orders recognized by the library.
Definition: bmconst.h:429
bm::sparse_vector_serial_layout::value_type
SV::value_type value_type
Definition: bmsparsevec_serial.h:59
bm::decoder_base::seek
void seek(int delta)
change current position
Definition: encoding.h:93
bmserial.h
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
bm::compressed_collection_deserializer::compressed_collection_type
CBC compressed_collection_type
Definition: bmsparsevec_serial.h:292
bm::sparse_vector_serial_layout::size
size_t size() const
return current serialized size
Definition: bmsparsevec_serial.h:84
bm::decoder::get_64
bm::id64_t get_64()
Reads 64-bit word from the decoding buffer.
Definition: encoding.h:628
bm::sparse_vector_serializer::bvector_type_ptr
bvector_type * bvector_type_ptr
Definition: bmsparsevec_serial.h:162
bm
Definition: bm.h:76
bm::sparse_vector_serial_layout::get_plain
const unsigned char * get_plain(unsigned i) const
Get plain pointer.
Definition: bmsparsevec_serial.h:103
bm::sparse_vector_serial_layout::plain_ptrs_
unsigned char * plain_ptrs_[SV::sv_plains]
pointers on serialized bit-plains
Definition: bmsparsevec_serial.h:113
bm::encoder::memcpy
void memcpy(const unsigned char *src, size_t count)
copy bytes into target buffer or just rewind if src is NULL
Definition: encoding.h:459
bm::decoder_base::get_8
unsigned char get_8()
Reads character from the decoding buffer.
Definition: encoding.h:87
bm::word_t
unsigned int word_t
Definition: bmconst.h:38
bm::bvector::blocks_manager_type
blocks_manager< Alloc > blocks_manager_type
Definition: bm.h:112
bm::bvector::size_type
bm::id_t size_type
Definition: bm.h:117
bm::encoder
Memory encoding.
Definition: encoding.h:49
bm::sparse_vector_deserializer::bvector_type
SV::bvector_type bvector_type
Definition: bmsparsevec_serial.h:195
bm::sparse_vector_serializer::sparse_vector_serializer
sparse_vector_serializer()
Definition: bmsparsevec_serial.h:480
bm::decoder
Class for decoding data from memory buffer.
Definition: encoding.h:112
bm::sparse_vector_serial_layout::freemem
void freemem()
free memory
Definition: bmsparsevec_serial.h:93
bm::sparse_vector_serializer
Definition: bmsparsevec_serial.h:157
bm::compressed_collection_deserializer::statistics_type
CBC::statistics statistics_type
Definition: bmsparsevec_serial.h:295
bm::sparse_vector_serializer::size_type
SV::size_type size_type
Definition: bmsparsevec_serial.h:164