src/persy.rs
changed.
Showing 2 of 2 files from the diff.
@@ -28,7 +28,10 @@
Loading
28 | 28 | discref::{Device, DiscRef, MemRef, PageOps, PAGE_METADATA_SIZE}, |
|
29 | 29 | error::{PRes, PersyError}, |
|
30 | 30 | id::{IndexId, PersyId, SegmentId, ToIndexId, ToSegmentId}, |
|
31 | - | io::{InfallibleReadFormat, InfallibleWrite, InfallibleWriteFormat}, |
|
31 | + | io::{ |
|
32 | + | InfallibleRead, InfallibleReadFormat, InfallibleReadVarInt, InfallibleWrite, InfallibleWriteFormat, |
|
33 | + | InfallibleWriteVarInt, |
|
34 | + | }, |
|
32 | 35 | journal::{Journal, JournalId, JOURNAL_PAGE_EXP}, |
|
33 | 36 | record_scanner::{SegmentRawIter, SegmentSnapshotRawIter, TxSegmentRawIter}, |
|
34 | 37 | snapshot::{release_snapshot, EntryCase, SegmentSnapshop, SnapshotId, Snapshots}, |
@@ -421,26 +424,37 @@
Loading
421 | 424 | } |
|
422 | 425 | } |
|
423 | 426 | ||
427 | + | pub fn write_record_metadata(len: u64, id: &RecRef) -> Vec<u8> { |
|
428 | + | let mut val = Vec::new(); |
|
429 | + | val.write_varint_u64(len); |
|
430 | + | id.write(&mut val); |
|
431 | + | val |
|
432 | + | } |
|
433 | + | pub fn read_record_metadata(meta: &mut dyn InfallibleRead) -> (u64, RecRef) { |
|
434 | + | let len = meta.read_varint_u64(); |
|
435 | + | let id = RecRef::read(meta); |
|
436 | + | (len, id) |
|
437 | + | } |
|
438 | + | ||
424 | 439 | pub fn insert_record(&self, tx: &mut Transaction, segment: impl ToSegmentId, rec: &[u8]) -> PRes<RecRef> { |
|
425 | 440 | let (segment_id, in_tx) = segment.to_segment_id_tx(self, tx)?; |
|
426 | 441 | let len = rec.len() as u64; |
|
427 | - | let allocation_exp = exp_from_content_size(len); |
|
428 | 442 | let allocator = &self.allocator; |
|
429 | 443 | let address = &self.address; |
|
430 | - | let mut pg = allocator.allocate(allocation_exp)?; |
|
431 | - | let page = pg.get_index(); |
|
432 | 444 | let (rec_ref, maybe_new_page) = if in_tx { |
|
433 | 445 | address.allocate_temp(segment_id.id) |
|
434 | 446 | } else { |
|
435 | 447 | address.allocate(segment_id.id) |
|
436 | 448 | }?; |
|
449 | + | let metadata = PersyImpl::write_record_metadata(len, &rec_ref); |
|
450 | + | let allocation_exp = exp_from_content_size(len + metadata.len() as u64); |
|
451 | + | let mut pg = allocator.allocate(allocation_exp)?; |
|
452 | + | let page = pg.get_index(); |
|
437 | 453 | tx.add_insert(&self.journal, segment_id.id, &rec_ref, page)?; |
|
438 | 454 | if let Some(new_page) = maybe_new_page { |
|
439 | 455 | tx.add_new_segment_page(&self.journal, segment_id.id, new_page.new_page, new_page.previus_page)?; |
|
440 | 456 | } |
|
441 | - | pg.write_u64(len); |
|
442 | - | pg.write_u64(rec_ref.page); |
|
443 | - | pg.write_u32(rec_ref.pos); |
|
457 | + | pg.write_all(&metadata); |
|
444 | 458 | pg.write_all(rec); |
|
445 | 459 | allocator.flush_page(pg)?; |
|
446 | 460 | Ok(rec_ref) |
@@ -486,10 +500,8 @@
Loading
486 | 500 | F: Fn(&[u8]) -> T, |
|
487 | 501 | { |
|
488 | 502 | if let Some(mut pg) = self.allocator.load_page_not_free(page)? { |
|
489 | - | let len = pg.read_u64(); |
|
490 | - | let page = pg.read_u64(); |
|
491 | - | let pos = pg.read_u32(); |
|
492 | - | if page == match_id.page && pos == match_id.pos { |
|
503 | + | let (len, id) = PersyImpl::read_record_metadata(&mut pg); |
|
504 | + | if id.page == match_id.page && id.pos == match_id.pos { |
|
493 | 505 | Ok(Some(f(pg.slice(len as usize)))) |
|
494 | 506 | } else { |
|
495 | 507 | Ok(None) |
@@ -632,13 +644,12 @@
Loading
632 | 644 | let allocator = &self.allocator; |
|
633 | 645 | let journal = &self.journal; |
|
634 | 646 | let len = rec.len(); |
|
635 | - | let allocation_exp = exp_from_content_size(len as u64); |
|
647 | + | let metadata = PersyImpl::write_record_metadata(len as u64, &rec_ref); |
|
648 | + | let allocation_exp = exp_from_content_size((len + metadata.len()) as u64); |
|
636 | 649 | let mut pg = allocator.allocate(allocation_exp)?; |
|
637 | 650 | let page = pg.get_index(); |
|
638 | 651 | tx.add_update(journal, segment, &rec_ref, page, version)?; |
|
639 | - | pg.write_u64(len as u64); |
|
640 | - | pg.write_u64(rec_ref.page); |
|
641 | - | pg.write_u32(rec_ref.pos); |
|
652 | + | pg.write_all(&metadata); |
|
642 | 653 | pg.write_all(rec); |
|
643 | 654 | allocator.flush_page(pg) |
|
644 | 655 | } else { |
@@ -933,7 +944,7 @@
Loading
933 | 944 | ||
934 | 945 | pub fn exp_from_content_size(size: u64) -> u8 { |
|
935 | 946 | // content + size + match_pointer:page+ match_pointer:pos + page_header |
|
936 | - | let final_size = size + 8 + 8 + 4 + u64::from(PAGE_METADATA_SIZE); |
|
947 | + | let final_size = size + u64::from(PAGE_METADATA_SIZE); |
|
937 | 948 | // Should be there a better way, so far is OK. |
|
938 | 949 | let mut res: u8 = 1; |
|
939 | 950 | loop { |
@@ -2,13 +2,15 @@
Loading
2 | 2 | address::Address, |
|
3 | 3 | error::{PRes, PersyError}, |
|
4 | 4 | index::config::{format_segment_name_data, format_segment_name_meta, index_name_from_meta_segment}, |
|
5 | + | io::{InfallibleRead, InfallibleReadVarInt, InfallibleWrite, InfallibleWriteVarInt}, |
|
5 | 6 | persy::PersyImpl, |
|
6 | 7 | snapshot::{SnapshotId, Snapshots}, |
|
7 | 8 | transaction::{Transaction, TxSegCheck}, |
|
8 | 9 | }; |
|
10 | + | ||
9 | 11 | use data_encoding::BASE32_DNSSEC; |
|
10 | 12 | pub use std::fs::OpenOptions; |
|
11 | - | use std::{fmt, io::Write, str}; |
|
13 | + | use std::{fmt, str}; |
|
12 | 14 | use unsigned_varint::{ |
|
13 | 15 | decode::{u32 as u32_vdec, u64 as u64_vdec}, |
|
14 | 16 | encode::{u32 as u32_venc, u32_buffer, u64 as u64_venc, u64_buffer}, |
@@ -16,9 +18,7 @@
Loading
16 | 18 | ||
17 | 19 | fn write_id(f: &mut fmt::Formatter, id: u32) -> fmt::Result { |
|
18 | 20 | let mut buffer = Vec::new(); |
|
19 | - | buffer |
|
20 | - | .write_all(u32_venc(id, &mut u32_buffer())) |
|
21 | - | .expect("no failure expected only memory allocation"); |
|
21 | + | buffer.write_all(u32_venc(id, &mut u32_buffer())); |
|
22 | 22 | buffer.push(0b0101_0101); |
|
23 | 23 | write!(f, "{}", BASE32_DNSSEC.encode(&buffer)) |
|
24 | 24 | } |
@@ -36,21 +36,26 @@
Loading
36 | 36 | ||
37 | 37 | impl RecRef { |
|
38 | 38 | #[inline] |
|
39 | - | pub fn new(page: u64, pos: u32) -> RecRef { |
|
39 | + | pub(crate) fn new(page: u64, pos: u32) -> RecRef { |
|
40 | 40 | RecRef { page, pos } |
|
41 | 41 | } |
|
42 | + | pub(crate) fn write(&self, write: &mut dyn InfallibleWrite) { |
|
43 | + | write.write_varint_u64(self.page); |
|
44 | + | write.write_varint_u32(self.pos); |
|
45 | + | } |
|
46 | + | pub(crate) fn read(read: &mut dyn InfallibleRead) -> RecRef { |
|
47 | + | let page = read.read_varint_u64(); |
|
48 | + | let pos = read.read_varint_u32(); |
|
49 | + | Self::new(page, pos) |
|
50 | + | } |
|
42 | 51 | } |
|
43 | 52 | ||
44 | 53 | impl fmt::Display for RecRef { |
|
45 | 54 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
|
46 | 55 | let mut buffer = Vec::new(); |
|
47 | - | buffer |
|
48 | - | .write_all(u64_venc(self.page, &mut u64_buffer())) |
|
49 | - | .expect("no failure expected only memory allocation"); |
|
56 | + | buffer.write_all(u64_venc(self.page, &mut u64_buffer())); |
|
50 | 57 | buffer.push(0b0101_0101); |
|
51 | - | buffer |
|
52 | - | .write_all(u32_venc(self.pos, &mut u32_buffer())) |
|
53 | - | .expect("no failure expected only memory allocation"); |
|
58 | + | buffer.write_all(u32_venc(self.pos, &mut u32_buffer())); |
|
54 | 59 | write!(f, "{}", BASE32_DNSSEC.encode(&buffer)) |
|
55 | 60 | } |
|
56 | 61 | } |
@@ -303,12 +308,8 @@
Loading
303 | 308 | impl fmt::Display for IndexId { |
|
304 | 309 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
|
305 | 310 | let mut buffer = Vec::new(); |
|
306 | - | buffer |
|
307 | - | .write_all(u32_venc(self.meta, &mut u32_buffer())) |
|
308 | - | .expect("no failure expected only memory allocation"); |
|
309 | - | buffer |
|
310 | - | .write_all(u32_venc(self.data, &mut u32_buffer())) |
|
311 | - | .expect("no failure expected only memory allocation"); |
|
311 | + | buffer.write_all(u32_venc(self.meta, &mut u32_buffer())); |
|
312 | + | buffer.write_all(u32_venc(self.data, &mut u32_buffer())); |
|
312 | 313 | buffer.push(0b0101_0101); |
|
313 | 314 | write!(f, "{}", BASE32_DNSSEC.encode(&buffer)) |
|
314 | 315 | } |
@@ -348,6 +349,12 @@
Loading
348 | 349 | assert_eq!(s.parse::<RecRef>().ok(), Some(id)); |
|
349 | 350 | } |
|
350 | 351 | ||
352 | + | #[test] |
|
353 | + | fn test_persy_id_fixed_parse() { |
|
354 | + | let id = RecRef::new(20, 30); |
|
355 | + | assert_eq!("2hahs".parse::<RecRef>().ok(), Some(id)); |
|
356 | + | } |
|
357 | + | ||
351 | 358 | #[test] |
|
352 | 359 | fn test_persy_id_parse_failure() { |
|
353 | 360 | let s = "ACCC"; |
705487723
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file.
The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files.
The size and color of each slice is representing the number of statements and the coverage, respectively.