pub mod config; pub mod fr; #[cfg(feature = "opera")] pub mod opera; pub mod pattern; pub mod prelude; pub mod tables; mod utils; pub mod wmo; use anyhow::Context; use memmap2::Mmap; use ph::fmph::GOFunction; use rkyv::api::high::HighValidator; use rkyv::bytecheck::CheckBytes; use rkyv::rancor::Error; use rkyv::{Archive, Deserialize, Serialize}; use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use std::fmt::Debug; use std::io::{Cursor, Write}; use std::path::Path; use crate::tables::{TableEntryFull, TableTypeTrait}; pub trait TableConverter { type OutputEntry: TableEntryFull; type TableType: TableTypeTrait; fn convert>(&self, path: P) -> anyhow::Result>; fn table_type(&self) -> crate::TableType { Self::TableType::TABLE_TYPE } } struct BufrTableMph { mphf: GOFunction, mmap: Mmap, _marker: std::marker::PhantomData, } #[derive(Archive, Deserialize, Serialize, PartialEq)] #[rkyv(compare(PartialEq))] struct BUFRTF where T: TableEntryFull, { pub function_header: Vec, pub entries: Vec, } impl BUFRTF where T: TableEntryFull, { fn new(entries: Vec) -> std::io::Result { let keys: Vec = entries.iter().map(|e| e.fxy()).collect(); let mphf = GOFunction::from_slice(&keys); let mut sorted_entries: Vec<(usize, T)> = entries .into_iter() .map(|e| (mphf.get(&(e.fxy())).unwrap() as usize, e)) .collect(); sorted_entries.sort_by_key(|(hash, _)| *hash); let mut mphf_bytes = Vec::new(); mphf.write(&mut mphf_bytes) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("{:?}", e)))?; Ok(Self { function_header: mphf_bytes, entries: sorted_entries.into_iter().map(|(_, e)| e).collect(), }) } fn write_to_disk>(&self, path: P) -> anyhow::Result<()> { let path = path.as_ref(); let mut file = std::fs::File::create(path)?; let bytes = rkyv::to_bytes::(self)?; file.write_all(&bytes)?; Ok(()) } } impl BufrTableMph where ::Archived: for<'a> CheckBytes>, { fn bufrtbl_path>(path: P) -> std::path::PathBuf { let mut path = path.as_ref().to_path_buf(); path.set_extension("bufrtbl"); path } fn build>(entries: Vec, output_path: P) -> anyhow::Result { let output_path = Self::bufrtbl_path(output_path); let bufrtf = BUFRTF::new(entries)?; bufrtf.write_to_disk(&output_path)?; Self::load(output_path) } fn load>(path: P) -> anyhow::Result { let path = Self::bufrtbl_path(path); let merged_file = std::fs::File::open(&path)?; let mmap = unsafe { Mmap::map(&merged_file)? }; let archived = rkyv::access::, Error>(&mmap)?; let function_reader = &archived.function_header[..]; let mut cursor = Cursor::new(function_reader); Ok(Self { mphf: GOFunction::read(&mut cursor)?, mmap, _marker: std::marker::PhantomData, }) } /// 获取拥有的版本 fn get(&self, fxy: FXY) -> Option<&::Archived> { let hash = self.mphf.get(&fxy)? as usize; self.archived().ok()?.entries.get(hash) } fn archived(&self) -> anyhow::Result<&ArchivedBUFRTF> { let archived = rkyv::access::, Error>(&self.mmap)?; Ok(archived) } /// 获取所有条目 fn get_all(&self) -> Vec<&::Archived> { if let Ok(archived) = self.archived() { let mut result = vec![]; archived.entries.iter().for_each(|entry| { result.push(entry); }); result } else { vec![] } } } #[derive( Archive, SerdeSerialize, SerdeDeserialize, rkyv::Serialize, rkyv::Deserialize, Debug, PartialEq, Eq, Clone, Copy, std::hash::Hash, )] #[rkyv(compare(PartialEq), derive(Debug, Clone, Copy))] pub struct FXY { pub f: i32, pub x: i32, pub y: i32, } impl FXY { pub fn new(f: i32, x: i32, y: i32) -> Self { FXY { f, x, y } } pub fn from_str(fxy_str: &str) -> anyhow::Result { // let bytes = fxy_str.as_bytes(); if fxy_str.len() != 6 { return Err(anyhow::anyhow!("Invalid FXY string length: {}", fxy_str)); } let f = fxy_str[0..2] .parse::() .with_context(|| format!("Failed to parse F from FXY: {}", fxy_str))?; let x = fxy_str[2..4] .parse::() .with_context(|| format!("Failed to parse X from FXY: {}", fxy_str))?; let y = fxy_str[4..6] .parse::() .with_context(|| format!("Failed to parse Y from FXY: {}", fxy_str))?; Ok(FXY { f, x, y }) } /// Convert FXY to u32 for use as hash key /// Format: F (2 bits) | X (6 bits) | Y (8 bits) = 16 bits total pub fn to_u32(&self) -> u32 { ((self.f as u32) << 14) | ((self.x as u32) << 8) | (self.y as u32) } // pub fn from_u32(value: u32) -> Self { // FXY { // f: ((value >> 14) & 0x3) as u16, // x: ((value >> 8) & 0x3F) as u16, // y: (value & 0xFF) as u16, // } // } } pub struct BUFRTableMPH { inner: BufrTableMph, } impl BUFRTableMPH where ::Archived: for<'a> CheckBytes>, { pub fn build_from_csv, L: TableConverter>( loader: L, path: P, output_path: P, ) -> anyhow::Result where L: TableConverter, L: TableConverter, ::Archived: for<'a> CheckBytes>, { let entries = loader.convert(path)?; let bhm = BufrTableMph::::build(entries, output_path)?; Ok(BUFRTableMPH { inner: bhm }) } pub fn get_all_entries(&self) -> Vec<&::Archived> { self.inner.get_all() } pub fn load_from_disk>(path: P) -> anyhow::Result { let bhm = BufrTableMph::::load(path)?; Ok(BUFRTableMPH { inner: bhm }) } pub fn lookup(&self, fxy: FXY) -> Option<&::Archived> { self.inner.get(fxy) } } #[repr(C)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TableType { B, D, BitMap, } #[cfg(test)] mod test { use crate::{ BUFRTableMPH, BufrTableMph, FXY, // wmo::{TableLoader, btable::BTableCsvLoader}, fr::{TableLoader, btable::BTableLoader as BTableCsvLoader}, prelude::{BUFRTableB, BUFRTableD}, }; #[test] fn test() { let table_loader = TableLoader::::default(); BUFRTableB::build_from_csv( table_loader, // "/Users/xiang.li1/projects/rbufr/BUFR4/BUFRCREX_TableB_en_42.csv", "/Users/xiang.li1/Downloads/tables 2/bufrtabb_16.csv", "./test.bufrtbl", ) .unwrap(); } #[test] fn load() { let table = BUFRTableD::load_from_disk( "/Users/xiang.li1/projects/rbufr/rbufr/tables/master/BUFR_TableD_16.bufrtbl", ) .unwrap(); let x = table.lookup(FXY::new(3, 21, 11)).unwrap(); println!("{:#?}", x); } }