use crate::error::ETWSError; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use chrono::{DateTime, NaiveDateTime, Utc}; use flate2::read::GzDecoder; use nom::{ bytes::complete::{tag, take}, multi::count, IResult, }; use nom_derive::*; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::{self, Read, Write}; use std::path::Path; pub enum ValueResult { I64(Vec), F64(Vec), I32(Vec), F32(Vec), I16(Vec), U64(Vec), U32(Vec), I8(Vec), U8(Vec), } enum ValueTypes { I64, F64, I32, F32, U64, U32, I16, I8, U8, } #[derive(Clone, Copy)] enum Order { BigEndian, LittleEndian, } pub struct Record { pub filetime: DateTime, pub blocks: Vec, // Fill in generic types appropriately } macro_rules! match_in_macro { ($block:ident,$len:ident,$input:ident,$offset:ident,$scale:ident,$fill_value:ident,$(($branch:path, $t:ty, $bigger:ty,$raw_result:path, $bigger_result:path)),+) => { { use std::mem; let need_trans = $offset != 0.0 || $scale != 1.0; let trans_to_bigger = $offset.trunc() != $offset || $scale != 1.0; match $block { $( $branch => { let ratio = mem::size_of::<$t>() / mem::size_of::(); let (input, result) = take($len * ratio)($input)?; let result = unsafe { let ptr = result.as_ptr() as *const $t; let slice = std::slice::from_raw_parts(ptr, $len); let slice = slice.to_vec(); if trans_to_bigger { let offset = $offset as $bigger; let scale = $scale as $bigger; $bigger_result( slice .into_iter() .map(|p| if (p as f64 - $fill_value).abs() < f64::EPSILON {p as $bigger} else {(p as $bigger - offset) / scale} ) .collect::>(), ) } else { $raw_result(if need_trans { let offset = $offset as $t; slice.into_iter().map(|p| if (p as f64 - $fill_value).abs() < f64::EPSILON {p} else {p - offset}).collect::>() } else { slice }) } }; Ok((input, result)) }, )+ } } }; } impl Record { pub fn parse_from_path(path: impl AsRef) -> Result { let path = path.as_ref(); if !(path.ends_with(".dat") || !path.ends_with(".dat.gz")) { return Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, "Invalid file extension", ) .into()); } if !path.exists() { return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "File not found").into()); } let mut file = File::open(path)?; let binary_data = if path.extension().unwrap() == "gz" { let mut result: Vec = Vec::new(); let mut decoder = GzDecoder::new(file); decoder.read_to_end(&mut result)?; result } else { let mut result = Vec::new(); file.read_to_end(&mut result)?; result }; let (_, parsed) = Self::_parse(binary_data.as_slice()).map_err(|_| anyhow::Error::msg("Parse error"))?; Ok(parsed) } fn _parse(binary_data: &[u8]) -> IResult<&[u8], Record> { let start_tag = b"UNI_DATA"; let (input, _) = tag(start_tag)(binary_data)?; let (input, order) = Self::_parse_split_fn(input, 8, 8, Self::_parse_order)?; let (input, hlen) = take(4usize)(input)?; let hlen = match order { Order::BigEndian => BigEndian::read_u32(hlen), _ => LittleEndian::read_u32(hlen), }; let (input, record_info) = Self::_parse_split_fn(input, 0, 8, |input| { let (input, p) = take(hlen)(input)?; let p: RecordInfo = serde_json::from_slice(p).unwrap(); Ok((input, p)) })?; let (input, blocks) = count(Self::_parse_block_fn(order), record_info.block_num as usize)(input)?; let data_time = NaiveDateTime::parse_from_str(&record_info.file_time, r"%Y%m%d%H%M%S").unwrap(); let filetime = data_time.and_utc(); Ok((input, Record { filetime, blocks })) } fn _parse_order(input: &[u8]) -> IResult<&[u8], Order> { let (input, order) = take(4usize)(input)?; let result = if order == b"LEND" { Order::LittleEndian } else { Order::BigEndian }; let (input, _) = take(4usize)(input)?; Ok((input, result)) } fn _parse_split(input: &[u8], s: usize, fore: usize, after: usize) -> IResult<&[u8], &[u8]> { let (input, _) = take(fore)(input)?; let (input, result) = take(s)(input)?; let (input, _) = take(after)(input)?; Ok((input, result)) } fn _parse_split_fn IResult<&[u8], E>>( input: &[u8], fore: usize, after: usize, f: F, ) -> IResult<&[u8], E> { let (input, _) = take(fore)(input)?; let (input, result) = (f)(input)?; let (input, _) = take(after)(input)?; Ok((input, result)) } fn _parse_block_fn(order: Order) -> impl FnMut(&[u8]) -> IResult<&[u8], ParsedBlock> { move |input| Self::_parse_split_fn(input, 0, 0, |input| Self::_parse_block(input, order)) } fn _parse_block(input: &[u8], order: Order) -> IResult<&[u8], ParsedBlock> { let (input, _) = take(8usize)(input)?; let (input, hlen1) = Self::_parse_u32(input, order)?; let (input, block_info) = Self::_parse_split_fn(input, 0, 8, |input| { let (input, p) = take(hlen1)(input)?; let p: BlockJsonInfo = serde_json::from_slice(p).unwrap(); Ok((input, p)) })?; let (input, _) = take(8usize)(input)?; // skip 8 bytes let (input, hlen2) = Self::_parse_u32(input, order)?; let dimension_size = block_info.dimension_size.clone(); let size = dimension_size.iter().fold(1, |acc, x| acc * x) as usize; let value_type = block_info.value_type.clone(); let value_type = Self::_parse_type(&value_type); let (input, data) = Self::_parse_matrix( input, value_type, order, size, block_info.value_offset, block_info.value_scale, block_info.fill_value, )?; Ok(( input, ParsedBlock { info: block_info, data, }, )) } fn _parse_i32(input: &[u8], order: Order) -> IResult<&[u8], i32> { let (input, hlen) = take(4usize)(input)?; let hlen = match order { Order::BigEndian => BigEndian::read_i32(hlen), _ => LittleEndian::read_i32(hlen), }; Ok((input, hlen)) } fn _parse_u32(input: &[u8], order: Order) -> IResult<&[u8], u32> { let (input, hlen) = take(4usize)(input)?; let hlen = match order { Order::BigEndian => BigEndian::read_u32(hlen), _ => LittleEndian::read_u32(hlen), }; Ok((input, hlen)) } fn _parse_i64(input: &[u8], order: Order) -> IResult<&[u8], i64> { let (input, hlen) = take(8usize)(input)?; let hlen = match order { Order::BigEndian => BigEndian::read_i64(hlen), _ => LittleEndian::read_i64(hlen), }; Ok((input, hlen)) } fn _parse_u64(input: &[u8], order: Order) -> IResult<&[u8], u64> { let (input, hlen) = take(8usize)(input)?; let hlen = match order { Order::BigEndian => BigEndian::read_u64(hlen), _ => LittleEndian::read_u64(hlen), }; Ok((input, hlen)) } fn _parse_type<'a>(type_str: &'a str) -> ValueTypes { match type_str { "b" => ValueTypes::I8, "B" => ValueTypes::U8, "i" => ValueTypes::I32, "I" => ValueTypes::U32, "f" => ValueTypes::F32, "d" => ValueTypes::F64, "h" => ValueTypes::I16, _ => panic!("Invalid type"), } } fn _parse_matrix( input: &[u8], type_: ValueTypes, order: Order, len: usize, offset: f32, scale: f32, fill_value: f64, ) -> IResult<&[u8], ValueResult> { match_in_macro!( type_, len, input, offset, scale, fill_value, ( ValueTypes::I64, i64, f64, ValueResult::I64, ValueResult::F64 ), ( ValueTypes::F64, f64, f64, ValueResult::F64, ValueResult::F64 ), ( ValueTypes::U64, u64, f64, ValueResult::U64, ValueResult::F64 ), ( ValueTypes::I32, i32, f32, ValueResult::I32, ValueResult::F32 ), ( ValueTypes::F32, f32, f32, ValueResult::F32, ValueResult::F32 ), ( ValueTypes::U32, u32, f32, ValueResult::U32, ValueResult::F32 ), ( ValueTypes::I16, i16, f32, ValueResult::I16, ValueResult::F32 ), (ValueTypes::I8, i8, f32, ValueResult::I8, ValueResult::F32), (ValueTypes::U8, u8, f32, ValueResult::U8, ValueResult::F32) ) } } #[derive(Serialize, Deserialize, Debug)] struct RecordInfo { file_time: String, block_num: i32, } #[derive(Serialize, Deserialize, Debug)] pub struct BlockJsonInfo { pub value_name: String, pub value_des: String, pub value_type: String, pub dimension_size: Vec, pub dimension: usize, pub dimension_des: Vec, pub dimension_start: Vec, pub dimension_end: Vec, pub dimension_values: Vec>, pub fill_value: f64, pub value_scale: f32, pub value_offset: f32, } pub struct ParsedBlock { pub info: BlockJsonInfo, pub data: ValueResult, } // fn _parse_matrix( // input: &[u8], // type_: ValueTypes, // order: Order, // len: usize, // offset: f32, // scale: f32, // fill_value: f64, // ) -> IResult<&[u8], ValueResult> { // use std::mem; // let need_trans = offset != 0.0 || scale != 1.0; // let trans_to_bigger = offset.trunc() != offset || scale != 1.0; // match type_ { // ValueTypes::I64 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const i64; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // if trans_to_bigger { // let offset = offset as f32; // ValueResult::F32( // slice // .into_iter() // .map(|p| (p as f32 - offset) / scale) // .collect::>(), // ) // } else { // ValueResult::I64(if need_trans { // let offset = offset as i64; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // } // }; // Ok((input, result)) // } // ValueTypes::F64 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const f64; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // ValueResult::F64(if need_trans { // let offset = offset as f64; // slice // .into_iter() // .map(|p| (p - offset) / scale as f64) // .collect::>() // } else { // slice // }) // }; // Ok((input, result)) // } // ValueTypes::I32 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const i32; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // if trans_to_bigger { // let offset = offset as f32; // ValueResult::F32( // slice // .into_iter() // .map(|p| (p as f32 - offset) / scale) // .collect::>(), // ) // } else { // ValueResult::I32(if need_trans { // let offset = offset as i32; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // } // }; // Ok((input, result)) // } // ValueTypes::U32 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const u32; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // if trans_to_bigger { // let offset = offset as f32; // ValueResult::F32( // slice // .into_iter() // .map(|p| (p as f32 - offset) / scale) // .collect::>(), // ) // } else { // ValueResult::U32(if need_trans { // let offset = offset as u32; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // } // }; // Ok((input, result)) // } // ValueTypes::F32 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const f32; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // ValueResult::F32(if need_trans { // let offset = offset as f32; // slice // .into_iter() // .map(|p| (p - offset) / scale) // .collect::>() // } else { // slice // }) // }; // Ok((input, result)) // } // ValueTypes::U64 => { // let ratio = mem::size_of::() / mem::size_of::(); // let (input, result) = take(len * ratio)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const u64; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // if trans_to_bigger { // let offset = offset as f64; // let scale = scale as f64; // ValueResult::F64( // slice // .into_iter() // .map(|p| (p as f64 - offset) / scale) // .collect::>(), // ) // } else { // ValueResult::U64(if need_trans { // let offset = offset as u64; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // } // }; // Ok((input, result)) // } // ValueTypes::I8 => { // let (input, result) = take(len)(input)?; // let result = unsafe { // let ptr = result.as_ptr() as *const i8; // let slice = std::slice::from_raw_parts(ptr, len); // let slice = slice.to_vec(); // if trans_to_bigger { // let offset = offset as f32; // ValueResult::F32( // slice // .into_iter() // .map(|p| (p as f32 - offset) / scale) // .collect::>(), // ) // } else { // ValueResult::I8(if need_trans { // let offset = offset as i8; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // } // }; // Ok((input, result)) // } // ValueTypes::U8 => { // let (input, slice) = take(len)(input)?; // let slice = slice.to_vec(); // let result = if trans_to_bigger { // let offset = offset as f32; // ValueResult::F32( // slice // .into_iter() // .map(|p| { // if p as f64 != fill_value { // (p as f32 - offset) / scale // } else { // p as f32 // } // }) // .collect::>(), // ) // } else { // ValueResult::U8(if need_trans { // let offset = offset as u8; // slice.into_iter().map(|p| p - offset).collect::>() // } else { // slice // }) // }; // Ok((input, result)) // } // } // }