601 lines
20 KiB
Rust
601 lines
20 KiB
Rust
use crate::error::ETWSError;
|
|
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
|
use chrono::{DateTime, NaiveDateTime, Utc};
|
|
use flate2::read::GzDecoder;
|
|
use nom::{
|
|
bytes::complete::{tag, take},
|
|
multi::count,
|
|
IResult,
|
|
};
|
|
use nom_derive::*;
|
|
use serde::{Deserialize, Serialize};
|
|
use std::fs::File;
|
|
use std::io::{self, Read, Write};
|
|
use std::path::Path;
|
|
|
|
pub enum ValueResult {
|
|
I64(Vec<i64>),
|
|
F64(Vec<f64>),
|
|
I32(Vec<i32>),
|
|
F32(Vec<f32>),
|
|
I16(Vec<i16>),
|
|
U64(Vec<u64>),
|
|
U32(Vec<u32>),
|
|
I8(Vec<i8>),
|
|
U8(Vec<u8>),
|
|
}
|
|
|
|
enum ValueTypes {
|
|
I64,
|
|
F64,
|
|
I32,
|
|
F32,
|
|
U64,
|
|
U32,
|
|
I16,
|
|
I8,
|
|
U8,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum Order {
|
|
BigEndian,
|
|
LittleEndian,
|
|
}
|
|
|
|
pub struct Record {
|
|
pub filetime: DateTime<Utc>,
|
|
pub blocks: Vec<ParsedBlock>, // Fill in generic types appropriately
|
|
}
|
|
|
|
macro_rules! match_in_macro {
|
|
($block:ident,$len:ident,$input:ident,$offset:ident,$scale:ident,$fill_value:ident,$(($branch:path, $t:ty, $bigger:ty,$raw_result:path, $bigger_result:path)),+) => {
|
|
{
|
|
use std::mem;
|
|
let need_trans = $offset != 0.0 || $scale != 1.0;
|
|
let trans_to_bigger = $offset.trunc() != $offset || $scale != 1.0;
|
|
match $block {
|
|
$(
|
|
$branch => {
|
|
let ratio = mem::size_of::<$t>() / mem::size_of::<u8>();
|
|
let (input, result) = take($len * ratio)($input)?;
|
|
let result = unsafe {
|
|
let ptr = result.as_ptr() as *const $t;
|
|
let slice = std::slice::from_raw_parts(ptr, $len);
|
|
let slice = slice.to_vec();
|
|
|
|
if trans_to_bigger {
|
|
let offset = $offset as $bigger;
|
|
let scale = $scale as $bigger;
|
|
$bigger_result(
|
|
slice
|
|
.into_iter()
|
|
.map(|p| if (p as f64 - $fill_value).abs() < f64::EPSILON {p as $bigger} else {(p as $bigger - offset) / scale} )
|
|
.collect::<Vec<$bigger>>(),
|
|
)
|
|
} else {
|
|
$raw_result(if need_trans {
|
|
let offset = $offset as $t;
|
|
slice.into_iter().map(|p| if (p as f64 - $fill_value).abs() < f64::EPSILON {p} else {p - offset}).collect::<Vec<$t>>()
|
|
} else {
|
|
slice
|
|
})
|
|
}
|
|
};
|
|
Ok((input, result))
|
|
},
|
|
)+
|
|
}
|
|
}
|
|
|
|
};
|
|
}
|
|
|
|
impl Record {
|
|
pub fn parse_from_path(path: impl AsRef<Path>) -> Result<Self, ETWSError> {
|
|
let path = path.as_ref();
|
|
if !(path.ends_with(".dat") || !path.ends_with(".dat.gz")) {
|
|
return Err(std::io::Error::new(
|
|
std::io::ErrorKind::InvalidInput,
|
|
"Invalid file extension",
|
|
)
|
|
.into());
|
|
}
|
|
|
|
if !path.exists() {
|
|
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "File not found").into());
|
|
}
|
|
|
|
let mut file = File::open(path)?;
|
|
let binary_data = if path.extension().unwrap() == "gz" {
|
|
let mut result: Vec<u8> = Vec::new();
|
|
let mut decoder = GzDecoder::new(file);
|
|
decoder.read_to_end(&mut result)?;
|
|
result
|
|
} else {
|
|
let mut result = Vec::new();
|
|
file.read_to_end(&mut result)?;
|
|
result
|
|
};
|
|
|
|
let (_, parsed) =
|
|
Self::_parse(binary_data.as_slice()).map_err(|_| anyhow::Error::msg("Parse error"))?;
|
|
|
|
Ok(parsed)
|
|
}
|
|
|
|
fn _parse(binary_data: &[u8]) -> IResult<&[u8], Record> {
|
|
let start_tag = b"UNI_DATA";
|
|
let (input, _) = tag(start_tag)(binary_data)?;
|
|
let (input, order) = Self::_parse_split_fn(input, 8, 8, Self::_parse_order)?;
|
|
let (input, hlen) = take(4usize)(input)?;
|
|
|
|
let hlen = match order {
|
|
Order::BigEndian => BigEndian::read_u32(hlen),
|
|
_ => LittleEndian::read_u32(hlen),
|
|
};
|
|
let (input, record_info) = Self::_parse_split_fn(input, 0, 8, |input| {
|
|
let (input, p) = take(hlen)(input)?;
|
|
let p: RecordInfo = serde_json::from_slice(p).unwrap();
|
|
Ok((input, p))
|
|
})?;
|
|
|
|
let (input, blocks) =
|
|
count(Self::_parse_block_fn(order), record_info.block_num as usize)(input)?;
|
|
|
|
let data_time =
|
|
NaiveDateTime::parse_from_str(&record_info.file_time, r"%Y%m%d%H%M%S").unwrap();
|
|
let filetime = data_time.and_utc();
|
|
|
|
Ok((input, Record { filetime, blocks }))
|
|
}
|
|
|
|
fn _parse_order(input: &[u8]) -> IResult<&[u8], Order> {
|
|
let (input, order) = take(4usize)(input)?;
|
|
let result = if order == b"LEND" {
|
|
Order::LittleEndian
|
|
} else {
|
|
Order::BigEndian
|
|
};
|
|
let (input, _) = take(4usize)(input)?;
|
|
Ok((input, result))
|
|
}
|
|
|
|
fn _parse_split(input: &[u8], s: usize, fore: usize, after: usize) -> IResult<&[u8], &[u8]> {
|
|
let (input, _) = take(fore)(input)?;
|
|
let (input, result) = take(s)(input)?;
|
|
let (input, _) = take(after)(input)?;
|
|
Ok((input, result))
|
|
}
|
|
|
|
fn _parse_split_fn<E, F: Fn(&[u8]) -> IResult<&[u8], E>>(
|
|
input: &[u8],
|
|
fore: usize,
|
|
after: usize,
|
|
f: F,
|
|
) -> IResult<&[u8], E> {
|
|
let (input, _) = take(fore)(input)?;
|
|
let (input, result) = (f)(input)?;
|
|
let (input, _) = take(after)(input)?;
|
|
Ok((input, result))
|
|
}
|
|
|
|
fn _parse_block_fn(order: Order) -> impl FnMut(&[u8]) -> IResult<&[u8], ParsedBlock> {
|
|
move |input| Self::_parse_split_fn(input, 0, 0, |input| Self::_parse_block(input, order))
|
|
}
|
|
|
|
fn _parse_block(input: &[u8], order: Order) -> IResult<&[u8], ParsedBlock> {
|
|
let (input, _) = take(8usize)(input)?;
|
|
let (input, hlen1) = Self::_parse_u32(input, order)?;
|
|
let (input, block_info) = Self::_parse_split_fn(input, 0, 8, |input| {
|
|
let (input, p) = take(hlen1)(input)?;
|
|
|
|
let p: BlockJsonInfo = serde_json::from_slice(p).unwrap();
|
|
Ok((input, p))
|
|
})?;
|
|
|
|
let (input, _) = take(8usize)(input)?; // skip 8 bytes
|
|
let (input, hlen2) = Self::_parse_u32(input, order)?;
|
|
let dimension_size = block_info.dimension_size.clone();
|
|
let size = dimension_size.iter().fold(1, |acc, x| acc * x) as usize;
|
|
let value_type = block_info.value_type.clone();
|
|
let value_type = Self::_parse_type(&value_type);
|
|
|
|
let (input, data) = Self::_parse_matrix(
|
|
input,
|
|
value_type,
|
|
order,
|
|
size,
|
|
block_info.value_offset,
|
|
block_info.value_scale,
|
|
block_info.fill_value,
|
|
)?;
|
|
|
|
Ok((
|
|
input,
|
|
ParsedBlock {
|
|
info: block_info,
|
|
data,
|
|
},
|
|
))
|
|
}
|
|
|
|
fn _parse_i32(input: &[u8], order: Order) -> IResult<&[u8], i32> {
|
|
let (input, hlen) = take(4usize)(input)?;
|
|
let hlen = match order {
|
|
Order::BigEndian => BigEndian::read_i32(hlen),
|
|
_ => LittleEndian::read_i32(hlen),
|
|
};
|
|
|
|
Ok((input, hlen))
|
|
}
|
|
|
|
fn _parse_u32(input: &[u8], order: Order) -> IResult<&[u8], u32> {
|
|
let (input, hlen) = take(4usize)(input)?;
|
|
let hlen = match order {
|
|
Order::BigEndian => BigEndian::read_u32(hlen),
|
|
_ => LittleEndian::read_u32(hlen),
|
|
};
|
|
|
|
Ok((input, hlen))
|
|
}
|
|
|
|
fn _parse_i64(input: &[u8], order: Order) -> IResult<&[u8], i64> {
|
|
let (input, hlen) = take(8usize)(input)?;
|
|
let hlen = match order {
|
|
Order::BigEndian => BigEndian::read_i64(hlen),
|
|
_ => LittleEndian::read_i64(hlen),
|
|
};
|
|
|
|
Ok((input, hlen))
|
|
}
|
|
|
|
fn _parse_u64(input: &[u8], order: Order) -> IResult<&[u8], u64> {
|
|
let (input, hlen) = take(8usize)(input)?;
|
|
let hlen = match order {
|
|
Order::BigEndian => BigEndian::read_u64(hlen),
|
|
_ => LittleEndian::read_u64(hlen),
|
|
};
|
|
|
|
Ok((input, hlen))
|
|
}
|
|
|
|
fn _parse_type<'a>(type_str: &'a str) -> ValueTypes {
|
|
match type_str {
|
|
"b" => ValueTypes::I8,
|
|
"B" => ValueTypes::U8,
|
|
"i" => ValueTypes::I32,
|
|
"I" => ValueTypes::U32,
|
|
"f" => ValueTypes::F32,
|
|
"d" => ValueTypes::F64,
|
|
"h" => ValueTypes::I16,
|
|
_ => panic!("Invalid type"),
|
|
}
|
|
}
|
|
|
|
fn _parse_matrix(
|
|
input: &[u8],
|
|
type_: ValueTypes,
|
|
order: Order,
|
|
len: usize,
|
|
offset: f32,
|
|
scale: f32,
|
|
fill_value: f64,
|
|
) -> IResult<&[u8], ValueResult> {
|
|
match_in_macro!(
|
|
type_,
|
|
len,
|
|
input,
|
|
offset,
|
|
scale,
|
|
fill_value,
|
|
(
|
|
ValueTypes::I64,
|
|
i64,
|
|
f64,
|
|
ValueResult::I64,
|
|
ValueResult::F64
|
|
),
|
|
(
|
|
ValueTypes::F64,
|
|
f64,
|
|
f64,
|
|
ValueResult::F64,
|
|
ValueResult::F64
|
|
),
|
|
(
|
|
ValueTypes::U64,
|
|
u64,
|
|
f64,
|
|
ValueResult::U64,
|
|
ValueResult::F64
|
|
),
|
|
(
|
|
ValueTypes::I32,
|
|
i32,
|
|
f32,
|
|
ValueResult::I32,
|
|
ValueResult::F32
|
|
),
|
|
(
|
|
ValueTypes::F32,
|
|
f32,
|
|
f32,
|
|
ValueResult::F32,
|
|
ValueResult::F32
|
|
),
|
|
(
|
|
ValueTypes::U32,
|
|
u32,
|
|
f32,
|
|
ValueResult::U32,
|
|
ValueResult::F32
|
|
),
|
|
(
|
|
ValueTypes::I16,
|
|
i16,
|
|
f32,
|
|
ValueResult::I16,
|
|
ValueResult::F32
|
|
),
|
|
(ValueTypes::I8, i8, f32, ValueResult::I8, ValueResult::F32),
|
|
(ValueTypes::U8, u8, f32, ValueResult::U8, ValueResult::F32)
|
|
)
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Debug)]
|
|
struct RecordInfo {
|
|
file_time: String,
|
|
block_num: i32,
|
|
}
|
|
|
|
#[derive(Serialize, Deserialize, Debug)]
|
|
pub struct BlockJsonInfo {
|
|
pub value_name: String,
|
|
pub value_des: String,
|
|
pub value_type: String,
|
|
pub dimension_size: Vec<u64>,
|
|
pub dimension: usize,
|
|
pub dimension_des: Vec<String>,
|
|
pub dimension_start: Vec<f64>,
|
|
pub dimension_end: Vec<f64>,
|
|
pub dimension_values: Vec<Vec<f64>>,
|
|
pub fill_value: f64,
|
|
pub value_scale: f32,
|
|
pub value_offset: f32,
|
|
}
|
|
|
|
pub struct ParsedBlock {
|
|
pub info: BlockJsonInfo,
|
|
pub data: ValueResult,
|
|
}
|
|
|
|
// fn _parse_matrix(
|
|
// input: &[u8],
|
|
// type_: ValueTypes,
|
|
// order: Order,
|
|
// len: usize,
|
|
// offset: f32,
|
|
// scale: f32,
|
|
// fill_value: f64,
|
|
// ) -> IResult<&[u8], ValueResult> {
|
|
// use std::mem;
|
|
// let need_trans = offset != 0.0 || scale != 1.0;
|
|
// let trans_to_bigger = offset.trunc() != offset || scale != 1.0;
|
|
// match type_ {
|
|
// ValueTypes::I64 => {
|
|
// let ratio = mem::size_of::<i64>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const i64;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// if trans_to_bigger {
|
|
// let offset = offset as f32;
|
|
// ValueResult::F32(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p as f32 - offset) / scale)
|
|
// .collect::<Vec<f32>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::I64(if need_trans {
|
|
// let offset = offset as i64;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<i64>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// }
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// ValueTypes::F64 => {
|
|
// let ratio = mem::size_of::<f64>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const f64;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// ValueResult::F64(if need_trans {
|
|
// let offset = offset as f64;
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p - offset) / scale as f64)
|
|
// .collect::<Vec<f64>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// ValueTypes::I32 => {
|
|
// let ratio = mem::size_of::<i32>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const i32;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// if trans_to_bigger {
|
|
// let offset = offset as f32;
|
|
// ValueResult::F32(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p as f32 - offset) / scale)
|
|
// .collect::<Vec<f32>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::I32(if need_trans {
|
|
// let offset = offset as i32;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<i32>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// }
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
|
|
// ValueTypes::U32 => {
|
|
// let ratio = mem::size_of::<u32>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const u32;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// if trans_to_bigger {
|
|
// let offset = offset as f32;
|
|
// ValueResult::F32(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p as f32 - offset) / scale)
|
|
// .collect::<Vec<f32>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::U32(if need_trans {
|
|
// let offset = offset as u32;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<u32>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// }
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// ValueTypes::F32 => {
|
|
// let ratio = mem::size_of::<f32>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const f32;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// ValueResult::F32(if need_trans {
|
|
// let offset = offset as f32;
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p - offset) / scale)
|
|
// .collect::<Vec<f32>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// ValueTypes::U64 => {
|
|
// let ratio = mem::size_of::<u64>() / mem::size_of::<u8>();
|
|
// let (input, result) = take(len * ratio)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const u64;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
// let slice = slice.to_vec();
|
|
|
|
// if trans_to_bigger {
|
|
// let offset = offset as f64;
|
|
// let scale = scale as f64;
|
|
// ValueResult::F64(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p as f64 - offset) / scale)
|
|
// .collect::<Vec<f64>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::U64(if need_trans {
|
|
// let offset = offset as u64;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<u64>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// }
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// ValueTypes::I8 => {
|
|
// let (input, result) = take(len)(input)?;
|
|
// let result = unsafe {
|
|
// let ptr = result.as_ptr() as *const i8;
|
|
// let slice = std::slice::from_raw_parts(ptr, len);
|
|
|
|
// let slice = slice.to_vec();
|
|
|
|
// if trans_to_bigger {
|
|
// let offset = offset as f32;
|
|
// ValueResult::F32(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| (p as f32 - offset) / scale)
|
|
// .collect::<Vec<f32>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::I8(if need_trans {
|
|
// let offset = offset as i8;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<i8>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// }
|
|
// };
|
|
// Ok((input, result))
|
|
// }
|
|
|
|
// ValueTypes::U8 => {
|
|
// let (input, slice) = take(len)(input)?;
|
|
// let slice = slice.to_vec();
|
|
// let result = if trans_to_bigger {
|
|
// let offset = offset as f32;
|
|
// ValueResult::F32(
|
|
// slice
|
|
// .into_iter()
|
|
// .map(|p| {
|
|
// if p as f64 != fill_value {
|
|
// (p as f32 - offset) / scale
|
|
// } else {
|
|
// p as f32
|
|
// }
|
|
// })
|
|
// .collect::<Vec<f32>>(),
|
|
// )
|
|
// } else {
|
|
// ValueResult::U8(if need_trans {
|
|
// let offset = offset as u8;
|
|
// slice.into_iter().map(|p| p - offset).collect::<Vec<u8>>()
|
|
// } else {
|
|
// slice
|
|
// })
|
|
// };
|
|
|
|
// Ok((input, result))
|
|
// }
|
|
// }
|
|
// }
|