tsuki/rsParser-r
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

close release debug, add new stat infomation

Browse Source
This commit is contained in:
Tsuki 2023-10-13 13:19:20 +08:00
parent b2febe579e
commit 7463a2c743
3 changed files with 75 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -25,4 +25,4 @@ clap-verbosity-flag = "2.0.1"
ndarray-stats = "0.5.1"
[profile.release]
debug = true
debug = false

3
src/main.rs
View File

@ -10,6 +10,7 @@ use app::{Cli, Commands};
use clap::Parser;
use cmds::parse::{ParseCmd, ParseCmdConfig};
use cmds::Cmd;
use std::time::Instant;
fn main() {
let args = Cli::parse();
@ -19,7 +20,9 @@ fn main() {
match args.command {
Commands::Parse { path, output_path, verbose } => {
let parse_conf = ParseCmdConfig::new(path, output_path, verbose);
let start = Instant::now();
ParseCmd.parse(&app, parse_conf).unwrap().unwrap();
println!("elapse time: {}", start.elapsed().as_secs());
}
}
}

107
src/parse.rs
View File

@ -11,6 +11,7 @@ use std::any::Any;
use std::str::from_utf8;
use ndarray::{ArrayD, ShapeBuilder};
use ndarray_stats::QuantileExt;
use ndarray::parallel::prelude::*;
#[derive(Deserialize, Serialize, Debug)]
pub struct HeaderJson {
@ -123,7 +124,7 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
if !only_header {
let c = j.dimension_size.iter().fold(1, |i, j| i * (*j)) as usize;
let mut converted_v: Option<ArrayD<f64>>;
// let mut converted_v: Option<ArrayD<f64>>;
d = match j.value_type.as_str() {
"b" => {
let (_, v) = count(
@ -134,17 +135,23 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c,
)(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size.clone()).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as i8 {
f64::NAN
} else {
i as f64
let (min_value, max_value) = v.iter().fold((i8::MAX, i8::MIN), |(min, max), i| {
if *i == j.fill_value as i8 {
return (min, max);
}
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
});
stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(),
max_value: f64_v.max_skipnan().clone(),
min_value: min_value as f64,
max_value: max_value as f64,
});
Some(Box::new(v))
@ -158,17 +165,24 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c,
)(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as u8 {
f64::NAN
} else {
i as f64
let (min_value, max_value) = v.iter().fold((u8::MAX, u8::MIN), |(min, max), i| {
if *i == j.fill_value as u8 {
return (min, max);
}
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
});
stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(),
max_value: f64_v.max_skipnan().clone(),
min_value: min_value as f64,
max_value: max_value as f64,
});
Some(Box::new(v))
}
@ -181,18 +195,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c,
)(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as i32 {
f64::NAN
} else {
i as f64
let (min_value, max_value) = v.iter().fold((i32::MAX, i32::MIN), |(min, max), i| {
if *i == j.fill_value as i32 {
return (min, max);
}
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
});
stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(),
max_value: f64_v.max_skipnan().clone(),
min_value: min_value as f64,
max_value: max_value as f64,
});
Some(Box::new(v))
}
"u" => {
@ -204,18 +225,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c,
)(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as u32 {
f64::NAN
} else {
i as f64
let (min_value, max_value) = v.iter().fold((u32::MAX, u32::MIN), |(min, max), i| {
if *i == j.fill_value as u32 {
return (min, max);
}
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
});
stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(),
max_value: f64_v.max_skipnan().clone(),
min_value: min_value as f64,
max_value: max_value as f64,
});
Some(Box::new(v))
}
"f" => {
@ -227,18 +255,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c,
)(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as f32 {
f64::NAN
} else {
i as f64
let (min_value, max_value) = v.iter().fold((f32::MAX, f32::MIN), |(min, max), i| {
if *i == j.fill_value as f32 {
return (min, max);
}
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
});
stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(),
max_value: f64_v.max_skipnan().clone(),
min_value: min_value as f64,
max_value: max_value as f64,
});
Some(Box::new(v))
}
_ => {