tsuki/rsParser-r
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

close release debug, add new stat infomation

Browse Source
This commit is contained in:
Tsuki 2023-10-13 13:19:20 +08:00
parent b2febe579e
commit 7463a2c743
3 changed files with 75 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@ -25,4 +25,4 @@ clap-verbosity-flag = "2.0.1"
ndarray-stats = "0.5.1" ndarray-stats = "0.5.1"
[profile.release] [profile.release]
debug = true debug = false

3
src/main.rs
View File

@ -10,6 +10,7 @@ use app::{Cli, Commands};
use clap::Parser; use clap::Parser;
use cmds::parse::{ParseCmd, ParseCmdConfig}; use cmds::parse::{ParseCmd, ParseCmdConfig};
use cmds::Cmd; use cmds::Cmd;
use std::time::Instant;
fn main() { fn main() {
let args = Cli::parse(); let args = Cli::parse();
@ -19,7 +20,9 @@ fn main() {
match args.command { match args.command {
Commands::Parse { path, output_path, verbose } => { Commands::Parse { path, output_path, verbose } => {
let parse_conf = ParseCmdConfig::new(path, output_path, verbose); let parse_conf = ParseCmdConfig::new(path, output_path, verbose);
let start = Instant::now();
ParseCmd.parse(&app, parse_conf).unwrap().unwrap(); ParseCmd.parse(&app, parse_conf).unwrap().unwrap();
println!("elapse time: {}", start.elapsed().as_secs());
} }
} }
} }

107
src/parse.rs
View File

@ -11,6 +11,7 @@ use std::any::Any;
use std::str::from_utf8; use std::str::from_utf8;
use ndarray::{ArrayD, ShapeBuilder}; use ndarray::{ArrayD, ShapeBuilder};
use ndarray_stats::QuantileExt; use ndarray_stats::QuantileExt;
use ndarray::parallel::prelude::*;
#[derive(Deserialize, Serialize, Debug)] #[derive(Deserialize, Serialize, Debug)]
pub struct HeaderJson { pub struct HeaderJson {
@ -123,7 +124,7 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
if !only_header { if !only_header {
let c = j.dimension_size.iter().fold(1, |i, j| i * (*j)) as usize; let c = j.dimension_size.iter().fold(1, |i, j| i * (*j)) as usize;
let mut converted_v: Option<ArrayD<f64>>; // let mut converted_v: Option<ArrayD<f64>>;
d = match j.value_type.as_str() { d = match j.value_type.as_str() {
"b" => { "b" => {
let (_, v) = count( let (_, v) = count(
@ -134,17 +135,23 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c, c,
)(_data)?; )(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size.clone()).clone().into_shape(), v).unwrap(); let v = ArrayD::from_shape_vec((&j.dimension_size.clone()).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| { let (min_value, max_value) = v.iter().fold((i8::MAX, i8::MIN), |(min, max), i| {
if i == j.fill_value as i8 { if *i == j.fill_value as i8 {
f64::NAN return (min, max);
} else {
i as f64
} }
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
}); });
stat = Some(StatInfo { stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(), min_value: min_value as f64,
max_value: f64_v.max_skipnan().clone(), max_value: max_value as f64,
}); });
Some(Box::new(v)) Some(Box::new(v))
@ -158,17 +165,24 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c, c,
)(_data)?; )(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| {
if i == j.fill_value as u8 { let (min_value, max_value) = v.iter().fold((u8::MAX, u8::MIN), |(min, max), i| {
f64::NAN if *i == j.fill_value as u8 {
} else { return (min, max);
i as f64
} }
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
}); });
stat = Some(StatInfo { stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(), min_value: min_value as f64,
max_value: f64_v.max_skipnan().clone(), max_value: max_value as f64,
}); });
Some(Box::new(v)) Some(Box::new(v))
} }
@ -181,18 +195,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c, c,
)(_data)?; )(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| { let (min_value, max_value) = v.iter().fold((i32::MAX, i32::MIN), |(min, max), i| {
if i == j.fill_value as i32 { if *i == j.fill_value as i32 {
f64::NAN return (min, max);
} else {
i as f64
} }
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
}); });
stat = Some(StatInfo { stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(), min_value: min_value as f64,
max_value: f64_v.max_skipnan().clone(), max_value: max_value as f64,
}); });
Some(Box::new(v)) Some(Box::new(v))
} }
"u" => { "u" => {
@ -204,18 +225,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c, c,
)(_data)?; )(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| { let (min_value, max_value) = v.iter().fold((u32::MAX, u32::MIN), |(min, max), i| {
if i == j.fill_value as u32 { if *i == j.fill_value as u32 {
f64::NAN return (min, max);
} else {
i as f64
} }
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
}); });
stat = Some(StatInfo { stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(), min_value: min_value as f64,
max_value: f64_v.max_skipnan().clone(), max_value: max_value as f64,
}); });
Some(Box::new(v)) Some(Box::new(v))
} }
"f" => { "f" => {
@ -227,18 +255,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
c, c,
)(_data)?; )(_data)?;
let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap();
let f64_v = v.mapv(|i| { let (min_value, max_value) = v.iter().fold((f32::MAX, f32::MIN), |(min, max), i| {
if i == j.fill_value as f32 { if *i == j.fill_value as f32 {
f64::NAN return (min, max);
} else {
i as f64
} }
let mut g = (min, max);
if *i < min {
g.0 = *i;
}
if *i > max {
g.1 = *i;
}
return g;
}); });
stat = Some(StatInfo { stat = Some(StatInfo {
min_value: f64_v.min_skipnan().clone(), min_value: min_value as f64,
max_value: f64_v.max_skipnan().clone(), max_value: max_value as f64,
}); });
Some(Box::new(v)) Some(Box::new(v))
} }
_ => { _ => {