From 7463a2c743979cb24252ad78d7c2dcb1b2f7698d Mon Sep 17 00:00:00 2001 From: Tsuki Date: Fri, 13 Oct 2023 13:19:20 +0800 Subject: [PATCH] close release debug, add new stat infomation --- Cargo.toml | 2 +- src/main.rs | 3 ++ src/parse.rs | 107 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 75 insertions(+), 37 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bf6ec47..438559d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,4 +25,4 @@ clap-verbosity-flag = "2.0.1" ndarray-stats = "0.5.1" [profile.release] -debug = true \ No newline at end of file +debug = false \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index cd7b535..eaf8eed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use app::{Cli, Commands}; use clap::Parser; use cmds::parse::{ParseCmd, ParseCmdConfig}; use cmds::Cmd; +use std::time::Instant; fn main() { let args = Cli::parse(); @@ -19,7 +20,9 @@ fn main() { match args.command { Commands::Parse { path, output_path, verbose } => { let parse_conf = ParseCmdConfig::new(path, output_path, verbose); + let start = Instant::now(); ParseCmd.parse(&app, parse_conf).unwrap().unwrap(); + println!("elapse time: {}", start.elapsed().as_secs()); } } } diff --git a/src/parse.rs b/src/parse.rs index 90da584..cab3358 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -11,6 +11,7 @@ use std::any::Any; use std::str::from_utf8; use ndarray::{ArrayD, ShapeBuilder}; use ndarray_stats::QuantileExt; +use ndarray::parallel::prelude::*; #[derive(Deserialize, Serialize, Debug)] pub struct HeaderJson { @@ -123,7 +124,7 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( if !only_header { let c = j.dimension_size.iter().fold(1, |i, j| i * (*j)) as usize; - let mut converted_v: Option>; + // let mut converted_v: Option>; d = match j.value_type.as_str() { "b" => { let (_, v) = count( @@ -134,17 +135,23 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( c, )(_data)?; let v = ArrayD::from_shape_vec((&j.dimension_size.clone()).clone().into_shape(), v).unwrap(); - let f64_v = v.mapv(|i| { - if i == j.fill_value as i8 { - f64::NAN - } else { - i as f64 + let (min_value, max_value) = v.iter().fold((i8::MAX, i8::MIN), |(min, max), i| { + if *i == j.fill_value as i8 { + return (min, max); } + let mut g = (min, max); + if *i < min { + g.0 = *i; + } + if *i > max { + g.1 = *i; + } + return g; }); stat = Some(StatInfo { - min_value: f64_v.min_skipnan().clone(), - max_value: f64_v.max_skipnan().clone(), + min_value: min_value as f64, + max_value: max_value as f64, }); Some(Box::new(v)) @@ -158,17 +165,24 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( c, )(_data)?; let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); - let f64_v = v.mapv(|i| { - if i == j.fill_value as u8 { - f64::NAN - } else { - i as f64 + + let (min_value, max_value) = v.iter().fold((u8::MAX, u8::MIN), |(min, max), i| { + if *i == j.fill_value as u8 { + return (min, max); } + let mut g = (min, max); + if *i < min { + g.0 = *i; + } + if *i > max { + g.1 = *i; + } + return g; }); stat = Some(StatInfo { - min_value: f64_v.min_skipnan().clone(), - max_value: f64_v.max_skipnan().clone(), + min_value: min_value as f64, + max_value: max_value as f64, }); Some(Box::new(v)) } @@ -181,18 +195,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( c, )(_data)?; let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); - let f64_v = v.mapv(|i| { - if i == j.fill_value as i32 { - f64::NAN - } else { - i as f64 + let (min_value, max_value) = v.iter().fold((i32::MAX, i32::MIN), |(min, max), i| { + if *i == j.fill_value as i32 { + return (min, max); } + let mut g = (min, max); + if *i < min { + g.0 = *i; + } + if *i > max { + g.1 = *i; + } + return g; }); stat = Some(StatInfo { - min_value: f64_v.min_skipnan().clone(), - max_value: f64_v.max_skipnan().clone(), + min_value: min_value as f64, + max_value: max_value as f64, }); + Some(Box::new(v)) } "u" => { @@ -204,18 +225,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( c, )(_data)?; let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); - let f64_v = v.mapv(|i| { - if i == j.fill_value as u32 { - f64::NAN - } else { - i as f64 + let (min_value, max_value) = v.iter().fold((u32::MAX, u32::MIN), |(min, max), i| { + if *i == j.fill_value as u32 { + return (min, max); } + let mut g = (min, max); + if *i < min { + g.0 = *i; + } + if *i > max { + g.1 = *i; + } + return g; }); stat = Some(StatInfo { - min_value: f64_v.min_skipnan().clone(), - max_value: f64_v.max_skipnan().clone(), + min_value: min_value as f64, + max_value: max_value as f64, }); + Some(Box::new(v)) } "f" => { @@ -227,18 +255,25 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>( c, )(_data)?; let v = ArrayD::from_shape_vec((&j.dimension_size).clone().into_shape(), v).unwrap(); - let f64_v = v.mapv(|i| { - if i == j.fill_value as f32 { - f64::NAN - } else { - i as f64 + let (min_value, max_value) = v.iter().fold((f32::MAX, f32::MIN), |(min, max), i| { + if *i == j.fill_value as f32 { + return (min, max); } + let mut g = (min, max); + if *i < min { + g.0 = *i; + } + if *i > max { + g.1 = *i; + } + return g; }); stat = Some(StatInfo { - min_value: f64_v.min_skipnan().clone(), - max_value: f64_v.max_skipnan().clone(), + min_value: min_value as f64, + max_value: max_value as f64, }); + Some(Box::new(v)) } _ => {