add rayon

Cargo.lock

@@ -56,6 +56,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -386,6 +392,7 @@ dependencies = [
 name = "rsParser-r"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "clap",
  "flate2",
  "indoc",
@@ -396,6 +403,7 @@ dependencies = [
  "rayon",
  "serde",
  "serde_json",
+ "thiserror",
 ]
 
 [[package]]
@@ -475,6 +483,26 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.33",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.12"

Cargo.toml

@@ -17,3 +17,5 @@ serde_json = "1.0.107"
 flate2 = "1.0.27"
 indoc = "2.0.3"
 num-traits = "0.2.16"
+thiserror = "1.0.48"
+anyhow = "1.0.75"

src/app.rs

@@ -1,11 +1,57 @@
-use num_traits::Num;
+use crate::parse::{Parsed, parse};
+use crate::error::{AResult, BrokenError};
+use crate::utils::is_gz;
+use rayon::prelude::{*, IndexedParallelIterator};
+use flate2::read::GzDecoder;
+use std::path::PathBuf;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
 
-use crate::parse::Parsed;
+type AAResult<O> = AResult<O,()>;
 
-pub struct App{
-    datas: Vec<Parsed>
+pub struct App {
+    paths: Vec<PathBuf>,
+    config: AppConfig,
+}
+
+#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)]
+pub struct AppConfig {
+    pub only_header: bool,
+    pub multi_threading: bool,
 }
 
 impl App {
+    pub fn new(paths: Vec<PathBuf>, config: AppConfig) -> Result<Self, BrokenError> {
+        Ok(Self { paths, config })
+    }
 
+    pub fn parse(&self) ->AAResult<Vec<Parsed>> {
+        let paths = &self.paths;
+        let config = &self.config;
+        let datas = paths.par_iter().map(|p| {
+            let d = Self::data_prepare(p);
+            match d {
+                Ok(_d) => parse(&_d, config.only_header).map(|(_,b)| b).map_err(|e| e.into()),
+                Err(e) => Err(e.into())
+            }
+        }).collect::<Result<Vec<_>, BrokenError>>()?;
+        Ok((datas,()))
+    }
+
+    fn data_prepare(path: &PathBuf) -> Result<Vec<u8>, std::io::Error> {
+        let mut f = File::open(path)?;
+        let mut buf = Vec::new();
+        {
+            let mut magic = [0; 2];
+            f.read_exact(&mut magic)?;
+            f.seek(SeekFrom::Start(0))?;
+            if is_gz(&magic) {
+                let mut d = GzDecoder::new(f);
+                d.read_to_end(&mut buf)?;
+            } else {
+                f.read_to_end(&mut buf)?;
+            }
+        }
+        Ok(buf)
+    }
 }

src/error.rs

@@ -0,0 +1,18 @@
+use thiserror::Error;
+use nom::Err;
+
+#[derive(Error, Debug)]
+pub enum BrokenError {
+    #[error("Can't Open file or file not exist.")]
+    IO(#[from] std::io::Error),
+    #[error("This file is not standard format.")]
+    ParseError,
+}
+
+impl<T> From<Err<T>> for BrokenError {
+    fn from(value: Err<T>) -> Self {
+        Self::ParseError
+    }
+}
+
+pub type AResult<O, E> = Result<(O, E), BrokenError>;

src/main.rs

@@ -2,6 +2,8 @@ mod parse;
 mod app;
 mod printer;
 mod utils;
+mod error;
+
 use std::io::prelude::*;
 use flate2::read::GzDecoder;
 use std::fs::File;
@@ -11,6 +13,8 @@ use std::path::PathBuf;
 use clap::{Args, command, Parser, Subcommand, ValueEnum};
 use parse::parse;
 
+use crate::app::{App, AppConfig};
+
 /// A fictional versioning CLI
 #[derive(Debug, Parser)] // requires `derive` feature
 #[command(name = "rsParser")]
@@ -76,12 +80,11 @@ fn main() {
     let args = Cli::parse();
     match args.command {
         Commands::Parse { path, header } => {
-            let f = File::open(path).unwrap();
-            let mut d = GzDecoder::new(f);
-            let mut buf = Vec::new();
-            d.read_to_end(&mut buf).unwrap();
-            let (_, parsed) = parse(&buf, header).unwrap();
-            println!("{}", parsed);
+            let config = AppConfig{
+                only_header:header,
+                multi_threading:true
+            };
+            let app = App::new(vec![path], config).unwrap();
         }
     }
 }

src/parse.rs

@@ -1,17 +1,12 @@
-use ndarray::ArrayD;
 use nom::bytes::complete::*;
-use nom::error::ParseError;
+use nom::error::{ParseError, ErrorKind};
 use nom::multi::count;
-use nom::number::complete::{
-    be_f32, be_f64, be_i16, be_i32, be_i8, be_u8, le_f32, le_f64, le_i16, le_i32, le_i8, le_u8,
-};
+use nom::number::complete::{be_f32, be_i32, be_i8, be_u32, be_u8, le_f32, le_i32, le_i8, le_u8, le_u32};
 use nom::sequence::tuple;
 use nom::IResult;
-use nom_derive;
 use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::str::from_utf8;
-use std::string;
 
 #[derive(Deserialize, Serialize, Debug)]
 pub struct HeaderJson {
@@ -51,7 +46,7 @@ pub struct CommonHeader {
 
 pub struct Block {
     pub block_info: BlockInfo,
-    pub data: Option<Box<dyn Any>>,
+    pub data: Option<Box<dyn Any + Sync + Send>>,
 }
 
 pub struct Parsed {
@@ -93,7 +88,8 @@ fn common_parse(input: &[u8]) -> IResult<&[u8], CommonHeader> {
 fn block_parse<'a, Error: ParseError<&'a [u8]>>(
     only_header: bool,
     endian: Endian,
-) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Block, Error> {
+) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Block, Error>
+{
     let l_p = match endian {
         Endian::Big => be_i32,
         Endian::Little => le_i32,
@@ -112,28 +108,57 @@ fn block_parse<'a, Error: ParseError<&'a [u8]>>(
         let (next, l2) = l_p(next)?;
         let (next, _data) = take(l2 as usize)(next)?;
 
+        let mut d: Option<Box<dyn Any + Send + Sync>> = None;
+
         if !only_header {
             let c = j.dimension_size.iter().fold(1, |i, j| i * (*j)) as usize;
-            match j.value_type.as_str() {
+            d = match j.value_type.as_str() {
                 "b" => {
-                    let (next,v) = count(match endian {
+                    let (_, v) = count(match endian {
                         Endian::Big => be_i8,
                         _ => le_i8
-                    },c)(_data)?;
+                    }, c)(_data)?;
+                    Some(Box::new(v))
                 }
-                "B" => {}
-                "h" => {}
-                "f" => {}
-                _ => {}
+                "B" => {
+                    let (_, v) = count(match endian {
+                        Endian::Big => be_u8,
+                        _ => le_u8
+                    }, c)(_data)?;
+                    Some(Box::new(v))
                 }
+                "i" => {
+                    let (_, v) = count(match endian {
+                        Endian::Big => be_i32,
+                        _ => le_i32
+                    }, c)(_data)?;
+                    Some(Box::new(v))
+                }
+                "u" => {
+                    let (_, v) = count(match endian {
+                        Endian::Big => be_u32,
+                        _ => le_u32
+                    }, c)(_data)?;
+                    Some(Box::new(v))
+                }
+                "f" => {
+                    let (_, v) = count(match endian {
+                        Endian::Big => be_f32,
+                        _ => le_f32
+                    }, c)(_data)?;
+                    Some(Box::new(v))
+                }
+                _ => {
+                    return Err(nom::Err::Failure(Error::from_error_kind(next, ErrorKind::Alpha)));
+                }
+            };
         }
-
         let (next, _) = take(8usize)(next)?;
         Ok((
             next,
             Block {
                 block_info: j,
-                data: None,
+                data: d,
             },
         ))
     }
@@ -151,7 +176,7 @@ fn blocks_parse(
 }
 
 pub fn parse<'a>(input: &'a [u8], only_header: bool) -> IResult<&'a [u8], Parsed> {
-    let (next, magic) = tag(b"UNI_DATA")(input)?;
+    let (next, _) = tag(b"UNI_DATA")(input)?;
     let (next, common_header) = common_parse(next)?;
     let (next, blocks) = blocks_parse(
         next,

src/utils.rs

@@ -1,3 +1,3 @@
-fn is_gz(data:&[u8;2]) -> bool{
+pub fn is_gz(data:&[u8;2]) -> bool{
     *data == [0x1f, 0x8b]
 }