From dce451befa970be561bd9cedde1806e73267b004 Mon Sep 17 00:00:00 2001 From: sleptworld Date: Mon, 28 Aug 2023 22:42:44 +0800 Subject: [PATCH] level2 update --- Cargo.lock | 68 +++++++++++++ Cargo.toml | 2 + src/error.rs | 14 ++- src/format/cc.rs | 36 +++++++ src/format/cinrad.rs | 222 +++++++++++++++++++++++++++++++++++++++++++ src/format/mod.rs | 8 ++ src/main.rs | 10 ++ src/utils.rs | 29 ++++++ tools.py | 108 +++++++++++++++++++++ 9 files changed, 494 insertions(+), 3 deletions(-) create mode 100644 src/format/cc.rs create mode 100644 src/format/cinrad.rs create mode 100644 src/format/mod.rs create mode 100644 src/utils.rs create mode 100644 tools.py diff --git a/Cargo.lock b/Cargo.lock index 8775120..d87501c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,9 @@ dependencies = [ "bzip2", "chrono", "flate2", + "ndarray", "nom", + "nom-derive", "num-integer", "num-traits", "thiserror", @@ -226,6 +228,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matrixmultiply" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090126dc04f95dc0d1c1c91f61bdd474b3930ca064c1edc8a849da2c6cbe1e77" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.5.0" @@ -247,6 +259,19 @@ dependencies = [ "adler", ] +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + [[package]] name = "nom" version = "7.1.3" @@ -257,6 +282,37 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff943d68b88d0b87a6e0d58615e8fa07f9fd5a1319fa0a72efc1f62275c79a7" +dependencies = [ + "nom", + "nom-derive-impl", + "rustversion", +] + +[[package]] +name = "nom-derive-impl" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b9a93a84b0d3ec3e70e02d332dc33ac6dfac9cde63e17fcb77172dededa62" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num-complex" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -306,6 +362,18 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "scratch" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 34ea1a2..acbfcb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,5 @@ thiserror = "1.0.40" bzip2 = "0.4.4" nom = "7.1.3" chrono = "0.4.24" +nom-derive = "0.10.1" +ndarray = "0.15.6" diff --git a/src/error.rs b/src/error.rs index 5d6563c..5495158 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,4 +1,3 @@ -use nom::error::Error; use std::{error, io}; use thiserror::Error; @@ -10,10 +9,19 @@ pub enum UnexpectedError { #[from] source: io::Error, }, - #[error("")] UnknownFormat, - #[error("")] DataParseError, } + +impl From> for UnexpectedError { + fn from(value: nom::error::Error) -> Self { + UnexpectedError::DataParseError + } +} +impl From> for UnexpectedError { + fn from(value: nom::Err) -> Self { + UnexpectedError::DataParseError + } +} diff --git a/src/format/cc.rs b/src/format/cc.rs new file mode 100644 index 0000000..e3f0767 --- /dev/null +++ b/src/format/cc.rs @@ -0,0 +1,36 @@ +struct CCHeader<'a> { + fs_file_id: &'a str, + ff_version_no: f32, + fl_file_header_length: i32, + fs_country: &'a str, + fs_province: &'a str, + fs_station: &'a str, + fs_station_number: &'a str, + fs_radar_type: &'a str, + fs_longitude: &'a str, + fs_latitude: &'a str, + fl_longitude_value: i32, + fl_latitude_value: i32, + fl_height: i32, + fsh_max_angle: i16, + fsh_opti_angle: i16, + fl_antenna_g: i32, + fus_beam_h: u16, + fus_beam_l: u16, + fuc_polarization: u8, + fus_sidelobe: u16, + fl_power: i32, + fl_wavelength: i32, + fus_log_a: u16, + fus_line_a: u16, + fus_agcp: u16, + fus_log_min_power: u16, + fus_line_min_power: u16, + fuc_clutter_t: u8, + fuc_velocity_p: u8, + fuc_filter_p: u8, + fuc_noise_t: u8, + fuc_sqit: u8, + fuc_intensity_c: u8, + fuc_intensity_r: u8, +} diff --git a/src/format/cinrad.rs b/src/format/cinrad.rs new file mode 100644 index 0000000..108b2f5 --- /dev/null +++ b/src/format/cinrad.rs @@ -0,0 +1,222 @@ +use std::fs::File; +use std::path::Path; +use std::str::FromStr; + +use chrono::prelude::*; +use chrono::Duration; +use ndarray::prelude::*; +use nom::branch::alt; +use nom::bytes::complete::*; +use nom::multi::many0; +use nom::sequence::tuple; +use nom::Err; +use nom::IResult; +use nom::Parser; +use nom_derive::NomLE; +use nom_derive::Parse; + +use crate::error::UnexpectedError; +use crate::utils::decompress; + +use super::Data; + +#[derive(Debug)] +struct CinradBlock { + t: RadarType, + info: Info, + res: Res, +} + +#[derive(Debug)] +pub enum RadarType { + SA, + SB, + SC, + CA, + CB, + CD, + CC, + CC2, +} + +impl FromStr for RadarType { + type Err = UnexpectedError; + fn from_str(s: &str) -> Result { + match s { + "SA" => Ok(RadarType::SA), + "SB" => Ok(RadarType::SB), + "SC" => Ok(RadarType::SC), + "CA" => Ok(RadarType::CA), + "CB" => Ok(RadarType::CB), + "CD" => Ok(RadarType::CD), + "CC" => Ok(RadarType::CC), + "CC2" => Ok(RadarType::CC2), + _ => Err(UnexpectedError::UnknownFormat), + } + } +} + +#[derive(Debug, NomLE)] +pub struct Info { + time: u32, + day: u16, + unambiguous_distance: u16, + azimuth: u16, + radial_num: u16, + radial_state: u16, + elevation: u16, + el_num: u16, + first_gate_r: u16, + first_gate_v: u16, + gate_length_r: u16, + gate_length_v: u16, + gate_num_r: u16, + gate_num_v: u16, + sector_num: u16, + system_coff: u32, + r_pointer: u16, + v_pointer: u16, + w_pointer: u16, + v_reso: u16, + vcp_mode: u16, + res2: [u16; 4], + r_pointer_2: u16, + v_pointer_2: u16, + w_pointer_2: u16, + nyquist_vel: u16, +} + +#[derive(Debug, NomLE)] +pub struct Res { + res_3: [u16; 19], +} + +#[derive(Debug, NomLE)] +struct SabData { + r: [u8; 460], + v: [u8; 920], + w: [u8; 920], + res4: [u16; 2], +} + +#[derive(Debug, NomLE)] +struct CabData { + r: [u8; 800], + v: [u8; 1600], + w: [u8; 1600], + res4: [u16; 2], +} + +#[derive(Debug, NomLE)] +struct SpecialData { + r: [u8; 1000], + v: [u8; 1000], + w: [u8; 1000], +} + +fn get_radar_type>( + input: &Vec, + file_name: S, +) -> Result { + let buf = &input[99..109]; + + match buf.as_ref() { + b"CINRAD/SC" => Ok(RadarType::SC), + b"CINRAD/CD" => Ok(RadarType::CD), + _ => { + let mut buf = &input[116..125]; + + if buf.as_ref() == b"CINRAD/CC" { + Ok(RadarType::CC) + } else { + let f_n = file_name.as_ref(); + if f_n.starts_with("RADA") { + let sp = f_n.split("-").collect::>(); + if sp.len() > 2 { + RadarType::from_str(sp[2]) + } else { + Err(UnexpectedError::UnknownFormat) + } + } else if f_n.starts_with("Z") { + let spart = f_n.split("_").collect::>(); + if spart.len() > 7 { + RadarType::from_str(spart[7]) + } else { + Err(UnexpectedError::UnknownFormat) + } + } else { + Err(UnexpectedError::UnknownFormat) + } + } + } + } +} + +fn parse(input: &[u8], radar_type: RadarType) -> Result<(), UnexpectedError> { + match radar_type { + RadarType::SA | RadarType::SB => { + let header_flag = take(28usize); + let (_, parsed_data) = many0(tuple(( + header_flag, + Info::parse, + Res::parse, + SabData::parse, + )))(input)?; + + parsed_data.iter().for_each(|x| { + let ele = x.1.elevation as f64 / 8.0 * (180.0 / 4096.0); + let azi = x.1.azimuth as f64 / 8.0 * (180.0 / 4096.0); + let ran = x.1.radial_num; + + if (ele - 0.5767822265625).abs() < f64::EPSILON { + println!("ele: {:?}, azi: {:?}, ran: {:?}", ele, azi, ran); + } + }); + + // let (header, info, res, _) = &parsed_data; + // let scan_time = Utc.with_ymd_and_hms(1969, 12, 31, 0, 0, 0).unwrap() + // + Duration::days(info.day as i64) + // + Duration::milliseconds(info.time as i64); + + // println!("scan time :{:?}", scan_time); + } + RadarType::CA | RadarType::CB => {} + RadarType::CC2 => {} + RadarType::CC => {} + RadarType::SC | RadarType::CD => {} + _ => {} + } + // match t { + // RadarType::SA | RadarType::SB => { + // let (_, parsed_data) = many0(tuple(( + // Header::parse, + // Info::parse, + // Res::parse, + // SabData::parse, + // )))(input)?; + // } + // RadarType::CA | RadarType::CB => { + // let (_, parsed_data) = + // tuple((Header::parse, Info::parse, Res::parse, CabData::parse))(input)?; + // } + // RadarType::CC2 => {} + // RadarType::CC => {} + // RadarType::SC | RadarType::CD => {} + // _ => {} + // } + Ok(()) +} +pub struct Cinrad {} + +impl Data for Cinrad { + fn parse(path: impl AsRef) -> Result { + let p = path.as_ref(); + let filename = p.file_name().unwrap().to_str().unwrap(); + + let mut f = File::open(p)?; + let depressed = decompress(&mut f)?; + let radar_type = get_radar_type(&depressed, filename)?; + parse(&depressed, radar_type)?; + Ok(Self {}) + } +} diff --git a/src/format/mod.rs b/src/format/mod.rs new file mode 100644 index 0000000..a5a5573 --- /dev/null +++ b/src/format/mod.rs @@ -0,0 +1,8 @@ +mod cc; +mod cinrad; +use crate::error::UnexpectedError; +pub use cinrad::Cinrad; +use std::path::Path; +pub trait Data: Sized { + fn parse(path: impl AsRef) -> Result; +} diff --git a/src/main.rs b/src/main.rs index 570ab00..7c0e9f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,15 @@ +use std::fs::File; + mod error; +mod format; mod product; +mod utils; + +use format::Cinrad; +use format::Data; fn main() { println!("Hello, world!"); + + let path = "/Users/ruomu/Library/Containers/com.tencent.WeWorkMac/Data/Documents/Profiles/8174C20601DD24B17112F7CC37A11CD9/Caches/Files/2023-04/e93ebe1aba50883b9bc661d7a44aee8e/Z_RADR_I_Z9595_20230324024700_O_DOR_SA_CAP.bin.bz2"; + let cinrad = Cinrad::parse(path).unwrap(); } diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..ec1eae9 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,29 @@ +use bzip2::read::{BzDecoder, BzEncoder}; +use bzip2::Decompress; +use flate2::bufread::DeflateDecoder; +use std::io::{prelude::*, SeekFrom}; +use std::{fs::File, io::BufReader, io::Read}; + +pub fn decompress(file: &mut File) -> Result, std::io::Error> { + let mut bf: [u8; 3] = [0, 0, 0]; + file.read_exact(&mut bf)?; + + let mut result = Vec::new(); + + match bf.as_ref() { + b"\x1f\x8b " => { + let reader = BufReader::new(file); + let mut decoder = DeflateDecoder::new(reader); + decoder.read_to_end(&mut result)?; + } + b"BZh" => { + file.seek(SeekFrom::Start(0))?; + let mut bz = BzDecoder::new(file); + bz.read_to_end(&mut result)?; + } + _ => { + file.read_to_end(&mut result)?; + } + } + Ok(result) +} diff --git a/tools.py b/tools.py new file mode 100644 index 0000000..30dc4e5 --- /dev/null +++ b/tools.py @@ -0,0 +1,108 @@ +import os +import re +table = { + "u": "u", + "i" : "i", + "b" : "i8", + "B" : "u8", + "S" : "&'a str", + "f" : "f", +} + +_S_INFO = [ + ("time", "u4"), + ("day", "u2"), + ("unambiguous_distance", "u2"), + ("azimuth", "u2"), + ("radial_num", "u2"), + ("radial_state", "u2"), + ("elevation", "u2"), + ("el_num", "u2"), + ("first_gate_r", "u2"), + ("first_gate_v", "u2"), + ("gate_length_r", "u2"), + ("gate_length_v", "u2"), + ("gate_num_r", "u2"), + ("gate_num_v", "u2"), + ("sector_num", "u2"), + ("system_coff", "u4"), + ("r_pointer", "u2"), + ("v_pointer", "u2"), + ("w_pointer", "u2"), + ("v_reso", "u2"), + ("vcp_mode", "u2"), + ("res2", "u2", 4), + ("r_pointer_2", "u2"), + ("v_pointer_2", "u2"), + ("w_pointer_2", "u2"), + ("nyquist_vel", "u2"), +] + +_S_INFO = [("r", "u1", 460), ("v", "u1", 920), ("w", "u1", 920), ("res4", "u2", 2)] + +_S_INFO = [("r", "u1", 800), ("v", "u1", 1600), ("w", "u1", 1600), ("res4", "u2", 2)] + +_S_INFO = [("r", "u1", 1000), ("v", "u1", 1000), ("w", "u1", 1000)] + +_S_INFO = [ + ("sFileID", "S4"), + ("fVersionNo", "f4"), + ("lFileHeaderLength", "i4"), + ("sCountry", "S30"), + ("sProvince", "S20"), + ("sStation", "S40"), + ("sStationNumber", "S10"), + ("sRadarType", "S20"), + ("sLongitude", "S16"), + ("sLatitude", "S16"), + ("lLongitudeValue", "i4"), + ("lLatitudeValue", "i4"), + ("lHeight", "i4"), + ("shMaxAngle", "i2"), + ("shOptiAngle", "i2"), + ("lAntennaG", "i4"), + ("usBeamH", "u2"), + ("usBeamL", "u2"), + ("ucPolarization", "B"), + ("usSidelobe", "u2"), + ("lPower", "i4"), + ("lWavelength", "i4"), + ("usLogA", "u2"), + ("usLineA", "u2"), + ("usAGCP", "u2"), + ("usLogMinPower", "u2"), + ("usLineMinPower", "u2"), + ("ucClutterT", "B"), + ("ucVelocityP", "B"), + ("ucFilterP", "B"), + ("ucNoiseT", "B"), + ("ucSQIT", "B"), + ("ucIntensityC", "B"), + ("ucIntensityR", "B"), +] + + +def transform(k): + f = k[0] + + tf = f[0] + flag = table.get(tf) + if len(k) == 1: + if tf in ['S','b','B']: + return flag + return f"{flag}{int(f[1:])*8}" + else: + l = int(k[1]) + if tf in ['S','b','B']: + return f"[{flag};{l}]" + else: + return f"[{flag}{int(f[1:])*8}; {l}]" + + +def camel_to_snake(name): + name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower() + +for i in _S_INFO: + print(f"f{camel_to_snake(i[0])}: {transform(i[1:])},") + \ No newline at end of file