diff --git a/Cargo.lock b/Cargo.lock index 1b37fe6..23b4779 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,42 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "ambassador" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68de4cdc6006162265d0957edb4a860fe4e711b1dc17a5746fd95f952f08285" +dependencies = [ + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.21" @@ -65,16 +101,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] -name = "boomphf" -version = "0.6.0" +name = "arbitrary-chunks" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617e2d952880a00583ddb9237ac3965732e8df6a92a8e7bcc054100ec467ec3b" +checksum = "2ad8689a486416c401ea15715a4694de30054248ec627edbf31f49cb64ee4086" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "crossbeam-utils", - "log", - "rayon", "serde", - "wyhash", +] + +[[package]] +name = "binout" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222fb4925a15bea6a68075021910e03d6aa2d04951d71ff1d956190a551d738f" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitm" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31468ea4a856000d83cb61960dfdc2980ecd96b15b61321c8c76cc96aea6e688" +dependencies = [ + "dyn_size_of", +] + +[[package]] +name = "block-pseudorand" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2097358495d244a0643746f4d13eedba4608137008cf9dec54e53a3b700115a6" +dependencies = [ + "chiapos-chacha8", + "nanorand", ] [[package]] @@ -103,9 +190,15 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -118,12 +211,66 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chiapos-chacha8" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f8be573a85f6c2bc1b8e43834c07e32f95e489b914bf856c0549c3c269cd0a" +dependencies = [ + "rayon", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.53" @@ -131,6 +278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -145,6 +293,18 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.5.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "clap_lex" version = "0.7.6" @@ -157,6 +317,38 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "common_traits" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af10ce0d9d58d055a641bb3c496cff570c4585b657807474621e91d85e0a082a" +dependencies = [ + "anyhow", + "half", + "impl-tools", +] + +[[package]] +name = "core_affinity" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -166,6 +358,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -191,6 +419,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "csv" version = "1.4.0" @@ -212,6 +446,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "dsi-progress-logger" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2f08728b2005dd95f078e2d775a91b5339839833057b6404b58f47e6ee6ac5" +dependencies = [ + "log", + "num-format", + "pluralizer", + "sysinfo", +] + +[[package]] +name = "dyn_size_of" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a742b95783b1f45b900129082cbc47717b6a77ee8d17eea70a8ea62462f5de3" + [[package]] name = "either" version = "1.15.0" @@ -227,12 +479,118 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "env_filter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "epserde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c40d342ff20a2ce62d9a85ce406e672dfa137f902ac9670034533184f1533976" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "common_traits", + "epserde-derive", + "maligned", + "mem_dbg", + "mmap-rs", + "sealed", + "thiserror 2.0.17", + "xxhash-rust", +] + +[[package]] +name = "epserde-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac80cc78b69765703f48ad93f33b8919cf5d907cda7459ad6ba2919cbbe605dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + [[package]] name = "flate2" version = "1.1.5" @@ -244,18 +602,51 @@ dependencies = [ ] [[package]] -name = "gen" +name = "gentools" version = "0.1.0" dependencies = [ "anyhow", - "boomphf", + "bincode", + "binout", "byteorder", "clap", "csv", + "glob", + "memmap2", + "ph", "rkyv", "serde", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -268,6 +659,42 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "impl-tools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae95c9095c2f1126d7db785955c73cdc5fc33e7c3fa911bd4a42931672029a7" +dependencies = [ + "autocfg", + "impl-tools-lib", + "proc-macro-error2", + "syn 2.0.111", +] + +[[package]] +name = "impl-tools-lib" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab699036df31c1f7d3561bfa6e9cb9bc3bb0fd2e2cd9bf121c31cb961d049ddf" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "indexmap" version = "2.12.1" @@ -278,18 +705,72 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ee5b5339afb4c41626dde77b7a611bd4f2c202b897852b4bcf5d03eddc61010" +[[package]] +name = "jiff" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + [[package]] name = "js-sys" version = "0.3.83" @@ -300,18 +781,107 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lender" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d82d5a5c23e4d70f79c4cdfaa1d0beb6b3c660080c05dafd236f0514ce51ba6" +dependencies = [ + "lender-derive", + "stable_try_trait_v2", +] + +[[package]] +name = "lender-derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556faf0c9adb22669a5f9f6b4ed804c8048d205dcd00fadd463affbb02931d26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + +[[package]] +name = "maligned" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e88c3cbe8288f77f293e48a28b3232e3defd203a6d839fa7f68ea4329e83464" + +[[package]] +name = "mem_dbg" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256baafaaed96f2358e588047add018b8e5f1fc58e1cb58deb026528dd42488e" +dependencies = [ + "bitflags 2.10.0", + "maligned", + "mem_dbg-derive", + "mmap-rs", +] + +[[package]] +name = "mem_dbg-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f266c6ba1558dee16c56363c3dc6093a97a2451ddf29f928d5da042ee330537b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "memchr" version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -322,6 +892,23 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mmap-rs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86968d85441db75203c34deefd0c88032f275aaa85cee19a1dcfff6ae9df56da" +dependencies = [ + "bitflags 1.3.2", + "combine", + "libc", + "mach2", + "nix", + "sysctl", + "thiserror 1.0.69", + "widestring", + "windows 0.48.0", +] + [[package]] name = "munge" version = "0.4.7" @@ -339,7 +926,26 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "nanorand" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958" + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", ] [[package]] @@ -351,6 +957,63 @@ dependencies = [ "memchr", ] +[[package]] +name = "ntapi" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.10.0", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -363,6 +1026,131 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "partition" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947f833aaa585cf12b8ec7c0476c98784c49f33b861376ffc84ed92adebf2aba" + +[[package]] +name = "ph" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f26b18647c1926dde1e041b99f69ea73bf88f8681341dcb7d9c61970c999dc" +dependencies = [ + "aligned-vec", + "arrayvec", + "binout", + "bitm", + "dyn_size_of", + "mem_dbg", + "rayon", + "seedable_hash", + "sux", + "voracious_radix_sort", +] + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "pluralizer" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3eba432a00a1f6c16f39147847a870e94e2e9b992759b503e330efec778cbe" +dependencies = [ + "once_cell", + "regex", +] + +[[package]] +name = "portable-atomic" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -389,7 +1177,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -401,6 +1189,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rancor" version = "0.1.1" @@ -411,10 +1205,33 @@ dependencies = [ ] [[package]] -name = "rand_core" -version = "0.6.4" +name = "rand" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] [[package]] name = "rayon" @@ -440,14 +1257,60 @@ dependencies = [ name = "rbufr" version = "0.1.0" dependencies = [ + "anyhow", "csv", "encoding_rs", "flate2", + "gentools", "nom", "serde", - "thiserror", + "thiserror 2.0.17", ] +[[package]] +name = "rdst" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7970b4e577b76a96d5e56b5f6662b66d1a4e1f5bb026ee118fc31b373c2752" +dependencies = [ + "arbitrary-chunks", + "block-pseudorand", + "criterion", + "partition", + "rayon", + "tikv-jemallocator", + "voracious_radix_sort", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + [[package]] name = "rend" version = "0.5.3" @@ -484,7 +1347,20 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] @@ -499,6 +1375,32 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62049b2877bf12821e8f9ad256ee38fdc31db7387ec2d3b3f403024de2034aea" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "sealed" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "seedable_hash" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2c8d98ed920f49590f063255962141de2122465004835b7032de3d21cac8cc9" + [[package]] name = "serde" version = "1.0.228" @@ -526,9 +1428,28 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] +[[package]] +name = "serde_json" +version = "1.0.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217ca874ae0207aac254aa02c957ded05585a90892cc8d87f9e5fa49669dadd8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simd-adler32" version = "0.3.8" @@ -541,12 +1462,60 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "stable_try_trait_v2" +version = "1.75.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4e48411f4db8ccca0470bfb67e3bb821af4227d455aa147917d8d109be0d13" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "sux" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495d3e5ad72523bb98a967e56aea75c8c6bcc2bfa9b8fa50b450dc9c8a8ce007" +dependencies = [ + "ambassador", + "anyhow", + "bitflags 2.10.0", + "bytemuck", + "common_traits", + "core_affinity", + "dsi-progress-logger", + "env_logger", + "epserde", + "flate2", + "impl-tools", + "lender", + "libc", + "log", + "mem_dbg", + "mmap-rs", + "num-traits", + "num_cpus", + "rand", + "rayon", + "rdst", + "tempfile", + "zstd", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.111" @@ -558,13 +1527,74 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.10.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "sysinfo" +version = "0.35.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", ] [[package]] @@ -575,7 +1605,37 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.5.4+5.3.0-patched" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", ] [[package]] @@ -615,6 +1675,34 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "voracious_radix_sort" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446e7ffcb6c27a71d05af7e51ef2ee5b71c48424b122a832f2439651e1914899" +dependencies = [ + "rayon", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.106" @@ -647,7 +1735,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -660,26 +1748,301 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-sys" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] -name = "wyhash" -version = "0.5.0" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "rand_core", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", ] diff --git a/gen/Cargo.toml b/gen/Cargo.toml index 1b200fa..3c066c8 100644 --- a/gen/Cargo.toml +++ b/gen/Cargo.toml @@ -1,13 +1,25 @@ [package] -name = "gen" +name = "gentools" version = "0.1.0" edition = "2024" +[lib] +name = "genlib" +path = "src/lib.rs" + +[[bin]] +name = "gen-ctl" +path = "src/main.rs" + [dependencies] anyhow = "1.0.100" -boomphf = { version = "0.6.0", features = ["serde"] } +bincode = "1.3.3" +ph = "0.10.0" byteorder = "1.5.0" -clap = "4.5.53" +clap = { version = "4.5.53", features = ["derive"] } csv = "1.4.0" +glob = "0.3" +memmap2 = "0.9" rkyv = { version = "0.8.12" } serde = { version = "1.0.228", features = ["derive"] } +binout = "0.3.1" diff --git a/gen/src/btable.rs b/gen/src/btable.rs deleted file mode 100644 index c4e4930..0000000 --- a/gen/src/btable.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::{FXY, TableEntryLoader}; -use anyhow::{Context, Result}; -use std::path::Path; - -pub struct BTableCsvLoader { - entries: Vec, -} - -#[derive(Debug, serde::Deserialize)] -pub struct RawBTableEntry { - #[serde(rename = "ClassNo")] - pub class_no: String, - #[serde(rename = "ClassName_en")] - pub class_name_en: String, - #[serde(rename = "FXY")] - pub fxy: String, - #[serde(rename = "ElementName_en")] - pub element_name_en: String, - #[serde(rename = "BUFR_Unit")] - pub bufr_unit: String, - #[serde(rename = "BUFR_Scale")] - pub bufr_scale: u32, - #[serde(rename = "BUFR_ReferenceValue")] - pub bufr_reference_value: u32, - #[serde(rename = "BUFR_DataWidth_Bits")] - pub bufr_datawidth_bits: u32, - #[serde(rename = "CREX_Unit")] - pub crex_unit: String, - #[serde(rename = "CREX_Scale")] - pub crex_scale: u32, - #[serde(rename = "CREX_DataWidth_Char")] - pub crex_datawidth_char: u32, - #[serde(rename = "Note_en")] - pub note_en: Option, - #[serde(rename = "noteIDs")] - pub note_ids: Option, - #[serde(rename = "Status")] - pub status: String, -} - -pub struct BTableEntry { - fxy: FXY, - class_name_en: String, - element_name_en: String, - bufr_unit: String, - bufr_scale: u32, - bufr_reference_value: u32, - bufr_datawidth_bits: u32, - note_en: Option, - note_ids: Option, - status: String, -} - -impl BTableCsvLoader { - pub fn new() -> Self { - BTableCsvLoader { - entries: Vec::new(), - } - } - - pub fn from_wmo_csv>(path: P) -> Result { - let mut loader = Self::new(); - let table_loader = crate::TableLoader::new(); - table_loader.load_table(path, &mut loader)?; - Ok(loader) - } - - pub fn entries(&self) -> &[BTableEntry] { - &self.entries - } -} - -impl TableEntryLoader for BTableCsvLoader { - type RawEntry = RawBTableEntry; - - fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result<()> { - // Parse FXY string (e.g., "001001") to u32 - let fxy = FXY::from_str(&raw.fxy)?; - - let entry = BTableEntry { - fxy, - class_name_en: raw.class_name_en, - element_name_en: raw.element_name_en, - bufr_unit: raw.bufr_unit, - bufr_scale: raw.bufr_scale, - bufr_reference_value: raw.bufr_reference_value, - bufr_datawidth_bits: raw.bufr_datawidth_bits, - note_en: raw.note_en, - note_ids: raw.note_ids, - status: raw.status, - }; - - self.entries.push(entry); - Ok(()) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_btable_csv_loader() { - let _loader = BTableCsvLoader::from_wmo_csv( - "/Users/tsuki/projects/rbufr/BUFR4/BUFRCREX_TableB_en_00.csv", - ) - .unwrap(); - } -} diff --git a/gen/src/fr/btable.rs b/gen/src/fr/btable.rs new file mode 100644 index 0000000..f3d4dec --- /dev/null +++ b/gen/src/fr/btable.rs @@ -0,0 +1,154 @@ +use crate::{FXY, TableEntry, TableEntryLoader}; +use rkyv::Archive; + +pub struct BTableCsvLoader; + +#[derive(Debug)] +pub struct RawBTableEntry { + pub f: u16, + pub x: u16, + pub y: u16, +} + +// Helper function to deserialize empty strings as None +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: String = serde::Deserialize::deserialize(deserializer)?; + if s.is_empty() { Ok(None) } else { Ok(Some(s)) } +} + +// Helper function to deserialize empty strings as None for u32 +fn deserialize_optional_u32<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: String = serde::Deserialize::deserialize(deserializer)?; + if s.is_empty() { + Ok(None) + } else { + s.parse::().map(Some).map_err(serde::de::Error::custom) + } +} + +#[derive( + Debug, Clone, serde::Deserialize, serde::Serialize, Archive, rkyv::Serialize, rkyv::Deserialize, +)] +#[rkyv(compare(PartialEq), derive(Debug))] +pub struct BTableEntry { + fxy: FXY, + class_name_en: String, + element_name_en: String, + bufr_unit: String, + bufr_scale: i32, + bufr_reference_value: i32, + bufr_datawidth_bits: u32, + note_en: Option, + note_ids: Option, + status: String, +} + +impl BTableEntry { + pub fn fxy(&self) -> FXY { + self.fxy + } + + pub fn class_name_en(&self) -> &str { + &self.class_name_en + } + + pub fn element_name_en(&self) -> &str { + &self.element_name_en + } + + pub fn bufr_unit(&self) -> &str { + &self.bufr_unit + } + + pub fn bufr_scale(&self) -> i32 { + self.bufr_scale + } + + pub fn bufr_reference_value(&self) -> i32 { + self.bufr_reference_value + } + + pub fn bufr_datawidth_bits(&self) -> u32 { + self.bufr_datawidth_bits + } + + pub fn note_en(&self) -> Option<&str> { + self.note_en.as_deref() + } + + pub fn note_ids(&self) -> Option<&str> { + self.note_ids.as_deref() + } + + pub fn status(&self) -> &str { + &self.status + } +} + +impl std::fmt::Display for BTableEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let element_name = if self.element_name_en.len() > 40 { + format!("{}...", &self.element_name_en[..37]) + } else { + self.element_name_en.clone() + }; + + let unit = if self.bufr_unit.len() > 15 { + format!("{}...", &self.bufr_unit[..12]) + } else { + self.bufr_unit.clone() + }; + + write!( + f, + "{:02}{:02}{:03} | {:<40} | {:<15} | {:>5} | {:>8} | {:>8} | {}", + self.fxy.f, + self.fxy.x, + self.fxy.y, + element_name, + unit, + self.bufr_scale, + self.bufr_reference_value, + self.bufr_datawidth_bits, + self.status + ) + } +} + +impl TableEntryLoader for BTableCsvLoader { + type RawEntry = RawBTableEntry; + type TableEntry = BTableEntry; + const TABLE_TYPE: crate::TableType = crate::TableType::B; + + fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result> { + // Parse FXY string (e.g., "001001") to u32 + let fxy = FXY::from_str(&raw.fxy)?; + + let entry = BTableEntry { + fxy, + class_name_en: raw.class_name_en, + element_name_en: raw.element_name_en, + bufr_unit: raw.bufr_unit, + bufr_scale: raw.bufr_scale, + bufr_reference_value: raw.bufr_reference_value, + bufr_datawidth_bits: raw.bufr_datawidth_bits, + note_en: raw.note_en, + note_ids: raw.note_ids, + status: raw.status, + }; + + Ok(Some(entry)) + } +} + +impl TableEntry for BTableEntry { + fn fxy(&self) -> FXY { + self.fxy + } +} diff --git a/gen/src/fr/dtable.rs b/gen/src/fr/dtable.rs new file mode 100644 index 0000000..10f6489 --- /dev/null +++ b/gen/src/fr/dtable.rs @@ -0,0 +1,151 @@ +use super::TableEntryLoader; +use csv::StringRecord; +use rkyv::Archive; + +#[derive(Debug, Clone, Default)] +pub struct DTableCsvLoader { + current_chain: Option, +} + +impl TableEntryLoader for DTableCsvLoader { + type TableEntry = DTableEntry; + const TABLE_TYPE: crate::TableType = crate::TableType::D; + + fn process_entry(&mut self, raw: &StringRecord) -> anyhow::Result> { + // Process the raw entry as needed + if self.current_chain.is_none() { + let entry = DTableEntry { + fxy: FXY::from_str(&raw.fxy1)?, + fxy_chain: vec![FXY::from_str(&raw.fxy2)?], + category: raw.category, + category_of_sequences_en: raw.category_of_sequences_en, + title_en: raw.title_en, + subtitle_en: raw.subtitle_en, + note_en: raw.note_en, + note_ids: raw.note_ids, + status: raw.status, + }; + self.current_chain = Some(entry); + return Ok(None); + } else { + let fxy = FXY::from_str(&raw.fxy1)?; + if self.current_chain.as_ref().unwrap().fxy != fxy { + // First take out the old completed chain + let finished = self.current_chain.take(); + + // Then create and save the new chain + let entry = DTableEntry { + fxy, + fxy_chain: vec![FXY::from_str(&raw.fxy2)?], + category: raw.category, + category_of_sequences_en: raw.category_of_sequences_en, + title_en: raw.title_en, + subtitle_en: raw.subtitle_en, + note_en: raw.note_en, + note_ids: raw.note_ids, + status: raw.status, + }; + self.current_chain = Some(entry); + + // Return the old completed chain + return Ok(finished); + } else { + self.current_chain + .as_mut() + .unwrap() + .fxy_chain + .push(FXY::from_str(&raw.fxy2)?); + + return Ok(None); + } + } + } + + fn finish(&mut self) -> anyhow::Result> { + Ok(self.current_chain.take()) + } +} + +#[derive( + Debug, Clone, serde::Deserialize, serde::Serialize, Archive, rkyv::Serialize, rkyv::Deserialize, +)] +#[rkyv(compare(PartialEq), derive(Debug))] +pub struct DTableEntry { + fxy: FXY, + fxy_chain: Vec, + category: String, + category_of_sequences_en: String, + title_en: Option, + subtitle_en: Option, + note_en: Option, + note_ids: String, + status: String, +} + +impl DTableEntry { + pub fn fxy(&self) -> FXY { + self.fxy + } + + pub fn fxy_chain(&self) -> &[FXY] { + &self.fxy_chain + } + + pub fn category(&self) -> &str { + &self.category + } + + pub fn category_of_sequences_en(&self) -> &str { + &self.category_of_sequences_en + } + + pub fn title_en(&self) -> Option<&str> { + self.title_en.as_deref() + } + + pub fn subtitle_en(&self) -> Option<&str> { + self.subtitle_en.as_deref() + } + + pub fn note_en(&self) -> Option<&str> { + self.note_en.as_deref() + } + + pub fn note_ids(&self) -> &str { + &self.note_ids + } + + pub fn status(&self) -> &str { + &self.status + } +} + +impl std::fmt::Display for DTableEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let fxy_chain_str: String = self + .fxy_chain + .iter() + .map(|fxy| format!("{:02}{:02}{:03}", fxy.f, fxy.x, fxy.y)) + .collect::>() + .join(", "); + + let title = self.title_en.as_deref().unwrap_or("N/A"); + let truncated_title = if title.len() > 50 { + format!("{}...", &title[..47]) + } else { + title.to_string() + }; + + write!( + f, + "{:02}{:02}{:03} | {:<50} | {:<12} | [{}]", + self.fxy.f, self.fxy.x, self.fxy.y, truncated_title, self.status, fxy_chain_str + ) + } +} + +impl TableEntry for DTableEntry { + fn fxy(&self) -> FXY { + self.fxy + } +} diff --git a/gen/src/fr/mod.rs b/gen/src/fr/mod.rs new file mode 100644 index 0000000..57c0dd6 --- /dev/null +++ b/gen/src/fr/mod.rs @@ -0,0 +1,61 @@ +use csv::{ReaderBuilder, StringRecord}; +use std::path::Path; +pub mod btable; +pub mod dtable; + +pub struct TableLoader; + +impl TableLoader { + pub fn load_table, T: TableEntryLoader>( + &self, + path: P, + loader: &mut T, + ) -> anyhow::Result> { + let mut entries = vec![]; + let mut rdr = ReaderBuilder::new() + .has_headers(false) + .delimiter(b';') + .flexible(false) // Allow variable number of fields + .from_path(path.as_ref())?; + + let mut line_num = 1; // Start at 1 for header + for result in rdr.records() { + line_num += 1; + match result { + Ok(record) => { + if let Some(processed_entry) = loader.process_entry(record)? { + entries.push(processed_entry); + } + } + Err(e) => { + // Log the error but continue processing + eprintln!( + "Warning: Skipping line {} in {}: {}", + line_num, + path.as_ref().display(), + e + ); + } + } + } + + if let Some(processed_entry) = loader.finish()? { + entries.push(processed_entry); + } + + Ok(entries) + } +} + +pub trait TableEntryLoader: Sized { + type TableEntry: TableEntry; + + const TABLE_TYPE: TableType; + + /// Process a single entry from the CSV file + fn process_entry(&mut self, raw: &StringRecord) -> anyhow::Result>; + + fn finish(&mut self) -> anyhow::Result> { + Ok(None) + } +} diff --git a/gen/src/lib.rs b/gen/src/lib.rs index 21c4437..ad68205 100644 --- a/gen/src/lib.rs +++ b/gen/src/lib.rs @@ -1,19 +1,20 @@ -mod btable; -mod dtable; +// pub mod fr; +pub mod prelude; mod utils; +pub mod wmo; use anyhow::Context; -use boomphf::Mphf; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use memmap2::Mmap; +use ph::fmph::GOFunction; use rkyv::Archive; -use rkyv::ser::serializers::AllocSerializer; +use rkyv::api::high::{HighDeserializer, HighSerializer}; +use rkyv::rancor::Error; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize as SerdeDeserialize, Serialize as SerdeSerialize}; use std::fmt::Debug; -use std::fs::File; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::Write; use std::path::Path; -use csv::{DeserializeRecordsIter, ReaderBuilder}; +use csv::ReaderBuilder; pub struct TableLoader; @@ -22,187 +23,265 @@ impl TableLoader { &self, path: P, loader: &mut T, - ) -> anyhow::Result<()> { + ) -> anyhow::Result> { + let mut entries = vec![]; let mut rdr = ReaderBuilder::new() .has_headers(true) .delimiter(b',') - .from_path(path)?; + .flexible(true) // Allow variable number of fields + .from_path(path.as_ref())?; + let mut line_num = 1; // Start at 1 for header for result in rdr.deserialize() { - let record: T::RawEntry = result?; - loader.process_entry(record)?; + line_num += 1; + match result { + Ok(record) => { + let record: T::RawEntry = record; + if let Some(processed_entry) = loader.process_entry(record)? { + entries.push(processed_entry); + } + } + Err(e) => { + // Log the error but continue processing + eprintln!( + "Warning: Skipping line {} in {}: {}", + line_num, + path.as_ref().display(), + e + ); + } + } } - Ok(()) + if let Some(processed_entry) = loader.finish()? { + entries.push(processed_entry); + } + + Ok(entries) } } -pub trait TableEntryLoader: Sized { +pub trait TableEntryLoader: Sized +where + Self::TableEntry: for<'a> rkyv::Serialize< + HighSerializer, Error>, + >, + ::Archived: + rkyv::Deserialize>, +{ /// The raw CSV entry type that will be deserialized type RawEntry: for<'de> serde::Deserialize<'de> + Debug; type TableEntry: TableEntry; - /// Process a single entry from the CSV file - fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result<()>; + const TABLE_TYPE: TableType; - fn finish(&mut self) -> anyhow::Result<()> { - Ok(()) + /// Process a single entry from the CSV file + fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result>; + + fn finish(&mut self) -> anyhow::Result> { + Ok(None) } } -pub trait TableEntry: Serialize + DeserializeOwned + Debug + Clone { +pub trait TableEntry: SerdeSerialize + DeserializeOwned + Debug + Clone + Archive { fn fxy(&self) -> FXY; } -#[derive(Serialize, Deserialize)] -struct MphMetadata { - mphf: Mphf, - offsets: Vec, -} - struct BufrTableMph { - mphf: Mphf, + mphf: GOFunction, offsets: Vec, - data_file: File, + mmap: Mmap, _marker: std::marker::PhantomData, } -impl BufrTableMph { - /// 构建 MPH 表 - fn build(entries: &Vec, output_path: &str) -> std::io::Result { +struct BUFRTableM { + mphf: GOFunction, +} + +impl BufrTableMph +where + for<'a> T::TableEntry: rkyv::Serialize< + HighSerializer, Error>, + >, + ::Archived: rkyv::Deserialize>, +{ + fn build(entries: Vec, output_path: &str) -> std::io::Result { + println!("Building MPH table with {} entries...", entries.len()); + let keys: Vec = entries.iter().map(|e| e.fxy()).collect(); - - let mphf = Mphf::new(2.0, &keys); - + let mphf = GOFunction::from_slice(&keys); let mut sorted_entries: Vec<(usize, T::TableEntry)> = entries .into_iter() - .map(|e| { - let hash = mphf.hash(&e.fxy()) as usize; - (hash, e) - }) + .map(|e| (mphf.get(&(e.fxy())).unwrap() as usize, e)) .collect(); sorted_entries.sort_by_key(|(hash, _)| *hash); - let data_path = format!("{}.data", output_path); - let mut data_file = File::create(&data_path)?; + // Serialize MPHF to bytes + let mut mphf_bytes = Vec::new(); + mphf.write(&mut mphf_bytes) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("{:?}", e)))?; + + // Prepare data entries + let mut data_bytes = Vec::new(); let mut offsets = Vec::with_capacity(sorted_entries.len()); for (_, entry) in sorted_entries { - let offset = data_file.stream_position()?; + let offset = data_bytes.len() as u64; offsets.push(offset); - // rkyv 序列化 - let mut serializer = AllocSerializer::<256>::default(); - serializer.serialize_value(&entry).unwrap(); - let bytes = serializer.into_serializer().into_inner(); + let bytes = rkyv::to_bytes::(&entry) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("{:?}", e)))?; - // 写入长度 + 数据 - data_file.write_u32::(bytes.len() as u32)?; - data_file.write_all(&bytes)?; + let len = bytes.len() as u32; + data_bytes.write_all(&len.to_le_bytes())?; + data_bytes.write_all(&bytes)?; } - // 5. 保存元数据 - let meta_path = format!("{}.meta", output_path); - let meta = MphMetadata { - mphf: mphf.clone(), - offsets: offsets.clone(), - }; + // Serialize offsets as raw bytes + let offsets_count = offsets.len() as u32; + let mut offsets_bytes = Vec::with_capacity(offsets.len() * 8); + for offset in &offsets { + offsets_bytes.extend_from_slice(&offset.to_le_bytes()); + } - // 元数据仍然用 bincode 或者也可以用 rkyv - let mut meta_serializer = AllocSerializer::<4096>::default(); - meta_serializer.serialize_value(&meta).unwrap(); - let meta_bytes = meta_serializer.into_serializer().into_inner(); - std::fs::write(meta_path, meta_bytes)?; + // Write merged file + let merged_path = format!("{}.bufrtbl", output_path); + let mut merged_file = std::fs::File::create(&merged_path)?; + + // Write header: mphf_size (u32) + offsets_count (u32) + merged_file.write_all(&(mphf_bytes.len() as u32).to_le_bytes())?; + merged_file.write_all(&offsets_count.to_le_bytes())?; + + // Write MPHF data + merged_file.write_all(&mphf_bytes)?; + + // Write offsets (already 8-byte aligned since each offset is u64) + merged_file.write_all(&offsets_bytes)?; + + // Write table entries data + merged_file.write_all(&data_bytes)?; + + println!("Built successfully!"); + println!( + " Merged file: {} bytes", + std::fs::metadata(&merged_path)?.len() + ); + + Self::load(output_path) + } + + fn load>(path: P) -> std::io::Result { + let mut path = path.as_ref().to_path_buf(); + path.set_extension("bufrtbl"); + let merged_file = std::fs::File::open(&path)?; + let mmap = unsafe { Mmap::map(&merged_file)? }; + + // Read header + let mphf_size = u32::from_le_bytes(mmap[0..4].try_into().unwrap()) as usize; + let offsets_count = u32::from_le_bytes(mmap[4..8].try_into().unwrap()) as usize; + + // Calculate section positions + let mphf_start = 8; + let mphf_end = mphf_start + mphf_size; + let offsets_start = mphf_end; + let offsets_size = offsets_count * 8; // each offset is u64 (8 bytes) + let offsets_end = offsets_start + offsets_size; + let data_start = offsets_end; + + // Load MPHF + let mphf_bytes = &mmap[mphf_start..mphf_end]; + let mphf: GOFunction = GOFunction::read(&mut &mphf_bytes[..]) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("{:?}", e)))?; + + // Load offsets + let offsets_bytes = &mmap[offsets_start..offsets_end]; + let mut offsets = Vec::with_capacity(offsets_count); + for i in 0..offsets_count { + let offset_bytes = &offsets_bytes[i * 8..(i + 1) * 8]; + let offset = u64::from_le_bytes(offset_bytes.try_into().unwrap()); + offsets.push(offset + data_start as u64); // Adjust offset for data section + } Ok(Self { mphf, offsets, - data_file: File::open(data_path)?, + mmap, _marker: std::marker::PhantomData, }) } - /// 从磁盘加载 - fn load(path: &str) -> std::io::Result { - let meta_bytes = std::fs::read(format!("{}.meta", path))?; + /// 获取拥有的版本 + fn get(&self, fxy: FXY) -> Option { + let hash = self.mphf.get(&fxy)? as usize; + let offset = *self.offsets.get(hash)? as usize; - // rkyv 反序列化元数据（零拷贝） - let archived = unsafe { rkyv::archived_root::(&meta_bytes) }; + let len_bytes = self.mmap.get(offset..offset + 4)?; + let len = u32::from_le_bytes(len_bytes.try_into().ok()?) as usize; - // 如果需要拥有的版本，可以 deserialize - let meta: MphMetadata = archived.deserialize(&mut rkyv::Infallible).unwrap(); + let data = self.mmap.get(offset + 4..offset + 4 + len)?; - let data_file = File::open(format!("{}.data", path))?; - - Ok(Self { - mphf: meta.mphf, - offsets: meta.offsets, - data_file, - _marker: std::marker::PhantomData, - }) + let archived = + unsafe { rkyv::access_unchecked::<::Archived>(data) }; + rkyv::deserialize::(archived).ok() } - /// 查找条目（零拷贝读取） - fn get(&mut self, fxy: FXY) -> std::io::Result> { - let hash = self.mphf.hash(&fxy) as usize; - let offset = match self.offsets.get(hash) { - Some(&off) => off, - None => return Ok(None), - }; - - // 读取数据 - self.data_file.seek(SeekFrom::Start(offset))?; - let len = self.data_file.read_u32::()? as usize; - let mut buffer = vec![0u8; len]; - self.data_file.read_exact(&mut buffer)?; - - // rkyv 零拷贝访问 - let archived = unsafe { rkyv::archived_root::(&buffer) }; - - // 校验归档数据（可选，生产环境推荐） - #[cfg(feature = "validation")] - { - use rkyv::validation::validators::DefaultValidator; - rkyv::check_archived_root::(&buffer) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + /// 获取所有条目 + fn get_all(&self) -> Vec { + let mut entries = Vec::new(); + for offset in &self.offsets { + let offset = *offset as usize; + if let Some(len_bytes) = self.mmap.get(offset..offset + 4) { + if let Ok(len_bytes_array) = len_bytes.try_into() { + let len = u32::from_le_bytes(len_bytes_array) as usize; + if let Some(data) = self.mmap.get(offset + 4..offset + 4 + len) { + let archived = unsafe { + rkyv::access_unchecked::<::Archived>(data) + }; + if let Ok(entry) = rkyv::deserialize::(archived) { + entries.push(entry); + } + } + } + } } - - // 反序列化为拥有的类型 - let entry: T::TableEntry = archived.deserialize(&mut rkyv::Infallible).unwrap(); - - if entry.fxy == fxy { - Ok(Some(entry)) - } else { - Ok(None) - } - } - - /// 零拷贝引用访问（更快！） - fn get_archived(&mut self, fxy: u32) -> std::io::Result>> { - let hash = self.mphf.hash(&fxy) as usize; - let offset = match self.offsets.get(hash) { - Some(&off) => off, - None => return Ok(None), - }; - - self.data_file.seek(SeekFrom::Start(offset))?; - let len = self.data_file.read_u32::()? as usize; - let mut buffer = vec![0u8; len]; - self.data_file.read_exact(&mut buffer)?; - - Ok(Some(buffer)) + entries } } -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone, Copy)] +#[derive( + Archive, + SerdeSerialize, + SerdeDeserialize, + rkyv::Serialize, + rkyv::Deserialize, + Debug, + PartialEq, + Eq, + Clone, + Copy, + std::hash::Hash, +)] +#[rkyv(compare(PartialEq), derive(Debug))] pub struct FXY { pub f: u16, pub x: u16, pub y: u16, } +// // Custom Hash implementation to work around boomphf's overflow bug +// // We implement Hash by converting to u32 first +// impl std::hash::Hash for FXY { +// fn hash(&self, state: &mut H) { +// // Convert FXY to a simple u32 value to avoid complex hashing +// self.to_u32().hash(state); +// } +// } + impl FXY { + pub fn new(f: u16, x: u16, y: u16) -> Self { + FXY { f, x, y } + } pub fn from_str(fxy_str: &str) -> anyhow::Result { // let bytes = fxy_str.as_bytes(); @@ -223,142 +302,72 @@ impl FXY { Ok(FXY { f, x, y }) } + + /// Convert FXY to u32 for use as hash key + /// Format: F (2 bits) | X (6 bits) | Y (8 bits) = 16 bits total + pub fn to_u32(&self) -> u32 { + ((self.f as u32) << 14) | ((self.x as u32) << 8) | (self.y as u32) + } + + /// Convert u32 back to FXY + pub fn from_u32(value: u32) -> Self { + FXY { + f: ((value >> 14) & 0x3) as u16, + x: ((value >> 8) & 0x3F) as u16, + y: (value & 0xFF) as u16, + } + } } -pub struct BUFRTableMPH {} +pub struct BUFRTableMPH { + inner: BufrTableMph, +} -impl BUFRTableMPH { - pub fn build_from_csv, T: TableEntryLoader>( +impl BUFRTableMPH { + pub fn build_from_csv>( mut loader: T, csv_path: P, output_path: P, ) -> anyhow::Result { - TableLoader.load_table(csv_path, &mut loader)?; - loader.finish()?; + let entries = TableLoader.load_table(csv_path, &mut loader)?; + let bhm = BufrTableMph::::build(entries, output_path.as_ref().to_str().unwrap())?; - Ok(BUFRTableMPH {}) + Ok(BUFRTableMPH { inner: bhm }) } pub fn load_from_disk>(path: P) -> anyhow::Result { - // Placeholder for loading MPH table from disk - Ok(BUFRTableMPH {}) + let bhm: BufrTableMph = BufrTableMph::load(path)?; + Ok(BUFRTableMPH { inner: bhm }) } - pub fn lookup(&self, fxy: FXY) -> anyhow::Result> { - // Placeholder for looking up an entry by FXY - Ok(None) + pub fn lookup(&self, fxy: FXY) -> anyhow::Result> { + Ok(self.inner.get(fxy)) + } + + pub fn get_all_entries(&self) -> Vec { + self.inner.get_all() } } -// #[derive(Serialize, Deserialize, Debug)] -// struct TableEntry { -// fxy: u32, // F-X-Y descriptor (e.g., 001001) -// name: String, // "WMO Block Number" -// unit: String, // "Numeric" -// scale: i8, -// reference: i32, -// data_width: u8, -// } +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TableType { + B, + D, +} -// struct BufrTableMph { -// mphf: Mphf, // The MPH function -// offsets: Vec, // Byte offsets into the data file -// data_file: File, // Memory-mapped or regular file -// } +#[cfg(test)] +mod test { + use crate::{BUFRTableMPH, dtable::DTableCsvLoader}; -// impl BufrTableMph { -// /// Build MPH table from entries -// fn build(entries: Vec, output_path: &str) -> std::io::Result { -// // 1. Extract keys (FXY descriptors) -// let keys: Vec = entries.iter().map(|e| e.fxy).collect(); - -// // 2. Build MPH function with gamma=2.0 (space/time tradeoff) -// let mphf = Mphf::new(2.0, &keys); - -// // 3. Create sorted entries by MPH hash -// let mut sorted_entries: Vec<(usize, TableEntry)> = entries -// .into_iter() -// .map(|e| { -// let hash = mphf.hash(&e.fxy) as usize; -// (hash, e) -// }) -// .collect(); -// sorted_entries.sort_by_key(|(hash, _)| *hash); - -// // 4. Write binary data file and collect offsets -// let data_path = format!("{}.data", output_path); -// let mut data_file = File::create(&data_path)?; -// let mut offsets = Vec::with_capacity(sorted_entries.len()); - -// for (_, entry) in sorted_entries { -// let offset = data_file.stream_position()?; -// offsets.push(offset); - -// // Write entry in compact binary format -// let serialized = bincode::serialize(&entry).unwrap(); -// data_file.write_u32::(serialized.len() as u32)?; -// data_file.write_all(&serialized)?; -// } - -// // 5. Write MPH metadata -// let meta_path = format!("{}.meta", output_path); -// let meta = MphMetadata { mphf, offsets }; -// let meta_bytes = bincode::serialize(&meta).unwrap(); -// std::fs::write(meta_path, meta_bytes)?; - -// Ok(Self { -// mphf: meta.mphf, -// offsets: meta.offsets, -// data_file: File::open(data_path)?, -// }) -// } - -// /// Load from disk -// fn load(path: &str) -> std::io::Result { -// let meta_bytes = std::fs::read(format!("{}.meta", path))?; -// let meta: MphMetadata = bincode::deserialize(&meta_bytes).unwrap(); -// let data_file = File::open(format!("{}.data", path))?; - -// Ok(Self { -// mphf: meta.mphf, -// offsets: meta.offsets, -// data_file, -// }) -// } - -// /// Lookup entry by FXY descriptor -// fn get(&mut self, fxy: u32) -> std::io::Result> { -// // 1. Hash the key to get index -// let hash = self.mphf.hash(&fxy) as usize; - -// // 2. Get offset from array -// let offset = match self.offsets.get(hash) { -// Some(&off) => off, -// None => return Ok(None), -// }; - -// // 3. Seek and read -// self.data_file.seek(SeekFrom::Start(offset))?; -// let len = self.data_file.read_u32::()? as usize; -// let mut buffer = vec![0u8; len]; -// self.data_file.read_exact(&mut buffer)?; - -// // 4. Deserialize -// let entry: TableEntry = bincode::deserialize(&buffer).unwrap(); - -// // Verify key matches (MPH guarantees unique index, but verify correctness) -// if entry.fxy == fxy { -// Ok(Some(entry)) -// } else { -// Ok(None) -// } -// } -// } - -// #[derive(Serialize, Deserialize)] -// struct MphMetadata { -// mphf: Mphf, -// offsets: Vec, -// } - -// use byteorder::{ReadBytesExt, WriteBytesExt}; + #[test] + fn test() { + let d_loader = DTableCsvLoader::default(); + BUFRTableMPH::build_from_csv( + d_loader, + "/Users/xiang.li1/projects/rbufr/BUFR4/BUFR_TableD_en_00.csv", + "test_table_d", + ) + .unwrap(); + } +} diff --git a/gen/src/main.rs b/gen/src/main.rs index e7a11a9..dd3d079 100644 --- a/gen/src/main.rs +++ b/gen/src/main.rs @@ -1,3 +1,289 @@ -fn main() { - println!("Hello, world!"); +use anyhow::{Context, Result}; +use clap::{Parser, Subcommand}; +use genlib::{BUFRTableMPH, btable::BTableCsvLoader, dtable::DTableCsvLoader}; +use std::path::{Path, PathBuf}; + +#[derive(Parser)] +#[command(name = "gen-ctl")] +#[command(about = "BUFR Table conversion tool", long_about = None)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Scan a directory and convert all BUFR tables to MPH format + Scan { + /// Input directory containing BUFR CSV files + #[arg(short, long)] + input: PathBuf, + + /// Output directory for generated .bufrtbl files + #[arg(short, long)] + output: PathBuf, + + /// Table type to process: "d", "b", or "all" + #[arg(short, long, default_value = "all")] + table_type: String, + }, + /// Convert a single BUFR table file + Convert { + /// Input CSV file + #[arg(short, long)] + input: PathBuf, + + /// Output path (without extension) + #[arg(short, long)] + output: PathBuf, + + /// Table type: "d" for Table D, "b" for Table B + #[arg(short, long)] + table_type: String, + }, + /// Print a BUFR table in formatted output + Print { + /// Path to .bufrtbl file (without extension) + #[arg(short, long)] + input: PathBuf, + + /// Table type: "d" for Table D, "b" for Table B + #[arg(short, long)] + table_type: String, + + /// Maximum number of entries to print (optional) + #[arg(short, long)] + limit: Option, + }, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.command { + Commands::Scan { + input, + output, + table_type, + } => { + scan_and_convert(&input, &output, &table_type)?; + } + Commands::Convert { + input, + output, + table_type, + } => { + convert_single_file(&input, &output, &table_type)?; + } + Commands::Print { + input, + table_type, + limit, + } => { + print_table(&input, &table_type, limit)?; + } + } + + Ok(()) +} + +fn scan_and_convert(input_dir: &Path, output_dir: &Path, table_type: &str) -> Result<()> { + // Create output directory if it doesn't exist + std::fs::create_dir_all(output_dir).context("Failed to create output directory")?; + + println!("Scanning directory: {}", input_dir.display()); + println!("Output directory: {}", output_dir.display()); + println!("Table type: {}", table_type); + println!(); + + let mut processed_count = 0; + let mut error_count = 0; + + // Scan for BUFR_TableD files + if table_type == "d" || table_type == "all" { + println!("Processing Table D files..."); + let pattern = input_dir.join("BUFR_TableD_*.csv"); + for entry in glob::glob(pattern.to_str().unwrap()).context("Failed to read glob pattern")? { + match entry { + Ok(path) => { + let mut filename = path.file_stem().unwrap().to_str().unwrap().to_string(); + if filename.contains("BUFRCREX") { + filename = filename.replace("BUFRCREX_", "BUFR_"); + } + let output_path = output_dir.join(filename); + + print!(" Converting {} ... ", path.display()); + match convert_table_d(&path, &output_path) { + Ok(_) => { + println!("OK"); + processed_count += 1; + } + Err(e) => { + println!("ERROR: {}", e); + error_count += 1; + } + } + } + Err(e) => { + println!("Error reading file: {}", e); + error_count += 1; + } + } + } + println!(); + } + + // Scan for BUFRCREX_TableB files + if table_type == "b" || table_type == "all" { + println!("Processing Table B files..."); + let pattern = input_dir.join("BUFRCREX_TableB_*.csv"); + + for entry in glob::glob(pattern.to_str().unwrap()).context("Failed to read glob pattern")? { + match entry { + Ok(path) => { + let mut filename = path.file_stem().unwrap().to_str().unwrap().to_string(); + if filename.contains("BUFRCREX") { + filename = filename.replace("BUFRCREX_", "BUFR_"); + } + let output_path = output_dir.join(filename); + + print!(" Converting {} ... ", path.display()); + match convert_table_b(&path, &output_path) { + Ok(_) => { + println!("OK"); + processed_count += 1; + } + Err(e) => { + eprintln!("ERROR: {}", e); + error_count += 1; + } + } + } + Err(e) => { + eprintln!("Error reading file: {}", e); + error_count += 1; + } + } + } + println!(); + } + + println!("Summary:"); + println!(" Successfully processed: {}", processed_count); + println!(" Errors: {}", error_count); + + if error_count > 0 { + anyhow::bail!("Conversion completed with {} errors", error_count); + } + + Ok(()) +} + +fn convert_single_file(input_path: &Path, output_path: &Path, table_type: &str) -> Result<()> { + println!( + "Converting {} to {}", + input_path.display(), + output_path.display() + ); + + match table_type.to_lowercase().as_str() { + "d" => convert_table_d(input_path, output_path)?, + "b" => convert_table_b(input_path, output_path)?, + _ => anyhow::bail!("Invalid table type: {}. Use 'd' or 'b'", table_type), + } + + println!("Conversion completed successfully!"); + Ok(()) +} + +fn convert_table_d(input_path: &Path, output_path: &Path) -> Result<()> { + let loader = DTableCsvLoader::default(); + BUFRTableMPH::build_from_csv(loader, input_path, output_path)?; + Ok(()) +} + +fn convert_table_b(input_path: &Path, output_path: &Path) -> Result<()> { + let loader = BTableCsvLoader; + BUFRTableMPH::build_from_csv(loader, input_path, output_path)?; + Ok(()) +} + +fn print_table(input_path: &Path, table_type: &str, limit: Option) -> Result<()> { + match table_type.to_lowercase().as_str() { + "d" => print_table_d(input_path, limit)?, + "b" => print_table_b(input_path, limit)?, + _ => anyhow::bail!("Invalid table type: {}. Use 'd' or 'b'", table_type), + } + + Ok(()) +} + +fn print_table_d(input_path: &Path, limit: Option) -> Result<()> { + use genlib::dtable::DTableEntry; + + println!("Loading Table D from: {}", input_path.display()); + + let table: BUFRTableMPH = BUFRTableMPH::load_from_disk(input_path)?; + let entries = table.get_all_entries(); + + println!("\nTable D Entries (Total: {})", entries.len()); + println!("{}", "=".repeat(140)); + println!( + "{:<7} | {:<50} | {:<12} | {}", + "FXY", "Title", "Status", "FXY Chain" + ); + println!("{}", "-".repeat(140)); + + let display_entries = if let Some(max) = limit { + &entries[..entries.len().min(max)] + } else { + &entries[..] + }; + + for entry in display_entries { + println!("{}", entry); + } + + if let Some(max) = limit { + if entries.len() > max { + println!("\n... ({} more entries omitted)", entries.len() - max); + } + } + + Ok(()) +} + +fn print_table_b(input_path: &Path, limit: Option) -> Result<()> { + use genlib::btable::BTableEntry; + + println!("Loading Table B from: {}", input_path.display()); + + let table: BUFRTableMPH = BUFRTableMPH::load_from_disk(input_path)?; + let entries = table.get_all_entries(); + + println!("\nTable B Entries (Total: {})", entries.len()); + println!("{}", "=".repeat(120)); + println!( + "{:<7} | {:<40} | {:<15} | {:<5} | {:<8} | {:<8} | {}", + "FXY", "Element Name", "Unit", "Scale", "Ref Val", "Width", "Status" + ); + println!("{}", "-".repeat(120)); + + let display_entries = if let Some(max) = limit { + &entries[..entries.len().min(max)] + } else { + &entries[..] + }; + + for entry in display_entries { + println!("{}", entry); + } + + if let Some(max) = limit { + if entries.len() > max { + println!("\n... ({} more entries omitted)", entries.len() - max); + } + } + + Ok(()) } diff --git a/gen/src/prelude.rs b/gen/src/prelude.rs new file mode 100644 index 0000000..7cdc85f --- /dev/null +++ b/gen/src/prelude.rs @@ -0,0 +1,6 @@ +pub use crate::wmo; +// pub type BUFRTableD = crate::BUFRTableMPH; +// pub type BUFRTableB = crate::BUFRTableMPH; +pub use crate::BUFRTableMPH; +pub use crate::FXY; +pub use crate::TableType; diff --git a/gen/src/wmo/btable.rs b/gen/src/wmo/btable.rs new file mode 100644 index 0000000..701e7b4 --- /dev/null +++ b/gen/src/wmo/btable.rs @@ -0,0 +1,179 @@ +use crate::{FXY, TableEntry, TableEntryLoader}; +use rkyv::Archive; + +pub struct BTableCsvLoader; + +#[derive(Debug, serde::Deserialize)] +pub struct RawBTableEntry { + #[serde(rename = "ClassNo")] + pub class_no: String, + #[serde(rename = "ClassName_en")] + pub class_name_en: String, + #[serde(rename = "FXY")] + pub fxy: String, + #[serde(rename = "ElementName_en")] + pub element_name_en: String, + #[serde(rename = "BUFR_Unit")] + pub bufr_unit: String, + #[serde(rename = "BUFR_Scale")] + pub bufr_scale: i32, + #[serde(rename = "BUFR_ReferenceValue")] + pub bufr_reference_value: i32, + #[serde(rename = "BUFR_DataWidth_Bits")] + pub bufr_datawidth_bits: u32, + #[serde(rename = "CREX_Unit")] + pub crex_unit: Option, + #[serde(rename = "CREX_Scale")] + pub crex_scale: Option, + #[serde(rename = "CREX_DataWidth_Char")] + pub crex_datawidth_char: Option, + #[serde(rename = "Note_en")] + pub note_en: Option, + #[serde(rename = "noteIDs")] + pub note_ids: Option, + #[serde(rename = "Status")] + pub status: String, +} + +// Helper function to deserialize empty strings as None +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: String = serde::Deserialize::deserialize(deserializer)?; + if s.is_empty() { Ok(None) } else { Ok(Some(s)) } +} + +// Helper function to deserialize empty strings as None for u32 +fn deserialize_optional_u32<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s: String = serde::Deserialize::deserialize(deserializer)?; + if s.is_empty() { + Ok(None) + } else { + s.parse::().map(Some).map_err(serde::de::Error::custom) + } +} + +#[derive( + Debug, Clone, serde::Deserialize, serde::Serialize, Archive, rkyv::Serialize, rkyv::Deserialize, +)] +#[rkyv(compare(PartialEq), derive(Debug))] +pub struct BTableEntry { + fxy: FXY, + class_name_en: String, + element_name_en: String, + bufr_unit: String, + bufr_scale: i32, + bufr_reference_value: i32, + bufr_datawidth_bits: u32, + note_en: Option, + note_ids: Option, + status: String, +} + +impl BTableEntry { + pub fn fxy(&self) -> FXY { + self.fxy + } + + pub fn class_name_en(&self) -> &str { + &self.class_name_en + } + + pub fn element_name_en(&self) -> &str { + &self.element_name_en + } + + pub fn bufr_unit(&self) -> &str { + &self.bufr_unit + } + + pub fn bufr_scale(&self) -> i32 { + self.bufr_scale + } + + pub fn bufr_reference_value(&self) -> i32 { + self.bufr_reference_value + } + + pub fn bufr_datawidth_bits(&self) -> u32 { + self.bufr_datawidth_bits + } + + pub fn note_en(&self) -> Option<&str> { + self.note_en.as_deref() + } + + pub fn note_ids(&self) -> Option<&str> { + self.note_ids.as_deref() + } + + pub fn status(&self) -> &str { + &self.status + } +} + +impl std::fmt::Display for BTableEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let element_name = if self.element_name_en.len() > 40 { + format!("{}...", &self.element_name_en[..37]) + } else { + self.element_name_en.clone() + }; + + let unit = if self.bufr_unit.len() > 15 { + format!("{}...", &self.bufr_unit[..12]) + } else { + self.bufr_unit.clone() + }; + + write!( + f, + "{:02}{:02}{:03} | {:<40} | {:<15} | {:>5} | {:>8} | {:>8} | {}", + self.fxy.f, + self.fxy.x, + self.fxy.y, + element_name, + unit, + self.bufr_scale, + self.bufr_reference_value, + self.bufr_datawidth_bits, + self.status + ) + } +} + +impl TableEntryLoader for BTableCsvLoader { + type RawEntry = RawBTableEntry; + type TableEntry = BTableEntry; + const TABLE_TYPE: crate::TableType = crate::TableType::B; + + fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result> { + // Parse FXY string (e.g., "001001") to u32 + let fxy = FXY::from_str(&raw.fxy)?; + + let entry = BTableEntry { + fxy, + class_name_en: raw.class_name_en, + element_name_en: raw.element_name_en, + bufr_unit: raw.bufr_unit, + bufr_scale: raw.bufr_scale, + bufr_reference_value: raw.bufr_reference_value, + bufr_datawidth_bits: raw.bufr_datawidth_bits, + note_en: raw.note_en, + note_ids: raw.note_ids, + status: raw.status, + }; + + Ok(Some(entry)) + } +} + +impl TableEntry for BTableEntry { + fn fxy(&self) -> FXY { + self.fxy + } +} diff --git a/gen/src/dtable.rs b/gen/src/wmo/dtable.rs similarity index 52% rename from gen/src/dtable.rs rename to gen/src/wmo/dtable.rs index b49425e..d9af9f8 100644 --- a/gen/src/dtable.rs +++ b/gen/src/wmo/dtable.rs @@ -1,11 +1,11 @@ -use crate::{FXY, TableEntryLoader}; +use crate::{FXY, TableEntry, TableEntryLoader}; +use rkyv::Archive; +#[derive(Debug, Clone, Default)] pub struct DTableCsvLoader { current_chain: Option, - entries: Vec, } -// Category,CategoryOfSequences_en,FXY1,Title_en,SubTitle_en,FXY2,ElementName_en,ElementDescription_en,Note_en,noteIDs,Status #[derive(Debug, serde::Deserialize)] pub struct RawDTableEntry { #[serde(rename = "Category")] @@ -21,9 +21,9 @@ pub struct RawDTableEntry { #[serde(rename = "FXY2")] pub fxy2: String, #[serde(rename = "ElementName_en")] - pub element_name_en: Option, + pub _element_name_en: Option, #[serde(rename = "ElementDescription_en")] - pub element_description_en: Option, + pub _element_description_en: Option, #[serde(rename = "Note_en")] pub note_en: Option, #[serde(rename = "noteIDs")] @@ -34,8 +34,10 @@ pub struct RawDTableEntry { impl TableEntryLoader for DTableCsvLoader { type RawEntry = RawDTableEntry; + type TableEntry = DTableEntry; + const TABLE_TYPE: crate::TableType = crate::TableType::D; - fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result<()> { + fn process_entry(&mut self, raw: Self::RawEntry) -> anyhow::Result> { // Process the raw entry as needed if self.current_chain.is_none() { let entry = DTableEntry { @@ -50,10 +52,14 @@ impl TableEntryLoader for DTableCsvLoader { status: raw.status, }; self.current_chain = Some(entry); + return Ok(None); } else { let fxy = FXY::from_str(&raw.fxy1)?; if self.current_chain.as_ref().unwrap().fxy != fxy { - self.entries.push(self.current_chain.take().unwrap()); + // First take out the old completed chain + let finished = self.current_chain.take(); + + // Then create and save the new chain let entry = DTableEntry { fxy, fxy_chain: vec![FXY::from_str(&raw.fxy2)?], @@ -66,25 +72,30 @@ impl TableEntryLoader for DTableCsvLoader { status: raw.status, }; self.current_chain = Some(entry); + + // Return the old completed chain + return Ok(finished); } else { self.current_chain .as_mut() .unwrap() .fxy_chain .push(FXY::from_str(&raw.fxy2)?); + + return Ok(None); } } - Ok(()) } - fn finish(&mut self) -> anyhow::Result<()> { - if let Some(entry) = self.current_chain.take() { - self.entries.push(entry); - } - Ok(()) + fn finish(&mut self) -> anyhow::Result> { + Ok(self.current_chain.take()) } } +#[derive( + Debug, Clone, serde::Deserialize, serde::Serialize, Archive, rkyv::Serialize, rkyv::Deserialize, +)] +#[rkyv(compare(PartialEq), derive(Debug))] pub struct DTableEntry { fxy: FXY, fxy_chain: Vec, @@ -96,3 +107,71 @@ pub struct DTableEntry { note_ids: String, status: String, } + +impl DTableEntry { + pub fn fxy(&self) -> FXY { + self.fxy + } + + pub fn fxy_chain(&self) -> &[FXY] { + &self.fxy_chain + } + + pub fn category(&self) -> &str { + &self.category + } + + pub fn category_of_sequences_en(&self) -> &str { + &self.category_of_sequences_en + } + + pub fn title_en(&self) -> Option<&str> { + self.title_en.as_deref() + } + + pub fn subtitle_en(&self) -> Option<&str> { + self.subtitle_en.as_deref() + } + + pub fn note_en(&self) -> Option<&str> { + self.note_en.as_deref() + } + + pub fn note_ids(&self) -> &str { + &self.note_ids + } + + pub fn status(&self) -> &str { + &self.status + } +} + +impl std::fmt::Display for DTableEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let fxy_chain_str: String = self + .fxy_chain + .iter() + .map(|fxy| format!("{:02}{:02}{:03}", fxy.f, fxy.x, fxy.y)) + .collect::>() + .join(", "); + + let title = self.title_en.as_deref().unwrap_or("N/A"); + let truncated_title = if title.len() > 50 { + format!("{}...", &title[..47]) + } else { + title.to_string() + }; + + write!( + f, + "{:02}{:02}{:03} | {:<50} | {:<12} | [{}]", + self.fxy.f, self.fxy.x, self.fxy.y, truncated_title, self.status, fxy_chain_str + ) + } +} + +impl TableEntry for DTableEntry { + fn fxy(&self) -> FXY { + self.fxy + } +} diff --git a/gen/src/wmo/mod.rs b/gen/src/wmo/mod.rs new file mode 100644 index 0000000..2b2c7b1 --- /dev/null +++ b/gen/src/wmo/mod.rs @@ -0,0 +1,2 @@ +pub mod btable; +pub mod dtable; diff --git a/rbufr/Cargo.toml b/rbufr/Cargo.toml index b2545e2..8804f60 100644 --- a/rbufr/Cargo.toml +++ b/rbufr/Cargo.toml @@ -14,3 +14,5 @@ flate2 = "1.1.5" nom = "8.0.0" serde = { version = "1.0.228", features = ["derive"] } thiserror = "2.0.17" +gentools = { path = "../gen" } +anyhow = "1.0.100" diff --git a/rbufr/tables/bufr/bufrtabb_11.csv b/rbufr/rtables/bufr/bufrtabb_11.csv similarity index 100% rename from rbufr/tables/bufr/bufrtabb_11.csv rename to rbufr/rtables/bufr/bufrtabb_11.csv diff --git a/rbufr/tables/bufr/bufrtabd_11.csv b/rbufr/rtables/bufr/bufrtabd_11.csv similarity index 100% rename from rbufr/tables/bufr/bufrtabd_11.csv rename to rbufr/rtables/bufr/bufrtabd_11.csv diff --git a/rbufr/src/block.rs b/rbufr/src/block.rs index 1a7e51d..02c13db 100644 --- a/rbufr/src/block.rs +++ b/rbufr/src/block.rs @@ -1,5 +1,8 @@ +use genlib::{BUFRTableMPH, TableEntryLoader}; + use crate::errors::Result; -use crate::{discriptor_table::*, structs::versions::BUFRMessage}; +use crate::structs::versions::BUFRMessage; +use crate::tables::*; pub struct MessageBlock { message: BUFRMessage, @@ -18,32 +21,47 @@ impl MessageBlock { pub fn load_data(&self) -> Result<()> { let table_info = self.message.table_info(); + let master_table_version = table_info.master_table_version; - let mut b_table_loader = TableLoader::::new(); - let master_b_sequences = - b_table_loader.load_table(TT::Standard, table_info.master_table_version)?; - - let mut d_table_loader = TableLoader::::new(); - let master_d_sequences = - d_table_loader.load_table(TT::Standard, table_info.master_table_version)?; + let master_b: BUFRTableB = self.load_first_validable_table(master_table_version)?; + let master_d: BUFRTableD = self.load_first_validable_table(master_table_version)?; let local_table_version = table_info.local_table_version as u32; if local_table_version > 0 { - let local_b_sequences = b_table_loader.load_table( - TT::Localized(local_table_version), + let local_b: BUFRTableB = TableLoader.load_table(LocalTable::new( + Some(table_info.subcenter_id), table_info.local_table_version, - )?; + ))?; - let local_d_sequences = d_table_loader.load_table( - TT::Localized(local_table_version), + let local_d: BUFRTableD = TableLoader.load_table(LocalTable::new( + Some(table_info.subcenter_id), table_info.local_table_version, - )?; + ))?; } Ok(()) // master_b_table.load_table(TT::Standard); } + + fn load_first_validable_table( + &self, + table_version: u8, + ) -> Result> { + (0..=table_version) + .rev() + .find_map(|version| { + TableLoader + .load_table(MasterTable::new(version)) + .ok() + .inspect(|_| { + if version != table_version { + eprintln!("Falling back to Master Table version {}", version); + } + }) + }) + .ok_or(crate::errors::Error::TableNotFoundEmpty) + } } pub struct BUFRFile { diff --git a/rbufr/src/errors.rs b/rbufr/src/errors.rs index a8b04d6..90e23b4 100644 --- a/rbufr/src/errors.rs +++ b/rbufr/src/errors.rs @@ -9,8 +9,11 @@ pub enum Error { #[error("CSV Error: {0}")] Csv(#[from] csv::Error), - #[error("Table not found for type {0:?}, sub_center {1:?}, version {2}")] - TableNotFound(crate::discriptor_table::TableType, Option, u8), + #[error("Table not found: {0}")] + TableNotFound(#[from] anyhow::Error), + + #[error("Table not found")] + TableNotFoundEmpty, #[error("Parse Error: {0}")] ParseError(String), diff --git a/rbufr/src/lib.rs b/rbufr/src/lib.rs index e00d30f..4558b27 100644 --- a/rbufr/src/lib.rs +++ b/rbufr/src/lib.rs @@ -1,5 +1,39 @@ mod block; -mod discriptor_table; +mod tables; +// mod discriptor_table; mod errors; pub mod parser; pub mod structs; + +#[cfg(test)] +mod test { + + #[test] + fn test() { + use genlib::prelude::*; + let bufr = BUFRTableB::load_from_disk( + "/Users/xiang.li1/projects/rbufr/tables/BUFRCREX_TableB_en_00.bufrtbl", + ) + .unwrap(); + + let entry = bufr.lookup(FXY::new(0, 0, 1)).unwrap().unwrap(); + + println!("{:#?}", entry); + } + + #[test] + fn test_rb() { + use crate::parser::Parser; + + let mut parser = Parser::new(); + let parsed_file = parser + .parse("/Users/xiang.li1/Downloads/36_2025-12-22T11_00_00.bufr") + .unwrap(); + + for msg in parsed_file.messages() { + println!("{}", msg); + + msg.load_data().unwrap(); + } + } +} diff --git a/rbufr/src/parser.rs b/rbufr/src/parser.rs index 10494c9..5c1b828 100644 --- a/rbufr/src/parser.rs +++ b/rbufr/src/parser.rs @@ -147,19 +147,3 @@ impl Parser { Ok(file_block) } } - -#[cfg(test)] -mod tests { - - use super::Parser; - - #[test] - fn test() { - let mut parser = Parser::new(); - if let Ok(file) = parser.parse("/Users/xiang.li1/Downloads/36_2025-12-22T11_00_00.bufr") { - for message in file.messages() { - println!("{}", message); - } - } - } -} diff --git a/rbufr/src/tables.rs b/rbufr/src/tables.rs new file mode 100644 index 0000000..6489f5b --- /dev/null +++ b/rbufr/src/tables.rs @@ -0,0 +1,102 @@ +use crate::errors::Result; +pub use genlib::prelude::{BUFRTableB, BUFRTableD, TableType}; +use genlib::{TableEntryLoader, prelude::*}; +use std::path::PathBuf; + +pub trait TableTrait { + fn file_path(&self, table_type: TableType) -> PathBuf; +} + +#[derive(Debug, Clone, Copy)] +pub struct MasterTable { + version: u8, +} + +impl MasterTable { + pub fn new(version: u8) -> Self { + MasterTable { version } + } +} +#[derive(Debug, Clone, Copy)] +pub struct LocalTable { + sub_center: Option, + version: u8, +} + +impl LocalTable { + pub fn new(sub_center: Option, version: u8) -> Self { + LocalTable { + sub_center, + version, + } + } +} +impl TableTrait for MasterTable { + fn file_path(&self, table_type: TableType) -> PathBuf { + match table_type { + TableType::B => { + let mut base_dir = PathBuf::new(); + base_dir.push("tables/master"); + let file_name = format!("BUFR_TableB_en_{:0>2}.bufrtbl", self.version); + base_dir.join(file_name) + } + TableType::D => { + let mut base_dir = PathBuf::new(); + base_dir.push("tables/master"); + let file_name = format!("BUFR_TableD_en_{:0>2}.bufrtbl", self.version); + base_dir.join(file_name) + } + _ => { + unreachable!("Table type not supported for MasterTable") + } + } + } +} + +impl TableTrait for LocalTable { + fn file_path(&self, table_type: TableType) -> PathBuf { + match table_type { + TableType::B => { + let mut base_dir = PathBuf::new(); + base_dir.push("tables/local"); + let sub_center_str = match self.sub_center { + Some(sc) => format!("sc{:0>2}", sc), + None => "sc00".to_string(), + }; + let file_name = format!( + "BUFR_TableB_{}_en_{:0>2}.bufrtbl", + sub_center_str, self.version + ); + base_dir.join(file_name) + } + TableType::D => { + let mut base_dir = PathBuf::new(); + base_dir.push("tables/local"); + let sub_center_str = match self.sub_center { + Some(sc) => format!("sc{:0>2}", sc), + None => "sc00".to_string(), + }; + let file_name = format!( + "BUFR_TableD_{}_en_{:0>2}.bufrtbl", + sub_center_str, self.version + ); + base_dir.join(file_name) + } + _ => { + unreachable!("Table type not supported for LocalTable") + } + } + } +} + +pub struct TableLoader; + +impl TableLoader { + pub fn load_table(&self, table_type: impl TableTrait) -> Result> + where + T: TableEntryLoader, + { + let path = table_type.file_path(T::TABLE_TYPE); + BUFRTableMPH::::load_from_disk(path).map_err(|e| e.into()) + } +} diff --git a/rbufr/tables/master/BUFR_TableB_en_00.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_00.bufrtbl new file mode 100644 index 0000000..babfa11 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_00.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_01.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_01.bufrtbl new file mode 100644 index 0000000..a4755b8 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_01.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_02.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_02.bufrtbl new file mode 100644 index 0000000..bb156f2 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_02.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_03.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_03.bufrtbl new file mode 100644 index 0000000..14bee96 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_03.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_04.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_04.bufrtbl new file mode 100644 index 0000000..8989c53 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_04.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_05.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_05.bufrtbl new file mode 100644 index 0000000..080a03c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_05.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_06.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_06.bufrtbl new file mode 100644 index 0000000..ffaeda8 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_06.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_07.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_07.bufrtbl new file mode 100644 index 0000000..3324fed Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_07.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_08.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_08.bufrtbl new file mode 100644 index 0000000..5e20f5c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_08.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_10.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_10.bufrtbl new file mode 100644 index 0000000..e3b0e4c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_10.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_11.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_11.bufrtbl new file mode 100644 index 0000000..eb500a0 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_11.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_12.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_12.bufrtbl new file mode 100644 index 0000000..5fe1d6a Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_12.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_13.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_13.bufrtbl new file mode 100644 index 0000000..0d52b9d Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_13.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_14.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_14.bufrtbl new file mode 100644 index 0000000..704b366 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_14.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_15.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_15.bufrtbl new file mode 100644 index 0000000..f2975db Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_15.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_19.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_19.bufrtbl new file mode 100644 index 0000000..fc3743c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_19.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_20.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_20.bufrtbl new file mode 100644 index 0000000..48f53e6 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_20.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_21.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_21.bufrtbl new file mode 100644 index 0000000..dfdf41a Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_21.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_22.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_22.bufrtbl new file mode 100644 index 0000000..96a240c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_22.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_23.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_23.bufrtbl new file mode 100644 index 0000000..f2a2cfa Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_23.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_24.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_24.bufrtbl new file mode 100644 index 0000000..e021675 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_24.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_25.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_25.bufrtbl new file mode 100644 index 0000000..67b16ab Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_25.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_26.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_26.bufrtbl new file mode 100644 index 0000000..27cb7b9 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_26.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_27.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_27.bufrtbl new file mode 100644 index 0000000..311b21f Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_27.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_28.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_28.bufrtbl new file mode 100644 index 0000000..76b0a41 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_28.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_29.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_29.bufrtbl new file mode 100644 index 0000000..de678b4 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_29.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_30.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_30.bufrtbl new file mode 100644 index 0000000..3f8fad2 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_30.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_31.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_31.bufrtbl new file mode 100644 index 0000000..1a289ce Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_31.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_33.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_33.bufrtbl new file mode 100644 index 0000000..e258bcb Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_33.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_35.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_35.bufrtbl new file mode 100644 index 0000000..dece8d2 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_35.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_40.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_40.bufrtbl new file mode 100644 index 0000000..236ecb2 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_40.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_41.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_41.bufrtbl new file mode 100644 index 0000000..1779d0a Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_41.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableB_en_42.bufrtbl b/rbufr/tables/master/BUFR_TableB_en_42.bufrtbl new file mode 100644 index 0000000..fa2f762 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableB_en_42.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_00.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_00.bufrtbl new file mode 100644 index 0000000..fef8285 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_00.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_01.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_01.bufrtbl new file mode 100644 index 0000000..9e195c1 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_01.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_02.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_02.bufrtbl new file mode 100644 index 0000000..b512e79 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_02.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_03.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_03.bufrtbl new file mode 100644 index 0000000..476b196 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_03.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_04.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_04.bufrtbl new file mode 100644 index 0000000..9a902d0 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_04.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_05.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_05.bufrtbl new file mode 100644 index 0000000..6ea92d6 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_05.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_06.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_06.bufrtbl new file mode 100644 index 0000000..367e14e Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_06.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_07.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_07.bufrtbl new file mode 100644 index 0000000..af93874 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_07.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_08.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_08.bufrtbl new file mode 100644 index 0000000..29960be Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_08.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_09.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_09.bufrtbl new file mode 100644 index 0000000..a55ee89 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_09.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_10.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_10.bufrtbl new file mode 100644 index 0000000..dfdfecf Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_10.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_11.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_11.bufrtbl new file mode 100644 index 0000000..3318af6 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_11.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_12.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_12.bufrtbl new file mode 100644 index 0000000..6885259 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_12.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_13.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_13.bufrtbl new file mode 100644 index 0000000..ab58def Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_13.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_15.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_15.bufrtbl new file mode 100644 index 0000000..f266c2e Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_15.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_16.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_16.bufrtbl new file mode 100644 index 0000000..6c4b93c Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_16.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_18.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_18.bufrtbl new file mode 100644 index 0000000..8c14c00 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_18.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_21.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_21.bufrtbl new file mode 100644 index 0000000..e5b0090 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_21.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_22.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_22.bufrtbl new file mode 100644 index 0000000..eb19ea4 Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_22.bufrtbl differ diff --git a/rbufr/tables/master/BUFR_TableD_en_40.bufrtbl b/rbufr/tables/master/BUFR_TableD_en_40.bufrtbl new file mode 100644 index 0000000..b6fbc6e Binary files /dev/null and b/rbufr/tables/master/BUFR_TableD_en_40.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_00.bufrtbl b/tables/master/BUFR_TableB_en_00.bufrtbl new file mode 100644 index 0000000..babfa11 Binary files /dev/null and b/tables/master/BUFR_TableB_en_00.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_01.bufrtbl b/tables/master/BUFR_TableB_en_01.bufrtbl new file mode 100644 index 0000000..a4755b8 Binary files /dev/null and b/tables/master/BUFR_TableB_en_01.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_02.bufrtbl b/tables/master/BUFR_TableB_en_02.bufrtbl new file mode 100644 index 0000000..bb156f2 Binary files /dev/null and b/tables/master/BUFR_TableB_en_02.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_03.bufrtbl b/tables/master/BUFR_TableB_en_03.bufrtbl new file mode 100644 index 0000000..14bee96 Binary files /dev/null and b/tables/master/BUFR_TableB_en_03.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_04.bufrtbl b/tables/master/BUFR_TableB_en_04.bufrtbl new file mode 100644 index 0000000..8989c53 Binary files /dev/null and b/tables/master/BUFR_TableB_en_04.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_05.bufrtbl b/tables/master/BUFR_TableB_en_05.bufrtbl new file mode 100644 index 0000000..080a03c Binary files /dev/null and b/tables/master/BUFR_TableB_en_05.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_06.bufrtbl b/tables/master/BUFR_TableB_en_06.bufrtbl new file mode 100644 index 0000000..ffaeda8 Binary files /dev/null and b/tables/master/BUFR_TableB_en_06.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_07.bufrtbl b/tables/master/BUFR_TableB_en_07.bufrtbl new file mode 100644 index 0000000..3324fed Binary files /dev/null and b/tables/master/BUFR_TableB_en_07.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_08.bufrtbl b/tables/master/BUFR_TableB_en_08.bufrtbl new file mode 100644 index 0000000..5e20f5c Binary files /dev/null and b/tables/master/BUFR_TableB_en_08.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_10.bufrtbl b/tables/master/BUFR_TableB_en_10.bufrtbl new file mode 100644 index 0000000..e3b0e4c Binary files /dev/null and b/tables/master/BUFR_TableB_en_10.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_11.bufrtbl b/tables/master/BUFR_TableB_en_11.bufrtbl new file mode 100644 index 0000000..eb500a0 Binary files /dev/null and b/tables/master/BUFR_TableB_en_11.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_12.bufrtbl b/tables/master/BUFR_TableB_en_12.bufrtbl new file mode 100644 index 0000000..5fe1d6a Binary files /dev/null and b/tables/master/BUFR_TableB_en_12.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_13.bufrtbl b/tables/master/BUFR_TableB_en_13.bufrtbl new file mode 100644 index 0000000..0d52b9d Binary files /dev/null and b/tables/master/BUFR_TableB_en_13.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_14.bufrtbl b/tables/master/BUFR_TableB_en_14.bufrtbl new file mode 100644 index 0000000..704b366 Binary files /dev/null and b/tables/master/BUFR_TableB_en_14.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_15.bufrtbl b/tables/master/BUFR_TableB_en_15.bufrtbl new file mode 100644 index 0000000..f2975db Binary files /dev/null and b/tables/master/BUFR_TableB_en_15.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_19.bufrtbl b/tables/master/BUFR_TableB_en_19.bufrtbl new file mode 100644 index 0000000..fc3743c Binary files /dev/null and b/tables/master/BUFR_TableB_en_19.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_20.bufrtbl b/tables/master/BUFR_TableB_en_20.bufrtbl new file mode 100644 index 0000000..48f53e6 Binary files /dev/null and b/tables/master/BUFR_TableB_en_20.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_21.bufrtbl b/tables/master/BUFR_TableB_en_21.bufrtbl new file mode 100644 index 0000000..dfdf41a Binary files /dev/null and b/tables/master/BUFR_TableB_en_21.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_22.bufrtbl b/tables/master/BUFR_TableB_en_22.bufrtbl new file mode 100644 index 0000000..96a240c Binary files /dev/null and b/tables/master/BUFR_TableB_en_22.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_23.bufrtbl b/tables/master/BUFR_TableB_en_23.bufrtbl new file mode 100644 index 0000000..f2a2cfa Binary files /dev/null and b/tables/master/BUFR_TableB_en_23.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_24.bufrtbl b/tables/master/BUFR_TableB_en_24.bufrtbl new file mode 100644 index 0000000..e021675 Binary files /dev/null and b/tables/master/BUFR_TableB_en_24.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_25.bufrtbl b/tables/master/BUFR_TableB_en_25.bufrtbl new file mode 100644 index 0000000..67b16ab Binary files /dev/null and b/tables/master/BUFR_TableB_en_25.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_26.bufrtbl b/tables/master/BUFR_TableB_en_26.bufrtbl new file mode 100644 index 0000000..27cb7b9 Binary files /dev/null and b/tables/master/BUFR_TableB_en_26.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_27.bufrtbl b/tables/master/BUFR_TableB_en_27.bufrtbl new file mode 100644 index 0000000..311b21f Binary files /dev/null and b/tables/master/BUFR_TableB_en_27.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_28.bufrtbl b/tables/master/BUFR_TableB_en_28.bufrtbl new file mode 100644 index 0000000..76b0a41 Binary files /dev/null and b/tables/master/BUFR_TableB_en_28.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_29.bufrtbl b/tables/master/BUFR_TableB_en_29.bufrtbl new file mode 100644 index 0000000..de678b4 Binary files /dev/null and b/tables/master/BUFR_TableB_en_29.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_30.bufrtbl b/tables/master/BUFR_TableB_en_30.bufrtbl new file mode 100644 index 0000000..3f8fad2 Binary files /dev/null and b/tables/master/BUFR_TableB_en_30.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_31.bufrtbl b/tables/master/BUFR_TableB_en_31.bufrtbl new file mode 100644 index 0000000..1a289ce Binary files /dev/null and b/tables/master/BUFR_TableB_en_31.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_33.bufrtbl b/tables/master/BUFR_TableB_en_33.bufrtbl new file mode 100644 index 0000000..e258bcb Binary files /dev/null and b/tables/master/BUFR_TableB_en_33.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_35.bufrtbl b/tables/master/BUFR_TableB_en_35.bufrtbl new file mode 100644 index 0000000..dece8d2 Binary files /dev/null and b/tables/master/BUFR_TableB_en_35.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_40.bufrtbl b/tables/master/BUFR_TableB_en_40.bufrtbl new file mode 100644 index 0000000..236ecb2 Binary files /dev/null and b/tables/master/BUFR_TableB_en_40.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_41.bufrtbl b/tables/master/BUFR_TableB_en_41.bufrtbl new file mode 100644 index 0000000..1779d0a Binary files /dev/null and b/tables/master/BUFR_TableB_en_41.bufrtbl differ diff --git a/tables/master/BUFR_TableB_en_42.bufrtbl b/tables/master/BUFR_TableB_en_42.bufrtbl new file mode 100644 index 0000000..fa2f762 Binary files /dev/null and b/tables/master/BUFR_TableB_en_42.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_00.bufrtbl b/tables/master/BUFR_TableD_en_00.bufrtbl new file mode 100644 index 0000000..fef8285 Binary files /dev/null and b/tables/master/BUFR_TableD_en_00.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_01.bufrtbl b/tables/master/BUFR_TableD_en_01.bufrtbl new file mode 100644 index 0000000..9e195c1 Binary files /dev/null and b/tables/master/BUFR_TableD_en_01.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_02.bufrtbl b/tables/master/BUFR_TableD_en_02.bufrtbl new file mode 100644 index 0000000..b512e79 Binary files /dev/null and b/tables/master/BUFR_TableD_en_02.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_03.bufrtbl b/tables/master/BUFR_TableD_en_03.bufrtbl new file mode 100644 index 0000000..476b196 Binary files /dev/null and b/tables/master/BUFR_TableD_en_03.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_04.bufrtbl b/tables/master/BUFR_TableD_en_04.bufrtbl new file mode 100644 index 0000000..9a902d0 Binary files /dev/null and b/tables/master/BUFR_TableD_en_04.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_05.bufrtbl b/tables/master/BUFR_TableD_en_05.bufrtbl new file mode 100644 index 0000000..6ea92d6 Binary files /dev/null and b/tables/master/BUFR_TableD_en_05.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_06.bufrtbl b/tables/master/BUFR_TableD_en_06.bufrtbl new file mode 100644 index 0000000..367e14e Binary files /dev/null and b/tables/master/BUFR_TableD_en_06.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_07.bufrtbl b/tables/master/BUFR_TableD_en_07.bufrtbl new file mode 100644 index 0000000..af93874 Binary files /dev/null and b/tables/master/BUFR_TableD_en_07.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_08.bufrtbl b/tables/master/BUFR_TableD_en_08.bufrtbl new file mode 100644 index 0000000..29960be Binary files /dev/null and b/tables/master/BUFR_TableD_en_08.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_09.bufrtbl b/tables/master/BUFR_TableD_en_09.bufrtbl new file mode 100644 index 0000000..a55ee89 Binary files /dev/null and b/tables/master/BUFR_TableD_en_09.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_10.bufrtbl b/tables/master/BUFR_TableD_en_10.bufrtbl new file mode 100644 index 0000000..dfdfecf Binary files /dev/null and b/tables/master/BUFR_TableD_en_10.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_11.bufrtbl b/tables/master/BUFR_TableD_en_11.bufrtbl new file mode 100644 index 0000000..3318af6 Binary files /dev/null and b/tables/master/BUFR_TableD_en_11.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_12.bufrtbl b/tables/master/BUFR_TableD_en_12.bufrtbl new file mode 100644 index 0000000..6885259 Binary files /dev/null and b/tables/master/BUFR_TableD_en_12.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_13.bufrtbl b/tables/master/BUFR_TableD_en_13.bufrtbl new file mode 100644 index 0000000..ab58def Binary files /dev/null and b/tables/master/BUFR_TableD_en_13.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_15.bufrtbl b/tables/master/BUFR_TableD_en_15.bufrtbl new file mode 100644 index 0000000..f266c2e Binary files /dev/null and b/tables/master/BUFR_TableD_en_15.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_16.bufrtbl b/tables/master/BUFR_TableD_en_16.bufrtbl new file mode 100644 index 0000000..6c4b93c Binary files /dev/null and b/tables/master/BUFR_TableD_en_16.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_18.bufrtbl b/tables/master/BUFR_TableD_en_18.bufrtbl new file mode 100644 index 0000000..8c14c00 Binary files /dev/null and b/tables/master/BUFR_TableD_en_18.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_21.bufrtbl b/tables/master/BUFR_TableD_en_21.bufrtbl new file mode 100644 index 0000000..e5b0090 Binary files /dev/null and b/tables/master/BUFR_TableD_en_21.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_22.bufrtbl b/tables/master/BUFR_TableD_en_22.bufrtbl new file mode 100644 index 0000000..eb19ea4 Binary files /dev/null and b/tables/master/BUFR_TableD_en_22.bufrtbl differ diff --git a/tables/master/BUFR_TableD_en_40.bufrtbl b/tables/master/BUFR_TableD_en_40.bufrtbl new file mode 100644 index 0000000..b6fbc6e Binary files /dev/null and b/tables/master/BUFR_TableD_en_40.bufrtbl differ