From a195be2e48db5b6638a76b3ab4ba18609991325c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 03:14:14 +0900 Subject: [PATCH 1/8] malloc --- Cargo.lock | 64 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 5 ++-- crates/cli/Cargo.toml | 1 + crates/cli/src/lib.rs | 2 ++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05d751cb1..ce1fff0ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -437,6 +437,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "dashmap" version = "5.5.3" @@ -500,6 +506,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dudy-malloc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7693457240fabab8f300db030cd86e969944089971c31654e4e2a9c255ebd369" +dependencies = [ + "mimalloc-rust", + "tikv-jemallocator", +] + [[package]] name = "dunce" version = "1.0.4" @@ -602,6 +618,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.28" @@ -1080,6 +1102,26 @@ dependencies = [ "syn", ] +[[package]] +name = "mimalloc-rust" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb726c8298efb4010b2c46d8050e4be36cf807b9d9e98cb112f830914fc9bbe" +dependencies = [ + "cty", + "mimalloc-rust-sys", +] + +[[package]] +name = "mimalloc-rust-sys" +version = "1.7.9-source" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6413e13241a9809f291568133eca6694572cf528c1a6175502d090adce5dd5db" +dependencies = [ + "cc", + "cty", +] + [[package]] name = "mime" version = "0.3.17" @@ -1305,6 +1347,7 @@ dependencies = [ "clap", "command-extra", "derive_more", + "dudy-malloc", "dunce", "home", "insta", @@ -2252,6 +2295,27 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.4.3+5.2.1-patched.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1792ccb507d955b46af42c123ea8863668fae24d03721e40cad6a41773dbb49" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b7bcecfafe4998587d636f9ae9d55eb9d0499877b88757767c346875067098" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinytemplate" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index 2b2109f7d..ac250fe62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ command-extra = { version = "1.0.0" } base64 = { version = "0.21.5" } dashmap = { version = "5.5.3" } derive_more = { version = "1.0.0-beta.3", features = ["full"] } +dudy-malloc = { version = "0.1.0" } dunce = { version = "1.0.4" } home = { version = "0.5.5" } insta = { version = "1.32.0", features = ["yaml", "glob", "walkdir"] } @@ -78,8 +79,8 @@ allow_branch = "main" opt-level = 3 lto = "fat" codegen-units = 1 -strip = "symbols" -debug = false +# strip = "symbols" +# debug = false panic = "abort" # Let it crash and force ourselves to write safe Rust. # Use the `--profile release-debug` flag to show symbols in release mode. diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index a5daff4e9..253f7bb12 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -32,6 +32,7 @@ miette = { workspace = true } reqwest = { workspace = true } pipe-trait = { workspace = true } tokio = { workspace = true } +dudy-malloc = { workspace = true } [dev-dependencies] pacquet-store-dir = { workspace = true } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 679893657..7be9525fa 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,3 +1,5 @@ +extern crate dudy_malloc; + mod cli_args; mod state; From 46b4ade3c1427c0a09ccf1c8571cf5afbcca6af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:24:50 +0900 Subject: [PATCH 2/8] swc_malloc --- Cargo.lock | 37 +++++++++++++++---------------------- Cargo.toml | 4 ++-- crates/cli/Cargo.toml | 2 +- crates/cli/src/lib.rs | 2 +- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce1fff0ec..ef2d5fca4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,16 +506,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dudy-malloc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7693457240fabab8f300db030cd86e969944089971c31654e4e2a9c255ebd369" -dependencies = [ - "mimalloc-rust", - "tikv-jemallocator", -] - [[package]] name = "dunce" version = "1.0.4" @@ -618,12 +608,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures" version = "0.3.28" @@ -1347,7 +1331,6 @@ dependencies = [ "clap", "command-extra", "derive_more", - "dudy-malloc", "dunce", "home", "insta", @@ -1367,6 +1350,7 @@ dependencies = [ "pretty_assertions", "reqwest", "serde_json", + "swc_malloc", "tempfile", "tokio", "walkdir", @@ -2197,6 +2181,16 @@ dependencies = [ "is-terminal", ] +[[package]] +name = "swc_malloc" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a279493814466a779ac93921b8a88fbd9f9162807e564d64dbbae2edee00c8a" +dependencies = [ + "mimalloc-rust", + "tikv-jemallocator", +] + [[package]] name = "syn" version = "2.0.28" @@ -2297,20 +2291,19 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.4.3+5.2.1-patched.2" +version = "0.5.4+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1792ccb507d955b46af42c123ea8863668fae24d03721e40cad6a41773dbb49" +checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" dependencies = [ "cc", - "fs_extra", "libc", ] [[package]] name = "tikv-jemallocator" -version = "0.4.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5b7bcecfafe4998587d636f9ae9d55eb9d0499877b88757767c346875067098" +checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" dependencies = [ "libc", "tikv-jemalloc-sys", diff --git a/Cargo.toml b/Cargo.toml index ac250fe62..b477a2a4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ command-extra = { version = "1.0.0" } base64 = { version = "0.21.5" } dashmap = { version = "5.5.3" } derive_more = { version = "1.0.0-beta.3", features = ["full"] } -dudy-malloc = { version = "0.1.0" } dunce = { version = "1.0.4" } home = { version = "0.5.5" } insta = { version = "1.32.0", features = ["yaml", "glob", "walkdir"] } @@ -55,6 +54,7 @@ sha2 = { version = "0.10.8" } split-first-char = { version = "0.0.0" } ssri = { version = "9.0.0" } strum = { version = "0.25.0", features = ["derive"] } +swc_malloc = { version = "0.5.9" } tar = { version = "0.4.40" } text-block-macros = { version = "0.1.1" } tracing = { version = "0.1.37" } @@ -81,7 +81,7 @@ lto = "fat" codegen-units = 1 # strip = "symbols" # debug = false -panic = "abort" # Let it crash and force ourselves to write safe Rust. +panic = "abort" # Let it crash and force ourselves to write safe Rust. # Use the `--profile release-debug` flag to show symbols in release mode. # e.g. `cargo build --profile release-debug` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 253f7bb12..dac5af3f3 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -32,7 +32,7 @@ miette = { workspace = true } reqwest = { workspace = true } pipe-trait = { workspace = true } tokio = { workspace = true } -dudy-malloc = { workspace = true } +swc_malloc = { workspace = true } [dev-dependencies] pacquet-store-dir = { workspace = true } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 7be9525fa..a286ff249 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,4 +1,4 @@ -extern crate dudy_malloc; +extern crate swc_malloc; mod cli_args; mod state; From 183971482c86e8fc416f4480cd2c30e188d00676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:33:48 +0900 Subject: [PATCH 3/8] Dep on rayon --- crates/cli/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index dac5af3f3..8c356ef65 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -33,6 +33,7 @@ reqwest = { workspace = true } pipe-trait = { workspace = true } tokio = { workspace = true } swc_malloc = { workspace = true } +rayon = { workspace = true } [dev-dependencies] pacquet-store-dir = { workspace = true } From 8aba776ab389580000f4b7b023366a13c6f41487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:33:51 +0900 Subject: [PATCH 4/8] cargo lockfile --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index ef2d5fca4..95368f7fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1348,6 +1348,7 @@ dependencies = [ "pacquet-testing-utils", "pipe-trait", "pretty_assertions", + "rayon", "reqwest", "serde_json", "swc_malloc", From af4045def6f6d99ce4e4b9c7c778b73898cd6b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:36:20 +0900 Subject: [PATCH 5/8] Dep on num_cpus --- Cargo.toml | 1 + crates/cli/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index b477a2a4a..373bfa956 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ reflink-copy = { version = "0.1.9" } junction = { version = "1.0.0" } reqwest = { version = "0.11", default-features = false, features = ["json", "native-tls-vendored"] } node-semver = { version = "2.1.0" } +num_cpus = { version = "1.16.0" } pipe-trait = { version = "0.4.0" } rayon = { version = "1.8.0" } serde = { version = "1.0.188", features = ["derive"] } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 8c356ef65..126d49ead 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -34,6 +34,7 @@ pipe-trait = { workspace = true } tokio = { workspace = true } swc_malloc = { workspace = true } rayon = { workspace = true } +num_cpus = { workspace = true } [dev-dependencies] pacquet-store-dir = { workspace = true } From 3297bdb83cf3b1a556b4ab8487f3edc3f57956d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:36:27 +0900 Subject: [PATCH 6/8] cargo lockfile --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 95368f7fc..27f7fb1ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1335,6 +1335,7 @@ dependencies = [ "home", "insta", "miette", + "num_cpus", "pacquet-diagnostics", "pacquet-executor", "pacquet-fs", From 3c3166c5b0a2920d7ba54642136ad7cf068ad0a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:37:07 +0900 Subject: [PATCH 7/8] Enlarge IO thread pool --- crates/cli/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index a286ff249..80eed3aab 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -10,6 +10,12 @@ use pacquet_diagnostics::enable_tracing_by_env; use state::State; pub async fn main() -> miette::Result<()> { + // We use rayon only for blocking syscalls, so we multiply the number of threads by 3. + rayon::ThreadPoolBuilder::new() + .num_threads(num_cpus::get() * 3) + .build_global() + .expect("build rayon thread pool"); + enable_tracing_by_env(); set_panic_hook(); CliArgs::parse().run().await From 8662d9482b7c5480cee97a15b5c3890f7df3c37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4=20=28Donny=29?= Date: Sun, 12 Nov 2023 05:38:12 +0900 Subject: [PATCH 8/8] doc --- crates/cli/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 80eed3aab..a858e548c 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -11,6 +11,10 @@ use state::State; pub async fn main() -> miette::Result<()> { // We use rayon only for blocking syscalls, so we multiply the number of threads by 3. + // + // If we are going to use rayon for CPU-bound tasks, + // we should create an extra threadpool for IO-bound tasks, + // and use the global theadpool for CPU-bound tasks. rayon::ThreadPoolBuilder::new() .num_threads(num_cpus::get() * 3) .build_global()