Vendor dependencies

notes-service/vendor/opendal/benches/ops/README.md

@@ -0,0 +1,26 @@
+# Ops Benchmark Tests
+
+Ops Benchmark Tests measure every operation's performance on the target platform.
+
+To support benching different backends simultaneously, we use `environment value` to carry the backend config.
+
+## Setup
+
+Please copy `.env.example` to `.env` and change the values on need.
+
+Take `fs` for example, we add config on `fs` on `/tmp`.
+
+```dotenv
+OPENDAL_FS_ROOT=/tmp
+```
+
+Notice: The default will skip benches if the env is not set.
+
+## Run
+
+Test specific backend, take s3 for example, first set the corresponding environment variables of s3, then:
+
+```shell
+OPENDAL_TEST=s3
+cargo bench ops --features tests
+```

notes-service/vendor/opendal/benches/ops/main.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod read;
+mod write;
+
+mod utils;
+
+fn main() {
+    divan::main();
+}

notes-service/vendor/opendal/benches/ops/read.rs

@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use divan::Bencher;
+use divan::counter::BytesCount;
+use opendal::tests::TEST_RUNTIME;
+use opendal::tests::init_test_service;
+use rand::rng;
+use size::Size;
+
+use super::utils::*;
+
+#[divan::bench(
+    args = [
+        Size::from_kib(4),
+        Size::from_kib(64),
+        Size::from_mib(1),
+        Size::from_mib(16),
+    ],
+    ignore = std::env::var("OPENDAL_TEST").is_err()
+)]
+fn whole(b: Bencher, size: Size) {
+    let op = init_test_service().unwrap().unwrap();
+    let mut rng = rng();
+    let content = gen_bytes(&mut rng, size.bytes() as usize);
+    let path = uuid::Uuid::new_v4().to_string();
+    let _temp_data = TempData::generate(op.clone(), &path, content.clone());
+
+    b.counter(BytesCount::from(size.bytes() as u64)).bench(|| {
+        let op = op.clone();
+        let path = path.clone();
+        TEST_RUNTIME.block_on(async move {
+            let _ = op.read(&path).await.unwrap();
+        })
+    })
+}
+
+mod concurrent {
+    use super::*;
+
+    #[divan::bench(
+        ignore = std::env::var("OPENDAL_TEST").is_err()
+    )]
+    fn baseline(b: Bencher) {
+        let op = init_test_service().unwrap().unwrap();
+        let mut rng = rng();
+        let content = gen_bytes(&mut rng, 1024 * 1024 * 1024);
+        let path = uuid::Uuid::new_v4().to_string();
+        let _temp_data = TempData::generate(op.clone(), &path, content.clone());
+
+        b.counter(BytesCount::from(1024usize * 1024 * 1024))
+            .bench(|| {
+                let op = op.clone();
+                let path = path.clone();
+                TEST_RUNTIME.block_on(async move {
+                    let _ = op.read(&path).await.unwrap();
+                })
+            })
+    }
+
+    #[divan::bench(
+        args = [1, 2, 4, 8, 16],
+        ignore = std::env::var("OPENDAL_TEST").is_err()
+    )]
+    fn concurrent(b: Bencher, concurrent: usize) {
+        let op = init_test_service().unwrap().unwrap();
+        let mut rng = rng();
+        let content = gen_bytes(&mut rng, 1024 * 1024 * 1024);
+        let path = uuid::Uuid::new_v4().to_string();
+        let _temp_data = TempData::generate(op.clone(), &path, content.clone());
+
+        b.counter(BytesCount::from(1024usize * 1024 * 1024))
+            .bench(|| {
+                let op = op.clone();
+                let path = path.clone();
+                TEST_RUNTIME.block_on(async move {
+                    let _ = op
+                        .read_with(&path)
+                        .chunk(16 * 1024 * 1024)
+                        .concurrent(concurrent)
+                        .await
+                        .unwrap();
+                })
+            })
+    }
+}

notes-service/vendor/opendal/benches/ops/utils.rs

@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Bytes;
+use opendal::tests::TEST_RUNTIME;
+use opendal::*;
+use rand::prelude::*;
+
+pub fn gen_bytes(rng: &mut ThreadRng, size: usize) -> Bytes {
+    let mut content = vec![0; size];
+    rng.fill_bytes(&mut content);
+
+    content.into()
+}
+
+pub struct TempData {
+    op: Operator,
+    path: String,
+}
+
+impl TempData {
+    pub fn existing(op: Operator, path: &str) -> Self {
+        Self {
+            op,
+            path: path.to_string(),
+        }
+    }
+
+    pub fn generate(op: Operator, path: &str, content: Bytes) -> Self {
+        TEST_RUNTIME.block_on(async { op.write(path, content).await.expect("create test data") });
+
+        Self {
+            op,
+            path: path.to_string(),
+        }
+    }
+}
+
+impl Drop for TempData {
+    fn drop(&mut self) {
+        TEST_RUNTIME.block_on(async {
+            self.op.delete(&self.path).await.expect("cleanup test data");
+        })
+    }
+}

notes-service/vendor/opendal/benches/ops/write.rs

@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use divan::Bencher;
+use divan::counter::BytesCount;
+use opendal::tests::TEST_RUNTIME;
+use opendal::tests::init_test_service;
+use rand::rng;
+use size::Size;
+
+use super::utils::*;
+
+#[divan::bench(
+    args = [
+        Size::from_kib(4),
+        Size::from_kib(64),
+        Size::from_mib(1),
+        Size::from_mib(16),
+    ],
+    ignore = std::env::var("OPENDAL_TEST").is_err()
+)]
+fn whole(b: Bencher, size: Size) {
+    let op = init_test_service().unwrap().unwrap();
+    let path = uuid::Uuid::new_v4().to_string();
+    let _temp_data = TempData::existing(op.clone(), &path);
+
+    b.counter(BytesCount::from(size.bytes() as u64))
+        .with_inputs(|| {
+            let mut rng = rng();
+            gen_bytes(&mut rng, size.bytes() as usize)
+        })
+        .bench_values(|content| {
+            let op = op.clone();
+            let path = path.clone();
+            TEST_RUNTIME.block_on(async move {
+                let _ = op.write(&path, content).await.unwrap();
+            })
+        })
+}
+
+mod concurrent {
+    use super::*;
+
+    /// Size of the data to be written.
+    const SIZE: usize = 64 * 1024 * 1024;
+
+    #[divan::bench(
+        ignore = std::env::var("OPENDAL_TEST").is_err()
+    )]
+    fn baseline(b: Bencher) {
+        let op = init_test_service().unwrap().unwrap();
+        let path = uuid::Uuid::new_v4().to_string();
+        let _temp_data = TempData::existing(op.clone(), &path);
+        let mut rng = rng();
+        let content = gen_bytes(&mut rng, SIZE);
+
+        b.counter(BytesCount::from(SIZE)).bench(|| {
+            let op = op.clone();
+            let path = path.clone();
+            let content = content.clone();
+            TEST_RUNTIME.block_on(async move {
+                let _ = op.write(&path, content).await.unwrap();
+            })
+        })
+    }
+
+    #[divan::bench(
+        args = [1, 2, 4, 8],
+        ignore = std::env::var("OPENDAL_TEST").is_err()
+    )]
+    fn concurrent(b: Bencher, concurrent: usize) {
+        let op = init_test_service().unwrap().unwrap();
+        let path = uuid::Uuid::new_v4().to_string();
+        let _temp_data = TempData::existing(op.clone(), &path);
+        let mut rng = rng();
+        let content = gen_bytes(&mut rng, SIZE);
+
+        b.counter(BytesCount::from(SIZE)).bench(|| {
+            let op = op.clone();
+            let path = path.clone();
+            let content = content.clone();
+            TEST_RUNTIME.block_on(async move {
+                let _ = op
+                    .write_with(&path, content)
+                    .chunk(8 * 1024 * 1024)
+                    .concurrent(concurrent)
+                    .await
+                    .unwrap();
+            })
+        })
+    }
+}