Vendor dependencies

This commit is contained in:
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
@@ -0,0 +1 @@
{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"f698785cad70e91ba7184988b14e41cd1d3cf570470b58d2ebc74f36f726d61a","Cargo.lock":"dbd1a5ea83bd87e570d724a753c5557f9af1a6ffb0b2c5c83894a003cdf0b7da","Cargo.toml":"7dd4526787e7aed5d8c69c25a6d648caa31a1de908168aba3efe18907986292e","Cargo.toml.orig":"1d22d3e4064d8680f4c9a338dd376a59cedf13013ba44f9fe6f948d7ed3cd145","src/backend.rs":"c0179516797d32a4432a5bb4bff11f599a491647e27ab466f79538d68ffde4a7","src/config.rs":"f85344d9239d3fae668568bbaf3c9891cc163edec8c82d8634b5b5ed73119583","src/core.rs":"d6cdd0caff3c48e7518c65ad9fb5e7f3e48bd89500d2d5647c75551e8691138b","src/deleter.rs":"ed9088b68c63c3217b67afc6110592fd1f5542a2bf3132385f8e492a317892cf","src/docs.md":"ff51be6e0c105085ccf8a6e9aee745441427ed94b24b487482e689a60b48888d","src/error.rs":"c4207706c45314285411f204227b54e5213a4625d5323363abd3ea3478f30514","src/lib.rs":"b11479fad52f8249f716ba43ee1019d4e43587f2bd3097bdeda423720f952239","src/lister.rs":"4f07741e9cb5cf77f584d25c774b38b7fda3b10e368a68c4afe740a27920a61f","src/reader.rs":"e4157a1350263b0a3016605f5fc50211802634987608e3949df1911ce28e40c3","src/writer.rs":"44b9e93e2146d11c39ea322dd80dda81370f037177abc6b0cf78b21baa0de88d"},"package":"22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c"}
@@ -0,0 +1,6 @@
{
"git": {
"sha1": "1e0fc73190afb7e221c7badda7ef43fe40592940"
},
"path_in_vcs": "core/services/fs"
}
File diff suppressed because it is too large Load Diff
+62
View File
@@ -0,0 +1,62 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2024"
rust-version = "1.85"
name = "opendal-service-fs"
version = "0.57.0"
authors = ["Apache OpenDAL <dev@opendal.apache.org>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Apache OpenDAL Fs service implementation"
homepage = "https://opendal.apache.org/"
readme = false
license = "Apache-2.0"
repository = "https://github.com/apache/opendal"
[package.metadata.docs.rs]
all-features = true
[lib]
name = "opendal_service_fs"
path = "src/lib.rs"
[dependencies.bytes]
version = "1.10"
[dependencies.log]
version = "0.4.29"
[dependencies.opendal-core]
version = "0.57.0"
features = ["internal-tokio-rt"]
default-features = false
[dependencies.serde]
version = "1"
features = ["derive"]
default-features = false
[dependencies.tokio]
version = "1.48"
features = [
"fs",
"rt-multi-thread",
]
default-features = false
[target."cfg(unix)".dependencies.xattr]
version = "1"
+43
View File
@@ -0,0 +1,43 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[package]
description = "Apache OpenDAL Fs service implementation"
name = "opendal-service-fs"
authors = { workspace = true }
edition = { workspace = true }
homepage = { workspace = true }
license = { workspace = true }
repository = { workspace = true }
rust-version = { workspace = true }
version = { workspace = true }
[package.metadata.docs.rs]
all-features = true
[dependencies]
bytes = { workspace = true }
log = { workspace = true }
opendal-core = { path = "../../core", version = "0.57.0", default-features = false, features = [
"internal-tokio-rt",
] }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
[target.'cfg(unix)'.dependencies]
xattr = "1"
+268
View File
@@ -0,0 +1,268 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::path::PathBuf;
use std::sync::Arc;
use log::debug;
use super::FS_SCHEME;
use super::config::FsConfig;
use super::core::*;
use super::deleter::FsDeleter;
use super::lister::FsLister;
use super::reader::FsReader;
use super::writer::FsWriter;
use super::writer::FsWriters;
use opendal_core::raw::*;
use opendal_core::*;
/// POSIX file system support.
#[doc = include_str!("docs.md")]
#[derive(Debug, Default)]
pub struct FsBuilder {
pub(super) config: FsConfig,
}
impl FsBuilder {
/// Set root for backend.
pub fn root(mut self, root: &str) -> Self {
self.config.root = if root.is_empty() {
None
} else {
Some(root.to_string())
};
self
}
/// Set temp dir for atomic write.
///
/// # Notes
///
/// - When append is enabled, we will not use atomic write
/// to avoid data loss and performance issue.
pub fn atomic_write_dir(mut self, dir: &str) -> Self {
if !dir.is_empty() {
self.config.atomic_write_dir = Some(dir.to_string());
}
self
}
}
impl Builder for FsBuilder {
type Config = FsConfig;
fn build(self) -> Result<impl Access> {
debug!("backend build started: {:?}", &self);
let root = match self.config.root.map(PathBuf::from) {
Some(root) => Ok(root),
None => Err(Error::new(
ErrorKind::ConfigInvalid,
"root is not specified",
)),
}?;
debug!("backend use root {}", root.to_string_lossy());
// If root dir is not exist, we must create it.
if let Err(e) = std::fs::metadata(&root) {
if e.kind() == std::io::ErrorKind::NotFound {
std::fs::create_dir_all(&root).map_err(|e| {
Error::new(ErrorKind::Unexpected, "create root dir failed")
.with_operation("Builder::build")
.with_context("root", root.to_string_lossy())
.set_source(e)
})?;
}
}
let atomic_write_dir = self.config.atomic_write_dir.map(PathBuf::from);
// If atomic write dir is not exist, we must create it.
if let Some(d) = &atomic_write_dir {
if let Err(e) = std::fs::metadata(d) {
if e.kind() == std::io::ErrorKind::NotFound {
std::fs::create_dir_all(d).map_err(|e| {
Error::new(ErrorKind::Unexpected, "create atomic write dir failed")
.with_operation("Builder::build")
.with_context("atomic_write_dir", d.to_string_lossy())
.set_source(e)
})?;
}
}
}
// Canonicalize the root directory. This should work since we already know that we can
// get the metadata of the path.
let root = root.canonicalize().map_err(|e| {
Error::new(
ErrorKind::Unexpected,
"canonicalize of root directory failed",
)
.set_source(e)
})?;
// Canonicalize the atomic_write_dir directory. This should work since we already know that
// we can get the metadata of the path.
let atomic_write_dir = atomic_write_dir
.map(|p| {
p.canonicalize().map(Some).map_err(|e| {
Error::new(
ErrorKind::Unexpected,
"canonicalize of atomic_write_dir directory failed",
)
.with_operation("Builder::build")
.with_context("root", root.to_string_lossy())
.set_source(e)
})
})
.unwrap_or(Ok(None))?;
Ok(FsBackend {
core: Arc::new(FsCore {
info: {
let am = AccessorInfo::default();
am.set_scheme(FS_SCHEME)
.set_root(&root.to_string_lossy())
.set_native_capability(Capability {
stat: true,
read: true,
write: true,
write_can_empty: true,
write_can_append: true,
write_can_multi: true,
write_with_if_not_exists: true,
#[cfg(unix)]
write_with_user_metadata: true,
create_dir: true,
delete: true,
delete_with_recursive: true,
list: true,
copy: true,
rename: true,
shared: true,
..Default::default()
});
am.into()
},
root,
atomic_write_dir,
buf_pool: oio::PooledBuf::new(16).with_initial_capacity(256 * 1024),
}),
})
}
}
/// Backend is used to serve `Accessor` support for posix-like fs.
#[derive(Debug, Clone)]
pub struct FsBackend {
core: Arc<FsCore>,
}
impl Access for FsBackend {
type Reader = FsReader<tokio::fs::File>;
type Writer = FsWriters;
type Lister = Option<FsLister<tokio::fs::ReadDir>>;
type Deleter = oio::OneShotDeleter<FsDeleter>;
type Copier = ();
fn info(&self) -> Arc<AccessorInfo> {
self.core.info.clone()
}
async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
self.core.fs_create_dir(path).await?;
Ok(RpCreateDir::default())
}
async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
let m = self.core.fs_stat(path).await?;
Ok(RpStat::new(m))
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let f = self.core.fs_read(path, &args).await?;
let r = FsReader::new(
self.core.clone(),
f,
args.range().size().unwrap_or(u64::MAX) as _,
);
Ok((RpRead::default(), r))
}
async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
let is_append = op.append();
let concurrent = op.concurrent();
let writer = FsWriter::create(self.core.clone(), path, op).await?;
let writer = if is_append {
FsWriters::One(writer)
} else {
FsWriters::Two(oio::PositionWriter::new(
self.info().clone(),
writer,
concurrent,
))
};
Ok((RpWrite::default(), writer))
}
async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
Ok((
RpDelete::default(),
oio::OneShotDeleter::new(FsDeleter::new(self.core.clone())),
))
}
async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
match self.core.fs_list(path).await? {
Some(f) => {
let rd = FsLister::new(&self.core.root, path, f);
Ok((RpList::default(), Some(rd)))
}
None => Ok((RpList::default(), None)),
}
}
async fn copy(
&self,
from: &str,
to: &str,
_args: OpCopy,
_opts: OpCopier,
) -> Result<(RpCopy, Self::Copier)> {
self.core.fs_copy(from, to).await?;
Ok((RpCopy::default(), ()))
}
async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
self.core.fs_rename(from, to).await?;
Ok(RpRename::default())
}
}
+74
View File
@@ -0,0 +1,74 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use serde::Deserialize;
use serde::Serialize;
use super::backend::FsBuilder;
/// config for file system
#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[serde(default)]
#[non_exhaustive]
pub struct FsConfig {
/// root dir for backend
pub root: Option<String>,
/// tmp dir for atomic write
pub atomic_write_dir: Option<String>,
}
impl opendal_core::Configurator for FsConfig {
type Builder = FsBuilder;
fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> {
let mut map = uri.options().clone();
if let Some(root) = uri.root().filter(|v| !v.is_empty()) {
map.entry("root".to_string())
.or_insert_with(|| format!("/{}", root));
}
Self::from_iter(map)
}
fn into_builder(self) -> Self::Builder {
FsBuilder { config: self }
}
}
#[cfg(test)]
mod tests {
use super::*;
use opendal_core::Configurator;
use opendal_core::OperatorUri;
#[test]
fn from_uri_extracts_root() {
let uri = OperatorUri::new("fs:///tmp/data", Vec::<(String, String)>::new()).unwrap();
let cfg = FsConfig::from_uri(&uri).unwrap();
assert_eq!(cfg.root.as_deref(), Some("/tmp/data"));
}
#[test]
fn from_uri_with_file_scheme() {
// "file" is an alias for "fs" to support standard file:// URIs
let uri = OperatorUri::new("file:///tmp/data", Vec::<(String, String)>::new()).unwrap();
let cfg = FsConfig::from_uri(&uri).unwrap();
assert_eq!(cfg.root.as_deref(), Some("/tmp/data"));
}
}
+279
View File
@@ -0,0 +1,279 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::collections::HashMap;
use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use super::error::*;
use opendal_core::raw::*;
use opendal_core::*;
#[derive(Debug)]
pub struct FsCore {
pub info: Arc<AccessorInfo>,
pub root: PathBuf,
pub atomic_write_dir: Option<PathBuf>,
pub buf_pool: oio::PooledBuf,
}
impl FsCore {
// Build write path and ensure the parent dirs created
pub async fn ensure_write_abs_path(&self, parent: &Path, path: &str) -> Result<PathBuf> {
let p = parent.join(path);
// Create dir before write path.
//
// TODO(xuanwo): There are many works to do here:
// - Is it safe to create dir concurrently?
// - Do we need to extract this logic as new util functions?
// - Is it better to check the parent dir exists before call mkdir?
let parent = PathBuf::from(&p)
.parent()
.ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"path should have parent but not, it must be malformed",
)
.with_context("input", p.to_string_lossy())
})?
.to_path_buf();
tokio::fs::create_dir_all(&parent)
.await
.map_err(new_std_io_error)?;
Ok(p)
}
pub async fn fs_create_dir(&self, path: &str) -> Result<()> {
let p = self.root.join(path.trim_end_matches('/'));
tokio::fs::create_dir_all(&p)
.await
.map_err(new_std_io_error)?;
Ok(())
}
pub async fn fs_stat(&self, path: &str) -> Result<Metadata> {
let p = self.root.join(path.trim_end_matches('/'));
let meta = tokio::fs::metadata(&p).await.map_err(new_std_io_error)?;
let mode = if meta.is_dir() {
EntryMode::DIR
} else if meta.is_file() {
EntryMode::FILE
} else {
EntryMode::Unknown
};
let mut m = Metadata::new(mode)
.with_content_length(meta.len())
.with_last_modified(Timestamp::try_from(
meta.modified().map_err(new_std_io_error)?,
)?);
// Read user metadata from xattr on Unix systems
#[cfg(unix)]
{
let user_metadata = Self::get_user_metadata(&p)?;
if !user_metadata.is_empty() {
m = m.with_user_metadata(user_metadata);
}
}
Ok(m)
}
pub async fn fs_read(&self, path: &str, args: &OpRead) -> Result<tokio::fs::File> {
let p = self.root.join(path.trim_end_matches('/'));
let mut f = tokio::fs::OpenOptions::new()
.read(true)
.open(&p)
.await
.map_err(new_std_io_error)?;
if args.range().offset() != 0 {
use tokio::io::AsyncSeekExt;
f.seek(SeekFrom::Start(args.range().offset()))
.await
.map_err(new_std_io_error)?;
}
Ok(f)
}
pub async fn fs_write(&self, path: &PathBuf, op: &OpWrite) -> Result<tokio::fs::File> {
let mut open_options = tokio::fs::OpenOptions::new();
if op.if_not_exists() {
open_options.create_new(true);
} else {
open_options.create(true);
}
open_options.write(true);
if op.append() {
open_options.append(true);
} else {
open_options.truncate(true);
}
let f = open_options.open(path).await.map_err(parse_error)?;
Ok(f)
}
/// This function is used to build a tempfile for writing.
///
/// We don't care about the OpWrite since every check should be performed on target path directly.
pub async fn fs_tempfile_write(
&self,
path: &str,
) -> Result<(tokio::fs::File, Option<PathBuf>)> {
let Some(atomic_write_dir) = self.atomic_write_dir.as_ref() else {
return Err(Error::new(
ErrorKind::Unexpected,
"fs didn't configure atomic_write_dir, but we're still entering the tempfile logic. This might be a bug.",
));
};
let tmp_path = self
.ensure_write_abs_path(atomic_write_dir, &build_tmp_path_of(path))
.await?;
let mut open_options = tokio::fs::OpenOptions::new();
// tempfile should always be new file.
open_options.create_new(true);
open_options.write(true);
open_options.truncate(true);
let f = open_options.open(&tmp_path).await.map_err(parse_error)?;
Ok((f, Some(tmp_path)))
}
pub async fn fs_list(&self, path: &str) -> Result<Option<tokio::fs::ReadDir>> {
let p = self.root.join(path.trim_end_matches('/'));
match tokio::fs::read_dir(&p).await {
Ok(rd) => Ok(Some(rd)),
Err(e) => {
match e.kind() {
// Return empty list if the directory not found
std::io::ErrorKind::NotFound => Ok(None),
// If the path is not a directory, return an empty list
//
// The path could be a file or a symbolic link in this case.
// Returning a NotADirectory error to the user isn't helpful; instead,
// providing an empty directory is a more user-friendly. In fact, the dir
// `path/` does not exist.
std::io::ErrorKind::NotADirectory => Ok(None),
_ => Err(new_std_io_error(e)),
}
}
}
}
pub async fn fs_copy(&self, from: &str, to: &str) -> Result<()> {
let from = self.root.join(from.trim_end_matches('/'));
// try to get the metadata of the source file to ensure it exists
tokio::fs::metadata(&from).await.map_err(new_std_io_error)?;
let to = self
.ensure_write_abs_path(&self.root, to.trim_end_matches('/'))
.await?;
tokio::fs::copy(from, to).await.map_err(new_std_io_error)?;
Ok(())
}
pub async fn fs_rename(&self, from: &str, to: &str) -> Result<()> {
let from = self.root.join(from.trim_end_matches('/'));
tokio::fs::metadata(&from).await.map_err(new_std_io_error)?;
let to = self
.ensure_write_abs_path(&self.root, to.trim_end_matches('/'))
.await?;
tokio::fs::rename(from, to)
.await
.map_err(new_std_io_error)?;
Ok(())
}
/// Get user metadata from extended attributes on Unix systems.
///
/// Reads xattr in the "user." namespace and strips the prefix, allowing
/// interoperability with other applications that set user xattr.
#[cfg(unix)]
pub fn get_user_metadata(path: &Path) -> Result<HashMap<String, String>> {
let mut user_metadata = HashMap::new();
let attrs = match xattr::list(path) {
Ok(attrs) => attrs,
Err(e) => {
// xattr may not be supported on some filesystems, ignore the error
if e.kind() == std::io::ErrorKind::Unsupported {
return Ok(user_metadata);
}
// Also ignore "operation not supported" errors (ENOTSUP/EOPNOTSUPP)
// which may occur on filesystems that don't support xattr
if let Some(os_error) = e.raw_os_error() {
// ENOTSUP (95) and EOPNOTSUPP (95) on Linux
if os_error == 95 {
return Ok(user_metadata);
}
}
return Err(new_std_io_error(e));
}
};
for attr in attrs {
let attr_name = attr.to_string_lossy();
// Only read xattr in the "user." namespace and strip the prefix
if let Some(key) = attr_name.strip_prefix(XATTR_USER_PREFIX) {
if let Ok(Some(value)) = xattr::get(path, &attr) {
if let Ok(v) = String::from_utf8(value) {
user_metadata.insert(key.to_string(), v);
}
}
}
}
Ok(user_metadata)
}
/// Set user metadata as extended attributes on Unix systems.
///
/// Keys are automatically prefixed with "user." to comply with Linux xattr
/// naming requirements. This allows interoperability with other applications.
#[cfg(unix)]
pub fn set_user_metadata(path: &Path, user_metadata: &HashMap<String, String>) -> Result<()> {
for (key, value) in user_metadata {
let attr_name = format!("{XATTR_USER_PREFIX}{key}");
xattr::set(path, &attr_name, value.as_bytes()).map_err(new_std_io_error)?;
}
Ok(())
}
}
/// Prefix for user xattr namespace on Linux.
/// Using "user." as the standard namespace for user-defined attributes.
#[cfg(unix)]
const XATTR_USER_PREFIX: &str = "user.";
+61
View File
@@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use super::core::*;
use opendal_core::raw::*;
use opendal_core::*;
pub struct FsDeleter {
core: Arc<FsCore>,
}
impl FsDeleter {
pub fn new(core: Arc<FsCore>) -> Self {
Self { core }
}
}
impl oio::OneShotDelete for FsDeleter {
async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
let p = self.core.root.join(path.trim_end_matches('/'));
let recursive = args.recursive();
let meta = tokio::fs::metadata(&p).await;
match meta {
Ok(meta) => {
if meta.is_dir() {
if recursive {
tokio::fs::remove_dir_all(&p)
.await
.map_err(new_std_io_error)?;
} else {
tokio::fs::remove_dir(&p).await.map_err(new_std_io_error)?;
}
} else {
tokio::fs::remove_file(&p).await.map_err(new_std_io_error)?;
}
Ok(())
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(new_std_io_error(err)),
}
}
}
+47
View File
@@ -0,0 +1,47 @@
## Capabilities
This service can be used to:
- [x] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [x] copy
- [x] rename
- [ ] ~~presign~~
## Configuration
- `root`: Set the work dir for backend.
-
You can refer to [`FsBuilder`]'s docs for more information
## Example
### Via Builder
```rust,ignore
use std::sync::Arc;
use anyhow::Result;
use opendal::services::Fs;
use opendal::Operator;
#[tokio::main]
async fn main() -> Result<()> {
// Create fs backend builder.
let mut builder = Fs::default()
// Set the root for fs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
.root("/tmp");
// `Accessor` provides the low level APIs, we will use `Operator` normally.
let op: Operator = Operator::new(builder)?.finish();
Ok(())
}
```
+31
View File
@@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use opendal_core::raw::*;
use opendal_core::*;
/// Parse error response into Error.
pub(super) fn parse_error(e: std::io::Error) -> Error {
match e.kind() {
std::io::ErrorKind::AlreadyExists => Error::new(
ErrorKind::ConditionNotMatch,
"The file already exists in the filesystem",
)
.set_source(e),
_ => new_std_io_error(e),
}
}
+43
View File
@@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#![cfg_attr(docsrs, feature(doc_cfg))]
//! Fs service implementation for Apache OpenDAL.
#![deny(missing_docs)]
mod backend;
mod config;
mod core;
mod deleter;
mod error;
mod lister;
mod reader;
mod writer;
pub use backend::FsBuilder as Fs;
pub use config::FsConfig;
/// Default scheme for fs service.
pub const FS_SCHEME: &str = "fs";
/// Alias scheme for fs service that follows the standard URI scheme.
pub const FILE_SCHEME: &str = "file";
/// Register this service into the given registry.
pub fn register_fs_service(registry: &opendal_core::OperatorRegistry) {
registry.register::<Fs>(FS_SCHEME);
registry.register::<Fs>(FILE_SCHEME);
}
+116
View File
@@ -0,0 +1,116 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::path::Path;
use std::path::PathBuf;
use opendal_core::EntryMode;
use opendal_core::Metadata;
use opendal_core::Result;
use opendal_core::raw::*;
pub struct FsLister<P> {
root: PathBuf,
current_path: Option<String>,
rd: P,
}
impl<P> FsLister<P> {
pub fn new(root: &Path, path: &str, rd: P) -> Self {
Self {
root: root.to_owned(),
current_path: Some(path.to_string()),
rd,
}
}
}
/// # Safety
///
/// We will only take `&mut Self` reference for FsLister.
unsafe impl<P> Sync for FsLister<P> {}
impl oio::List for FsLister<tokio::fs::ReadDir> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
loop {
// since list should return path itself, we return it first
if let Some(path) = self.current_path.take() {
let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR));
return Ok(Some(e));
}
let de = match self.rd.next_entry().await {
Ok(Some(de)) => de,
Ok(None) => return Ok(None),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// Directory can be removed during listing; stop gracefully.
return Ok(None);
}
Err(e) => return Err(new_std_io_error(e)),
};
let entry_path = de.path();
let rel_path = normalize_path(
&entry_path
.strip_prefix(&self.root)
.expect("cannot fail because the prefix is iterated")
.to_string_lossy()
.replace('\\', "/"),
);
let ft = match de.file_type().await {
Ok(ft) => ft,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// Entry can be deleted between readdir and stat calls.
continue;
}
Err(e) => return Err(new_std_io_error(e)),
};
let (path, mode) = if ft.is_dir() {
// Make sure we are returning the correct path.
(&format!("{rel_path}/"), EntryMode::DIR)
} else if ft.is_file() {
(&rel_path, EntryMode::FILE)
} else {
(&rel_path, EntryMode::Unknown)
};
let de_metadata = match de.metadata().await {
Ok(de_metadata) => de_metadata,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// Entry can be deleted between readdir and metadata calls.
continue;
}
Err(e) => return Err(new_std_io_error(e)),
};
let last_modified = match de_metadata.modified() {
Ok(v) => v,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
continue;
}
Err(e) => return Err(new_std_io_error(e)),
};
let metadata = Metadata::new(mode)
.with_content_length(de_metadata.len())
.with_last_modified(Timestamp::try_from(last_modified)?);
return Ok(Some(oio::Entry::new(path, metadata)));
}
}
}
+83
View File
@@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use tokio::io::AsyncReadExt;
use tokio::io::ReadBuf;
use super::core::*;
use opendal_core::raw::*;
use opendal_core::*;
pub struct FsReader<F> {
core: Arc<FsCore>,
f: F,
read: usize,
size: usize,
buf_size: usize,
}
impl<F> FsReader<F> {
pub fn new(core: Arc<FsCore>, f: F, size: usize) -> Self {
Self {
core,
f,
read: 0,
size,
// Use 2 MiB as default value.
buf_size: 2 * 1024 * 1024,
}
}
}
impl oio::Read for FsReader<tokio::fs::File> {
async fn read(&mut self) -> Result<Buffer> {
if self.read >= self.size {
return Ok(Buffer::new());
}
let mut bs = self.core.buf_pool.get();
bs.reserve(self.buf_size);
let size = (self.size - self.read).min(self.buf_size);
let buf = &mut bs.spare_capacity_mut()[..size];
let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
// SAFETY: Read at most `limit` bytes into `read_buf`.
unsafe {
read_buf.assume_init(size);
}
let n = self
.f
.read_buf(&mut read_buf)
.await
.map_err(new_std_io_error)?;
self.read += n;
// Safety: We make sure that bs contains `n` more bytes.
let filled = read_buf.filled().len();
unsafe { bs.set_len(filled) }
let frozen = bs.split().freeze();
// Return the buffer to the pool.
self.core.buf_pool.put(bs);
Ok(Buffer::from(frozen))
}
}
+240
View File
@@ -0,0 +1,240 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::collections::HashMap;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use bytes::Buf;
use tokio::io::AsyncWriteExt;
use super::core::FsCore;
use opendal_core::raw::*;
use opendal_core::*;
pub type FsWriters = TwoWays<FsWriter, oio::PositionWriter<FsWriter>>;
pub struct FsWriter {
target_path: PathBuf,
/// The temp_path is used to specify whether we should move to target_path after the file has been closed.
temp_path: Option<PathBuf>,
f: tokio::fs::File,
/// User metadata to be written to xattr on Unix systems.
#[cfg(unix)]
user_metadata: Option<HashMap<String, String>>,
}
impl FsWriter {
pub async fn create(core: Arc<FsCore>, path: &str, op: OpWrite) -> Result<Self> {
let target_path = core.ensure_write_abs_path(&core.root, path).await?;
// Store user metadata for later use on Unix systems.
#[cfg(unix)]
let user_metadata = op.user_metadata().cloned();
// Quick path while atomic_write_dir is not set.
if core.atomic_write_dir.is_none() {
let target_file = core.fs_write(&target_path, &op).await?;
return Ok(Self {
target_path,
temp_path: None,
f: target_file,
#[cfg(unix)]
user_metadata,
});
}
let is_append = op.append();
let is_exist = tokio::fs::try_exists(&target_path)
.await
.map_err(new_std_io_error)?;
if op.if_not_exists() && is_exist {
return Err(Error::new(
ErrorKind::ConditionNotMatch,
"file already exists, doesn't match the condition if_not_exists",
));
}
// The only case we allow write in place is the file
// exists and users request for append writing.
let (f, temp_path) = if !(is_append && is_exist) {
core.fs_tempfile_write(path).await?
} else {
let f = core.fs_write(&target_path, &op).await?;
(f, None)
};
Ok(Self {
target_path,
temp_path,
f,
#[cfg(unix)]
user_metadata,
})
}
}
/// # Safety
///
/// We will only take `&mut Self` reference for FsWriter.
unsafe impl Sync for FsWriter {}
impl oio::Write for FsWriter {
async fn write(&mut self, mut bs: Buffer) -> Result<()> {
while bs.has_remaining() {
let n = self.f.write(bs.chunk()).await.map_err(new_std_io_error)?;
bs.advance(n);
}
Ok(())
}
async fn close(&mut self) -> Result<Metadata> {
self.f.flush().await.map_err(new_std_io_error)?;
self.f.sync_all().await.map_err(new_std_io_error)?;
if let Some(temp_path) = &self.temp_path {
tokio::fs::rename(temp_path, &self.target_path)
.await
.map_err(new_std_io_error)?;
}
// Write user metadata to xattr on Unix systems.
#[cfg(unix)]
if let Some(ref user_metadata) = self.user_metadata {
FsCore::set_user_metadata(&self.target_path, user_metadata)?;
}
let file_meta = self.f.metadata().await.map_err(new_std_io_error)?;
let meta = Metadata::new(EntryMode::FILE)
.with_content_length(file_meta.len())
.with_last_modified(Timestamp::try_from(
file_meta.modified().map_err(new_std_io_error)?,
)?);
Ok(meta)
}
async fn abort(&mut self) -> Result<()> {
if let Some(temp_path) = &self.temp_path {
tokio::fs::remove_file(temp_path)
.await
.map_err(new_std_io_error)
} else {
Err(Error::new(
ErrorKind::Unsupported,
"Fs doesn't support abort if atomic_write_dir is not set",
))
}
}
}
impl oio::PositionWrite for FsWriter {
async fn write_all_at(&self, offset: u64, buf: Buffer) -> Result<()> {
let f = self
.f
.try_clone()
.await
.map_err(new_std_io_error)?
.into_std()
.await;
tokio::task::spawn_blocking(move || {
let mut buf = buf;
let mut offset = offset;
while !buf.is_empty() {
match write_at(&f, buf.chunk(), offset) {
Ok(n) => {
buf.advance(n);
offset += n as u64
}
Err(e) => return Err(e),
}
}
Ok(())
})
.await
.map_err(new_task_join_error)?
}
async fn close(&self, _size: u64) -> Result<Metadata> {
let mut f = self
.f
.try_clone()
.await
.map_err(new_std_io_error)?
.into_std()
.await;
f.flush().map_err(new_std_io_error)?;
f.sync_all().map_err(new_std_io_error)?;
if let Some(temp_path) = &self.temp_path {
tokio::fs::rename(temp_path, &self.target_path)
.await
.map_err(new_std_io_error)?;
}
// Write user metadata to xattr on Unix systems.
#[cfg(unix)]
if let Some(ref user_metadata) = self.user_metadata {
FsCore::set_user_metadata(&self.target_path, user_metadata)?;
}
let file_meta = f.metadata().map_err(new_std_io_error)?;
let mode = if file_meta.is_file() {
EntryMode::FILE
} else if file_meta.is_dir() {
EntryMode::DIR
} else {
EntryMode::Unknown
};
let meta = Metadata::new(mode)
.with_content_length(file_meta.len())
.with_last_modified(Timestamp::try_from(
file_meta.modified().map_err(new_std_io_error)?,
)?);
Ok(meta)
}
async fn abort(&self) -> Result<()> {
if let Some(temp_path) = &self.temp_path {
tokio::fs::remove_file(temp_path)
.await
.map_err(new_std_io_error)
} else {
Err(Error::new(
ErrorKind::Unsupported,
"Fs doesn't support abort if atomic_write_dir is not set",
))
}
}
}
#[cfg(windows)]
fn write_at(f: &File, buf: &[u8], offset: u64) -> Result<usize> {
use std::os::windows::fs::FileExt;
f.seek_write(buf, offset).map_err(new_std_io_error)
}
#[cfg(unix)]
fn write_at(f: &File, buf: &[u8], offset: u64) -> Result<usize> {
use std::os::unix::fs::FileExt;
f.write_at(buf, offset).map_err(new_std_io_error)
}