Vendor dependencies

This commit is contained in:
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
File diff suppressed because one or more lines are too long
+8
View File
@@ -0,0 +1,8 @@
[target.armv5te-unknown-linux-musleabi]
rustflags = ["--cfg", "armv5te"]
[target.mips-unknown-linux-musl]
rustflags = ["--cfg", "mips"]
[target.mipsel-unknown-linux-musl]
rustflags = ["--cfg", "mips"]
+6
View File
@@ -0,0 +1,6 @@
{
"git": {
"sha1": "616473ee923f4cd1429b3d8eb3be7df3eb9906b1"
},
"path_in_vcs": ""
}
+2218
View File
File diff suppressed because it is too large Load Diff
+209
View File
@@ -0,0 +1,209 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
rust-version = "1.71.1"
name = "moka"
version = "0.12.15"
build = false
exclude = [
".ci_extras/",
".codecov.yml",
".devcontainer",
".github",
".gitignore",
".gitpod.yml",
".markdownlint.json",
".vscode",
"CHANGELOG.md",
"examples",
"MIGRATION-GUIDE.md",
"tests/compile_tests/",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A fast and concurrent cache library inspired by Java Caffeine"
documentation = "https://docs.rs/moka/"
readme = "README.md"
keywords = [
"cache",
"concurrent",
]
categories = [
"caching",
"concurrency",
]
license = "(MIT OR Apache-2.0) AND Apache-2.0"
repository = "https://github.com/moka-rs/moka"
[package.metadata.docs.rs]
features = [
"future",
"sync",
]
rustdoc-args = [
"--cfg",
"docsrs",
]
[features]
atomic64 = []
default = []
future = [
"dep:async-lock",
"dep:event-listener",
"dep:futures-util",
]
logging = ["dep:log"]
quanta = ["dep:quanta"]
sync = []
unstable-debug-counters = ["future"]
[lib]
name = "moka"
path = "src/lib.rs"
[[test]]
name = "and_compute_with_race"
path = "tests/and_compute_with_race.rs"
[[test]]
name = "entry_api_actix_rt2"
path = "tests/entry_api_actix_rt2.rs"
[[test]]
name = "entry_api_sync"
path = "tests/entry_api_sync.rs"
[[test]]
name = "entry_api_tokio"
path = "tests/entry_api_tokio.rs"
[[test]]
name = "runtime_actix_rt2"
path = "tests/runtime_actix_rt2.rs"
[[test]]
name = "runtime_tokio"
path = "tests/runtime_tokio.rs"
[[test]]
name = "timer_wheel_panic_test"
path = "tests/timer_wheel_panic_test.rs"
[dependencies.async-lock]
version = "3.3"
optional = true
[dependencies.crossbeam-channel]
version = "0.5.15"
[dependencies.crossbeam-epoch]
version = "0.9.18"
[dependencies.crossbeam-utils]
version = "0.8.21"
[dependencies.equivalent]
version = "1.0"
[dependencies.event-listener]
version = "5.3"
optional = true
[dependencies.futures-util]
version = "0.3.17"
optional = true
[dependencies.log]
version = "0.4"
optional = true
[dependencies.parking_lot]
version = "0.12"
[dependencies.portable-atomic]
version = "1.6"
[dependencies.quanta]
version = "0.12.2"
optional = true
[dependencies.smallvec]
version = "1.8"
[dependencies.tagptr]
version = "0.2"
[dependencies.uuid]
version = "1.1"
features = ["v4"]
[dev-dependencies.actix-rt]
version = "2.8"
[dev-dependencies.ahash]
version = "0.8.3"
[dev-dependencies.anyhow]
version = "1.0.19"
[dev-dependencies.env_logger]
version = "0.10.0"
[dev-dependencies.getrandom]
version = "0.2"
[dev-dependencies.once_cell]
version = "1.7"
[dev-dependencies.rand]
version = "0.8.5"
[dev-dependencies.reqwest]
version = "0.12"
features = ["rustls-tls"]
default-features = false
[dev-dependencies.tokio]
version = "1.19"
features = [
"fs",
"io-util",
"macros",
"rt-multi-thread",
"sync",
"time",
]
[target."cfg(moka_loom)".dev-dependencies.loom]
version = "0.7"
[target."cfg(trybuild)".dev-dependencies.trybuild]
version = "1.0"
[lints.rust.unexpected_cfgs]
level = "warn"
priority = 0
check-cfg = [
"cfg(armv5te)",
"cfg(beta_clippy)",
"cfg(kani)",
"cfg(moka_loom)",
"cfg(mips)",
"cfg(run_flaky_tests)",
"cfg(skip_large_mem_tests)",
"cfg(trybuild)",
]
+193
View File
@@ -0,0 +1,193 @@
[package]
name = "moka"
version = "0.12.15"
edition = "2021"
# Rust 1.71.1 was released on August 3, 2023.
rust-version = "1.71.1"
description = "A fast and concurrent cache library inspired by Java Caffeine"
license = "(MIT OR Apache-2.0) AND Apache-2.0"
# homepage = "https://"
documentation = "https://docs.rs/moka/"
repository = "https://github.com/moka-rs/moka"
keywords = ["cache", "concurrent"]
categories = ["caching", "concurrency"]
readme = "README.md"
exclude = [
".ci_extras/",
".codecov.yml",
".devcontainer",
".github",
".gitignore",
".gitpod.yml",
".markdownlint.json",
".vscode",
"CHANGELOG.md",
"examples",
"MIGRATION-GUIDE.md",
"tests/compile_tests/",
]
[features]
default = []
# Enable this feature to use `moka::sync::{Cache, SegmentedCache}`
sync = []
# Enable this feature to use `moka::future::Cache`.
future = ["dep:async-lock", "dep:event-listener", "dep:futures-util"]
# Enable this feature to activate optional logging from caches.
# Currently cache will emit log only when it encounters a panic in user provided
# callback closure.
logging = ["dep:log"]
# Enable this feature to use `quanta::Instant` for some performance critical
# operations in the cache instead of `std::time::Instant`. As of v0.12.10, this
# feature will not make any noticeable performance difference, but in the future
# when cache metrics are added, it will be useful to have this feature enabled.
quanta = ["dep:quanta"]
# This is an old feature and has no effect in v0.12.10 or newer. It is kept for
# backward compatibility and will be removed in v0.13.0.
atomic64 = []
# This unstable feature adds `GlobalDebugCounters::current` function, which returns
# counters of internal object construction and destruction. It will have some
# performance impacts and is intended for debugging.
unstable-debug-counters = ["future"]
[dependencies]
crossbeam-channel = "0.5.15"
crossbeam-epoch = "0.9.18"
crossbeam-utils = "0.8.21"
equivalent = "1.0"
parking_lot = "0.12"
portable-atomic = "1.6"
smallvec = "1.8"
tagptr = "0.2"
uuid = { version = "1.1", features = ["v4"] }
# Optional dependencies (quanta)
quanta = { version = "0.12.2", optional = true }
# Optional dependencies (future)
async-lock = { version = "3.3", optional = true }
event-listener = { version = "5.3", optional = true }
futures-util = { version = "0.3.17", optional = true }
# Optional dependencies (logging)
log = { version = "0.4", optional = true }
[dev-dependencies]
actix-rt = "2.8"
ahash = "0.8.3"
anyhow = "1.0.19"
env_logger = "0.10.0"
getrandom = "0.2"
once_cell = "1.7"
rand = "0.8.5"
reqwest = { version = "0.12", default-features = false, features = [
"rustls-tls",
] }
tokio = { version = "1.19", features = [
"fs",
"io-util",
"macros",
"rt-multi-thread",
"sync",
"time",
] }
# We cannot use `cfg(loom)` here because an indirect dependency `concurrent-queue`
# uses it.
[target.'cfg(moka_loom)'.dev-dependencies]
loom = "0.7"
[target.'cfg(trybuild)'.dev-dependencies]
trybuild = "1.0"
[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = [
"cfg(armv5te)",
"cfg(beta_clippy)",
"cfg(kani)",
"cfg(moka_loom)",
"cfg(mips)",
"cfg(run_flaky_tests)",
"cfg(skip_large_mem_tests)",
"cfg(trybuild)",
] }
# https://docs.rs/about/metadata
[package.metadata.docs.rs]
# Build the doc at docs.rs with some features enabled.
#
# You can test locally with:
# ```
# cargo +nightly -Z unstable-options --config 'build.rustdocflags="--cfg docsrs"' \
# doc --no-deps --features 'future, sync'
# ```
features = ["future", "sync"]
rustdoc-args = ["--cfg", "docsrs"]
# Examples
[[example]]
name = "append_value_async"
required-features = ["future"]
[[example]]
name = "append_value_sync"
required-features = ["sync"]
[[example]]
name = "basics_async"
required-features = ["future"]
[[example]]
name = "basics_sync"
required-features = ["sync"]
[[example]]
name = "bounded_counter_async"
required-features = ["future"]
[[example]]
name = "bounded_counter_sync"
required-features = ["sync"]
[[example]]
name = "cascading_drop_async"
required-features = ["future"]
[[example]]
name = "counter_async"
required-features = ["future"]
[[example]]
name = "counter_sync"
required-features = ["sync"]
[[example]]
name = "eviction_listener_sync"
required-features = ["sync"]
[[example]]
name = "jittered_expiry_policy_sync"
required-features = ["sync"]
[[example]]
name = "reinsert_expired_entries_sync"
required-features = ["sync"]
[[example]]
name = "size_aware_eviction_sync"
required-features = ["sync"]
[[example]]
name = "try_append_value_async"
required-features = ["future"]
[[example]]
name = "try_append_value_sync"
required-features = ["sync"]
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2020 - 2026 Tatsuya Kawano
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 - 2026 Tatsuya Kawano
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+14
View File
@@ -0,0 +1,14 @@
Additional Notices for Moka
The majority of the Moka library is dual-licensed under the MIT and Apache 2.0
licenses.
However, the following files are an exception and are licensed solely under
the Apache License 2.0:
- src/common/frequency_sketch.rs
- src/common/timer_wheel.rs
These files were ported from the Java Caffeine library and are not dual-licensed.
Please refer to the LICENSE-APACHE file for more details on the Apache License 2.0.
+655
View File
@@ -0,0 +1,655 @@
# Moka
[![GitHub Actions][gh-actions-badge]][gh-actions]
[![crates.io release][release-badge]][crate]
[![docs][docs-badge]][docs]
[![DeepWiki][deepwiki-badge]][deepwiki]
[![dependency status][deps-rs-badge]][deps-rs]
[![codecov][codecov-badge]][codecov]
[![license][license-badge]](#license)
> [!NOTE]
> If you have any questions about Moka's APIs or internal design, you can ask the
> AI chatbot at DeepWiki in a natural language:
> <https://deepwiki.com/moka-rs/moka>
> [!NOTE]
> `v0.12.0` had major breaking changes on the API and internal behavior. Please read
> the [MIGRATION-GUIDE.md][migration-guide-v012] for the details.
* * *
Moka is a fast, concurrent cache library for Rust. Moka is inspired by the
[Caffeine][caffeine-git] library for Java.
Moka provides cache implementations on top of hash maps. They support full
concurrency of retrievals and a high expected concurrency for updates.
All caches perform a best-effort bounding of a hash map using an entry replacement
algorithm to determine which entries to evict when the capacity is exceeded.
[gh-actions-badge]: https://github.com/moka-rs/moka/workflows/CI/badge.svg
[release-badge]: https://img.shields.io/crates/v/moka.svg
[docs-badge]: https://docs.rs/moka/badge.svg
[deepwiki-badge]: https://deepwiki.com/badge.svg
[deps-rs-badge]: https://deps.rs/repo/github/moka-rs/moka/status.svg
[codecov-badge]: https://codecov.io/gh/moka-rs/moka/graph/badge.svg?token=7GYZNS7O67
[license-badge]: https://img.shields.io/crates/l/moka.svg
[gh-actions]: https://github.com/moka-rs/moka/actions?query=workflow%3ACI
[crate]: https://crates.io/crates/moka
[docs]: https://docs.rs/moka
[deepwiki]: https://deepwiki.com/moka-rs/moka
[deps-rs]: https://deps.rs/repo/github/moka-rs/moka
[codecov]: https://codecov.io/gh/moka-rs/moka
[caffeine-git]: https://github.com/ben-manes/caffeine
## Features
Moka provides a rich and flexible feature set while maintaining high hit ratio and a
high level of concurrency for concurrent access.
- Thread-safe, highly concurrent in-memory cache implementations:
- Synchronous caches that can be shared across OS threads.
- An asynchronous (futures aware) cache.
- A cache can be bounded by one of the followings:
- The maximum number of entries.
- The total weighted size of entries. (Size aware eviction)
- Maintains near optimal hit ratio by using an entry replacement algorithms inspired
by Caffeine:
- Admission to a cache is controlled by the Least Frequently Used (LFU) policy.
- Eviction from a cache is controlled by the Least Recently Used (LRU) policy.
- [More details and some benchmark results are available here][tiny-lfu].
- Supports expiration policies:
- Time to live.
- Time to idle.
- Per-entry variable expiration.
- Supports eviction listener, a callback function that will be called when an entry
is removed from the cache.
### Choosing the right cache for your use case
No cache implementation is perfect for every use cases. Moka is a complex software
and can be overkill for your use case. Sometimes simpler caches like
[Mini Moka][mini-moka-crate] or [Quick Cache][quick-cache] might be a better fit.
The following table shows the trade-offs between the different cache implementations:
| Feature | Moka v0.12 | Mini Moka v0.10 | Quick Cache v0.6 |
|:------- |:---- |:--------- |:----------- |
| Thread-safe, sync cache | ✅ | ✅ | ✅ |
| Thread-safe, async cache | ✅ | ❌ | ✅ |
| Non-concurrent cache | ❌ | ✅ | ✅ |
| Bounded by the maximum number of entries | ✅ | ✅ | ✅ |
| Bounded by the total weighted size of entries | ✅ | ✅ | ✅ |
| Near optimal hit ratio | ✅ TinyLFU | ✅ TinyLFU | ✅ S3-FIFO |
| Per-key, atomic insertion. (e.g. `get_with` method) | ✅ | ❌ | ✅ |
| Cache-level expiration policies (time-to-live and time-to-idle) | ✅ | ✅ | ❌ |
| Per-entry variable expiration | ✅ | ❌ | ❌ |
| Eviction listener | ✅ | ❌ | ✅ (via lifecycle hook) |
| Lock-free, concurrent iterator | ✅ | ❌ | ❌ |
| Lock-per-shard, concurrent iterator | ❌ | ✅ | ✅ |
| Performance, etc. | Moka v0.12 | Mini Moka v0.10 | Quick Cache v0.6 |
|:------- |:---- |:--------- |:----------- |
| Small overhead compared to a concurrent hash table | ❌ | ❌ | ✅ |
| Does not use background threads | ❌ → ✅ Removed from v0.12 | ✅ | ✅ |
| Small dependency tree | ❌ | ✅ | ✅ |
[tiny-lfu]: https://github.com/moka-rs/moka/wiki#admission-and-eviction-policies
[quick-cache]: https://crates.io/crates/quick_cache
[mini-moka-crate]: https://crates.io/crates/mini-moka
## Moka in Production
Moka is powering production services as well as embedded Linux devices like home
routers. Here are some highlights:
- [crates.io](https://crates.io/): The official crate registry has been using Moka in
its API service to reduce the loads on PostgreSQL. Moka is maintaining
[cache hit rates of ~85%][gh-discussions-51] for the high-traffic download endpoint.
(Moka used: Nov 2021 &mdash; present)
- [aliyundrive-webdav][aliyundrive-webdav-git]: This WebDAV gateway for a cloud drive
may have been deployed in hundreds of home Wi-Fi routers, including inexpensive
models with 32-bit MIPS or ARMv5TE-based SoCs. Moka is used to cache the metadata
of remote files. (Moka used: Aug 2021 &mdash; present)
[gh-discussions-51]: https://github.com/moka-rs/moka/discussions/51
[aliyundrive-webdav-git]: https://github.com/messense/aliyundrive-webdav
## Recent Changes
> [!NOTE]
> `v0.12.0` had major breaking changes on the API and internal behavior. Please read
> the [MIGRATION-GUIDE.md][migration-guide-v012] for the details.
- [MIGRATION-GUIDE.md][migration-guide-v012]
- [CHANGELOG.md](https://github.com/moka-rs/moka/blob/main/CHANGELOG.md)
[migration-guide-v012]: https://github.com/moka-rs/moka/blob/main/MIGRATION-GUIDE.md
## Table of Contents
- [Features](#features)
- [Choosing the right cache for your use case](#choosing-the-right-cache-for-your-use-case)
- [Moka in Production](#moka-in-production)
- [Change Log](#change-log)
- [Supported Platforms](#supported-platforms)
- [Usage](#usage)
- Examples (Part 1)
- [Synchronous Cache](#example-synchronous-cache)
- [Asynchronous Cache](#example-asynchronous-cache)
- [Avoiding to clone the value at `get`](#avoiding-to-clone-the-value-at-get)
- Example (Part 2)
- [Size Aware Eviction](#example-size-aware-eviction)
- [Expiration Policies](#expiration-policies)
- [Minimum Supported Rust Versions](#minimum-supported-rust-versions)
- Troubleshooting
- [Compile Errors on Some 32-bit Platforms](#compile-errors-on-some-32-bit-platforms)
- [Developing Moka](#developing-moka)
- [Road Map](#road-map)
- [About the Name](#about-the-name)
- [Credits](#credits)
- [License](#license)
## Supported Platforms
Moka should work on most 64-bit and 32-bit platforms if Rust `std` library is
available with threading support. However, WebAssembly (Wasm) and WASI targets are
not supported.
The following platforms are tested on CI:
- Linux 64-bit (x86_64, arm aarch64)
- Linux 32-bit (i646, armv7, armv5, mips)
- If you get compile errors on 32-bit platforms, see
[troubleshooting](#compile-errors-on-some-32-bit-platforms).
The following platforms are not tested on CI but should work:
- macOS (arm64)
- Windows (x86_64 msvc and gnu)
- iOS (arm64)
The following platforms are _not_ supported:
- WebAssembly (Wasm) and WASI targets are not supported.
(See [this project task][gh-proj-49877487])
- `nostd` environment (platforms without `std` library) are not supported.
- 16-bit platforms are not supported.
[gh-proj-49877487]: https://github.com/orgs/moka-rs/projects/1?pane=issue&itemId=49877487
## Usage
To add Moka to your dependencies, run `cargo add` as the followings:
```console
# To use the synchronous cache:
cargo add moka --features sync
# To use the asynchronous cache:
cargo add moka --features future
```
If you want to use the cache under an async runtime such as `tokio` or `async-std`, you should specify the `future` feature. Otherwise, specify the `sync` feature.
## Example: Synchronous Cache
The thread-safe, synchronous caches are defined in the `sync` module.
Cache entries are manually added using `insert` or `get_with` method, and
are stored in the cache until either evicted or manually invalidated.
Here's an example of reading and updating a cache by using multiple threads:
```rust
// Use the synchronous cache.
use moka::sync::Cache;
use std::thread;
fn value(n: usize) -> String {
format!("value {n}")
}
fn main() {
const NUM_THREADS: usize = 16;
const NUM_KEYS_PER_THREAD: usize = 64;
// Create a cache that can store up to 10,000 entries.
let cache = Cache::new(10_000);
// Spawn threads and read and update the cache simultaneously.
let threads: Vec<_> = (0..NUM_THREADS)
.map(|i| {
// To share the same cache across the threads, clone it.
// This is a cheap operation.
let my_cache = cache.clone();
let start = i * NUM_KEYS_PER_THREAD;
let end = (i + 1) * NUM_KEYS_PER_THREAD;
thread::spawn(move || {
// Insert 64 entries. (NUM_KEYS_PER_THREAD = 64)
for key in start..end {
my_cache.insert(key, value(key));
// get() returns Option<String>, a clone of the stored value.
assert_eq!(my_cache.get(&key), Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
my_cache.invalidate(&key);
}
})
})
.collect();
// Wait for all threads to complete.
threads.into_iter().for_each(|t| t.join().expect("Failed"));
// Verify the result.
for key in 0..(NUM_THREADS * NUM_KEYS_PER_THREAD) {
if key % 4 == 0 {
assert_eq!(cache.get(&key), None);
} else {
assert_eq!(cache.get(&key), Some(value(key)));
}
}
}
```
You can try the synchronous example by cloning the repository and running the
following cargo instruction:
```console
$ cargo run --example sync_example
```
If you want to atomically initialize and insert a value when the key is not present,
you might want to check [the document][doc-sync-cache] for other insertion methods
`get_with` and `try_get_with`.
[doc-sync-cache]: https://docs.rs/moka/*/moka/sync/struct.Cache.html#method.get_with
## Example: Asynchronous Cache
The asynchronous (futures aware) cache is defined in the `future` module.
It works with asynchronous runtime such as [Tokio][tokio-crate],
[async-std][async-std-crate] or [actix-rt][actix-rt-crate].
To use the asynchronous cache, [enable a crate feature called "future"](#usage).
[tokio-crate]: https://crates.io/crates/tokio
[async-std-crate]: https://crates.io/crates/async-std
[actix-rt-crate]: https://crates.io/crates/actix-rt
Cache entries are manually added using an insert method, and are stored in the cache
until either evicted or manually invalidated:
- Inside an async context (`async fn` or `async` block), use `insert` or `invalidate`
method for updating the cache and `await` them.
- Outside any async context, use `blocking` method to access blocking version of
`insert` or `invalidate` methods.
Here is a similar program to the previous example, but using asynchronous cache with
[Tokio][tokio-crate] runtime:
```rust,ignore
// Cargo.toml
//
// [dependencies]
// moka = { version = "0.12", features = ["future"] }
// tokio = { version = "1", features = ["rt-multi-thread", "macros" ] }
// futures-util = "0.3"
// Use the asynchronous cache.
use moka::future::Cache;
#[tokio::main]
async fn main() {
const NUM_TASKS: usize = 16;
const NUM_KEYS_PER_TASK: usize = 64;
fn value(n: usize) -> String {
format!("value {n}")
}
// Create a cache that can store up to 10,000 entries.
let cache = Cache::new(10_000);
// Spawn async tasks and write to and read from the cache.
let tasks: Vec<_> = (0..NUM_TASKS)
.map(|i| {
// To share the same cache across the async tasks, clone it.
// This is a cheap operation.
let my_cache = cache.clone();
let start = i * NUM_KEYS_PER_TASK;
let end = (i + 1) * NUM_KEYS_PER_TASK;
tokio::spawn(async move {
// Insert 64 entries. (NUM_KEYS_PER_TASK = 64)
for key in start..end {
// insert() is an async method, so await it.
my_cache.insert(key, value(key)).await;
// get() returns Option<String>, a clone of the stored value.
assert_eq!(my_cache.get(&key).await, Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
// invalidate() is an async method, so await it.
my_cache.invalidate(&key).await;
}
})
})
.collect();
// Wait for all tasks to complete.
futures_util::future::join_all(tasks).await;
// Verify the result.
for key in 0..(NUM_TASKS * NUM_KEYS_PER_TASK) {
if key % 4 == 0 {
assert_eq!(cache.get(&key).await, None);
} else {
assert_eq!(cache.get(&key).await, Some(value(key)));
}
}
}
```
You can try the asynchronous example by cloning the repository and running the
following cargo instruction:
```console
$ cargo run --example async_example --features future
```
If you want to atomically initialize and insert a value when the key is not present,
you might want to check [the document][doc-future-cache] for other insertion methods
`get_with` and `try_get_with`.
[doc-future-cache]: https://docs.rs/moka/*/moka/future/struct.Cache.html#method.get_with
## Avoiding to clone the value at `get`
For the concurrent caches (`sync` and `future` caches), the return type of `get`
method is `Option<V>` instead of `Option<&V>`, where `V` is the value type. Every
time `get` is called for an existing key, it creates a clone of the stored value `V`
and returns it. This is because the `Cache` allows concurrent updates from threads so
a value stored in the cache can be dropped or replaced at any time by any other
thread. `get` cannot return a reference `&V` as it is impossible to guarantee the
value outlives the reference.
If you want to store values that will be expensive to clone, wrap them by
`std::sync::Arc` before storing in a cache. [`Arc`][rustdoc-std-arc] is a thread-safe
reference-counted pointer and its `clone()` method is cheap.
[rustdoc-std-arc]: https://doc.rust-lang.org/stable/std/sync/struct.Arc.html
```rust,ignore
use std::sync::Arc;
let key = ...
let large_value = vec![0u8; 2 * 1024 * 1024]; // 2 MiB
// When insert, wrap the large_value by Arc.
cache.insert(key.clone(), Arc::new(large_value));
// get() will call Arc::clone() on the stored value, which is cheap.
cache.get(&key);
```
## Example: Size Aware Eviction
If different cache entries have different "weights" &mdash; e.g. each entry has
different memory footprints &mdash; you can specify a `weigher` closure at the cache
creation time. The closure should return a weighted size (relative size) of an entry
in `u32`, and the cache will evict entries when the total weighted size exceeds its
`max_capacity`.
```rust
use moka::sync::Cache;
fn main() {
let cache = Cache::builder()
// A weigher closure takes &K and &V and returns a u32 representing the
// relative size of the entry. Here, we use the byte length of the value
// String as the size.
.weigher(|_key, value: &String| -> u32 {
value.len().try_into().unwrap_or(u32::MAX)
})
// This cache will hold up to 32MiB of values.
.max_capacity(32 * 1024 * 1024)
.build();
cache.insert(0, "zero".to_string());
}
```
Note that weighted sizes are not used when making eviction selections.
You can try the size aware eviction example by cloning the repository and running the
following cargo instruction:
```console
$ cargo run --example size_aware_eviction
```
## Expiration Policies
Moka supports the following expiration policies:
- **Cache-level expiration policies:**
- Cache-level policies are applied to all entries in the cache.
- **Time to live (TTL)**: A cached entry will be expired after the specified
duration past from `insert`.
- **Time to idle (TTI)**: A cached entry will be expired after the specified
duration past from `get` or `insert`.
- **Per-entry expiration policy:**
- The per-entry expiration lets you sets a different expiration time for each
entry.
For details and examples of above policies, see the "Example: Time-based Expiration"
section ([`sync::Cache`][doc-sync-cache-expiration],
[`future::Cache`][doc-future-cache-expiration]) of the document.
[doc-sync-cache-expiration]: https://docs.rs/moka/latest/moka/sync/struct.Cache.html#example-time-based-expirations
[doc-future-cache-expiration]: https://docs.rs/moka/latest/moka/future/struct.Cache.html#example-time-based-expirations
## Minimum Supported Rust Versions
Moka's minimum supported Rust versions (MSRV) are the followings:
| Feature | MSRV |
|:---------|:----------------------------:|
| `future` | Rust 1.71.1 (August 3, 2023) |
| `sync` | Rust 1.71.1 (August 3, 2023) |
It will keep a rolling MSRV policy of at least 6 months. If the default features with
a mandatory features (`future` or `sync`) are enabled, MSRV will be updated
conservatively. When using other features, MSRV might be updated more frequently, up
to the latest stable.
In both cases, increasing MSRV is _not_ considered a semver-breaking change.
<!--
> Be conservative with MSRV, dependency versions and semver breaks
https://rust-team.pages.debian.net/book/upstream.html
https://salsa.debian.org/rust-team/debcargo-conf/-/tree/master/src/moka/debian
-->
## Troubleshooting
### Compile Errors on Some 32-bit Platforms
#### Symptoms
When using Moka v0.12.9 or earlier on some 32-bit platforms, you may get compile
errors:
```console
error[E0432]: unresolved import `std::sync::atomic::AtomicU64`
--> ... /moka-0.5.3/src/sync.rs:10:30
|
10 | atomic::{AtomicBool, AtomicU64, Ordering},
| ^^^^^^^^^
| |
| no `AtomicU64` in `sync::atomic`
```
or
```console
error[E0583]: file not found for module `atomic_time`
--> ... /moka-0.12.9/src/common/concurrent.rs:23:1
|
23 | pub(crate) mod atomic_time;
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
```
#### How to Fix
You can fix these compilation errors by one of the following:
1. (Recommended) Upgrade Moka to v0.12.10 or later. (`cargo update -p moka`)
2. Or, keep using Moka v0.12.9 or earlier but disable the default features in
`Cargo.toml`. (`default-features = false`)
- The default features include the `atomic64` feature, which need to be disabled.
These error messages are caused by the absence of `std::sync::atomic::AtomicU64` on
some 32-bit platforms. Moka v0.12.10 and later will automatically use a fallback
implementation when `AtomicU64` is not available. With v0.12.9 and earlier, you must
manually disable the `atomic64` feature to use the fallback implementation.
## Developing Moka
**Running All Tests**
To run all tests including `future` feature and doc tests on the README, use the
following command:
```console
$ RUSTFLAGS='--cfg trybuild' cargo test --all-features
```
**Running All Tests without Default Features**
```console
$ RUSTFLAGS='--cfg trybuild' cargo test \
--no-default-features --features 'future, sync'
```
**Generating the Doc**
```console
$ cargo +nightly -Z unstable-options --config 'build.rustdocflags="--cfg docsrs"' \
doc --no-deps --features 'future, sync'
```
## Roadmap
See the [project roadmap][gh-proj-1] for the updated and detailed plans.
But here are some highlights:
[gh-proj-1]: https://github.com/orgs/moka-rs/projects/1/views/1
- [x] Size-aware eviction. (`v0.7.0` via [#24][gh-pull-024])
- [x] API stabilization. (Smaller core API, shorter names for frequently used
methods) (`v0.8.0` via [#105][gh-pull-105])
- e.g.
- `get_or_insert_with(K, F)` → `get_with(K, F)`
- `get_or_try_insert_with(K, F)` → `try_get_with(K, F)`
- `time_to_live()` → `policy().time_to_live()`
- [x] Notifications on eviction. (`v0.9.0` via [#145][gh-pull-145])
- [x] Variable (per-entry) expiration, using hierarchical timer wheels.
(`v0.11.0` via [#248][gh-pull-248])
- [x] Remove background threads. (`v0.12.0` via [#294][gh-pull-294] and
[#316][gh-pull-316])
- [x] Add upsert and compute methods. (`v0.12.3` via [#370][gh-pull-370])
- [ ] Cache statistics (Hit rate, etc.). ([details][cache-stats])
- [ ] Upgrade TinyLFU to Window-TinyLFU. ([details][tiny-lfu])
- [ ] Restore cache from a snapshot. ([details][restore])
[gh-pull-024]: https://github.com/moka-rs/moka/pull/24
[gh-pull-105]: https://github.com/moka-rs/moka/pull/105
[gh-pull-145]: https://github.com/moka-rs/moka/pull/145
[gh-pull-248]: https://github.com/moka-rs/moka/pull/248
[gh-pull-294]: https://github.com/moka-rs/moka/pull/294
[gh-pull-316]: https://github.com/moka-rs/moka/pull/316
[gh-pull-370]: https://github.com/moka-rs/moka/pull/370
[cache-stats]: https://github.com/moka-rs/moka/issues/234
[restore]: https://github.com/moka-rs/moka/issues/314
## About the Name
Moka is named after the [moka pot][moka-pot-wikipedia], a stove-top coffee maker that
brews espresso-like coffee using boiling water pressurized by steam.
This name would imply the following facts and hopes:
- Moka is a part of the Java Caffeine cache family.
- It is written in Rust. (Many moka pots are made of aluminum alloy or stainless
steel. We know they don't rust though)
- It should be fast. ("Espresso" in Italian means express)
- It should be easy to use, like a moka pot.
[moka-pot-wikipedia]: https://en.wikipedia.org/wiki/Moka_pot
## Credits
### Caffeine
Moka's architecture is heavily inspired by the [Caffeine][caffeine-git] library for
Java. Thanks go to Ben Manes and all contributors of Caffeine.
### cht
The source files of the concurrent hash table under `moka::cht` module were copied
from the [cht crate v0.4.1][cht-v041] and modified by us. We did so for better
integration. cht v0.4.1 and earlier are licensed under the MIT license.
Thanks go to Gregory Meyer.
[cht-v041]: https://github.com/Gregory-Meyer/cht/tree/v0.4.1
## License
Moka is distributed under either of
- The MIT license
- The Apache License 2.0
at your option.
See [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) for details.
**Note on Licensing:**
Certain components, specifically
[`src/common/frequency_sketch.rs`](src/common/frequency_sketch.rs) and
[`src/common/timer_wheel.rs`](src/common/timer_wheel.rs), are distributed solely
under the Apache License 2.0. These files were ported from the [Caffeine][caffeine-git]
library and are not dual-licensed.
<!--
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fmoka-rs%2Fmoka.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fmoka-rs%2Fmoka?ref=badge_large)
-->
+82
View File
@@ -0,0 +1,82 @@
//! Lock-free hash tables.
//!
//! The hash tables in this crate are, at their core, open addressing hash
//! tables implemented using open addressing and boxed buckets. The core of
//! these hash tables are bucket arrays, which consist of a vector of atomic
//! pointers to buckets, an atomic pointer to the next bucket array, and an
//! epoch number. In the context of this crate, an atomic pointer is a nullable
//! pointer that is accessed and manipulated using atomic memory operations.
//! Each bucket consists of a key and a possibly-uninitialized value.
//!
//! The key insight into making the hash table resizable is to incrementally
//! copy buckets from the old bucket array to the new bucket array. As buckets
//! are copied between bucket arrays, their pointers in the old bucket array are
//! CAS'd with a null pointer that has a sentinel bit set. If the CAS fails,
//! that thread must read the bucket pointer again and retry copying it into the
//! new bucket array. If at any time a thread reads a bucket pointer with the
//! sentinel bit set, that thread knows that a new (larger) bucket array has
//! been allocated. That thread will then immediately attempt to copy all
//! buckets to the new bucket array. It is possible to implement an algorithm in
//! which a subset of buckets are relocated per-thread; such an algorithm has
//! not been implemented for the sake of simplicity.
//!
//! Bucket pointers that have been copied from an old bucket array into a new
//! bucket array are marked with a borrowed bit. If a thread copies a bucket
//! from an old bucket array into a new bucket array, fails to CAS the bucket
//! pointer in the old bucket array, it attempts to CAS the bucket pointer in
//! the new bucket array that it previously inserted to. If the bucket pointer
//! in the new bucket array does *not* have the borrowed tag bit set, that
//! thread knows that the value in the new bucket array was modified more
//! recently than the value in the old bucket array. To avoid discarding updates
//! to the new bucket array, a thread will never replace a bucket pointer that
//! has the borrowed tag bit set with one that does not. To see why this is
//! necessary, consider the case where a bucket pointer is copied into the new
//! array, removed from the new array by a second thread, then copied into the
//! new array again by a third thread.
//!
//! Mutating operations are, at their core, an atomic compare-and-swap (CAS) on
//! a bucket pointer. Insertions CAS null pointers and bucket pointers with
//! matching keys, modifications CAS bucket pointers with matching keys, and
//! removals CAS non-tombstone bucket pointers. Tombstone bucket pointers are
//! bucket pointers with a tombstone bit set as part of a removal; this
//! indicates that the bucket's value has been moved from and will be destroyed
//! if it has not been already.
//!
//! As previously mentioned, removing an entry from the hash table results in
//! that bucket pointer having a tombstone bit set. Insertions cannot
//! displace a tombstone bucket unless their key compares equal, so once an
//! entry is inserted into the hash table, the specific index it is assigned to
//! will only ever hold entries whose keys compare equal. Without this
//! restriction, resizing operations could result in the old and new bucket
//! arrays being temporarily inconsistent. Consider the case where one thread,
//! as part of a resizing operation, copies a bucket into a new bucket array
//! while another thread removes and replaces that bucket from the old bucket
//! array. If the new bucket has a non-matching key, what happens to the bucket
//! that was just copied into the new bucket array?
//!
//! Tombstone bucket pointers are typically not copied into new bucket arrays.
//! The exception is the case where a bucket pointer was copied to the new
//! bucket array, then CAS on the old bucket array fails because that bucket has
//! been replaced with a tombstone. In this case, the tombstone bucket pointer
//! will be copied over to reflect the update without displacing a key from its
//! bucket.
//!
//! This hash table algorithm was inspired by [a blog post by Jeff Phreshing]
//! that describes the implementation of the Linear hash table in [Junction], a
//! C++ library of concurrent data structures. Additional inspiration was drawn
//! from the lock-free hash table described by Cliff Click in [a tech talk] given
//! at Google in 2007.
//!
//! [a blog post by Jeff Phreshing]: https://preshing.com/20160222/a-resizable-concurrent-map/
//! [Junction]: https://github.com/preshing/junction
//! [a tech talk]: https://youtu.be/HJ-719EGIts
pub(crate) mod iter;
pub(crate) mod map;
pub(crate) mod segment;
#[cfg(test)]
#[macro_use]
pub(crate) mod test_util;
pub(crate) use segment::HashMap as SegmentedHashMap;
+89
View File
@@ -0,0 +1,89 @@
use std::hash::Hash;
pub(crate) trait ScanningGet<K, V>
where
K: Clone,
V: Clone,
{
/// Returns a _clone_ of the value corresponding to the key.
fn scanning_get(&self, key: &K) -> Option<V>;
/// Returns a vec of keys in a specified segment of the concurrent hash table.
fn keys(&self, cht_segment: usize) -> Option<Vec<K>>;
}
pub(crate) struct Iter<'i, K, V> {
keys: Option<Vec<K>>,
map: &'i dyn ScanningGet<K, V>,
num_segments: usize,
seg_index: usize,
is_done: bool,
}
impl<'i, K, V> Iter<'i, K, V> {
pub(crate) fn with_single_cache_segment(
map: &'i dyn ScanningGet<K, V>,
num_segments: usize,
) -> Self {
Self {
keys: None,
map,
num_segments,
seg_index: 0,
is_done: false,
}
}
}
impl<K, V> Iterator for Iter<'_, K, V>
where
K: Eq + Hash + Clone + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
type Item = (K, V);
fn next(&mut self) -> Option<Self::Item> {
if self.is_done {
return None;
}
while let Some(key) = self.next_key() {
if let Some(v) = self.map.scanning_get(&key) {
return Some((key, v));
}
}
self.is_done = true;
None
}
}
impl<K, V> Iter<'_, K, V>
where
K: Eq + Hash + Clone + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
fn next_key(&mut self) -> Option<K> {
while let Some(keys) = self.current_keys() {
if let key @ Some(_) = keys.pop() {
return key;
}
}
None
}
fn current_keys(&mut self) -> Option<&mut Vec<K>> {
// If keys is none or some but empty, try to get next keys.
while self.keys.as_ref().map_or(true, Vec::is_empty) {
// Adjust indices.
if self.seg_index >= self.num_segments {
return None;
}
self.keys = self.map.keys(self.seg_index);
self.seg_index += 1;
}
self.keys.as_mut()
}
}
+10
View File
@@ -0,0 +1,10 @@
//! A lock-free hash map implemented with bucket pointer arrays, open addressing,
//! and linear probing.
pub(crate) mod bucket;
pub(crate) mod bucket_array_ref;
use std::collections::hash_map::RandomState;
/// Default hasher for `HashMap`.
pub type DefaultHashBuilder = RandomState;
+916
View File
@@ -0,0 +1,916 @@
use std::{
hash::{BuildHasher, Hash},
mem::{self, MaybeUninit},
ptr,
sync::{
atomic::{self, AtomicUsize, Ordering},
Arc, Mutex, TryLockError,
},
};
#[cfg(feature = "unstable-debug-counters")]
use crate::common::concurrent::debug_counters;
use crossbeam_epoch::{Atomic, CompareExchangeError, Guard, Owned, Shared};
pub(crate) const BUCKET_ARRAY_DEFAULT_LENGTH: usize = 128;
pub(crate) struct BucketArray<K, V> {
pub(crate) buckets: Box<[Atomic<Bucket<K, V>>]>,
pub(crate) next: Atomic<BucketArray<K, V>>,
pub(crate) epoch: usize,
pub(crate) rehash_lock: Arc<Mutex<()>>,
pub(crate) tombstone_count: AtomicUsize,
}
impl<K, V> Default for BucketArray<K, V> {
fn default() -> Self {
Self::with_length(0, BUCKET_ARRAY_DEFAULT_LENGTH)
}
}
impl<K, V> BucketArray<K, V> {
pub(crate) fn with_length(epoch: usize, length: usize) -> Self {
assert!(length.is_power_of_two());
let mut buckets = Vec::with_capacity(length);
unsafe {
ptr::write_bytes(buckets.as_mut_ptr(), 0, length);
buckets.set_len(length);
}
let buckets = buckets.into_boxed_slice();
#[cfg(feature = "unstable-debug-counters")]
{
use debug_counters::InternalGlobalDebugCounters as Counters;
let size = (buckets.len() * std::mem::size_of::<Atomic<Bucket<K, V>>>()) as u64;
Counters::bucket_array_created(size);
}
Self {
buckets,
next: Atomic::null(),
epoch,
rehash_lock: Arc::new(Mutex::new(())),
tombstone_count: AtomicUsize::default(),
}
}
pub(crate) fn capacity(&self) -> usize {
assert!(self.buckets.len().is_power_of_two());
self.buckets.len() / 2
}
}
#[cfg(feature = "unstable-debug-counters")]
impl<K, V> Drop for BucketArray<K, V> {
fn drop(&mut self) {
use debug_counters::InternalGlobalDebugCounters as Counters;
let size = (self.buckets.len() * std::mem::size_of::<Atomic<Bucket<K, V>>>()) as u64;
Counters::bucket_array_dropped(size);
}
}
impl<'g, K: 'g + Eq, V: 'g> BucketArray<K, V> {
pub(crate) fn get(
&self,
guard: &'g Guard,
hash: u64,
mut eq: impl FnMut(&K) -> bool,
) -> Result<Shared<'g, Bucket<K, V>>, RelocatedError> {
for bucket in self.probe(guard, hash) {
let Ok((_, _, this_bucket_ptr)) = bucket else {
return Err(RelocatedError);
};
let Some(this_bucket_ref) = (unsafe { this_bucket_ptr.as_ref() }) else {
// Not found.
return Ok(Shared::null());
};
if !eq(&this_bucket_ref.key) {
// Different key. Try next bucket
continue;
}
if is_tombstone(this_bucket_ptr) {
// Not found. (It has been removed)
return Ok(Shared::null());
} else {
// Found.
return Ok(this_bucket_ptr);
}
}
Ok(Shared::null())
}
pub(crate) fn remove_if<F>(
&self,
guard: &'g Guard,
hash: u64,
mut eq: impl FnMut(&K) -> bool,
mut condition: F,
) -> Result<Shared<'g, Bucket<K, V>>, F>
where
F: FnMut(&K, &V) -> bool,
{
let mut probe = self.probe(guard, hash);
while let Some(bucket) = probe.next() {
let Ok((_, this_bucket, this_bucket_ptr)) = bucket else {
return Err(condition);
};
let Some(this_bucket_ref) = (unsafe { this_bucket_ptr.as_ref() }) else {
// Nothing to remove.
return Ok(Shared::null());
};
let this_key = &this_bucket_ref.key;
if !eq(this_key) {
// Different key. Try next bucket.
continue;
}
if is_tombstone(this_bucket_ptr) {
// Already removed.
return Ok(Shared::null());
}
let this_value = unsafe { &*this_bucket_ref.maybe_value.as_ptr() };
if !condition(this_key, this_value) {
// Found but the condition is false. Do not remove.
return Ok(Shared::null());
}
// Found and the condition is true. Remove it. (Make it a tombstone)
let new_bucket_ptr = this_bucket_ptr.with_tag(TOMBSTONE_TAG);
match this_bucket.compare_exchange_weak(
this_bucket_ptr,
new_bucket_ptr,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
// Succeeded. Return the removed value. (can be null)
Ok(_) => return Ok(new_bucket_ptr),
// Failed. Reload to retry.
Err(_) => probe.reload(),
}
}
Ok(Shared::null())
}
pub(crate) fn insert_if_not_present<F>(
&self,
guard: &'g Guard,
hash: u64,
mut state: InsertOrModifyState<K, V, F>,
) -> Result<InsertionResult<'g, K, V>, InsertOrModifyState<K, V, F>>
where
F: FnOnce() -> V,
{
let mut probe = self.probe(guard, hash);
while let Some(Ok((_, this_bucket, this_bucket_ptr))) = probe.next() {
if let Some(this_bucket_ref) = unsafe { this_bucket_ptr.as_ref() } {
if &this_bucket_ref.key != state.key() {
// Different key. Try next bucket.
continue;
}
if !is_tombstone(this_bucket_ptr) {
// Found. Return it.
return Ok(InsertionResult::AlreadyPresent(this_bucket_ptr));
}
}
// Not found or found a tombstone. Insert it.
let new_bucket = state.into_insert_bucket();
if let Err(CompareExchangeError { new, .. }) = this_bucket.compare_exchange_weak(
this_bucket_ptr,
new_bucket,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
state = InsertOrModifyState::from_bucket_value(new, None);
probe.reload();
} else if unsafe { this_bucket_ptr.as_ref() }.is_some() {
// Inserted by replacing a tombstone.
return Ok(InsertionResult::ReplacedTombstone(this_bucket_ptr));
} else {
// Inserted.
return Ok(InsertionResult::Inserted);
}
}
Err(state)
}
// https://rust-lang.github.io/rust-clippy/master/index.html#type_complexity
#[allow(clippy::type_complexity)]
pub(crate) fn insert_or_modify<F, G>(
&self,
guard: &'g Guard,
hash: u64,
mut state: InsertOrModifyState<K, V, F>,
mut modifier: G,
) -> Result<Shared<'g, Bucket<K, V>>, (InsertOrModifyState<K, V, F>, G)>
where
F: FnOnce() -> V,
G: FnMut(&K, &V) -> V,
{
let mut probe = self.probe(guard, hash);
while let Some(bucket) = probe.next() {
let Ok((_, this_bucket, this_bucket_ptr)) = bucket else {
return Err((state, modifier));
};
let (new_bucket, maybe_insert_value) =
if let Some(this_bucket_ref) = unsafe { this_bucket_ptr.as_ref() } {
let this_key = &this_bucket_ref.key;
if this_key != state.key() {
// Different key. Try next bucket.
continue;
}
if is_tombstone(this_bucket_ptr) {
// Found a tombstone for this key. Replace it.
(state.into_insert_bucket(), None)
} else {
// Found. Modify it.
let this_value = unsafe { &*this_bucket_ref.maybe_value.as_ptr() };
let new_value = modifier(this_key, this_value);
let (new_bucket, insert_value) = state.into_modify_bucket(new_value);
(new_bucket, Some(insert_value))
}
} else {
// Not found. Insert it.
(state.into_insert_bucket(), None)
};
if let Err(CompareExchangeError { new, .. }) = this_bucket.compare_exchange_weak(
this_bucket_ptr,
new_bucket,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
// Failed. Reload to retry.
state = InsertOrModifyState::from_bucket_value(new, maybe_insert_value);
probe.reload();
} else {
// Succeeded. Return the previous value. (can be null)
return Ok(this_bucket_ptr);
}
}
Err((state, modifier))
}
fn insert_for_grow(
&self,
guard: &'g Guard,
hash: u64,
bucket_ptr: Shared<'g, Bucket<K, V>>,
) -> Option<usize> {
assert!(!bucket_ptr.is_null());
assert!(!is_sentinel(bucket_ptr));
assert!(is_borrowed(bucket_ptr));
let key = &unsafe { bucket_ptr.deref() }.key;
let mut probe = self.probe(guard, hash);
while let Some(bucket) = probe.next() {
let Ok((i, this_bucket, this_bucket_ptr)) = bucket else {
return None;
};
if let Some(Bucket { key: this_key, .. }) = unsafe { this_bucket_ptr.as_ref() } {
if this_bucket_ptr == bucket_ptr {
return None;
} else if this_key != key {
continue;
} else if !is_borrowed(this_bucket_ptr) {
return None;
}
}
if this_bucket_ptr.is_null() && is_tombstone(bucket_ptr) {
return None;
} else if this_bucket
.compare_exchange_weak(
this_bucket_ptr,
bucket_ptr,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
)
.is_ok()
{
return Some(i);
} else {
probe.reload();
}
}
None
}
pub(crate) fn keys<F, T>(
&self,
guard: &'g Guard,
with_key: &mut F,
) -> Result<Vec<T>, RelocatedError>
where
F: FnMut(&K) -> T,
{
let mut keys = Vec::new();
for bucket in self.buckets.iter() {
let bucket_ptr = bucket.load_consume(guard);
if is_sentinel(bucket_ptr) {
return Err(RelocatedError);
}
if let Some(bucket_ref) = unsafe { bucket_ptr.as_ref() } {
if !is_tombstone(bucket_ptr) {
keys.push(with_key(&bucket_ref.key));
}
}
}
Ok(keys)
}
}
struct Probe<'b, 'g, K: 'g, V: 'g> {
buckets: &'b [Atomic<Bucket<K, V>>],
guard: &'g Guard,
this_bucket: (usize, &'b Atomic<Bucket<K, V>>),
offset: usize,
i: usize,
reload: bool,
}
impl<'g, K: 'g, V: 'g> Probe<'_, 'g, K, V> {
fn reload(&mut self) {
self.reload = true;
}
}
impl<'b, 'g, K: 'g, V: 'g> Iterator for Probe<'b, 'g, K, V> {
type Item = Result<(usize, &'b Atomic<Bucket<K, V>>, Shared<'g, Bucket<K, V>>), ()>;
fn next(&mut self) -> Option<Self::Item> {
if !self.reload {
let max = self.buckets.len() - 1;
if self.i >= max {
return None;
}
self.i += 1;
let i = self.i.wrapping_add(self.offset) & max;
self.this_bucket = (i, &self.buckets[i]);
}
self.reload = false;
let this_bucket_ptr = self.this_bucket.1.load_consume(self.guard);
if is_sentinel(this_bucket_ptr) {
return Some(Err(()));
}
let val = (self.this_bucket.0, self.this_bucket.1, this_bucket_ptr);
Some(Ok(val))
}
}
impl<'g, K: 'g, V: 'g> BucketArray<K, V> {
fn probe(&self, guard: &'g Guard, hash: u64) -> Probe<'_, 'g, K, V> {
let buckets = &self.buckets;
let offset = hash as usize & (buckets.len() - 1);
// SAFETY: `len()` is never be 0 so this index access will never panic.
// This invariant is ensured by the `assert!()` at the beginning of
// `with_length()` because 0 is not a power of two.
let this_bucket = (offset, &buckets[offset]);
Probe {
buckets,
guard,
this_bucket,
offset,
i: 0,
reload: true,
}
}
pub(crate) fn rehash<H>(
&self,
guard: &'g Guard,
build_hasher: &H,
rehash_op: RehashOp,
) -> Option<&'g BucketArray<K, V>>
where
K: Hash + Eq,
H: BuildHasher,
{
// Ensure that the rehashing is not performed concurrently.
let lock = match self.rehash_lock.try_lock() {
Ok(lk) => lk,
Err(TryLockError::WouldBlock) => {
// Wait until the lock become available.
std::mem::drop(self.rehash_lock.lock());
// We need to return here to see if rehashing is still needed.
return None;
}
Err(e @ TryLockError::Poisoned(_)) => panic!("{e:?}"),
};
let next_array = self.next_array(guard, rehash_op);
for this_bucket in self.buckets.iter() {
let mut maybe_state: Option<(usize, Shared<'g, Bucket<K, V>>)> = None;
loop {
let this_bucket_ptr = this_bucket.load_consume(guard);
if is_sentinel(this_bucket_ptr) {
break;
}
let to_put_ptr = this_bucket_ptr.with_tag(this_bucket_ptr.tag() | BORROWED_TAG);
if let Some((index, mut next_bucket_ptr)) = maybe_state {
assert!(!this_bucket_ptr.is_null());
let next_bucket = &next_array.buckets[index];
while is_borrowed(next_bucket_ptr)
&& next_bucket
.compare_exchange_weak(
next_bucket_ptr,
to_put_ptr,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
)
.is_err()
{
next_bucket_ptr = next_bucket.load_consume(guard);
}
} else if let Some(this_bucket_ref) = unsafe { this_bucket_ptr.as_ref() } {
let key = &this_bucket_ref.key;
let hash = hash(build_hasher, key);
if let Some(index) = next_array.insert_for_grow(guard, hash, to_put_ptr) {
maybe_state = Some((index, to_put_ptr));
}
}
if this_bucket
.compare_exchange_weak(
this_bucket_ptr,
Shared::null().with_tag(SENTINEL_TAG),
Ordering::AcqRel,
Ordering::Relaxed,
guard,
)
.is_ok()
{
// TODO: If else, we may need to count tombstone.
if !this_bucket_ptr.is_null()
&& is_tombstone(this_bucket_ptr)
&& maybe_state.is_none()
{
unsafe { defer_destroy_bucket(guard, this_bucket_ptr) };
}
break;
}
}
}
guard.flush();
std::mem::drop(lock);
Some(next_array)
}
fn next_array(&self, guard: &'g Guard, rehash_op: RehashOp) -> &'g BucketArray<K, V> {
let mut maybe_new_next = None;
loop {
let next_ptr = self.next.load_consume(guard);
if let Some(next_ref) = unsafe { next_ptr.as_ref() } {
return next_ref;
}
let new_length = rehash_op.new_len(self.buckets.len());
let new_next = maybe_new_next.unwrap_or_else(|| {
Owned::new(BucketArray::with_length(self.epoch + 1, new_length))
});
match self.next.compare_exchange_weak(
Shared::null(),
new_next,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
Ok(p) => return unsafe { p.deref() },
Err(CompareExchangeError { new, .. }) => {
maybe_new_next = Some(new);
}
}
}
}
}
#[repr(align(8))]
#[derive(Debug)]
pub(crate) struct Bucket<K, V> {
pub(crate) key: K,
pub(crate) maybe_value: MaybeUninit<V>,
}
impl<K, V> Bucket<K, V> {
pub(crate) fn new(key: K, value: V) -> Bucket<K, V> {
#[cfg(feature = "unstable-debug-counters")]
debug_counters::InternalGlobalDebugCounters::bucket_created();
Self {
key,
maybe_value: MaybeUninit::new(value),
}
}
}
#[cfg(feature = "unstable-debug-counters")]
impl<K, V> Drop for Bucket<K, V> {
fn drop(&mut self) {
debug_counters::InternalGlobalDebugCounters::bucket_dropped();
}
}
#[derive(Debug, Eq, PartialEq)]
pub(crate) struct RelocatedError;
pub(crate) enum InsertOrModifyState<K, V, F: FnOnce() -> V> {
New(K, F),
AttemptedInsertion(Owned<Bucket<K, V>>),
AttemptedModification(Owned<Bucket<K, V>>, ValueOrFunction<V, F>),
}
impl<K, V, F: FnOnce() -> V> InsertOrModifyState<K, V, F> {
fn from_bucket_value(
bucket: Owned<Bucket<K, V>>,
value_or_function: Option<ValueOrFunction<V, F>>,
) -> Self {
if let Some(value_or_function) = value_or_function {
Self::AttemptedModification(bucket, value_or_function)
} else {
Self::AttemptedInsertion(bucket)
}
}
fn key(&self) -> &K {
match self {
InsertOrModifyState::New(k, _) => k,
InsertOrModifyState::AttemptedInsertion(b)
| InsertOrModifyState::AttemptedModification(b, _) => &b.key,
}
}
fn into_insert_bucket(self) -> Owned<Bucket<K, V>> {
match self {
InsertOrModifyState::New(k, f) => Owned::new(Bucket::new(k, f())),
InsertOrModifyState::AttemptedInsertion(b) => b,
InsertOrModifyState::AttemptedModification(mut b, v_or_f) => {
unsafe {
mem::drop(
mem::replace(&mut b.maybe_value, MaybeUninit::new(v_or_f.into_value()))
.assume_init(),
);
};
b
}
}
}
fn into_modify_bucket(self, value: V) -> (Owned<Bucket<K, V>>, ValueOrFunction<V, F>) {
match self {
InsertOrModifyState::New(k, f) => (
Owned::new(Bucket::new(k, value)),
ValueOrFunction::Function(f),
),
InsertOrModifyState::AttemptedInsertion(mut b) => {
let insert_value = unsafe {
mem::replace(&mut b.maybe_value, MaybeUninit::new(value)).assume_init()
};
(b, ValueOrFunction::Value(insert_value))
}
InsertOrModifyState::AttemptedModification(mut b, v_or_f) => {
unsafe {
mem::drop(
mem::replace(&mut b.maybe_value, MaybeUninit::new(value)).assume_init(),
);
}
(b, v_or_f)
}
}
}
}
pub(crate) enum ValueOrFunction<V, F: FnOnce() -> V> {
Value(V),
Function(F),
}
impl<V, F: FnOnce() -> V> ValueOrFunction<V, F> {
fn into_value(self) -> V {
match self {
ValueOrFunction::Value(v) => v,
ValueOrFunction::Function(f) => f(),
}
}
}
pub(crate) fn hash<K, H>(build_hasher: &H, key: &K) -> u64
where
K: ?Sized + Hash,
H: BuildHasher,
{
build_hasher.hash_one(key)
}
pub(crate) enum InsertionResult<'g, K, V> {
AlreadyPresent(Shared<'g, Bucket<K, V>>),
Inserted,
ReplacedTombstone(Shared<'g, Bucket<K, V>>),
}
pub(crate) unsafe fn defer_destroy_bucket<'g, K, V>(
guard: &'g Guard,
mut ptr: Shared<'g, Bucket<K, V>>,
) {
assert!(!ptr.is_null());
guard.defer_unchecked(move || {
atomic::fence(Ordering::Acquire);
if !is_tombstone(ptr) {
ptr::drop_in_place(ptr.deref_mut().maybe_value.as_mut_ptr());
}
mem::drop(ptr.into_owned());
});
}
pub(crate) unsafe fn defer_destroy_tombstone<'g, K, V>(
guard: &'g Guard,
mut ptr: Shared<'g, Bucket<K, V>>,
) {
assert!(!ptr.is_null());
assert!(is_tombstone(ptr));
atomic::fence(Ordering::Acquire);
// read the value now, but defer its destruction for later
let value = ptr::read(ptr.deref_mut().maybe_value.as_ptr());
// to be entirely honest, i don't know what order deferred functions are
// called in crossbeam-epoch. in the case that the deferred functions are
// called out of order, this prevents that from being an issue.
guard.defer_unchecked(move || mem::drop(value));
}
pub(crate) unsafe fn defer_acquire_destroy<'g, T>(guard: &'g Guard, ptr: Shared<'g, T>) {
assert!(!ptr.is_null());
guard.defer_unchecked(move || {
atomic::fence(Ordering::Acquire);
mem::drop(ptr.into_owned());
});
}
#[derive(Clone, Copy)]
pub(crate) enum RehashOp {
Expand,
Shrink,
GcOnly,
Skip,
}
impl RehashOp {
pub(crate) fn new(cap: usize, tombstone_count: &AtomicUsize, len: &AtomicUsize) -> Self {
let real_cap = cap as f64 * 2.0;
let quarter_cap = real_cap / 4.0;
let tbc = tombstone_count.load(Ordering::Relaxed) as f64;
let len = len.load(Ordering::Relaxed) as f64;
if tbc >= 25_000.0 || tbc / real_cap >= 0.1 {
if len - tbc < quarter_cap && quarter_cap as usize >= BUCKET_ARRAY_DEFAULT_LENGTH {
return Self::Shrink;
} else {
return Self::GcOnly;
}
}
if len > real_cap * 0.7 {
return Self::Expand;
}
Self::Skip
}
pub(crate) fn is_skip(self) -> bool {
matches!(self, Self::Skip)
}
fn new_len(self, current_len: usize) -> usize {
match self {
Self::Expand => current_len * 2,
Self::Shrink => current_len / 2,
Self::GcOnly => current_len,
Self::Skip => unreachable!(),
}
}
}
pub(crate) const SENTINEL_TAG: usize = 0b001; // set on old table buckets when copied into a new table
pub(crate) const TOMBSTONE_TAG: usize = 0b010; // set when the value has been destroyed
pub(crate) const BORROWED_TAG: usize = 0b100; // set on new table buckets when copied from an old table
#[inline]
pub(crate) fn is_sentinel<K, V>(bucket_ptr: Shared<'_, Bucket<K, V>>) -> bool {
bucket_ptr.tag() & SENTINEL_TAG != 0
}
#[inline]
pub(crate) fn is_tombstone<K, V>(bucket_ptr: Shared<'_, Bucket<K, V>>) -> bool {
bucket_ptr.tag() & TOMBSTONE_TAG != 0
}
#[inline]
pub(crate) fn is_borrowed<K, V>(bucket_ptr: Shared<'_, Bucket<K, V>>) -> bool {
bucket_ptr.tag() & BORROWED_TAG != 0
}
#[cfg(test)]
mod tests {
use super::{
defer_destroy_bucket, defer_destroy_tombstone, hash, is_tombstone, Bucket, BucketArray,
InsertOrModifyState, InsertionResult, RelocatedError,
};
use crossbeam_epoch::{Guard, Shared};
use std::{collections::hash_map::RandomState, sync::atomic::Ordering};
#[test]
fn get_insert_remove() {
let build_hasher = RandomState::new();
let buckets = BucketArray::with_length(0, 16);
let guard = unsafe { crossbeam_epoch::unprotected() };
let k1 = "foo";
let h1 = hash(&build_hasher, k1);
let v1 = 5;
let k2 = "bar";
let h2 = hash(&build_hasher, k2);
let v2 = 10;
let k3 = "baz";
let h3 = hash(&build_hasher, k3);
let v3 = 15;
assert_eq!(buckets.get(guard, h1, |&k| k == k1), Ok(Shared::null()));
assert_eq!(buckets.get(guard, h2, |&k| k == k2), Ok(Shared::null()));
assert_eq!(buckets.get(guard, h3, |&k| k == k3), Ok(Shared::null()));
assert!(matches!(
insert(&buckets, guard, k1, h1, || v1),
Ok(InsertionResult::Inserted)
));
assert_eq!(
into_value(buckets.get(guard, h1, |&k| k == k1)),
Ok(Some(v1))
);
assert_eq!(buckets.get(guard, h2, |&k| k == k2), Ok(Shared::null()));
assert_eq!(buckets.get(guard, h3, |&k| k == k3), Ok(Shared::null()));
assert!(matches!(
insert(&buckets, guard, k2, h2, || v2),
Ok(InsertionResult::Inserted)
));
assert_eq!(
into_value(buckets.get(guard, h1, |&k| k == k1)),
Ok(Some(v1))
);
assert_eq!(
into_value(buckets.get(guard, h2, |&k| k == k2)),
Ok(Some(v2))
);
assert_eq!(buckets.get(guard, h3, |&k| k == k3), Ok(Shared::null()));
assert!(matches!(
insert(&buckets, guard, k3, h3, || v3),
Ok(InsertionResult::Inserted)
));
assert_eq!(
into_value(buckets.get(guard, h1, |&k| k == k1)),
Ok(Some(v1))
);
assert_eq!(
into_value(buckets.get(guard, h2, |&k| k == k2)),
Ok(Some(v2))
);
assert_eq!(
into_value(buckets.get(guard, h3, |&k| k == k3)),
Ok(Some(v3))
);
let b1 = buckets
.remove_if(guard, h1, |&k| k == k1, |_, _| true)
.ok()
.unwrap();
assert!(is_tombstone(b1));
unsafe { defer_destroy_tombstone(guard, b1) };
let b2 = buckets
.remove_if(guard, h2, |&k| k == k2, |_, _| true)
.ok()
.unwrap();
assert!(is_tombstone(b2));
unsafe { defer_destroy_tombstone(guard, b2) };
let b3 = buckets
.remove_if(guard, h3, |&k| k == k3, |_, _| true)
.ok()
.unwrap();
assert!(is_tombstone(b3));
unsafe { defer_destroy_tombstone(guard, b3) };
assert_eq!(buckets.get(guard, h1, |&k| k == k1), Ok(Shared::null()));
assert_eq!(buckets.get(guard, h2, |&k| k == k2), Ok(Shared::null()));
assert_eq!(buckets.get(guard, h3, |&k| k == k3), Ok(Shared::null()));
for this_bucket in buckets.buckets.iter() {
let this_bucket_ptr = this_bucket.swap(Shared::null(), Ordering::Relaxed, guard);
if this_bucket_ptr.is_null() {
continue;
}
unsafe {
defer_destroy_bucket(guard, this_bucket_ptr);
}
}
}
fn insert<'g, K, V, F>(
buckets: &BucketArray<K, V>,
guard: &'g Guard,
key: K,
hash: u64,
value_init: F,
) -> Result<InsertionResult<'g, K, V>, InsertOrModifyState<K, V, F>>
where
K: Eq,
F: FnOnce() -> V,
{
let state = InsertOrModifyState::New(key, value_init);
buckets.insert_if_not_present(guard, hash, state)
}
fn into_value<K, V>(
maybe_bucket_ptr: Result<Shared<'_, Bucket<K, V>>, RelocatedError>,
) -> Result<Option<V>, RelocatedError>
where
V: Clone,
{
maybe_bucket_ptr
.map(|p| unsafe { p.as_ref() }.map(|b| unsafe { &*b.maybe_value.as_ptr() }.clone()))
}
}
@@ -0,0 +1,357 @@
use super::bucket::{self, Bucket, BucketArray, InsertOrModifyState, RehashOp};
use std::{
hash::{BuildHasher, Hash},
sync::atomic::{AtomicUsize, Ordering},
};
use crossbeam_epoch::{Atomic, CompareExchangeError, Guard, Owned, Shared};
pub(crate) struct BucketArrayRef<'a, K, V, S> {
pub(crate) bucket_array: &'a Atomic<BucketArray<K, V>>,
pub(crate) build_hasher: &'a S,
pub(crate) len: &'a AtomicUsize,
}
impl<K, V, S> BucketArrayRef<'_, K, V, S>
where
K: Hash + Eq,
S: BuildHasher,
{
pub(crate) fn get_key_value_and_then<T>(
&self,
hash: u64,
mut eq: impl FnMut(&K) -> bool,
with_entry: impl FnOnce(&K, &V) -> Option<T>,
) -> Option<T> {
let guard = &crossbeam_epoch::pin();
let current_ref = self.get(guard);
let mut bucket_array_ref = current_ref;
let result;
loop {
match bucket_array_ref
.get(guard, hash, &mut eq)
.map(|p| unsafe { p.as_ref() })
{
Ok(Some(Bucket {
key,
maybe_value: value,
})) => {
result = with_entry(key, unsafe { &*value.as_ptr() });
break;
}
Ok(None) => {
result = None;
break;
}
Err(_) => {
if let Some(r) =
bucket_array_ref.rehash(guard, self.build_hasher, RehashOp::Expand)
{
bucket_array_ref = r;
}
}
}
}
self.swing(guard, current_ref, bucket_array_ref);
result
}
pub(crate) fn remove_entry_if_and<T>(
&self,
hash: u64,
mut eq: impl FnMut(&K) -> bool,
mut condition: impl FnMut(&K, &V) -> bool,
with_previous_entry: impl FnOnce(&K, &V) -> T,
) -> Option<T> {
let guard = &crossbeam_epoch::pin();
let current_ref = self.get(guard);
let mut bucket_array_ref = current_ref;
let result;
loop {
loop {
let rehash_op = RehashOp::new(
bucket_array_ref.capacity(),
&bucket_array_ref.tombstone_count,
self.len,
);
if rehash_op.is_skip() {
break;
}
if let Some(r) = bucket_array_ref.rehash(guard, self.build_hasher, rehash_op) {
bucket_array_ref = r;
}
}
match bucket_array_ref.remove_if(guard, hash, &mut eq, condition) {
Ok(previous_bucket_ptr) => {
if let Some(previous_bucket_ref) = unsafe { previous_bucket_ptr.as_ref() } {
let Bucket {
key,
maybe_value: value,
} = previous_bucket_ref;
self.len.fetch_sub(1, Ordering::Relaxed);
bucket_array_ref
.tombstone_count
.fetch_add(1, Ordering::Relaxed);
result = Some(with_previous_entry(key, unsafe { &*value.as_ptr() }));
unsafe { bucket::defer_destroy_tombstone(guard, previous_bucket_ptr) };
} else {
result = None;
}
break;
}
Err(c) => {
condition = c;
if let Some(r) =
bucket_array_ref.rehash(guard, self.build_hasher, RehashOp::Expand)
{
bucket_array_ref = r;
}
}
}
}
self.swing(guard, current_ref, bucket_array_ref);
result
}
pub(crate) fn insert_if_not_present_and<T>(
&self,
key: K,
hash: u64,
on_insert: impl FnOnce() -> V,
with_existing_entry: impl FnOnce(&K, &V) -> T,
) -> Option<T> {
use bucket::InsertionResult;
let guard = &crossbeam_epoch::pin();
let current_ref = self.get(guard);
let mut bucket_array_ref = current_ref;
let mut state = InsertOrModifyState::New(key, on_insert);
let result;
loop {
loop {
let rehash_op = RehashOp::new(
bucket_array_ref.capacity(),
&bucket_array_ref.tombstone_count,
self.len,
);
if rehash_op.is_skip() {
break;
}
if let Some(r) = bucket_array_ref.rehash(guard, self.build_hasher, rehash_op) {
bucket_array_ref = r;
}
}
match bucket_array_ref.insert_if_not_present(guard, hash, state) {
Ok(InsertionResult::AlreadyPresent(current_bucket_ptr)) => {
let current_bucket_ref = unsafe { current_bucket_ptr.as_ref() }.unwrap();
assert!(!bucket::is_tombstone(current_bucket_ptr));
let Bucket {
key,
maybe_value: value,
} = current_bucket_ref;
result = Some(with_existing_entry(key, unsafe { &*value.as_ptr() }));
break;
}
Ok(InsertionResult::Inserted) => {
self.len.fetch_add(1, Ordering::Relaxed);
result = None;
break;
}
Ok(InsertionResult::ReplacedTombstone(previous_bucket_ptr)) => {
assert!(bucket::is_tombstone(previous_bucket_ptr));
self.len.fetch_add(1, Ordering::Relaxed);
unsafe { bucket::defer_destroy_bucket(guard, previous_bucket_ptr) };
result = None;
break;
}
Err(s) => {
state = s;
if let Some(r) =
bucket_array_ref.rehash(guard, self.build_hasher, RehashOp::Expand)
{
bucket_array_ref = r;
}
}
}
}
self.swing(guard, current_ref, bucket_array_ref);
result
}
pub(crate) fn insert_with_or_modify_entry_and<T>(
&self,
key: K,
hash: u64,
on_insert: impl FnOnce() -> V,
mut on_modify: impl FnMut(&K, &V) -> V,
with_old_entry: impl FnOnce(&K, &V) -> T,
) -> Option<T> {
let guard = &crossbeam_epoch::pin();
let current_ref = self.get(guard);
let mut bucket_array_ref = current_ref;
let mut state = InsertOrModifyState::New(key, on_insert);
let result;
loop {
loop {
let rehash_op = RehashOp::new(
bucket_array_ref.capacity(),
&bucket_array_ref.tombstone_count,
self.len,
);
if rehash_op.is_skip() {
break;
}
if let Some(r) = bucket_array_ref.rehash(guard, self.build_hasher, rehash_op) {
bucket_array_ref = r;
}
}
match bucket_array_ref.insert_or_modify(guard, hash, state, on_modify) {
Ok(previous_bucket_ptr) => {
if let Some(previous_bucket_ref) = unsafe { previous_bucket_ptr.as_ref() } {
if bucket::is_tombstone(previous_bucket_ptr) {
self.len.fetch_add(1, Ordering::Relaxed);
result = None;
} else {
let Bucket {
key,
maybe_value: value,
} = previous_bucket_ref;
result = Some(with_old_entry(key, unsafe { &*value.as_ptr() }));
}
unsafe { bucket::defer_destroy_bucket(guard, previous_bucket_ptr) };
} else {
self.len.fetch_add(1, Ordering::Relaxed);
result = None;
}
break;
}
Err((s, f)) => {
state = s;
on_modify = f;
if let Some(r) =
bucket_array_ref.rehash(guard, self.build_hasher, RehashOp::Expand)
{
bucket_array_ref = r;
}
}
}
}
self.swing(guard, current_ref, bucket_array_ref);
result
}
pub(crate) fn keys<T>(&self, mut with_key: impl FnMut(&K) -> T) -> Vec<T> {
let guard = &crossbeam_epoch::pin();
let current_ref = self.get(guard);
let mut bucket_array_ref = current_ref;
let result;
loop {
match bucket_array_ref.keys(guard, &mut with_key) {
Ok(keys) => {
result = keys;
break;
}
Err(_) => {
if let Some(r) =
bucket_array_ref.rehash(guard, self.build_hasher, RehashOp::Expand)
{
bucket_array_ref = r;
}
}
}
}
self.swing(guard, current_ref, bucket_array_ref);
result
}
}
impl<'g, K, V, S> BucketArrayRef<'_, K, V, S> {
fn get(&self, guard: &'g Guard) -> &'g BucketArray<K, V> {
let mut maybe_new_bucket_array = None;
loop {
let bucket_array_ptr = self.bucket_array.load_consume(guard);
if let Some(bucket_array_ref) = unsafe { bucket_array_ptr.as_ref() } {
return bucket_array_ref;
}
let new_bucket_array =
maybe_new_bucket_array.unwrap_or_else(|| Owned::new(BucketArray::default()));
match self.bucket_array.compare_exchange_weak(
Shared::null(),
new_bucket_array,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
Ok(b) => return unsafe { b.as_ref() }.unwrap(),
Err(CompareExchangeError { new, .. }) => maybe_new_bucket_array = Some(new),
}
}
}
fn swing(
&self,
guard: &'g Guard,
mut current_ref: &'g BucketArray<K, V>,
min_ref: &'g BucketArray<K, V>,
) {
let min_epoch = min_ref.epoch;
let mut current_ptr = (current_ref as *const BucketArray<K, V>).into();
let min_ptr: Shared<'g, _> = (min_ref as *const BucketArray<K, V>).into();
loop {
if current_ref.epoch >= min_epoch {
return;
}
match self.bucket_array.compare_exchange_weak(
current_ptr,
min_ptr,
Ordering::AcqRel,
Ordering::Relaxed,
guard,
) {
Ok(_) => unsafe { bucket::defer_acquire_destroy(guard, current_ptr) },
Err(_) => {
let new_ptr = self.bucket_array.load_consume(guard);
assert!(!new_ptr.is_null());
current_ptr = new_ptr;
current_ref = unsafe { new_ptr.as_ref() }.unwrap();
}
}
}
}
}
File diff suppressed because it is too large Load Diff
+114
View File
@@ -0,0 +1,114 @@
use std::{
borrow::{Borrow, BorrowMut},
hash::{Hash, Hasher},
ops::{Deref, DerefMut},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use crossbeam_epoch::Owned;
#[derive(Clone, Debug)]
pub(crate) struct NoisyDropper<T: ?Sized> {
parent: Arc<DropNotifier>,
pub elem: T,
}
impl<T> NoisyDropper<T> {
pub(crate) fn new(parent: Arc<DropNotifier>, elem: T) -> Self {
Self { parent, elem }
}
}
impl<T: ?Sized> Drop for NoisyDropper<T> {
fn drop(&mut self) {
assert!(!self.parent.dropped.swap(true, Ordering::Relaxed));
}
}
impl<T: ?Sized + PartialEq> PartialEq for NoisyDropper<T> {
fn eq(&self, other: &Self) -> bool {
self.elem == other.elem
}
}
impl<T: ?Sized + PartialEq> PartialEq<T> for NoisyDropper<T> {
fn eq(&self, other: &T) -> bool {
&self.elem == other
}
}
impl<T: ?Sized + Eq> Eq for NoisyDropper<T> {}
impl<T: ?Sized + Hash> Hash for NoisyDropper<T> {
fn hash<H: Hasher>(&self, hasher: &mut H) {
self.elem.hash(hasher);
}
}
impl<T: ?Sized> AsRef<T> for NoisyDropper<T> {
fn as_ref(&self) -> &T {
&self.elem
}
}
impl<T: ?Sized> AsMut<T> for NoisyDropper<T> {
fn as_mut(&mut self) -> &mut T {
&mut self.elem
}
}
impl<T: ?Sized> Borrow<T> for NoisyDropper<T> {
fn borrow(&self) -> &T {
&self.elem
}
}
impl<T: ?Sized> BorrowMut<T> for NoisyDropper<T> {
fn borrow_mut(&mut self) -> &mut T {
&mut self.elem
}
}
impl<T: ?Sized> Deref for NoisyDropper<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.elem
}
}
impl<T: ?Sized> DerefMut for NoisyDropper<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.elem
}
}
#[derive(Debug)]
pub(crate) struct DropNotifier {
dropped: AtomicBool,
}
impl DropNotifier {
pub(crate) fn new() -> Self {
Self {
dropped: AtomicBool::new(false),
}
}
pub(crate) fn was_dropped(&self) -> bool {
self.dropped.load(Ordering::Relaxed)
}
}
pub(crate) fn run_deferred() {
for _ in 0..65536 {
let guard = crossbeam_epoch::pin();
unsafe { guard.defer_destroy(Owned::new(0).into_shared(&guard)) };
guard.flush();
}
}
+125
View File
@@ -0,0 +1,125 @@
use std::time::Duration;
pub(crate) mod builder_utils;
pub(crate) mod concurrent;
pub(crate) mod deque;
pub(crate) mod entry;
pub(crate) mod error;
pub(crate) mod frequency_sketch;
pub(crate) mod iter;
pub(crate) mod time;
pub(crate) mod timer_wheel;
#[cfg(test)]
pub(crate) mod test_utils;
use self::concurrent::constants::{
DEFAULT_EVICTION_BATCH_SIZE, DEFAULT_MAINTENANCE_TASK_TIMEOUT_MILLIS,
DEFAULT_MAX_LOG_SYNC_REPEATS,
};
// Note: `CacheRegion` cannot have more than four enum variants. This is because
// `crate::{sync,unsync}::DeqNodes` uses a `tagptr::TagNonNull<DeqNode<T>, 2>`
// pointer, where the 2-bit tag is `CacheRegion`.
#[derive(Clone, Copy, Debug, Eq)]
pub(crate) enum CacheRegion {
Window = 0,
MainProbation = 1,
MainProtected = 2,
Other = 3,
}
impl From<usize> for CacheRegion {
fn from(n: usize) -> Self {
match n {
0 => Self::Window,
1 => Self::MainProbation,
2 => Self::MainProtected,
3 => Self::Other,
_ => panic!("No such CacheRegion variant for {n}"),
}
}
}
impl CacheRegion {
pub(crate) fn name(self) -> &'static str {
match self {
Self::Window => "window",
Self::MainProbation => "main probation",
Self::MainProtected => "main protected",
Self::Other => "other",
}
}
}
impl PartialEq<Self> for CacheRegion {
fn eq(&self, other: &Self) -> bool {
core::mem::discriminant(self) == core::mem::discriminant(other)
}
}
impl PartialEq<usize> for CacheRegion {
fn eq(&self, other: &usize) -> bool {
*self as usize == *other
}
}
#[derive(Clone, Debug)]
pub(crate) struct HousekeeperConfig {
/// The timeout duration for the `run_pending_tasks` method. This is a safe-guard
/// to prevent cache read/write operations (that may call `run_pending_tasks`
/// internally) from being blocked for a long time when the user wrote a slow
/// eviction listener closure.
///
/// Used only when the eviction listener closure is set for the cache instance.
///
/// Default: `DEFAULT_MAINTENANCE_TASK_TIMEOUT_MILLIS`
pub(crate) maintenance_task_timeout: Duration,
/// The maximum repeat count for receiving operation logs from the read and write
/// log channels. Default: `MAX_LOG_SYNC_REPEATS`.
pub(crate) max_log_sync_repeats: u32,
/// The batch size of entries to be processed by each internal eviction method.
/// Default: `EVICTION_BATCH_SIZE`.
pub(crate) eviction_batch_size: u32,
}
impl Default for HousekeeperConfig {
fn default() -> Self {
Self {
maintenance_task_timeout: Duration::from_millis(
DEFAULT_MAINTENANCE_TASK_TIMEOUT_MILLIS,
),
max_log_sync_repeats: DEFAULT_MAX_LOG_SYNC_REPEATS as u32,
eviction_batch_size: DEFAULT_EVICTION_BATCH_SIZE,
}
}
}
impl HousekeeperConfig {
#[cfg(test)]
pub(crate) fn new(
maintenance_task_timeout: Option<Duration>,
max_log_sync_repeats: Option<u32>,
eviction_batch_size: Option<u32>,
) -> Self {
Self {
maintenance_task_timeout: maintenance_task_timeout.unwrap_or(Duration::from_millis(
DEFAULT_MAINTENANCE_TASK_TIMEOUT_MILLIS,
)),
max_log_sync_repeats: max_log_sync_repeats
.unwrap_or(DEFAULT_MAX_LOG_SYNC_REPEATS as u32),
eviction_batch_size: eviction_batch_size.unwrap_or(DEFAULT_EVICTION_BATCH_SIZE),
}
}
}
// Ensures the value fits in a range of `128u32..=u32::MAX`.
pub(crate) fn sketch_capacity(max_capacity: u64) -> u32 {
max_capacity.try_into().unwrap_or(u32::MAX).max(128)
}
#[cfg(test)]
pub(crate) fn available_parallelism() -> usize {
use std::{num::NonZeroUsize, thread::available_parallelism};
available_parallelism().map(NonZeroUsize::get).unwrap_or(1)
}
+16
View File
@@ -0,0 +1,16 @@
use std::time::Duration;
const YEAR_SECONDS: u64 = 365 * 24 * 3600;
pub(crate) fn ensure_expirations_or_panic(
time_to_live: Option<Duration>,
time_to_idle: Option<Duration>,
) {
let max_duration = Duration::from_secs(1_000 * YEAR_SECONDS);
if let Some(d) = time_to_live {
assert!(d <= max_duration, "time_to_live is longer than 1000 years");
}
if let Some(d) = time_to_idle {
assert!(d <= max_duration, "time_to_idle is longer than 1000 years");
}
}
+400
View File
@@ -0,0 +1,400 @@
use crate::common::{concurrent::arc::MiniArc, deque::DeqNode, time::Instant};
use parking_lot::Mutex;
use std::{fmt, ptr::NonNull, sync::Arc};
use tagptr::TagNonNull;
pub(crate) mod arc;
pub(crate) mod constants;
pub(crate) mod deques;
pub(crate) mod entry_info;
#[cfg(feature = "sync")]
pub(crate) mod housekeeper;
#[cfg(feature = "unstable-debug-counters")]
pub(crate) mod debug_counters;
use self::entry_info::EntryInfo;
use super::timer_wheel::TimerNode;
pub(crate) type Weigher<K, V> = Arc<dyn Fn(&K, &V) -> u32 + Send + Sync + 'static>;
pub(crate) trait AccessTime {
fn last_accessed(&self) -> Option<Instant>;
fn set_last_accessed(&self, timestamp: Instant);
fn last_modified(&self) -> Option<Instant>;
fn set_last_modified(&self, timestamp: Instant);
}
#[derive(Debug)]
pub(crate) struct KeyHash<K> {
pub(crate) key: Arc<K>,
pub(crate) hash: u64,
}
impl<K> KeyHash<K> {
pub(crate) fn new(key: Arc<K>, hash: u64) -> Self {
Self { key, hash }
}
}
impl<K> Clone for KeyHash<K> {
fn clone(&self) -> Self {
Self {
key: Arc::clone(&self.key),
hash: self.hash,
}
}
}
pub(crate) struct KeyHashDate<K> {
entry_info: MiniArc<EntryInfo<K>>,
}
impl<K> KeyHashDate<K> {
pub(crate) fn new(entry_info: &MiniArc<EntryInfo<K>>) -> Self {
Self {
entry_info: MiniArc::clone(entry_info),
}
}
pub(crate) fn key(&self) -> &Arc<K> {
&self.entry_info.key_hash().key
}
pub(crate) fn hash(&self) -> u64 {
self.entry_info.key_hash().hash
}
pub(crate) fn entry_info(&self) -> &EntryInfo<K> {
&self.entry_info
}
pub(crate) fn last_modified(&self) -> Option<Instant> {
self.entry_info.last_modified()
}
pub(crate) fn last_accessed(&self) -> Option<Instant> {
self.entry_info.last_accessed()
}
pub(crate) fn is_dirty(&self) -> bool {
self.entry_info.is_dirty()
}
}
pub(crate) struct KvEntry<K, V> {
pub(crate) key: Arc<K>,
pub(crate) entry: MiniArc<ValueEntry<K, V>>,
}
impl<K, V> KvEntry<K, V> {
pub(crate) fn new(key: Arc<K>, entry: MiniArc<ValueEntry<K, V>>) -> Self {
Self { key, entry }
}
}
impl<K, V> Clone for KvEntry<K, V> {
fn clone(&self) -> Self {
Self {
key: Arc::clone(&self.key),
entry: MiniArc::clone(&self.entry),
}
}
}
impl<K> AccessTime for DeqNode<KeyHashDate<K>> {
#[inline]
fn last_accessed(&self) -> Option<Instant> {
self.element.entry_info.last_accessed()
}
#[inline]
fn set_last_accessed(&self, timestamp: Instant) {
self.element.entry_info.set_last_accessed(timestamp);
}
#[inline]
fn last_modified(&self) -> Option<Instant> {
self.element.entry_info.last_modified()
}
#[inline]
fn set_last_modified(&self, timestamp: Instant) {
self.element.entry_info.set_last_modified(timestamp);
}
}
// DeqNode for an access order queue.
type KeyDeqNodeAo<K> = TagNonNull<DeqNode<KeyHashDate<K>>, 2>;
// DeqNode for the write order queue.
type KeyDeqNodeWo<K> = NonNull<DeqNode<KeyHashDate<K>>>;
// DeqNode for the timer wheel.
type DeqNodeTimer<K> = NonNull<DeqNode<TimerNode<K>>>;
pub(crate) struct DeqNodes<K> {
access_order_q_node: Option<KeyDeqNodeAo<K>>,
write_order_q_node: Option<KeyDeqNodeWo<K>>,
timer_node: Option<DeqNodeTimer<K>>,
/// The expiry generation when timer_node was set.
/// Used to validate the timer_node hasn't become stale.
timer_node_expiry_gen: u32,
}
impl<K> Default for DeqNodes<K> {
fn default() -> Self {
Self {
access_order_q_node: None,
write_order_q_node: None,
timer_node: None,
timer_node_expiry_gen: 0,
}
}
}
// We need this `unsafe impl` as DeqNodes have NonNull pointers.
unsafe impl<K> Send for DeqNodes<K> {}
impl<K> DeqNodes<K> {
pub(crate) fn set_timer_node(&mut self, timer_node: Option<DeqNodeTimer<K>>, expiry_gen: u32) {
self.timer_node = timer_node;
self.timer_node_expiry_gen = expiry_gen;
}
pub(crate) fn timer_node_with_expiry_gen(&self) -> (Option<DeqNodeTimer<K>>, u32) {
(self.timer_node, self.timer_node_expiry_gen)
}
}
pub(crate) struct ValueEntry<K, V> {
pub(crate) value: V,
info: MiniArc<EntryInfo<K>>,
nodes: MiniArc<Mutex<DeqNodes<K>>>,
}
impl<K, V> ValueEntry<K, V> {
pub(crate) fn new(value: V, entry_info: MiniArc<EntryInfo<K>>) -> Self {
#[cfg(feature = "unstable-debug-counters")]
self::debug_counters::InternalGlobalDebugCounters::value_entry_created();
Self {
value,
info: entry_info,
nodes: MiniArc::new(Mutex::new(DeqNodes::default())),
}
}
pub(crate) fn new_from(value: V, entry_info: MiniArc<EntryInfo<K>>, other: &Self) -> Self {
#[cfg(feature = "unstable-debug-counters")]
self::debug_counters::InternalGlobalDebugCounters::value_entry_created();
Self {
value,
info: entry_info,
nodes: MiniArc::clone(&other.nodes),
}
}
pub(crate) fn entry_info(&self) -> &MiniArc<EntryInfo<K>> {
&self.info
}
pub(crate) fn is_admitted(&self) -> bool {
self.info.is_admitted()
}
pub(crate) fn set_admitted(&self, value: bool) {
self.info.set_admitted(value);
}
pub(crate) fn is_dirty(&self) -> bool {
self.info.is_dirty()
}
#[inline]
pub(crate) fn policy_weight(&self) -> u32 {
self.info.policy_weight()
}
pub(crate) fn deq_nodes(&self) -> &MiniArc<Mutex<DeqNodes<K>>> {
&self.nodes
}
pub(crate) fn access_order_q_node(&self) -> Option<KeyDeqNodeAo<K>> {
self.nodes.lock().access_order_q_node
}
pub(crate) fn set_access_order_q_node(&self, node: Option<KeyDeqNodeAo<K>>) {
self.nodes.lock().access_order_q_node = node;
}
pub(crate) fn take_access_order_q_node(&self) -> Option<KeyDeqNodeAo<K>> {
self.nodes.lock().access_order_q_node.take()
}
pub(crate) fn write_order_q_node(&self) -> Option<KeyDeqNodeWo<K>> {
self.nodes.lock().write_order_q_node
}
pub(crate) fn set_write_order_q_node(&self, node: Option<KeyDeqNodeWo<K>>) {
self.nodes.lock().write_order_q_node = node;
}
pub(crate) fn take_write_order_q_node(&self) -> Option<KeyDeqNodeWo<K>> {
self.nodes.lock().write_order_q_node.take()
}
/// Returns the timer node and its expected expiry generation for validation.
pub(crate) fn timer_node_with_expiry_gen(&self) -> (Option<DeqNodeTimer<K>>, u32) {
self.nodes.lock().timer_node_with_expiry_gen()
}
pub(crate) fn set_timer_node(&self, node: Option<DeqNodeTimer<K>>, expiry_gen: u32) {
self.nodes.lock().set_timer_node(node, expiry_gen);
}
/// Takes the timer node and returns it along with its stored expiry generation.
pub(crate) fn take_timer_node(&self) -> (Option<DeqNodeTimer<K>>, u32) {
let mut nodes = self.nodes.lock();
let expiry_gen = nodes.timer_node_expiry_gen;
nodes.timer_node_expiry_gen = 0;
(nodes.timer_node.take(), expiry_gen)
}
pub(crate) fn unset_q_nodes(&self) {
let mut nodes = self.nodes.lock();
nodes.access_order_q_node = None;
nodes.write_order_q_node = None;
}
}
#[cfg(feature = "unstable-debug-counters")]
impl<K, V> Drop for ValueEntry<K, V> {
fn drop(&mut self) {
self::debug_counters::InternalGlobalDebugCounters::value_entry_dropped();
}
}
impl<K, V> AccessTime for MiniArc<ValueEntry<K, V>> {
#[inline]
fn last_accessed(&self) -> Option<Instant> {
self.info.last_accessed()
}
#[inline]
fn set_last_accessed(&self, timestamp: Instant) {
self.info.set_last_accessed(timestamp);
}
#[inline]
fn last_modified(&self) -> Option<Instant> {
self.info.last_modified()
}
#[inline]
fn set_last_modified(&self, timestamp: Instant) {
self.info.set_last_modified(timestamp);
}
}
pub(crate) enum ReadOp<K, V> {
Hit {
value_entry: MiniArc<ValueEntry<K, V>>,
is_expiry_modified: bool,
},
// u64 is the hash of the key.
Miss(u64),
}
pub(crate) enum WriteOp<K, V> {
Upsert {
key_hash: KeyHash<K>,
value_entry: MiniArc<ValueEntry<K, V>>,
/// Entry generation after the operation.
entry_gen: u16,
old_weight: u32,
new_weight: u32,
},
Remove {
kv_entry: KvEntry<K, V>,
entry_gen: u16,
},
}
/// Cloning a `WriteOp` is safe and cheap because it uses `Arc` and `MiniArc` pointers to
/// the actual data.
impl<K, V> Clone for WriteOp<K, V> {
fn clone(&self) -> Self {
match self {
Self::Upsert {
key_hash,
value_entry,
entry_gen,
old_weight,
new_weight,
} => Self::Upsert {
key_hash: key_hash.clone(),
value_entry: MiniArc::clone(value_entry),
entry_gen: *entry_gen,
old_weight: *old_weight,
new_weight: *new_weight,
},
Self::Remove {
kv_entry,
entry_gen,
} => Self::Remove {
kv_entry: kv_entry.clone(),
entry_gen: *entry_gen,
},
}
}
}
impl<K, V> fmt::Debug for WriteOp<K, V> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Upsert { .. } => f.debug_struct("Upsert").finish(),
Self::Remove { .. } => f.debug_tuple("Remove").finish(),
}
}
}
impl<K, V> WriteOp<K, V> {
pub(crate) fn new_upsert(
key: &Arc<K>,
hash: u64,
value_entry: &MiniArc<ValueEntry<K, V>>,
entry_generation: u16,
old_weight: u32,
new_weight: u32,
) -> Self {
let key_hash = KeyHash::new(Arc::clone(key), hash);
let value_entry = MiniArc::clone(value_entry);
Self::Upsert {
key_hash,
value_entry,
entry_gen: entry_generation,
old_weight,
new_weight,
}
}
}
pub(crate) struct OldEntryInfo<K, V> {
pub(crate) entry: MiniArc<ValueEntry<K, V>>,
pub(crate) last_accessed: Option<Instant>,
pub(crate) last_modified: Option<Instant>,
}
impl<K, V> OldEntryInfo<K, V> {
pub(crate) fn new(entry: &MiniArc<ValueEntry<K, V>>) -> Self {
Self {
entry: MiniArc::clone(entry),
last_accessed: entry.last_accessed(),
last_modified: entry.last_modified(),
}
}
}
+331
View File
@@ -0,0 +1,331 @@
// This module's source code was written by us, the `moka` developers, referring to
// the following book and code:
//
// - Chapter 6. Building Our Own "Arc" of the Rust Atomics and Locks book.
// - Rust Atomics and Locks by Mara Bos (O’Reilly). Copyright 2023 Mara Bos,
// ISBN: 978-1-098-11944-7
// - https://marabos.nl/atomics/
// - The `triomphe` crate v0.1.13 and v0.1.11 by Manish Goregaokar (Manishearth)
// - MIT or Apache-2.0 License
// - https://github.com/Manishearth/triomphe
// - `std::sync::Arc` in the Rust Standard Library (1.81.0).
// - MIT or Apache-2.0 License
use std::{
fmt,
hash::{Hash, Hasher},
ops::Deref,
ptr::NonNull,
};
#[cfg(not(moka_loom))]
use std::sync::atomic::{self, AtomicU32};
#[cfg(moka_loom)]
use loom::sync::atomic::{self, AtomicU32};
/// A thread-safe reference-counting pointer. `MiniArc` is similar to
/// `std::sync::Arc`, Atomically Reference Counted shared pointer, but with a few
/// differences:
///
/// - Smaller memory overhead:
/// - `MiniArc` does not support weak references, so it does not need to store a
/// weak reference count.
/// - `MiniArc` uses `AtomicU32` for the reference count, while `std::sync::Arc`
/// uses `AtomicUsize`. On a 64-bit system, `AtomicU32` is half the size of
/// `AtomicUsize`.
/// - Note: Depending on the value type `T`, the Rust compiler may add
/// padding to the internal struct of `MiniArc<T>`, so the actual memory
/// overhead may vary.
/// - Smaller code size:
/// - Only about 100 lines of code.
/// - This is because `MiniArc` provides only the methods needed for the
/// `moka` and `mini-moka` crates.
/// - Smaller code size means less chance of bugs.
pub(crate) struct MiniArc<T: ?Sized> {
ptr: NonNull<ArcData<T>>,
}
struct ArcData<T: ?Sized> {
ref_count: AtomicU32,
data: T,
}
/// A soft limit on the amount of references that may be made to an `MiniArc`.
///
/// Going above this limit will abort your program (although not necessarily)
/// at _exactly_ `MAX_REFCOUNT + 1` references.
const MAX_REFCOUNT: u32 = (i32::MAX) as u32;
unsafe impl<T: ?Sized + Send + Sync> Send for MiniArc<T> {}
unsafe impl<T: ?Sized + Send + Sync> Sync for MiniArc<T> {}
impl<T> MiniArc<T> {
pub(crate) fn new(data: T) -> MiniArc<T> {
MiniArc {
ptr: NonNull::from(Box::leak(Box::new(ArcData {
ref_count: AtomicU32::new(1),
data,
}))),
}
}
}
impl<T: ?Sized> MiniArc<T> {
/// Gets the number of [`MiniArc`] pointers to this allocation
pub(crate) fn count(this: &Self) -> u32 {
use atomic::Ordering::Acquire;
this.data().ref_count.load(Acquire)
}
/// Returns `true` if the two `MiniArc`s point to the same allocation in a
/// vein similar to [`ptr::eq`].
///
/// # Safety
///
/// This function is unreliable when `T` is a `dyn Trait`. Currently
/// coercing `MiniArc<SomeTime>` to `MiniArc<dyn Trait>` is not possible, so
/// this is not a problem in practice. However, if this coercion becomes
/// possible in the future, this function may return incorrect results when
/// comparing `MiniArc<dyn Trait>` instances.
///
/// To fix this, we must rise the minimum supported Rust version (MSRV) to
/// 1.76 and use `std::ptr::addr_eq` internally instead of `eq` (`==`).
/// `addr_eq` compares the _addresses_ of the pointers for equality,
/// ignoring any metadata in fat pointers.
///
/// See the following `triomphe` issue for more information:
/// https://github.com/Manishearth/triomphe/pull/84
///
/// Note that `triomphe` has a feature called `unsize`, which enables the
/// coercion by using the `unsize` crate. `MiniArc` does not have such a
/// feature, so we are safe for now.
#[inline]
#[allow(ambiguous_wide_pointer_comparisons)] // Remove this when MSRV is 1.76 or newer.
#[allow(clippy::ptr_eq)] // Remove this when MSRV is 1.76 or newer.
pub(crate) fn ptr_eq(this: &Self, other: &Self) -> bool {
// `addr_eq` requires Rust 1.76 or newer.
// ptr::addr_eq(this.ptr.as_ptr(), other.ptr.as_ptr())
this.ptr.as_ptr() == other.ptr.as_ptr()
}
#[inline]
fn data(&self) -> &ArcData<T> {
unsafe { self.ptr.as_ref() }
}
}
impl<T: ?Sized> Deref for MiniArc<T> {
type Target = T;
fn deref(&self) -> &T {
&self.data().data
}
}
impl<T: ?Sized> Clone for MiniArc<T> {
fn clone(&self) -> Self {
use atomic::Ordering::Relaxed;
if self.data().ref_count.fetch_add(1, Relaxed) > MAX_REFCOUNT {
std::process::abort();
}
MiniArc { ptr: self.ptr }
}
}
impl<T: ?Sized> Drop for MiniArc<T> {
fn drop(&mut self) {
use std::sync::atomic::Ordering::{Acquire, Release};
if self.data().ref_count.fetch_sub(1, Release) == 1 {
atomic::fence(Acquire);
unsafe {
drop(Box::from_raw(self.ptr.as_ptr()));
}
}
}
}
impl<T: Default> Default for MiniArc<T> {
/// Creates a new `MiniArc<T>`, with the `Default` value for `T`.
fn default() -> MiniArc<T> {
MiniArc::new(Default::default())
}
}
impl<T: ?Sized + PartialEq> PartialEq for MiniArc<T> {
fn eq(&self, other: &MiniArc<T>) -> bool {
// TODO: pointer equality is incorrect if `T` is not `Eq`.
// See: https://github.com/Manishearth/triomphe/pull/88
Self::ptr_eq(self, other) || *(*self) == *(*other)
}
#[allow(clippy::partialeq_ne_impl)]
fn ne(&self, other: &MiniArc<T>) -> bool {
!Self::ptr_eq(self, other) && *(*self) != *(*other)
}
}
impl<T: ?Sized + Eq> Eq for MiniArc<T> {}
impl<T: ?Sized + fmt::Display> fmt::Display for MiniArc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&**self, f)
}
}
impl<T: ?Sized + fmt::Debug> fmt::Debug for MiniArc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&**self, f)
}
}
impl<T: ?Sized> fmt::Pointer for MiniArc<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Pointer::fmt(&self.ptr.as_ptr(), f)
}
}
impl<T: ?Sized + Hash> Hash for MiniArc<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
(**self).hash(state)
}
}
#[cfg(all(test, not(moka_loom)))]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
use super::*;
#[test]
fn test_drop() {
static NUM_DROPS: AtomicUsize = AtomicUsize::new(0);
struct DetectDrop;
impl Drop for DetectDrop {
fn drop(&mut self) {
NUM_DROPS.fetch_add(1, Relaxed);
}
}
// Create two MiniArcs sharing an object containing a string
// and a DetectDrop, to detect when it is dropped.
let x = MiniArc::new(("hello", DetectDrop));
let y = x.clone();
// Send x to another thread, and use it there.
let t = std::thread::spawn(move || {
assert_eq!(x.0, "hello");
});
// In parallel, y should still be usable here.
assert_eq!(y.0, "hello");
assert!(MiniArc::count(&y) >= 1);
// Wait for the thread to finish.
t.join().unwrap();
// One MiniArc, x, should be dropped by now.
// We still have y, so the object should not have been dropped yet.
assert_eq!(NUM_DROPS.load(Relaxed), 0);
assert_eq!(MiniArc::count(&y), 1);
// Drop the remaining `MiniArc`.
drop(y);
// Now that `y` is dropped too,
// the object should have been dropped.
assert_eq!(NUM_DROPS.load(Relaxed), 1);
}
#[test]
fn test_eq() {
let w = MiniArc::new(6502);
let x = w.clone();
let y = MiniArc::new(6502);
let z = MiniArc::new(8086);
assert_eq!(w, x);
assert_eq!(x, w);
assert_eq!(w, y);
assert_eq!(y, w);
assert_ne!(y, z);
assert_ne!(z, y);
}
#[test]
fn test_partial_eq_bug() {
let float = f32::NAN;
assert_ne!(float, float);
let arc = MiniArc::new(f32::NAN);
// TODO: this is a bug.
// See: https://github.com/Manishearth/triomphe/pull/88
assert_eq!(arc, arc);
}
#[allow(dead_code)]
const fn is_partial_eq<T: ?Sized + PartialEq>() {}
#[allow(dead_code)]
const fn is_eq<T: ?Sized + Eq>() {}
// compile-time check that PartialEq/Eq is correctly derived
const _: () = is_partial_eq::<MiniArc<i32>>();
const _: () = is_eq::<MiniArc<i32>>();
}
#[cfg(all(test, moka_loom))]
mod loom_tests {
use super::*;
#[test]
fn test_drop() {
use loom::sync::atomic::{AtomicUsize, Ordering::Relaxed};
struct DetectDrop(loom::sync::Arc<AtomicUsize>);
impl Drop for DetectDrop {
fn drop(&mut self) {
self.0.fetch_add(1, Relaxed);
}
}
loom::model(move || {
let num_drops = loom::sync::Arc::new(AtomicUsize::new(0));
// Create two MiniArcs sharing an object containing a string
// and a DetectDrop, to detect when it is dropped.
let x = MiniArc::new(("hello", DetectDrop(loom::sync::Arc::clone(&num_drops))));
let y = x.clone();
// Send x to another thread, and use it there.
let t = loom::thread::spawn(move || {
assert_eq!(x.0, "hello");
});
// In parallel, y should still be usable here.
assert_eq!(y.0, "hello");
assert!(MiniArc::count(&y) >= 1);
// Wait for the thread to finish.
t.join().unwrap();
// One MiniArc, x, should be dropped by now.
// We still have y, so the object should not have been dropped yet.
assert_eq!(num_drops.load(Relaxed), 0);
assert_eq!(MiniArc::count(&y), 1);
// Drop the remaining `MiniArc`.
drop(y);
// Now that `y` is dropped too,
// the object should have been dropped.
assert_eq!(num_drops.load(Relaxed), 1);
});
}
}
@@ -0,0 +1,23 @@
pub(crate) const DEFAULT_MAX_LOG_SYNC_REPEATS: usize = 4;
pub(crate) const LOG_SYNC_INTERVAL_MILLIS: u64 = 300;
pub(crate) const READ_LOG_FLUSH_POINT: usize = 64;
pub(crate) const WRITE_LOG_FLUSH_POINT: usize = 64;
// 384 elements
pub(crate) const READ_LOG_CH_SIZE: usize =
READ_LOG_FLUSH_POINT * (DEFAULT_MAX_LOG_SYNC_REPEATS + 2);
// 384 elements
pub(crate) const WRITE_LOG_CH_SIZE: usize =
WRITE_LOG_FLUSH_POINT * (DEFAULT_MAX_LOG_SYNC_REPEATS + 2);
// TODO: Calculate the batch size based on the number of entries in the cache (or an
// estimated number of entries to evict)
pub(crate) const DEFAULT_EVICTION_BATCH_SIZE: u32 = WRITE_LOG_CH_SIZE as u32;
/// The default timeout duration for the `run_pending_tasks` method.
pub(crate) const DEFAULT_MAINTENANCE_TASK_TIMEOUT_MILLIS: u64 = 100;
#[cfg(feature = "sync")]
pub(crate) const WRITE_RETRY_INTERVAL_MICROS: u64 = 50;
@@ -0,0 +1,142 @@
use crossbeam_utils::atomic::AtomicCell;
#[derive(Clone, Debug)]
pub struct GlobalDebugCounters {
pub bucket_array_creation_count: u64,
pub bucket_array_allocation_bytes: u64,
pub bucket_array_drop_count: u64,
pub bucket_array_release_bytes: u64,
pub bucket_creation_count: u64,
pub bucket_drop_count: u64,
pub value_entry_creation_count: u64,
pub value_entry_drop_count: u64,
pub entry_info_creation_count: u64,
pub entry_info_drop_count: u64,
pub deq_node_creation_count: u64,
pub deq_node_drop_count: u64,
}
impl GlobalDebugCounters {
pub fn current() -> Self {
InternalGlobalDebugCounters::current()
}
}
static COUNTERS: InternalGlobalDebugCounters = InternalGlobalDebugCounters {
bucket_array_creation_count: AtomicCell::new(0),
bucket_array_allocation_bytes: AtomicCell::new(0),
bucket_array_drop_count: AtomicCell::new(0),
bucket_array_release_bytes: AtomicCell::new(0),
bucket_creation_count: AtomicCell::new(0),
bucket_drop_count: AtomicCell::new(0),
value_entry_creation_count: AtomicCell::new(0),
value_entry_drop_count: AtomicCell::new(0),
entry_info_creation_count: AtomicCell::new(0),
entry_info_drop_count: AtomicCell::new(0),
deq_node_creation_count: AtomicCell::new(0),
deq_node_drop_count: AtomicCell::new(0),
};
#[derive(Default)]
pub(crate) struct InternalGlobalDebugCounters {
bucket_array_creation_count: AtomicCell<u64>,
bucket_array_allocation_bytes: AtomicCell<u64>,
bucket_array_drop_count: AtomicCell<u64>,
bucket_array_release_bytes: AtomicCell<u64>,
bucket_creation_count: AtomicCell<u64>,
bucket_drop_count: AtomicCell<u64>,
value_entry_creation_count: AtomicCell<u64>,
value_entry_drop_count: AtomicCell<u64>,
entry_info_creation_count: AtomicCell<u64>,
entry_info_drop_count: AtomicCell<u64>,
deq_node_creation_count: AtomicCell<u64>,
deq_node_drop_count: AtomicCell<u64>,
}
impl InternalGlobalDebugCounters {
fn current() -> GlobalDebugCounters {
let c = &COUNTERS;
GlobalDebugCounters {
bucket_array_creation_count: c.bucket_array_creation_count.load(),
bucket_array_allocation_bytes: c.bucket_array_allocation_bytes.load(),
bucket_array_drop_count: c.bucket_array_drop_count.load(),
bucket_array_release_bytes: c.bucket_array_release_bytes.load(),
bucket_creation_count: c.bucket_creation_count.load(),
bucket_drop_count: c.bucket_drop_count.load(),
value_entry_creation_count: c.value_entry_creation_count.load(),
value_entry_drop_count: c.value_entry_drop_count.load(),
entry_info_creation_count: c.entry_info_creation_count.load(),
entry_info_drop_count: c.entry_info_drop_count.load(),
deq_node_creation_count: c.deq_node_creation_count.load(),
deq_node_drop_count: c.deq_node_drop_count.load(),
}
}
pub(crate) fn bucket_array_created(byte_size: u64) {
COUNTERS.bucket_array_creation_count.fetch_add(1);
COUNTERS.bucket_array_allocation_bytes.fetch_add(byte_size);
}
pub(crate) fn bucket_array_dropped(byte_size: u64) {
COUNTERS.bucket_array_drop_count.fetch_add(1);
COUNTERS.bucket_array_release_bytes.fetch_add(byte_size);
}
pub(crate) fn bucket_created() {
COUNTERS.bucket_creation_count.fetch_add(1);
}
pub(crate) fn bucket_dropped() {
COUNTERS.bucket_drop_count.fetch_add(1);
}
pub(crate) fn value_entry_created() {
COUNTERS.value_entry_creation_count.fetch_add(1);
}
pub(crate) fn value_entry_dropped() {
COUNTERS.value_entry_drop_count.fetch_add(1);
}
pub(crate) fn entry_info_created() {
COUNTERS.entry_info_creation_count.fetch_add(1);
}
pub(crate) fn entry_info_dropped() {
COUNTERS.entry_info_drop_count.fetch_add(1);
}
pub(crate) fn deq_node_created() {
COUNTERS.deq_node_creation_count.fetch_add(1);
}
pub(crate) fn deq_node_dropped() {
COUNTERS.deq_node_drop_count.fetch_add(1);
}
}
#[derive(Clone, Debug)]
pub struct CacheDebugStats {
pub entry_count: u64,
pub weighted_size: u64,
// bytes
pub freq_sketch_size: u64,
// max entries
pub hashmap_capacity: u64,
}
impl CacheDebugStats {
pub(crate) fn new(
entry_count: u64,
weighted_size: u64,
hashmap_capacity: u64,
freq_sketch_size: u64,
) -> Self {
Self {
entry_count,
weighted_size,
freq_sketch_size,
hashmap_capacity,
}
}
}
@@ -0,0 +1,206 @@
use super::{arc::MiniArc, KeyHashDate, ValueEntry};
use crate::common::{
deque::{DeqNode, Deque},
CacheRegion,
};
use std::ptr::NonNull;
use tagptr::TagNonNull;
pub(crate) struct Deques<K> {
pub(crate) window: Deque<KeyHashDate<K>>, // Not used yet.
pub(crate) probation: Deque<KeyHashDate<K>>,
pub(crate) protected: Deque<KeyHashDate<K>>, // Not used yet.
pub(crate) write_order: Deque<KeyHashDate<K>>,
}
#[cfg(feature = "future")]
// TODO: https://github.com/moka-rs/moka/issues/54
#[allow(clippy::non_send_fields_in_send_ty)]
// Multi-threaded async runtimes require base_cache::Inner to be Send, but it will
// not be without this `unsafe impl`. This is because DeqNodes have NonNull
// pointers.
unsafe impl<K> Send for Deques<K> {}
impl<K> Default for Deques<K> {
fn default() -> Self {
Self {
window: Deque::new(CacheRegion::Window),
probation: Deque::new(CacheRegion::MainProbation),
protected: Deque::new(CacheRegion::MainProtected),
write_order: Deque::new(CacheRegion::Other),
}
}
}
impl<K> Deques<K> {
pub(crate) fn select_mut(
&mut self,
selector: CacheRegion,
) -> (&mut Deque<KeyHashDate<K>>, &mut Deque<KeyHashDate<K>>) {
match selector {
CacheRegion::Window => (&mut self.window, &mut self.write_order),
CacheRegion::MainProbation => (&mut self.probation, &mut self.write_order),
CacheRegion::MainProtected => (&mut self.protected, &mut self.write_order),
CacheRegion::Other => unreachable!(),
}
}
pub(crate) fn push_back_ao<V>(
&mut self,
region: CacheRegion,
khd: KeyHashDate<K>,
entry: &MiniArc<ValueEntry<K, V>>,
) {
let node = Box::new(DeqNode::new(khd));
let node = match region {
CacheRegion::Window => self.window.push_back(node),
CacheRegion::MainProbation => self.probation.push_back(node),
CacheRegion::MainProtected => self.protected.push_back(node),
CacheRegion::Other => unreachable!(),
};
let tagged_node = TagNonNull::compose(node, region as usize);
entry.set_access_order_q_node(Some(tagged_node));
}
pub(crate) fn push_back_wo<V>(
&mut self,
kd: KeyHashDate<K>,
entry: &MiniArc<ValueEntry<K, V>>,
) {
let node = Box::new(DeqNode::new(kd));
let node = self.write_order.push_back(node);
entry.set_write_order_q_node(Some(node));
}
pub(crate) fn move_to_back_ao<V>(&mut self, entry: &MiniArc<ValueEntry<K, V>>) {
if let Some(tagged_node) = entry.access_order_q_node() {
let (node, tag) = tagged_node.decompose();
let p = unsafe { node.as_ref() };
match tag.into() {
CacheRegion::Window if self.window.contains(p) => {
unsafe { self.window.move_to_back(node) };
}
CacheRegion::MainProbation if self.probation.contains(p) => {
unsafe { self.probation.move_to_back(node) };
}
CacheRegion::MainProtected if self.protected.contains(p) => {
unsafe { self.protected.move_to_back(node) };
}
_ => unreachable!(),
}
}
}
pub(crate) fn move_to_back_ao_in_deque<V>(
deq_name: &str,
deq: &mut Deque<KeyHashDate<K>>,
entry: &MiniArc<ValueEntry<K, V>>,
) {
if let Some(tagged_node) = entry.access_order_q_node() {
let (node, tag) = tagged_node.decompose();
let p = unsafe { node.as_ref() };
assert_eq!(
deq.region(),
tag,
"move_to_back_ao_in_deque - node is not a member of {deq_name} deque. {p:?}"
);
if deq.contains(p) {
unsafe { deq.move_to_back(node) };
}
}
}
pub(crate) fn move_to_back_wo<V>(&mut self, entry: &MiniArc<ValueEntry<K, V>>) {
if let Some(node) = entry.write_order_q_node() {
let p = unsafe { node.as_ref() };
if self.write_order.contains(p) {
unsafe { self.write_order.move_to_back(node) };
}
}
}
pub(crate) fn move_to_back_wo_in_deque<V>(
deq: &mut Deque<KeyHashDate<K>>,
entry: &MiniArc<ValueEntry<K, V>>,
) {
if let Some(node) = entry.write_order_q_node() {
let p = unsafe { node.as_ref() };
if deq.contains(p) {
unsafe { deq.move_to_back(node) };
}
}
}
pub(crate) fn unlink_ao<V>(&mut self, entry: &MiniArc<ValueEntry<K, V>>) {
if let Some(node) = entry.take_access_order_q_node() {
self.unlink_node_ao(node);
}
}
pub(crate) fn unlink_ao_from_deque<V>(
deq_name: &str,
deq: &mut Deque<KeyHashDate<K>>,
entry: &MiniArc<ValueEntry<K, V>>,
) {
if let Some(node) = entry.take_access_order_q_node() {
unsafe { Self::unlink_node_ao_from_deque(deq_name, deq, node) };
}
}
pub(crate) fn unlink_wo<V>(deq: &mut Deque<KeyHashDate<K>>, entry: &MiniArc<ValueEntry<K, V>>) {
if let Some(node) = entry.take_write_order_q_node() {
Self::unlink_node_wo(deq, node);
}
}
pub(crate) fn unlink_node_ao(&mut self, tagged_node: TagNonNull<DeqNode<KeyHashDate<K>>, 2>) {
unsafe {
match tagged_node.decompose_tag().into() {
CacheRegion::Window => {
Self::unlink_node_ao_from_deque("window", &mut self.window, tagged_node);
}
CacheRegion::MainProbation => {
Self::unlink_node_ao_from_deque("probation", &mut self.probation, tagged_node);
}
CacheRegion::MainProtected => {
Self::unlink_node_ao_from_deque("protected", &mut self.protected, tagged_node);
}
CacheRegion::Other => unreachable!(),
}
}
}
unsafe fn unlink_node_ao_from_deque(
deq_name: &str,
deq: &mut Deque<KeyHashDate<K>>,
tagged_node: TagNonNull<DeqNode<KeyHashDate<K>>, 2>,
) {
let (node, tag) = tagged_node.decompose();
let p = node.as_ref();
assert_eq!(
deq.region(),
tag,
"unlink_node - node is not a member of {deq_name} deque. {p:?}"
);
if deq.contains(p) {
// https://github.com/moka-rs/moka/issues/64
deq.unlink_and_drop(node);
}
}
pub(crate) fn unlink_node_wo(
deq: &mut Deque<KeyHashDate<K>>,
node: NonNull<DeqNode<KeyHashDate<K>>>,
) {
unsafe {
let p = node.as_ref();
if deq.contains(p) {
// https://github.com/moka-rs/moka/issues/64
deq.unlink_and_drop(node);
}
}
}
}
// TODO: Add tests and run Miri with them.
@@ -0,0 +1,233 @@
use std::sync::atomic::{self, AtomicBool, AtomicU16, AtomicU32, Ordering};
use super::{AccessTime, KeyHash};
use crate::common::time::{AtomicInstant, Instant};
use portable_atomic::AtomicU64;
#[derive(Debug)]
pub(crate) struct EntryInfo<K> {
key_hash: KeyHash<K>,
/// `is_admitted` indicates that the entry has been admitted to the cache. When
/// `false`, it means the entry is _temporary_ admitted to the cache or evicted
/// from the cache (so it should not have LRU nodes).
is_admitted: AtomicBool,
/// `entry_gen` (entry generation) is incremented every time the entry is updated
/// in the concurrent hash table.
entry_gen: AtomicU16,
/// `policy_gen` (policy generation) is incremented every time entry's `WriteOp`
/// is applied to the cache policies including the access-order queue (the LRU
/// deque).
policy_gen: AtomicU16,
/// Packed expiration state: contains both `expiration_time` (upper 52 bits) and
/// `expiry_gen` (lower 12 bits) in a single atomic u64 for consistent reads.
///
/// Encoding:
/// - Bits 0-11: 12-bit expiry_gen (wraps from 4095 to 0)
/// - Bits 12-63: 52-bit expiration timestamp (nanoseconds from a monotonic clock/Instant)
/// - Sentinel: When time field (bits 12-63) is all 1s (0xFFFF_FFFF_FFFF_F000),
/// represents "None" (no expiration). Gen bits are always preserved.
/// - Valid timestamps are clamped to 52-bit range to avoid collisions with sentinel.
///
/// NOTE: The time value is relative to the runtime's monotonic clock (not Unix epoch).
/// It is obtained from `Instant` and should not be interpreted as an absolute time
/// suitable for serialization or cross-process comparison.
expiration_state: AtomicU64,
last_accessed: AtomicInstant,
last_modified: AtomicInstant,
policy_weight: AtomicU32,
}
impl<K> EntryInfo<K> {
#[inline]
pub(crate) fn new(key_hash: KeyHash<K>, timestamp: Instant, policy_weight: u32) -> Self {
#[cfg(feature = "unstable-debug-counters")]
super::debug_counters::InternalGlobalDebugCounters::entry_info_created();
Self {
key_hash,
is_admitted: AtomicBool::default(),
// `entry_gen` starts at 1 and `policy_gen` start at 0.
entry_gen: AtomicU16::new(1),
policy_gen: AtomicU16::new(0),
// Initial state: None (time field all 1s), gen=0
expiration_state: AtomicU64::new(0xFFFF_FFFF_FFFF_F000),
last_accessed: AtomicInstant::new(timestamp),
last_modified: AtomicInstant::new(timestamp),
policy_weight: AtomicU32::new(policy_weight),
}
}
#[inline]
pub(crate) fn key_hash(&self) -> &KeyHash<K> {
&self.key_hash
}
#[inline]
pub(crate) fn is_admitted(&self) -> bool {
self.is_admitted.load(Ordering::Acquire)
}
#[inline]
pub(crate) fn set_admitted(&self, value: bool) {
self.is_admitted.store(value, Ordering::Release);
}
/// Returns `true` if the `ValueEntry` having this `EntryInfo` is dirty.
///
/// Dirty means that the entry has been updated in the concurrent hash table but
/// not yet in the cache policies such as access-order queue.
#[inline]
pub(crate) fn is_dirty(&self) -> bool {
let result =
self.entry_gen.load(Ordering::Relaxed) != self.policy_gen.load(Ordering::Relaxed);
atomic::fence(Ordering::Acquire);
result
}
#[inline]
pub(crate) fn entry_gen(&self) -> u16 {
self.entry_gen.load(Ordering::Acquire)
}
/// Increments the entry generation and returns the new value.
#[inline]
pub(crate) fn incr_entry_gen(&self) -> u16 {
// NOTE: This operation wraps around on overflow.
let prev = self.entry_gen.fetch_add(1, Ordering::AcqRel);
// Need to add `1` to the previous value to get the current value.
prev.wrapping_add(1)
}
/// Sets the policy generation to the given value.
#[inline]
pub(crate) fn set_policy_gen(&self, value: u16) {
let g = &self.policy_gen;
loop {
let current = g.load(Ordering::Acquire);
// Do not set the given value if it is smaller than the current value of
// `policy_gen`. Note that the current value may have been wrapped
// around. If the value is much larger than the current value, it is
// likely that the value of `policy_gen` has been wrapped around.
if current >= value || value.wrapping_sub(current) > u16::MAX / 2 {
break;
}
// Try to set the value.
if g.compare_exchange_weak(current, value, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
break;
}
}
}
#[inline]
pub(crate) fn policy_weight(&self) -> u32 {
self.policy_weight.load(Ordering::Acquire)
}
pub(crate) fn set_policy_weight(&self, size: u32) {
self.policy_weight.store(size, Ordering::Release);
}
/// Atomically reads both `expiration_time` and `expiry_gen` as a single unit.
/// Returns `(expiration_time, expiry_gen)` where expiration_time is None if unset.
///
/// This provides a consistent snapshot of the expiration state, avoiding TOCTOU
/// issues where the time and generation could be read separately while being
/// modified by another thread.
#[inline]
pub(crate) fn expiration_state(&self) -> (Option<Instant>, u32) {
const GEN_MASK: u64 = 0xFFF;
const TIME_MASK: u64 = 0xFFFF_FFFF_FFFF_F000;
let packed = self.expiration_state.load(Ordering::Acquire);
// Extract time field and gen bits
let time_nanos = packed & TIME_MASK;
let gen = (packed & GEN_MASK) as u32;
// Check if time field (upper 52 bits) is all 1s (sentinel for None)
if time_nanos == TIME_MASK {
(None, gen)
} else {
(Some(Instant::from_nanos(time_nanos)), gen)
}
}
/// Sets the expiration time and returns the new expiry generation.
pub(crate) fn set_expiration_time(&self, time: Option<Instant>) -> u32 {
const GEN_MASK: u64 = 0xFFF;
const TIME_MASK: u64 = 0xFFFF_FFFF_FFFF_F000;
// Use compare_exchange to atomically update the expiration state.
// This prevents race conditions where multiple threads try to update
// the expiration time simultaneously.
loop {
let prev_packed = self.expiration_state.load(Ordering::Acquire);
// Extract previous generation (always preserved, even for None state)
let prev_gen = (prev_packed & GEN_MASK) as u32;
let new_gen = prev_gen.wrapping_add(1) & GEN_MASK as u32;
// Pack the new state
let new_packed = if let Some(t) = time {
// Clamp timestamp to 52-bit range. Ensure it's strictly less than TIME_MASK
// to avoid collision with the sentinel (None) value.
let mut nanos = t.as_nanos() & TIME_MASK;
// If nanos equals TIME_MASK, adjust it down by one time unit (4096 nanos)
// to avoid corrupting the generation counter bits (should never happen in practice).
if nanos == TIME_MASK {
nanos = TIME_MASK - 0x1000; // Subtract one 12-bit unit to keep lower bits clear
}
debug_assert!(nanos < TIME_MASK, "Timestamp value collides with sentinel");
// Pack: store nanos in upper 52 bits, gen in lower 12 bits
nanos | (new_gen as u64)
} else {
// Sentinel: time field all 1s, gen preserved in lower bits
TIME_MASK | (new_gen as u64)
};
// Try to atomically update if the state hasn't changed
match self.expiration_state.compare_exchange_weak(
prev_packed,
new_packed,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return new_gen, // Successfully updated
Err(_) => continue, // State changed, retry
}
}
}
}
#[cfg(feature = "unstable-debug-counters")]
impl<K> Drop for EntryInfo<K> {
fn drop(&mut self) {
super::debug_counters::InternalGlobalDebugCounters::entry_info_dropped();
}
}
impl<K> AccessTime for EntryInfo<K> {
#[inline]
fn last_accessed(&self) -> Option<Instant> {
self.last_accessed.instant()
}
#[inline]
fn set_last_accessed(&self, timestamp: Instant) {
self.last_accessed.set_instant(timestamp);
}
#[inline]
fn last_modified(&self) -> Option<Instant> {
self.last_modified.instant()
}
#[inline]
fn set_last_modified(&self, timestamp: Instant) {
self.last_modified.set_instant(timestamp);
}
}
@@ -0,0 +1,140 @@
use super::constants::LOG_SYNC_INTERVAL_MILLIS;
use super::constants::{READ_LOG_FLUSH_POINT, WRITE_LOG_FLUSH_POINT};
use crate::common::time::{AtomicInstant, Instant};
use crate::common::HousekeeperConfig;
use parking_lot::{Mutex, MutexGuard};
use std::{
sync::atomic::{AtomicBool, Ordering},
time::Duration,
};
pub(crate) trait InnerSync {
/// Runs the pending tasks. Returns `true` if there are more entries to evict in
/// next run.
fn run_pending_tasks(
&self,
timeout: Option<Duration>,
max_log_sync_repeats: u32,
eviction_batch_size: u32,
) -> bool;
fn now(&self) -> Instant;
}
pub(crate) struct Housekeeper {
run_lock: Mutex<()>,
run_after: AtomicInstant,
/// A flag to indicate if the last call on `run_pending_tasks` method left some
/// entries to evict.
///
/// Used only when the eviction listener closure is set for this cache instance
/// because, if not, `run_pending_tasks` will never leave entries to evict.
more_entries_to_evict: Option<AtomicBool>,
/// The timeout duration for the `run_pending_tasks` method. This is a safe-guard
/// to prevent cache read/write operations (that may call `run_pending_tasks`
/// internally) from being blocked for a long time when the user wrote a slow
/// eviction listener closure.
///
/// Used only when the eviction listener closure is set for this cache instance.
maintenance_task_timeout: Option<Duration>,
/// The maximum repeat count for receiving operation logs from the read and write
/// log channels. Default: `MAX_LOG_SYNC_REPEATS`.
max_log_sync_repeats: u32,
/// The batch size of entries to be processed by each internal eviction method.
/// Default: `EVICTION_BATCH_SIZE`.
eviction_batch_size: u32,
auto_run_enabled: AtomicBool,
}
impl Housekeeper {
pub(crate) fn new(
is_eviction_listener_enabled: bool,
config: HousekeeperConfig,
now: Instant,
) -> Self {
let (more_entries_to_evict, maintenance_task_timeout) = if is_eviction_listener_enabled {
(
Some(AtomicBool::new(false)),
Some(config.maintenance_task_timeout),
)
} else {
(None, None)
};
Self {
run_lock: Mutex::default(),
run_after: AtomicInstant::new(Self::sync_after(now)),
more_entries_to_evict,
maintenance_task_timeout,
max_log_sync_repeats: config.max_log_sync_repeats,
eviction_batch_size: config.eviction_batch_size,
auto_run_enabled: AtomicBool::new(true),
}
}
pub(crate) fn should_apply_reads(&self, ch_len: usize, now: Instant) -> bool {
self.more_entries_to_evict() || self.should_apply(ch_len, READ_LOG_FLUSH_POINT, now)
}
pub(crate) fn should_apply_writes(&self, ch_len: usize, now: Instant) -> bool {
self.more_entries_to_evict() || self.should_apply(ch_len, WRITE_LOG_FLUSH_POINT, now)
}
#[inline]
fn more_entries_to_evict(&self) -> bool {
self.more_entries_to_evict
.as_ref()
.map(|v| v.load(Ordering::Acquire))
.unwrap_or(false)
}
fn set_more_entries_to_evict(&self, v: bool) {
if let Some(flag) = &self.more_entries_to_evict {
flag.store(v, Ordering::Release);
}
}
#[inline]
fn should_apply(&self, ch_len: usize, ch_flush_point: usize, now: Instant) -> bool {
self.auto_run_enabled.load(Ordering::Relaxed)
&& (ch_len >= ch_flush_point || now >= self.run_after.instant().unwrap())
}
pub(crate) fn run_pending_tasks<T: InnerSync>(&self, cache: &T) {
let lock = self.run_lock.lock();
self.do_run_pending_tasks(cache, lock);
}
pub(crate) fn try_run_pending_tasks<T: InnerSync>(&self, cache: &T) -> bool {
if let Some(lock) = self.run_lock.try_lock() {
self.do_run_pending_tasks(cache, lock);
true
} else {
false
}
}
fn do_run_pending_tasks<T: InnerSync>(&self, cache: &T, _lock: MutexGuard<'_, ()>) {
let now = cache.now();
self.run_after.set_instant(Self::sync_after(now));
let timeout = self.maintenance_task_timeout;
let repeats = self.max_log_sync_repeats;
let batch_size = self.eviction_batch_size;
let more_to_evict = cache.run_pending_tasks(timeout, repeats, batch_size);
self.set_more_entries_to_evict(more_to_evict);
}
fn sync_after(now: Instant) -> Instant {
let dur = Duration::from_millis(LOG_SYNC_INTERVAL_MILLIS);
now.saturating_add(dur)
}
}
#[cfg(test)]
impl Housekeeper {
pub(crate) fn disable_auto_run(&self) {
self.auto_run_enabled.store(false, Ordering::Relaxed);
}
}
+788
View File
@@ -0,0 +1,788 @@
// License and Copyright Notice:
//
// Some of the code and doc comments in this module were copied from
// `std::collections::LinkedList` in the Rust standard library.
// https://github.com/rust-lang/rust/blob/master/src/liballoc/collections/linked_list.rs
//
// The original code/comments from LinkedList are dual-licensed under
// the Apache License, Version 2.0 <https://github.com/rust-lang/rust/blob/master/LICENSE-APACHE>
// or the MIT license <https://github.com/rust-lang/rust/blob/master/LICENSE-MIT>
//
// Copyrights of the original code/comments are retained by their contributors.
// For full authorship information, see the version control history of
// https://github.com/rust-lang/rust/ or https://thanks.rust-lang.org
use std::{marker::PhantomData, ptr::NonNull};
use super::CacheRegion;
#[cfg(feature = "unstable-debug-counters")]
use crate::common::concurrent::debug_counters;
// `crate::{sync,unsync}::DeqNodes` uses a `tagptr::TagNonNull<DeqNode<T>, 2>`
// pointer. To reserve the space for the 2-bit tag, use 4 bytes as the *minimum*
// alignment.
// https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers
#[repr(align(4))]
#[derive(PartialEq, Eq)]
pub(crate) struct DeqNode<T> {
next: Option<NonNull<DeqNode<T>>>,
prev: Option<NonNull<DeqNode<T>>>,
pub(crate) element: T,
}
impl<T> std::fmt::Debug for DeqNode<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeqNode")
.field("next", &self.next)
.field("prev", &self.prev)
.finish()
}
}
impl<T> DeqNode<T> {
pub(crate) fn new(element: T) -> Self {
#[cfg(feature = "unstable-debug-counters")]
debug_counters::InternalGlobalDebugCounters::deq_node_created();
Self {
next: None,
prev: None,
element,
}
}
pub(crate) fn next_node_ptr(this: NonNull<Self>) -> Option<NonNull<DeqNode<T>>> {
unsafe { this.as_ref() }.next
}
}
#[cfg(feature = "unstable-debug-counters")]
impl<T> Drop for DeqNode<T> {
fn drop(&mut self) {
debug_counters::InternalGlobalDebugCounters::deq_node_dropped();
}
}
/// Cursor is used to remember the current iterating position.
enum DeqCursor<T> {
Node(NonNull<DeqNode<T>>),
Done,
}
pub(crate) struct Deque<T> {
region: CacheRegion,
len: usize,
head: Option<NonNull<DeqNode<T>>>,
tail: Option<NonNull<DeqNode<T>>>,
cursor: Option<DeqCursor<T>>,
marker: PhantomData<Box<DeqNode<T>>>,
}
impl<T> Drop for Deque<T> {
fn drop(&mut self) {
struct DropGuard<'a, T>(&'a mut Deque<T>);
impl<T> Drop for DropGuard<'_, T> {
fn drop(&mut self) {
// Continue the same loop we do below. This only runs when a destructor has
// panicked. If another one panics this will abort.
while self.0.pop_front().is_some() {}
}
}
while let Some(node) = self.pop_front() {
let guard = DropGuard(self);
drop(node);
std::mem::forget(guard);
}
}
}
// Inner crate public function/methods
impl<T> Deque<T> {
pub(crate) fn new(region: CacheRegion) -> Self {
Self {
region,
len: 0,
head: None,
tail: None,
cursor: None,
marker: PhantomData,
}
}
pub(crate) fn region(&self) -> CacheRegion {
self.region
}
pub(crate) fn len(&self) -> usize {
self.len
}
pub(crate) fn contains(&self, node: &DeqNode<T>) -> bool {
node.prev.is_some() || self.is_head(node)
}
pub(crate) fn peek_front(&self) -> Option<&DeqNode<T>> {
self.head.as_ref().map(|node| unsafe { node.as_ref() })
}
pub(crate) fn peek_front_ptr(&self) -> Option<NonNull<DeqNode<T>>> {
self.head.as_ref().copied()
}
/// Removes and returns the node at the front of the list.
pub(crate) fn pop_front(&mut self) -> Option<Box<DeqNode<T>>> {
// This method takes care not to create mutable references to whole nodes,
// to maintain validity of aliasing pointers into `element`.
self.head.map(|node| unsafe {
if self.is_at_cursor(node.as_ref()) {
self.advance_cursor();
}
let mut node = Box::from_raw(node.as_ptr());
self.head = node.next;
match self.head {
None => self.tail = None,
// Not creating new mutable (unique!) references overlapping `element`.
Some(head) => (*head.as_ptr()).prev = None,
}
self.len -= 1;
node.prev = None;
node.next = None;
node
})
}
pub(crate) fn peek_back(&self) -> Option<&DeqNode<T>> {
self.tail.as_ref().map(|node| unsafe { node.as_ref() })
}
/// Adds the given node to the back of the list.
pub(crate) fn push_back(&mut self, mut node: Box<DeqNode<T>>) -> NonNull<DeqNode<T>> {
// This method takes care not to create mutable references to whole nodes,
// to maintain validity of aliasing pointers into `element`.
unsafe {
node.next = None;
node.prev = self.tail;
let node = NonNull::new(Box::into_raw(node)).expect("Got a null ptr");
match self.tail {
None => self.head = Some(node),
// Not creating new mutable (unique!) references overlapping `element`.
Some(tail) => (*tail.as_ptr()).next = Some(node),
}
self.tail = Some(node);
self.len += 1;
node
}
}
pub(crate) unsafe fn move_to_back(&mut self, mut node: NonNull<DeqNode<T>>) {
if self.is_tail(node.as_ref()) {
// Already at the tail. Nothing to do.
return;
}
if self.is_at_cursor(node.as_ref()) {
self.advance_cursor();
}
// Extract the prev and next pointers before we start modifying the node
let (prev, next) = {
let node_ref = node.as_ref();
(node_ref.prev, node_ref.next)
};
// Not creating new mutable (unique!) references overlapping `element`.
match prev {
Some(prev_node) if next.is_some() => (*prev_node.as_ptr()).next = next,
Some(..) => (),
// This node is the head node.
None => self.head = next,
};
// This node is not the tail node.
if let Some(next_node) = next {
(*next_node.as_ptr()).prev = prev;
// Update the node's pointers directly without creating conflicting references
let node_mut = node.as_mut();
node_mut.prev = self.tail;
node_mut.next = None;
match self.tail {
// Not creating new mutable (unique!) references overlapping `element`.
Some(tail) => (*tail.as_ptr()).next = Some(node),
None => unreachable!(),
}
self.tail = Some(node);
}
}
pub(crate) fn move_front_to_back(&mut self) {
if let Some(node) = self.head {
unsafe { self.move_to_back(node) };
}
}
/// Unlinks the specified node from the current list.
///
/// This method takes care not to create mutable references to `element`, to
/// maintain validity of aliasing pointers.
///
/// IMPORTANT: This method does not drop the node. If the node is no longer
/// needed, use `unlink_and_drop` instead, or drop it at the caller side.
/// Otherwise, the node will leak.
pub(crate) unsafe fn unlink(&mut self, mut node: NonNull<DeqNode<T>>) {
if self.is_at_cursor(node.as_ref()) {
self.advance_cursor();
}
let node = node.as_mut(); // this one is ours now, we can create an &mut.
// Not creating new mutable (unique!) references overlapping `element`.
match node.prev {
Some(prev) => (*prev.as_ptr()).next = node.next,
// this node is the head node
None => self.head = node.next,
};
match node.next {
Some(next) => (*next.as_ptr()).prev = node.prev,
// this node is the tail node
None => self.tail = node.prev,
};
node.prev = None;
node.next = None;
self.len -= 1;
}
/// Unlinks the specified node from the current list, and then drop the node.
///
/// This method takes care not to create mutable references to `element`, to
/// maintain validity of aliasing pointers.
///
/// Panics:
pub(crate) unsafe fn unlink_and_drop(&mut self, node: NonNull<DeqNode<T>>) {
self.unlink(node);
std::mem::drop(Box::from_raw(node.as_ptr()));
}
pub(crate) fn reset_cursor(&mut self) {
self.cursor = None;
}
}
impl<'a, T> Iterator for &'a mut Deque<T> {
type Item = &'a T;
fn next(&mut self) -> Option<Self::Item> {
if self.cursor.is_none() {
if let Some(head) = self.head {
self.cursor = Some(DeqCursor::Node(head));
}
}
let elem = if let Some(DeqCursor::Node(node)) = self.cursor {
unsafe { Some(&(*node.as_ptr()).element) }
} else {
None
};
self.advance_cursor();
elem
}
}
// Private function/methods
impl<T> Deque<T> {
fn is_head(&self, node: &DeqNode<T>) -> bool {
if let Some(head) = self.head {
std::ptr::eq(unsafe { head.as_ref() }, node)
} else {
false
}
}
fn is_tail(&self, node: &DeqNode<T>) -> bool {
if let Some(tail) = self.tail {
std::ptr::eq(unsafe { tail.as_ref() }, node)
} else {
false
}
}
fn is_at_cursor(&self, node: &DeqNode<T>) -> bool {
if let Some(DeqCursor::Node(cur_node)) = self.cursor {
std::ptr::eq(unsafe { cur_node.as_ref() }, node)
} else {
false
}
}
fn advance_cursor(&mut self) {
match self.cursor.take() {
None => (),
Some(DeqCursor::Node(node)) => unsafe {
if let Some(next) = (*node.as_ptr()).next {
self.cursor = Some(DeqCursor::Node(next));
} else {
self.cursor = Some(DeqCursor::Done);
}
},
Some(DeqCursor::Done) => {
self.cursor = None;
}
}
}
}
#[cfg(test)]
mod tests {
use super::{CacheRegion::MainProbation, DeqNode, Deque};
#[test]
#[allow(clippy::cognitive_complexity)]
fn basics() {
let mut deque: Deque<String> = Deque::new(MainProbation);
assert_eq!(deque.len(), 0);
assert!(deque.peek_front().is_none());
assert!(deque.peek_back().is_none());
// push_back(node1)
let node1 = DeqNode::new("a".to_string());
assert!(!deque.contains(&node1));
let node1 = Box::new(node1);
let node1_ptr = deque.push_back(node1);
assert_eq!(deque.len(), 1);
// peek_front() -> node1
let head_a = deque.peek_front().unwrap();
assert!(deque.contains(head_a));
assert!(deque.is_head(head_a));
assert!(deque.is_tail(head_a));
assert_eq!(head_a.element, "a".to_string());
// move_to_back(node1)
unsafe { deque.move_to_back(node1_ptr) };
assert_eq!(deque.len(), 1);
// peek_front() -> node1
let head_b = deque.peek_front().unwrap();
assert!(deque.contains(head_b));
assert!(deque.is_head(head_b));
assert!(deque.is_tail(head_b));
assert!(std::ptr::eq(head_b, node1_ptr.as_ptr()));
assert!(head_b.prev.is_none());
assert!(head_b.next.is_none());
// peek_back() -> node1
let tail_a = deque.peek_back().unwrap();
assert!(deque.contains(tail_a));
assert!(deque.is_head(tail_a));
assert!(deque.is_tail(tail_a));
assert!(std::ptr::eq(tail_a, node1_ptr.as_ptr()));
assert!(tail_a.prev.is_none());
assert!(tail_a.next.is_none());
// push_back(node2)
let node2 = DeqNode::new("b".to_string());
assert!(!deque.contains(&node2));
let node2_ptr = deque.push_back(Box::new(node2));
assert_eq!(deque.len(), 2);
// peek_front() -> node1
let head_c = deque.peek_front().unwrap();
assert!(deque.contains(head_c));
assert!(deque.is_head(head_c));
assert!(!deque.is_tail(head_c));
assert!(std::ptr::eq(head_c, node1_ptr.as_ptr()));
assert!(head_c.prev.is_none());
assert!(std::ptr::eq(
head_c.next.unwrap().as_ptr(),
node2_ptr.as_ptr()
));
// move_to_back(node2)
unsafe { deque.move_to_back(node2_ptr) };
assert_eq!(deque.len(), 2);
// peek_front() -> node1
let head_d = deque.peek_front().unwrap();
assert!(deque.contains(head_d));
assert!(deque.is_head(head_d));
assert!(!deque.is_tail(head_d));
assert!(std::ptr::eq(head_d, node1_ptr.as_ptr()));
assert!(head_d.prev.is_none());
assert!(std::ptr::eq(
head_d.next.unwrap().as_ptr(),
node2_ptr.as_ptr()
));
// peek_back() -> node2
let tail_b = deque.peek_back().unwrap();
assert!(deque.contains(tail_b));
assert!(!deque.is_head(tail_b));
assert!(deque.is_tail(tail_b));
assert!(std::ptr::eq(tail_b, node2_ptr.as_ptr()));
assert!(std::ptr::eq(
tail_b.prev.unwrap().as_ptr(),
node1_ptr.as_ptr()
));
assert_eq!(tail_b.element, "b".to_string());
assert!(tail_b.next.is_none());
// move_to_back(node1)
unsafe { deque.move_to_back(node1_ptr) };
assert_eq!(deque.len(), 2);
// peek_front() -> node2
let head_e = deque.peek_front().unwrap();
assert!(deque.contains(head_e));
assert!(deque.is_head(head_e));
assert!(!deque.is_tail(head_e));
assert!(std::ptr::eq(head_e, node2_ptr.as_ptr()));
assert!(head_e.prev.is_none());
assert!(std::ptr::eq(
head_e.next.unwrap().as_ptr(),
node1_ptr.as_ptr()
));
// peek_back() -> node1
let tail_c = deque.peek_back().unwrap();
assert!(deque.contains(tail_c));
assert!(!deque.is_head(tail_c));
assert!(deque.is_tail(tail_c));
assert!(std::ptr::eq(tail_c, node1_ptr.as_ptr()));
assert!(std::ptr::eq(
tail_c.prev.unwrap().as_ptr(),
node2_ptr.as_ptr()
));
assert!(tail_c.next.is_none());
// push_back(node3)
let node3 = DeqNode::new("c".to_string());
assert!(!deque.contains(&node3));
let node3_ptr = deque.push_back(Box::new(node3));
assert_eq!(deque.len(), 3);
// peek_front() -> node2
let head_f = deque.peek_front().unwrap();
assert!(deque.contains(head_f));
assert!(deque.is_head(head_f));
assert!(!deque.is_tail(head_f));
assert!(std::ptr::eq(head_f, node2_ptr.as_ptr()));
assert!(head_f.prev.is_none());
assert!(std::ptr::eq(
head_f.next.unwrap().as_ptr(),
node1_ptr.as_ptr()
));
// peek_back() -> node3
let tail_d = deque.peek_back().unwrap();
assert!(std::ptr::eq(tail_d, node3_ptr.as_ptr()));
assert_eq!(tail_d.element, "c".to_string());
assert!(deque.contains(tail_d));
assert!(!deque.is_head(tail_d));
assert!(deque.is_tail(tail_d));
assert!(std::ptr::eq(tail_d, node3_ptr.as_ptr()));
assert!(std::ptr::eq(
tail_d.prev.unwrap().as_ptr(),
node1_ptr.as_ptr()
));
assert!(tail_d.next.is_none());
// move_to_back(node1)
unsafe { deque.move_to_back(node1_ptr) };
assert_eq!(deque.len(), 3);
// peek_front() -> node2
let head_g = deque.peek_front().unwrap();
assert!(deque.contains(head_g));
assert!(deque.is_head(head_g));
assert!(!deque.is_tail(head_g));
assert!(std::ptr::eq(head_g, node2_ptr.as_ptr()));
assert!(head_g.prev.is_none());
assert!(std::ptr::eq(
head_g.next.unwrap().as_ptr(),
node3_ptr.as_ptr()
));
// peek_back() -> node1
let tail_e = deque.peek_back().unwrap();
assert!(deque.contains(tail_e));
assert!(!deque.is_head(tail_e));
assert!(deque.is_tail(tail_e));
assert!(std::ptr::eq(tail_e, node1_ptr.as_ptr()));
assert!(std::ptr::eq(
tail_e.prev.unwrap().as_ptr(),
node3_ptr.as_ptr()
));
assert!(tail_e.next.is_none());
// unlink(node3)
unsafe { deque.unlink(node3_ptr) };
assert_eq!(deque.len(), 2);
let node3_ref = unsafe { node3_ptr.as_ref() };
assert!(!deque.contains(node3_ref));
assert!(node3_ref.next.is_none());
assert!(node3_ref.next.is_none());
std::mem::drop(unsafe { Box::from_raw(node3_ptr.as_ptr()) });
// peek_front() -> node2
let head_h = deque.peek_front().unwrap();
assert!(deque.contains(head_h));
assert!(deque.is_head(head_h));
assert!(!deque.is_tail(head_h));
assert!(std::ptr::eq(head_h, node2_ptr.as_ptr()));
assert!(head_h.prev.is_none());
assert!(std::ptr::eq(
head_h.next.unwrap().as_ptr(),
node1_ptr.as_ptr()
));
// peek_back() -> node1
let tail_f = deque.peek_back().unwrap();
assert!(deque.contains(tail_f));
assert!(!deque.is_head(tail_f));
assert!(deque.is_tail(tail_f));
assert!(std::ptr::eq(tail_f, node1_ptr.as_ptr()));
assert!(std::ptr::eq(
tail_f.prev.unwrap().as_ptr(),
node2_ptr.as_ptr()
));
assert!(tail_f.next.is_none());
// unlink(node2)
unsafe { deque.unlink(node2_ptr) };
assert_eq!(deque.len(), 1);
let node2_ref = unsafe { node2_ptr.as_ref() };
assert!(!deque.contains(node2_ref));
assert!(node2_ref.next.is_none());
assert!(node2_ref.next.is_none());
std::mem::drop(unsafe { Box::from_raw(node2_ptr.as_ptr()) });
// peek_front() -> node1
let head_g = deque.peek_front().unwrap();
assert!(deque.contains(head_g));
assert!(deque.is_head(head_g));
assert!(deque.is_tail(head_g));
assert!(std::ptr::eq(head_g, node1_ptr.as_ptr()));
assert!(head_g.prev.is_none());
assert!(head_g.next.is_none());
// peek_back() -> node1
let tail_g = deque.peek_back().unwrap();
assert!(deque.contains(tail_g));
assert!(deque.is_head(tail_g));
assert!(deque.is_tail(tail_g));
assert!(std::ptr::eq(tail_g, node1_ptr.as_ptr()));
assert!(tail_g.next.is_none());
assert!(tail_g.next.is_none());
// unlink(node1)
unsafe { deque.unlink(node1_ptr) };
assert_eq!(deque.len(), 0);
let node1_ref = unsafe { node1_ptr.as_ref() };
assert!(!deque.contains(node1_ref));
assert!(node1_ref.next.is_none());
assert!(node1_ref.next.is_none());
std::mem::drop(unsafe { Box::from_raw(node1_ptr.as_ptr()) });
// peek_front() -> node1
let head_h = deque.peek_front();
assert!(head_h.is_none());
// peek_back() -> node1
let tail_e = deque.peek_back();
assert!(tail_e.is_none());
}
#[test]
fn iter() {
let mut deque: Deque<String> = Deque::new(MainProbation);
assert!((&mut deque).next().is_none());
let node1 = DeqNode::new("a".into());
deque.push_back(Box::new(node1));
let node2 = DeqNode::new("b".into());
let node2_ptr = deque.push_back(Box::new(node2));
let node3 = DeqNode::new("c".into());
let node3_ptr = deque.push_back(Box::new(node3));
// -------------------------------------------------------
// First iteration.
assert_eq!((&mut deque).next(), Some(&"a".into()));
assert_eq!((&mut deque).next(), Some(&"b".into()));
assert_eq!((&mut deque).next(), Some(&"c".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Ensure the iterator restarts.
assert_eq!((&mut deque).next(), Some(&"a".into()));
assert_eq!((&mut deque).next(), Some(&"b".into()));
assert_eq!((&mut deque).next(), Some(&"c".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Ensure reset_cursor works.
assert_eq!((&mut deque).next(), Some(&"a".into()));
assert_eq!((&mut deque).next(), Some(&"b".into()));
deque.reset_cursor();
assert_eq!((&mut deque).next(), Some(&"a".into()));
assert_eq!((&mut deque).next(), Some(&"b".into()));
assert_eq!((&mut deque).next(), Some(&"c".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Try to move_to_back during iteration.
assert_eq!((&mut deque).next(), Some(&"a".into()));
// Next will be "b", but we move it to the back.
unsafe { deque.move_to_back(node2_ptr) };
// Now, next should be "c", and then "b".
assert_eq!((&mut deque).next(), Some(&"c".into()));
assert_eq!((&mut deque).next(), Some(&"b".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Try to unlink during iteration.
assert_eq!((&mut deque).next(), Some(&"a".into()));
// Next will be "c", but we unlink it.
unsafe { deque.unlink_and_drop(node3_ptr) };
// Now, next should be "b".
assert_eq!((&mut deque).next(), Some(&"b".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Try pop_front during iteration.
let node3 = DeqNode::new("c".into());
deque.push_back(Box::new(node3));
assert_eq!((&mut deque).next(), Some(&"a".into()));
// Next will be "b", but we call pop_front twice to remove "a" and "b".
deque.pop_front(); // "a"
deque.pop_front(); // "b"
// Now, next should be "c".
assert_eq!((&mut deque).next(), Some(&"c".into()));
assert!((&mut deque).next().is_none());
// -------------------------------------------------------
// Check iterating on an empty deque.
deque.pop_front(); // "c"
assert!((&mut deque).next().is_none());
assert!((&mut deque).next().is_none());
}
#[test]
fn next_node() {
let mut deque: Deque<String> = Deque::new(MainProbation);
let node1 = DeqNode::new("a".into());
deque.push_back(Box::new(node1));
let node2 = DeqNode::new("b".into());
let node2_ptr = deque.push_back(Box::new(node2));
let node3 = DeqNode::new("c".into());
let node3_ptr = deque.push_back(Box::new(node3));
// -------------------------------------------------------
// First iteration.
// peek_front() -> node1
let node1a = deque.peek_front_ptr().unwrap();
assert_eq!(unsafe { node1a.as_ref() }.element, "a".to_string());
let node2a = DeqNode::next_node_ptr(node1a).unwrap();
assert_eq!(unsafe { node2a.as_ref() }.element, "b".to_string());
let node3a = DeqNode::next_node_ptr(node2a).unwrap();
assert_eq!(unsafe { node3a.as_ref() }.element, "c".to_string());
assert!(DeqNode::next_node_ptr(node3a).is_none());
// -------------------------------------------------------
// Iterate after a move_to_back.
// Move "b" to the back. So now "a" -> "c" -> "b".
unsafe { deque.move_to_back(node2_ptr) };
let node1a = deque.peek_front_ptr().unwrap();
assert_eq!(unsafe { node1a.as_ref() }.element, "a".to_string());
let node3a = DeqNode::next_node_ptr(node1a).unwrap();
assert_eq!(unsafe { node3a.as_ref() }.element, "c".to_string());
let node2a = DeqNode::next_node_ptr(node3a).unwrap();
assert_eq!(unsafe { node2a.as_ref() }.element, "b".to_string());
assert!(DeqNode::next_node_ptr(node2a).is_none());
// -------------------------------------------------------
// Iterate after an unlink.
// Unlink the second node "c". Now "a" -> "c".
unsafe { deque.unlink_and_drop(node3_ptr) };
let node1a = deque.peek_front_ptr().unwrap();
assert_eq!(unsafe { node1a.as_ref() }.element, "a".to_string());
let node2a = DeqNode::next_node_ptr(node1a).unwrap();
assert_eq!(unsafe { node2a.as_ref() }.element, "b".to_string());
assert!(DeqNode::next_node_ptr(node2a).is_none());
}
#[test]
fn peek_and_move_to_back() {
let mut deque: Deque<String> = Deque::new(MainProbation);
let node1 = DeqNode::new("a".into());
deque.push_back(Box::new(node1));
let node2 = DeqNode::new("b".into());
let _ = deque.push_back(Box::new(node2));
let node3 = DeqNode::new("c".into());
let _ = deque.push_back(Box::new(node3));
// "a" -> "b" -> "c"
let node1a = deque.peek_front_ptr().unwrap();
assert_eq!(unsafe { node1a.as_ref() }.element, "a".to_string());
unsafe { deque.move_to_back(node1a) };
// "b" -> "c" -> "a"
let node2a = deque.peek_front_ptr().unwrap();
assert_eq!(unsafe { node2a.as_ref() }.element, "b".to_string());
let node3a = DeqNode::next_node_ptr(node2a).unwrap();
assert_eq!(unsafe { node3a.as_ref() }.element, "c".to_string());
unsafe { deque.move_to_back(node3a) };
// "b" -> "a" -> "c"
deque.move_front_to_back();
// "a" -> "c" -> "b"
let node1b = deque.peek_front().unwrap();
assert_eq!(node1b.element, "a".to_string());
}
#[test]
fn drop() {
use std::{cell::RefCell, rc::Rc};
struct X(u32, Rc<RefCell<Vec<u32>>>);
impl Drop for X {
fn drop(&mut self) {
self.1.borrow_mut().push(self.0)
}
}
let mut deque: Deque<X> = Deque::new(MainProbation);
let dropped = Rc::new(RefCell::new(Vec::default()));
let node1 = DeqNode::new(X(1, Rc::clone(&dropped)));
let node2 = DeqNode::new(X(2, Rc::clone(&dropped)));
let node3 = DeqNode::new(X(3, Rc::clone(&dropped)));
let node4 = DeqNode::new(X(4, Rc::clone(&dropped)));
deque.push_back(Box::new(node1));
deque.push_back(Box::new(node2));
deque.push_back(Box::new(node3));
deque.push_back(Box::new(node4));
assert_eq!(deque.len(), 4);
std::mem::drop(deque);
assert_eq!(*dropped.borrow(), &[1, 2, 3, 4]);
}
}
+92
View File
@@ -0,0 +1,92 @@
use std::{fmt::Debug, sync::Arc};
/// A snapshot of a single entry in the cache.
///
/// `Entry` is constructed from the methods like `or_insert` on the struct returned
/// by cache's `entry` or `entry_by_ref` methods. `Entry` holds the cached key and
/// value at the time it was constructed. It also carries extra information about the
/// entry; [`is_fresh`](#method.is_fresh) method returns `true` if the value was not
/// cached and was freshly computed.
///
/// See the followings for more information about `entry` and `entry_by_ref` methods:
///
/// - `sync::Cache`:
/// - [`entry`](./sync/struct.Cache.html#method.entry)
/// - [`entry_by_ref`](./sync/struct.Cache.html#method.entry_by_ref)
/// - `future::Cache`:
/// - [`entry`](./future/struct.Cache.html#method.entry)
/// - [`entry_by_ref`](./future/struct.Cache.html#method.entry_by_ref)
///
pub struct Entry<K, V> {
key: Option<Arc<K>>,
value: V,
is_fresh: bool,
is_old_value_replaced: bool,
}
impl<K, V> Debug for Entry<K, V>
where
K: Debug,
V: Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Entry")
.field("key", self.key())
.field("value", &self.value)
.field("is_fresh", &self.is_fresh)
.field("is_old_value_replaced", &self.is_old_value_replaced)
.finish()
}
}
impl<K, V> Entry<K, V> {
pub(crate) fn new(
key: Option<Arc<K>>,
value: V,
is_fresh: bool,
is_old_value_replaced: bool,
) -> Self {
Self {
key,
value,
is_fresh,
is_old_value_replaced,
}
}
/// Returns a reference to the wrapped key.
pub fn key(&self) -> &K {
self.key.as_ref().expect("Bug: Key is None")
}
/// Returns a reference to the wrapped value.
///
/// Note that the returned reference is _not_ pointing to the original value in
/// the cache. Instead, it is pointing to the cloned value in this `Entry`.
pub fn value(&self) -> &V {
&self.value
}
/// Consumes this `Entry`, returning the wrapped value.
///
/// Note that the returned value is a clone of the original value in the cache.
/// It was cloned when this `Entry` was constructed.
pub fn into_value(self) -> V {
self.value
}
/// Returns `true` if the value in this `Entry` was not cached and was freshly
/// computed.
pub fn is_fresh(&self) -> bool {
self.is_fresh
}
/// Returns `true` if an old value existed in the cache and was replaced by the
/// value in this `Entry`.
///
/// Note that the new value can be the same as the old value. This method still
/// returns `true` in that case.
pub fn is_old_value_replaced(&self) -> bool {
self.is_old_value_replaced
}
}
+31
View File
@@ -0,0 +1,31 @@
use std::{error::Error, fmt::Display};
/// The error type for the functionalities around
/// [`Cache::invalidate_entries_if`][invalidate-if] method.
///
/// [invalidate-if]: ./sync/struct.Cache.html#method.invalidate_entries_if
#[derive(Debug)]
pub enum PredicateError {
/// This cache does not have a necessary configuration enabled to support
/// invalidating entries with a closure.
///
/// To enable the configuration, call
/// [`CacheBuilder::support_invalidation_closures`][support-invalidation-closures]
/// method at the cache creation time.
///
/// [support-invalidation-closures]: ./sync/struct.CacheBuilder.html#method.support_invalidation_closures
InvalidationClosuresDisabled,
}
impl Display for PredicateError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Support for invalidation closures is disabled in this cache. \
Please enable it by calling the support_invalidation_closures \
method of the builder at the cache creation time",
)
}
}
impl Error for PredicateError {}
+397
View File
@@ -0,0 +1,397 @@
// License and Copyright Notice:
//
// Some of the code and doc comments in this module were ported or copied from
// a Java class `com.github.benmanes.caffeine.cache.FrequencySketch` of Caffeine.
// https://github.com/ben-manes/caffeine/blob/master/caffeine/src/main/java/com/github/benmanes/caffeine/cache/FrequencySketch.java
//
// The original code/comments from Caffeine are licensed under the Apache License,
// Version 2.0 <https://github.com/ben-manes/caffeine/blob/master/LICENSE>
//
// Copyrights of the original code/comments are retained by their contributors.
// For full authorship information, see the version control history of
// https://github.com/ben-manes/caffeine/
/// A probabilistic multi-set for estimating the popularity of an element within
/// a time window. The maximum frequency of an element is limited to 15 (4-bits)
/// and an aging process periodically halves the popularity of all elements.
#[derive(Default)]
pub(crate) struct FrequencySketch {
sample_size: u32,
table_mask: u64,
table: Box<[u64]>,
size: u32,
}
// A mixture of seeds from FNV-1a, CityHash, and Murmur3. (Taken from Caffeine)
static SEED: [u64; 4] = [
0xc3a5_c85c_97cb_3127,
0xb492_b66f_be98_f273,
0x9ae1_6a3b_2f90_404f,
0xcbf2_9ce4_8422_2325,
];
static RESET_MASK: u64 = 0x7777_7777_7777_7777;
static ONE_MASK: u64 = 0x1111_1111_1111_1111;
// -------------------------------------------------------------------------------
// Some of the code and doc comments in this module were ported or copied from
// a Java class `com.github.benmanes.caffeine.cache.FrequencySketch` of Caffeine.
// https://github.com/ben-manes/caffeine/blob/master/caffeine/src/main/java/com/github/benmanes/caffeine/cache/FrequencySketch.java
// -------------------------------------------------------------------------------
//
// FrequencySketch maintains a 4-bit CountMinSketch [1] with periodic aging to
// provide the popularity history for the TinyLfu admission policy [2].
// The time and space efficiency of the sketch allows it to cheaply estimate the
// frequency of an entry in a stream of cache access events.
//
// The counter matrix is represented as a single dimensional array holding 16
// counters per slot. A fixed depth of four balances the accuracy and cost,
// resulting in a width of four times the length of the array. To retain an
// accurate estimation the array's length equals the maximum number of entries
// in the cache, increased to the closest power-of-two to exploit more efficient
// bit masking. This configuration results in a confidence of 93.75% and error
// bound of e / width.
//
// The frequency of all entries is aged periodically using a sampling window
// based on the maximum number of entries in the cache. This is referred to as
// the reset operation by TinyLfu and keeps the sketch fresh by dividing all
// counters by two and subtracting based on the number of odd counters
// found. The O(n) cost of aging is amortized, ideal for hardware pre-fetching,
// and uses inexpensive bit manipulations per array location.
//
// [1] An Improved Data Stream Summary: The Count-Min Sketch and its Applications
// http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf
// [2] TinyLFU: A Highly Efficient Cache Admission Policy
// https://dl.acm.org/citation.cfm?id=3149371
//
// -------------------------------------------------------------------------------
impl FrequencySketch {
/// Initializes and increases the capacity of this `FrequencySketch` instance,
/// if necessary, to ensure that it can accurately estimate the popularity of
/// elements given the maximum size of the cache. This operation forgets all
/// previous counts when resizing.
pub(crate) fn ensure_capacity(&mut self, cap: u32) {
// The max byte size of the table, Box<[u64; table_size]>
//
// | Pointer width | Max size |
// |:-----------------|---------:|
// | 16 bit | 8 KiB |
// | 32 bit | 128 MiB |
// | 64 bit or bigger | 8 GiB |
let maximum = if cfg!(target_pointer_width = "16") {
cap.min(1024)
} else if cfg!(target_pointer_width = "32") {
cap.min(2u32.pow(24)) // about 16 millions
} else {
// Same to Caffeine's limit:
// `Integer.MAX_VALUE >>> 1` with `ceilingPowerOfTwo()` applied.
cap.min(2u32.pow(30)) // about 1 billion
};
let table_size = if maximum == 0 {
1
} else {
maximum.next_power_of_two()
};
if self.table.len() as u32 >= table_size {
return;
}
self.table = vec![0; table_size as usize].into_boxed_slice();
self.table_mask = table_size.saturating_sub(1) as u64;
self.sample_size = if cap == 0 {
10
} else {
maximum.saturating_mul(10).min(i32::MAX as u32)
};
}
/// Takes the hash value of an element, and returns the estimated number of
/// occurrences of the element, up to the maximum (15).
pub(crate) fn frequency(&self, hash: u64) -> u8 {
if self.table.is_empty() {
return 0;
}
let start = ((hash & 3) << 2) as u8;
let mut frequency = u8::MAX;
for i in 0..4 {
let index = self.index_of(hash, i);
let shift = (start + i) << 2;
let count = ((self.table[index] >> shift) & 0xF) as u8;
frequency = frequency.min(count);
}
frequency
}
/// Take a hash value of an element and increments the popularity of the
/// element if it does not exceed the maximum (15). The popularity of all
/// elements will be periodically down sampled when the observed events
/// exceeds a threshold. This process provides a frequency aging to allow
/// expired long term entries to fade away.
pub(crate) fn increment(&mut self, hash: u64) {
if self.table.is_empty() {
return;
}
let start = ((hash & 3) << 2) as u8;
let mut added = false;
for i in 0..4 {
let index = self.index_of(hash, i);
added |= self.increment_at(index, start + i);
}
if added {
self.size += 1;
if self.size >= self.sample_size {
self.reset();
}
}
}
/// Takes a table index (each entry has 16 counters) and counter index, and
/// increments the counter by 1 if it is not already at the maximum value
/// (15). Returns `true` if incremented.
fn increment_at(&mut self, table_index: usize, counter_index: u8) -> bool {
let offset = (counter_index as usize) << 2;
let mask = 0xF_u64 << offset;
if self.table[table_index] & mask != mask {
self.table[table_index] += 1u64 << offset;
true
} else {
false
}
}
/// Reduces every counter by half of its original value.
fn reset(&mut self) {
let mut count = 0u32;
for entry in self.table.iter_mut() {
// Count number of odd numbers.
count += (*entry & ONE_MASK).count_ones();
*entry = (*entry >> 1) & RESET_MASK;
}
self.size = (self.size >> 1) - (count >> 2);
}
/// Returns the table index for the counter at the specified depth.
fn index_of(&self, hash: u64, depth: u8) -> usize {
let i = depth as usize;
let mut hash = hash.wrapping_add(SEED[i]).wrapping_mul(SEED[i]);
hash = hash.wrapping_add(hash >> 32);
(hash & self.table_mask) as usize
}
#[cfg(feature = "unstable-debug-counters")]
pub(crate) fn table_size(&self) -> u64 {
(self.table.len() * std::mem::size_of::<u64>()) as u64
}
}
// Methods only available for testing.
#[cfg(test)]
impl FrequencySketch {
pub(crate) fn table_len(&self) -> usize {
self.table.len()
}
}
// Some test cases were ported from Caffeine at:
// https://github.com/ben-manes/caffeine/blob/master/caffeine/src/test/java/com/github/benmanes/caffeine/cache/FrequencySketchTest.java
//
// To see the debug prints, run test as `cargo test -- --nocapture`
#[cfg(test)]
mod tests {
use super::FrequencySketch;
use once_cell::sync::Lazy;
use std::hash::{BuildHasher, Hash};
static ITEM: Lazy<u32> = Lazy::new(|| {
let mut buf = [0; 4];
getrandom::getrandom(&mut buf).unwrap();
u32::from_ne_bytes(buf)
});
// This test was ported from Caffeine.
#[test]
fn increment_once() {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(512);
let hasher = hasher();
let item_hash = hasher(*ITEM);
sketch.increment(item_hash);
assert_eq!(sketch.frequency(item_hash), 1);
}
// This test was ported from Caffeine.
#[test]
fn increment_max() {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(512);
let hasher = hasher();
let item_hash = hasher(*ITEM);
for _ in 0..20 {
sketch.increment(item_hash);
}
assert_eq!(sketch.frequency(item_hash), 15);
}
// This test was ported from Caffeine.
#[test]
fn increment_distinct() {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(512);
let hasher = hasher();
sketch.increment(hasher(*ITEM));
sketch.increment(hasher(ITEM.wrapping_add(1)));
assert_eq!(sketch.frequency(hasher(*ITEM)), 1);
assert_eq!(sketch.frequency(hasher(ITEM.wrapping_add(1))), 1);
assert_eq!(sketch.frequency(hasher(ITEM.wrapping_add(2))), 0);
}
// This test was ported from Caffeine.
#[test]
fn index_of_around_zero() {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(512);
let mut indexes = std::collections::HashSet::new();
let hashes = [u64::MAX, 0, 1];
for hash in hashes.iter() {
for depth in 0..4 {
indexes.insert(sketch.index_of(*hash, depth));
}
}
assert_eq!(indexes.len(), 4 * hashes.len())
}
// This test was ported from Caffeine.
#[test]
fn reset() {
let mut reset = false;
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(64);
let hasher = hasher();
for i in 1..(20 * sketch.table.len() as u32) {
sketch.increment(hasher(i));
if sketch.size != i {
reset = true;
break;
}
}
assert!(reset);
assert!(sketch.size <= sketch.sample_size / 2);
}
// This test was ported from Caffeine.
#[test]
fn heavy_hitters() {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(65_536);
let hasher = hasher();
for i in 100..100_000 {
sketch.increment(hasher(i));
}
for i in (0..10).step_by(2) {
for _ in 0..i {
sketch.increment(hasher(i));
}
}
// A perfect popularity count yields an array [0, 0, 2, 0, 4, 0, 6, 0, 8, 0]
let popularity = (0..10)
.map(|i| sketch.frequency(hasher(i)))
.collect::<Vec<_>>();
for (i, freq) in popularity.iter().enumerate() {
match i {
2 => assert!(freq <= &popularity[4]),
4 => assert!(freq <= &popularity[6]),
6 => assert!(freq <= &popularity[8]),
8 => (),
_ => assert!(freq <= &popularity[2]),
}
}
}
fn hasher<K: Hash>() -> impl Fn(K) -> u64 {
let build_hasher = std::collections::hash_map::RandomState::default();
move |key| build_hasher.hash_one(&key)
}
}
// Verify that some properties hold such as no panic occurs on any possible inputs.
#[cfg(kani)]
mod kani {
use super::FrequencySketch;
const CAPACITIES: &[u32] = &[
0,
1,
1024,
1025,
2u32.pow(24),
2u32.pow(24) + 1,
2u32.pow(30),
2u32.pow(30) + 1,
u32::MAX,
];
#[kani::proof]
fn verify_ensure_capacity() {
// Check for arbitrary capacities.
let capacity = kani::any();
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(capacity);
}
#[kani::proof]
fn verify_frequency() {
// Check for some selected capacities.
for capacity in CAPACITIES {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(*capacity);
// Check for arbitrary hashes.
let hash = kani::any();
let frequency = sketch.frequency(hash);
assert!(frequency <= 15);
}
}
#[kani::proof]
fn verify_increment() {
// Only check for small capacities. Because Kani Rust Verifier is a model
// checking tool, it will take much longer time (exponential) to check larger
// capacities here.
for capacity in &[0, 1, 128] {
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(*capacity);
// Check for arbitrary hashes.
let hash = kani::any();
sketch.increment(hash);
}
}
#[kani::proof]
fn verify_index_of() {
// Check for arbitrary capacities.
let capacity = kani::any();
let mut sketch = FrequencySketch::default();
sketch.ensure_capacity(capacity);
// Check for arbitrary hashes.
let hash = kani::any();
for i in 0..4 {
let index = sketch.index_of(hash, i);
assert!(index < sketch.table.len());
}
}
}
+149
View File
@@ -0,0 +1,149 @@
use std::{hash::Hash, sync::Arc};
// This trait is implemented by `sync::BaseCache` and `sync::Cache`.
pub(crate) trait ScanningGet<K, V> {
/// Returns the number of segments in the concurrent hash table.
fn num_cht_segments(&self) -> usize;
/// Returns a _clone_ of the value corresponding to the key.
///
/// Unlike the `get` method of cache, this method is not considered a cache read
/// operation, so it does not update the historic popularity estimator or reset
/// the idle timer for the key.
fn scanning_get(&self, key: &Arc<K>) -> Option<V>;
/// Returns a vec of keys in a specified segment of the concurrent hash table.
fn keys(&self, cht_segment: usize) -> Option<Vec<Arc<K>>>;
}
/// Iterator visiting all key-value pairs in a cache in arbitrary order.
///
/// Call [`Cache::iter`](./struct.Cache.html#method.iter) method to obtain an `Iter`.
pub struct Iter<'i, K, V> {
keys: Option<Vec<Arc<K>>>,
cache_segments: Box<[&'i dyn ScanningGet<K, V>]>,
num_cht_segments: usize,
cache_seg_index: usize,
cht_seg_index: usize,
is_done: bool,
}
impl<'i, K, V> Iter<'i, K, V> {
pub(crate) fn with_single_cache_segment(
cache: &'i dyn ScanningGet<K, V>,
num_cht_segments: usize,
) -> Self {
Self {
keys: None,
cache_segments: Box::new([cache]),
num_cht_segments,
cache_seg_index: 0,
cht_seg_index: 0,
is_done: false,
}
}
#[cfg(feature = "sync")]
pub(crate) fn with_multiple_cache_segments(
cache_segments: Box<[&'i dyn ScanningGet<K, V>]>,
num_cht_segments: usize,
) -> Self {
Self {
keys: None,
cache_segments,
num_cht_segments,
cache_seg_index: 0,
cht_seg_index: 0,
is_done: false,
}
}
}
impl<K, V> Iterator for Iter<'_, K, V>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
type Item = (Arc<K>, V);
fn next(&mut self) -> Option<Self::Item> {
if self.is_done {
return None;
}
while let Some(key) = self.next_key() {
if let Some(v) = self.cache().scanning_get(&key) {
return Some((key, v));
}
}
self.is_done = true;
None
}
}
impl<'i, K, V> Iter<'i, K, V>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
fn cache(&self) -> &'i dyn ScanningGet<K, V> {
self.cache_segments[self.cache_seg_index]
}
fn next_key(&mut self) -> Option<Arc<K>> {
while let Some(keys) = self.current_keys() {
if let key @ Some(_) = keys.pop() {
return key;
}
}
None
}
fn current_keys(&mut self) -> Option<&mut Vec<Arc<K>>> {
// If keys is none or some but empty, try to get next keys.
while self.keys.as_ref().map_or(true, Vec::is_empty) {
// Adjust indices.
if self.cht_seg_index >= self.num_cht_segments {
self.cache_seg_index += 1;
self.cht_seg_index = 0;
if self.cache_seg_index >= self.cache_segments.len() {
// No more cache segments left.
return None;
}
}
let cache_segment = self.cache_segments[self.cache_seg_index];
self.keys = cache_segment.keys(self.cht_seg_index);
self.num_cht_segments = cache_segment.num_cht_segments();
self.cht_seg_index += 1;
}
self.keys.as_mut()
}
}
// Clippy beta 0.1.83 (f41c7ed9889 2024-10-31) warns about unused lifetimes on 'a.
// This seems a false positive. The lifetimes are used in the Send and Sync impls.
// Let's suppress the warning.
// https://rust-lang.github.io/rust-clippy/master/index.html#extra_unused_lifetimes
#[allow(clippy::extra_unused_lifetimes)]
unsafe impl<'a, K, V> Send for Iter<'_, K, V>
where
K: 'a + Eq + Hash + Send,
V: 'a + Send,
{
}
// Clippy beta 0.1.83 (f41c7ed9889 2024-10-31) warns about unused lifetimes on 'a.
// This seems a false positive. The lifetimes are used in the Send and Sync impls.
// Let's suppress the warning.
// https://rust-lang.github.io/rust-clippy/master/index.html#extra_unused_lifetimes
#[allow(clippy::extra_unused_lifetimes)]
unsafe impl<'a, K, V> Sync for Iter<'_, K, V>
where
K: 'a + Eq + Hash + Sync,
V: 'a + Sync,
{
}
+81
View File
@@ -0,0 +1,81 @@
use std::sync::{
atomic::{AtomicU32, Ordering},
Arc,
};
#[derive(Debug, Default)]
pub(crate) struct Counters {
inserted: AtomicU32,
evicted: AtomicU32,
invalidated: AtomicU32,
value_created: AtomicU32,
value_dropped: AtomicU32,
}
impl Counters {
pub(crate) fn inserted(&self) -> u32 {
self.inserted.load(Ordering::Acquire)
}
pub(crate) fn evicted(&self) -> u32 {
self.evicted.load(Ordering::Acquire)
}
pub(crate) fn invalidated(&self) -> u32 {
self.invalidated.load(Ordering::Acquire)
}
pub(crate) fn value_created(&self) -> u32 {
self.value_created.load(Ordering::Acquire)
}
pub(crate) fn value_dropped(&self) -> u32 {
self.value_dropped.load(Ordering::Acquire)
}
pub(crate) fn incl_inserted(&self) {
self.inserted.fetch_add(1, Ordering::AcqRel);
}
pub(crate) fn incl_evicted(&self) {
self.evicted.fetch_add(1, Ordering::AcqRel);
}
pub(crate) fn incl_invalidated(&self) {
self.invalidated.fetch_add(1, Ordering::AcqRel);
}
pub(crate) fn incl_value_created(&self) {
self.value_created.fetch_add(1, Ordering::AcqRel);
}
pub(crate) fn incl_value_dropped(&self) {
self.value_dropped.fetch_add(1, Ordering::AcqRel);
}
}
#[derive(Debug)]
pub(crate) struct Value {
// blob: Vec<u8>,
counters: Arc<Counters>,
}
impl Value {
pub(crate) fn new(_blob: Vec<u8>, counters: &Arc<Counters>) -> Self {
counters.incl_value_created();
Self {
// blob,
counters: Arc::clone(counters),
}
}
// pub(crate) fn blob(&self) -> &[u8] {
// &self.blob
// }
}
impl Drop for Value {
fn drop(&mut self) {
self.counters.incl_value_dropped();
}
}
+10
View File
@@ -0,0 +1,10 @@
mod atomic_time;
mod clock;
mod instant;
pub(crate) use atomic_time::AtomicInstant;
pub(crate) use clock::Clock;
pub(crate) use instant::Instant;
#[cfg(test)]
pub(crate) use clock::Mock;
@@ -0,0 +1,57 @@
use crate::common::time::Instant;
use portable_atomic::AtomicU64;
use std::sync::atomic::Ordering;
/// `AtomicInstant` is a wrapper around `AtomicU64` that provides thread-safe access
/// to an `Instant`.
///
/// `u64::MAX` is used to represent an unset `Instant`.
#[derive(Debug)]
pub(crate) struct AtomicInstant {
instant: AtomicU64,
}
impl Default for AtomicInstant {
/// Creates a new `AtomicInstant` with an unset `Instant`.
fn default() -> Self {
Self {
instant: AtomicU64::new(u64::MAX),
}
}
}
impl AtomicInstant {
/// Creates a new `AtomicInstant` with the given `Instant`.
pub(crate) fn new(instant: Instant) -> Self {
// Ensure the `Instant` is not `u64::MAX`, which means unset.
debug_assert!(instant.as_nanos() != u64::MAX);
Self {
instant: AtomicU64::new(instant.as_nanos()),
}
}
/// Returns `true` if the `Instant` is set.
pub(crate) fn is_set(&self) -> bool {
self.instant.load(Ordering::Acquire) != u64::MAX
}
/// Returns the `Instant` if it is set, otherwise `None`.
pub(crate) fn instant(&self) -> Option<Instant> {
let ts = self.instant.load(Ordering::Acquire);
if ts == u64::MAX {
None
} else {
Some(Instant::from_nanos(ts))
}
}
/// Sets the `Instant`.
pub(crate) fn set_instant(&self, instant: Instant) {
// Ensure the `Instant` is not `u64::MAX`, which means unset.
debug_assert!(instant.as_nanos() != u64::MAX);
self.instant.store(instant.as_nanos(), Ordering::Release);
}
}
+166
View File
@@ -0,0 +1,166 @@
use std::time::{Duration, Instant as StdInstant};
#[cfg(test)]
use std::sync::Arc;
#[cfg(test)]
use parking_lot::RwLock;
// This is `moka`'s `Instant` struct.
use super::Instant;
#[derive(Default, Clone)]
pub(crate) struct Clock {
ty: ClockType,
}
#[derive(Clone)]
enum ClockType {
/// A clock that uses `std::time::Instant` as the source of time.
Standard { origin: StdInstant },
#[cfg(feature = "quanta")]
/// A clock that uses both `std::time::Instant` and `quanta::Instant` as the
/// sources of time.
Hybrid {
std_origin: StdInstant,
quanta_origin: quanta::Instant,
},
#[cfg(test)]
/// A clock that uses a mocked source of time.
Mocked { mock: Arc<Mock> },
}
impl Default for ClockType {
/// Create a new `ClockType` with the current time as the origin.
///
/// If the `quanta` feature is enabled, `Hybrid` will be used. Otherwise,
/// `Standard` will be used.
fn default() -> Self {
#[cfg(feature = "quanta")]
{
return ClockType::Hybrid {
std_origin: StdInstant::now(),
quanta_origin: quanta::Instant::now(),
};
}
#[allow(unreachable_code)]
ClockType::Standard {
origin: StdInstant::now(),
}
}
}
impl Clock {
#[cfg(test)]
/// Creates a new `Clock` with a mocked source of time.
pub(crate) fn mock() -> (Clock, Arc<Mock>) {
let mock = Arc::new(Mock::default());
let clock = Clock {
ty: ClockType::Mocked {
mock: Arc::clone(&mock),
},
};
(clock, mock)
}
/// Returns the current time using a reliable source of time.
///
/// When the type is `Standard` or `Hybrid`, the time is based on
/// `std::time::Instant`. When the type is `Mocked`, the time is based on the
/// mocked source of time.
pub(crate) fn now(&self) -> Instant {
match &self.ty {
ClockType::Standard { origin } => {
Instant::from_duration_since_clock_start(origin.elapsed())
}
#[cfg(feature = "quanta")]
ClockType::Hybrid { std_origin, .. } => {
Instant::from_duration_since_clock_start(std_origin.elapsed())
}
#[cfg(test)]
ClockType::Mocked { mock } => Instant::from_duration_since_clock_start(mock.elapsed()),
}
}
/// Returns the current time _maybe_ using a fast but less reliable source of
/// time. The time may drift from the time returned by `now`, or not be
/// monotonically increasing.
///
/// This is useful for performance critical code that does not require the same
/// level of precision as `now`. (e.g. measuring the time between two events for
/// metrics)
///
/// When the type is `Standard` or `Mocked`, `now` is internally called. So there
/// is no performance benefit.
///
/// When the type is `Hybrid`, the time is based on `quanta::Instant`, which can
/// be faster than `std::time::Instant`, depending on the CPU architecture.
pub(crate) fn fast_now(&self) -> Instant {
match &self.ty {
#[cfg(feature = "quanta")]
ClockType::Hybrid { quanta_origin, .. } => {
Instant::from_duration_since_clock_start(quanta_origin.elapsed())
}
ClockType::Standard { .. } => self.now(),
#[cfg(test)]
ClockType::Mocked { .. } => self.now(),
}
}
/// Converts an `Instant` to a `std::time::Instant`.
///
/// **IMPORTANT**: The caller must ensure that the `Instant` was created by this
/// `Clock`, otherwise the resulting `std::time::Instant` will be incorrect.
pub(crate) fn to_std_instant(&self, instant: Instant) -> StdInstant {
match &self.ty {
ClockType::Standard { origin } => {
let duration = Duration::from_nanos(instant.as_nanos());
*origin + duration
}
#[cfg(feature = "quanta")]
ClockType::Hybrid { std_origin, .. } => {
let duration = Duration::from_nanos(instant.as_nanos());
*std_origin + duration
}
#[cfg(test)]
ClockType::Mocked { mock } => {
let duration = Duration::from_nanos(instant.as_nanos());
// https://github.com/moka-rs/moka/issues/487
//
// This `dbg!` will workaround an incorrect compilation by Rust
// 1.84.0 for the armv7-unknown-linux-musleabihf target in the
// release build of the tests.
dbg!(mock.origin + duration)
}
}
}
}
#[cfg(test)]
pub(crate) struct Mock {
origin: StdInstant,
now: RwLock<StdInstant>,
}
#[cfg(test)]
impl Default for Mock {
fn default() -> Self {
let origin = StdInstant::now();
Self {
origin,
now: RwLock::new(origin),
}
}
}
#[cfg(test)]
impl Mock {
pub(crate) fn increment(&self, amount: Duration) {
*self.now.write() += amount;
}
pub(crate) fn elapsed(&self) -> Duration {
self.now.read().duration_since(self.origin)
}
}
+80
View File
@@ -0,0 +1,80 @@
use std::time::Duration;
pub(crate) const MAX_NANOS: u64 = u64::MAX - 1;
/// `Instant` represents a point in time since the `Clock` was created. It has
/// nanosecond precision.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) struct Instant {
elapsed_ns: u64,
}
impl Instant {
pub(crate) fn from_nanos(nanos: u64) -> Instant {
debug_assert!(nanos <= MAX_NANOS);
Instant { elapsed_ns: nanos }
}
pub(crate) fn from_duration_since_clock_start(duration: Duration) -> Instant {
Instant::from_nanos(Self::duration_to_saturating_nanoseconds(duration))
}
pub(crate) fn as_nanos(&self) -> u64 {
self.elapsed_ns
}
/// Converts a `std::time::Duration` to nanoseconds, saturating to
/// `MAX_NANOSECONDS` (`u64::MAX - 1`) if the duration is too large.
/// (`Duration::as_nanos` returns `u128`)
///
/// Note that `u64::MAX - 1` is used here instead of `u64::MAX` because
/// `u64::MAX` is used by `moka`'s `AtomicTime` to indicate the time is unset.
pub(crate) fn duration_to_saturating_nanoseconds(duration: Duration) -> u64 {
u64::try_from(duration.as_nanos())
.map(|n| n.min(MAX_NANOS))
.unwrap_or(MAX_NANOS)
}
pub(crate) fn saturating_add(&self, duration: Duration) -> Instant {
let dur_ms = Self::duration_to_saturating_nanoseconds(duration);
Instant::from_nanos(self.elapsed_ns.saturating_add(dur_ms).min(MAX_NANOS))
}
pub(crate) fn saturating_duration_since(&self, earlier: Self) -> Duration
where
Self: Sized,
{
Duration::from_nanos(self.elapsed_ns.saturating_sub(earlier.elapsed_ns))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_saturating_add() {
let instant = Instant::from_nanos(100_000);
let duration = Duration::from_nanos(50_000);
let result = instant.saturating_add(duration);
assert_eq!(result, Instant::from_nanos(150_000));
let instant = Instant::from_nanos(u64::MAX - 10_000);
let duration = Duration::from_nanos(12_000);
let result = instant.saturating_add(duration);
assert_eq!(result, Instant::from_nanos(u64::MAX - 1));
}
#[test]
fn test_saturating_duration_since() {
let instant = Instant::from_nanos(100_000);
let earlier = Instant::from_nanos(60_000);
let result = instant.saturating_duration_since(earlier);
assert_eq!(result, Duration::from_nanos(40_000));
let instant = Instant::from_nanos(60_000);
let earlier = Instant::from_nanos(100_000);
let result = instant.saturating_duration_since(earlier);
assert_eq!(result, Duration::ZERO);
}
}
+907
View File
@@ -0,0 +1,907 @@
// License and Copyright Notice:
//
// Some of the code and doc comments in this module were ported or copied from
// a Java class `com.github.benmanes.caffeine.cache.TimerWheel` of Caffeine.
// https://github.com/ben-manes/caffeine/blob/master/caffeine/src/main/java/com/github/benmanes/caffeine/cache/TimerWheel.java
//
// The original code/comments from Caffeine are licensed under the Apache License,
// Version 2.0 <https://github.com/ben-manes/caffeine/blob/master/LICENSE>
//
// Copyrights of the original code/comments are retained by their contributors.
// For full authorship information, see the version control history of
// https://github.com/ben-manes/caffeine/
use std::{ptr::NonNull, time::Duration};
use super::{
concurrent::{arc::MiniArc, entry_info::EntryInfo, DeqNodes},
deque::{DeqNode, Deque},
time::Instant,
};
use parking_lot::Mutex;
const BUCKET_COUNTS: &[u64] = &[
64, // roughly seconds
64, // roughly minutes
32, // roughly hours
4, // roughly days
1, // overflow (> ~6.5 days)
];
const OVERFLOW_QUEUE_INDEX: usize = BUCKET_COUNTS.len() - 1;
const NUM_LEVELS: usize = OVERFLOW_QUEUE_INDEX - 1;
const DAY: Duration = Duration::from_secs(60 * 60 * 24);
const SPANS: &[u64] = &[
aligned_duration(Duration::from_secs(1)), // 1.07s
aligned_duration(Duration::from_secs(60)), // 1.14m
aligned_duration(Duration::from_secs(60 * 60)), // 1.22h
aligned_duration(DAY), // 1.63d
BUCKET_COUNTS[3] * aligned_duration(DAY), // 6.5d
BUCKET_COUNTS[3] * aligned_duration(DAY), // 6.5d
];
const SHIFT: &[u64] = &[
SPANS[0].trailing_zeros() as u64,
SPANS[1].trailing_zeros() as u64,
SPANS[2].trailing_zeros() as u64,
SPANS[3].trailing_zeros() as u64,
SPANS[4].trailing_zeros() as u64,
];
/// Returns the next power of two of the duration in nanoseconds.
const fn aligned_duration(duration: Duration) -> u64 {
// NOTE: as_nanos() returns u128, so convert it to u64 by using `as`.
// We cannot call TryInto::try_into() here because it is not a const fn.
(duration.as_nanos() as u64).next_power_of_two()
}
/// A timer node stored in a bucket of a timer wheel.
pub(crate) enum TimerNode<K> {
/// A sentinel node that is used to mark the end of a timer wheel bucket.
Sentinel,
/// A timer entry that is holding Arc pointers to the data structures in a cache
/// entry.
Entry {
/// The position (level and index) of the timer wheel bucket.
pos: Option<(u8, u8)>,
/// The expiry generation when this timer node was created.
/// Used to detect stale timer node pointers.
expiry_gen: u32,
/// An Arc pointer to the `EntryInfo` of the cache entry (`ValueEntry`).
entry_info: MiniArc<EntryInfo<K>>,
/// An Arc pointer to the `DeqNodes` of the cache entry (`ValueEntry`).
deq_nodes: MiniArc<Mutex<DeqNodes<K>>>,
},
}
impl<K> TimerNode<K> {
fn new(
entry_info: MiniArc<EntryInfo<K>>,
deq_nodes: MiniArc<Mutex<DeqNodes<K>>>,
expiry_gen: u32,
level: usize,
index: usize,
) -> Self {
Self::Entry {
pos: Some((level as u8, index as u8)),
expiry_gen,
entry_info,
deq_nodes,
}
}
/// Returns the position (level and index) of the timer wheel bucket.
fn position(&self) -> Option<(usize, usize)> {
if let Self::Entry { pos, .. } = &self {
pos.map(|(level, index)| (level as usize, index as usize))
} else {
unreachable!()
}
}
fn set_position(&mut self, level: usize, index: usize) {
if let Self::Entry { pos, .. } = self {
*pos = Some((level as u8, index as u8));
} else {
unreachable!()
}
}
fn unset_position(&mut self) {
if let Self::Entry { pos, .. } = self {
*pos = None;
} else {
unreachable!()
}
}
fn is_sentinel(&self) -> bool {
matches!(self, Self::Sentinel)
}
pub(crate) fn entry_info(&self) -> &MiniArc<EntryInfo<K>> {
if let Self::Entry { entry_info, .. } = &self {
entry_info
} else {
unreachable!()
}
}
fn unset_timer_node_in_deq_nodes(&self) {
if let Self::Entry { deq_nodes, .. } = &self {
deq_nodes.lock().set_timer_node(None, 0);
} else {
unreachable!();
}
}
/// Returns the expiry generation stored in this timer node.
/// Used to validate the timer node hasn't become stale.
pub(crate) fn expiry_gen(&self) -> u32 {
if let Self::Entry { expiry_gen, .. } = &self {
*expiry_gen
} else {
0 // Sentinel has no expiry_gen
}
}
}
type Bucket<K> = Deque<TimerNode<K>>;
#[must_use = "this `ReschedulingResult` may be an `Removed` variant, which should be handled"]
pub(crate) enum ReschedulingResult<K> {
/// The timer event was rescheduled.
Rescheduled,
/// The timer event was not rescheduled because the entry has no expiration time.
Removed(Box<DeqNode<TimerNode<K>>>),
}
/// A hierarchical timer wheel to add, remove, and fire expiration events in
/// amortized O(1) time.
///
/// The expiration events are deferred until the timer is advanced, which is
/// performed as part of the cache's housekeeping cycle.
pub(crate) struct TimerWheel<K> {
/// The hierarchical timer wheels.
wheels: Box<[Box<[Bucket<K>]>]>,
/// The time when this `TimerWheel` was created.
origin: Instant,
/// The time when this `TimerWheel` was last advanced.
current: Instant,
}
#[cfg(feature = "future")]
// TODO: https://github.com/moka-rs/moka/issues/54
#[allow(clippy::non_send_fields_in_send_ty)]
// Multi-threaded async runtimes require base_cache::Inner to be Send, but it will
// not be without this `unsafe impl`. This is because DeqNodes have NonNull
// pointers.
unsafe impl<K> Send for TimerWheel<K> {}
impl<K> TimerWheel<K> {
pub(crate) fn new(now: Instant) -> Self {
Self {
wheels: Box::default(), // Empty.
origin: now,
current: now,
}
}
pub(crate) fn is_enabled(&self) -> bool {
!self.wheels.is_empty()
}
pub(crate) fn enable(&mut self) {
assert!(!self.is_enabled());
// Populate each bucket with a queue having a sentinel node.
self.wheels = BUCKET_COUNTS
.iter()
.map(|b| {
(0..*b)
.map(|_| {
let mut deq = Deque::new(super::CacheRegion::Other);
deq.push_back(Box::new(DeqNode::new(TimerNode::Sentinel)));
deq
})
.collect::<Vec<_>>()
.into_boxed_slice()
})
.collect::<Vec<_>>()
.into_boxed_slice();
}
/// Schedules a timer event for the node.
/// The `expiry_gen` is the current expiry generation from EntryInfo, used for
/// validation when operating on the timer node later.
pub(crate) fn schedule(
&mut self,
entry_info: MiniArc<EntryInfo<K>>,
deq_nodes: MiniArc<Mutex<DeqNodes<K>>>,
expiry_gen: u32,
) -> Option<NonNull<DeqNode<TimerNode<K>>>> {
debug_assert!(self.is_enabled());
// Capture a single snapshot of the expiration state to ensure consistency
let (opt_time, current_gen) = entry_info.expiration_state();
// Validate expiry_gen before proceeding to prevent operating on stale entries
if current_gen != expiry_gen {
// The entry's expiry generation has changed since this method was called
#[cfg(feature = "logging")]
log::debug!(
"schedule: expiry generation mismatch. Expected: {expiry_gen}, Actual: {current_gen}. Entry state changed."
);
return None;
}
if let Some(t) = opt_time {
let (level, index) = self.bucket_indices(t);
let node = Box::new(DeqNode::new(TimerNode::new(
entry_info, deq_nodes, expiry_gen, level, index,
)));
let node = self.wheels[level][index].push_back(node);
Some(node)
} else {
None
}
}
fn schedule_existing_node(
&mut self,
mut node: NonNull<DeqNode<TimerNode<K>>>,
) -> ReschedulingResult<K> {
debug_assert!(self.is_enabled());
// Since cache entry's ValueEntry has a pointer to this node, we must reuse
// the node.
//
// SAFETY on `node.as_mut()`: The self (`TimerWheel`) is the only owner of
// the node, and we have `&mut self` here. We are the only one who can mutate
// the node.
if let entry @ TimerNode::Entry { .. } = &mut unsafe { node.as_mut() }.element {
if let Some(t) = entry.entry_info().expiration_state().0 {
let (level, index) = self.bucket_indices(t);
entry.set_position(level, index);
let node = unsafe { Box::from_raw(node.as_ptr()) };
self.wheels[level][index].push_back(node);
ReschedulingResult::Rescheduled
} else {
entry.unset_position();
entry.unset_timer_node_in_deq_nodes();
ReschedulingResult::Removed(unsafe { Box::from_raw(node.as_ptr()) })
}
} else {
unreachable!()
}
}
/// Reschedules an active timer event for the node.
///
/// This method validates that the timer node's expiry generation matches the
/// expected generation before operating on it. This prevents use-after-free
/// when a stale timer_node pointer is used.
///
/// Returns `Some` with the result if successful, `None` if validation failed
/// or the node was invalid.
pub(crate) fn reschedule(
&mut self,
node: NonNull<DeqNode<TimerNode<K>>>,
expected_expiry_gen: u32,
) -> Option<ReschedulingResult<K>> {
debug_assert!(self.is_enabled());
unsafe {
// First validate the node by checking if expiry generation matches
let p = node.as_ref();
let actual_gen = p.element.expiry_gen();
if actual_gen != expected_expiry_gen {
// The timer node is stale (expiry changed since it was scheduled)
#[cfg(feature = "logging")]
log::error!(
"reschedule: expiry generation mismatch. \
Expected: {expected_expiry_gen}, Actual: {actual_gen}. Timer node is stale.",
);
return None;
}
}
if unsafe { self.unlink_timer(node) } {
Some(self.schedule_existing_node(node))
} else {
// Node was invalid, cannot reschedule
None
}
}
/// Removes a timer event for this node if present.
///
/// This method validates that the timer node's expiry generation matches the
/// expected generation before operating on it. This prevents use-after-free
/// when a stale timer_node pointer is used.
///
/// Returns `true` if the node was successfully descheduled, `false` if the node
/// was invalid or validation failed.
pub(crate) fn deschedule(
&mut self,
node: NonNull<DeqNode<TimerNode<K>>>,
expected_expiry_gen: u32,
) -> bool {
debug_assert!(self.is_enabled());
unsafe {
// First validate the node by checking if expiry generation matches
let p = node.as_ref();
let actual_gen = p.element.expiry_gen();
if actual_gen != expected_expiry_gen {
// The timer node is stale (expiry changed since it was scheduled)
#[cfg(feature = "logging")]
log::error!(
"deschedule: expiry generation mismatch. \
Expected: {expected_expiry_gen}, Actual: {actual_gen}. Timer node is stale.",
);
return false;
}
if self.unlink_timer(node) {
Self::drop_node(node);
true
} else {
// Node was invalid, do not attempt to drop
false
}
}
}
/// Removes a timer event for this node if present.
///
/// IMPORTANT: This method does not drop the node.
///
/// Returns `true` if the node was a valid `TimerNode::Entry`, `false` otherwise.
/// The caller should check the return value and handle the case where the node
/// is not a valid entry (e.g., due to use-after-free or memory corruption).
unsafe fn unlink_timer(&mut self, mut node: NonNull<DeqNode<TimerNode<K>>>) -> bool {
// SAFETY: The self (`TimerWheel`) is the only owner of the node, and we have
// `&mut self` here. We are the only one who can mutate the node.
let p = node.as_mut();
if let entry @ TimerNode::Entry { .. } = &mut p.element {
if let Some((level, index)) = entry.position() {
self.wheels[level][index].unlink(node);
entry.unset_position();
}
true
} else {
// This should not happen in normal operation. If we reach here, it means
// the node pointer is invalid (possibly use-after-free or memory corruption).
// Log the error and return false to allow the caller to handle gracefully.
#[cfg(feature = "logging")]
log::error!(
"unlink_timer: expected TimerNode::Entry but found Sentinel. \
This indicates a bug in the timer wheel or use-after-free."
);
false
}
}
unsafe fn drop_node(node: NonNull<DeqNode<TimerNode<K>>>) {
std::mem::drop(Box::from_raw(node.as_ptr()));
}
/// Advances the timer wheel to the current time, and returns an iterator over
/// timer events.
pub(crate) fn advance(
&mut self,
current_time: Instant,
) -> impl Iterator<Item = TimerEvent<K>> + '_ {
debug_assert!(self.is_enabled());
let previous_time = self.current;
self.current = current_time;
TimerEventsIter::new(self, previous_time, current_time)
}
/// Returns a pointer to the timer event (cache entry) at the front of the queue.
/// Returns `None` if the front node is a sentinel.
fn pop_timer_node(&mut self, level: usize, index: usize) -> Option<Box<DeqNode<TimerNode<K>>>> {
let deque = &mut self.wheels[level][index];
if let Some(node) = deque.peek_front() {
if node.element.is_sentinel() {
return None;
}
}
let mut popped = deque.pop_front();
if let Some(node) = &mut popped {
node.element.unset_position();
}
popped
}
/// Reset the positions of the nodes in the queue at the given level and index.
/// When done, the sentinel is at the back of the queue.
fn reset_timer_node_positions(&mut self, level: usize, index: usize) {
let deque = &mut self.wheels[level][index];
debug_assert!(
deque.len() > 0,
"BUG: The queue is empty. level: {level}, index: {index}"
);
// Rotate the nodes in the queue until we see the sentinel at the back of the
// queue.
while !deque.peek_back().unwrap().element.is_sentinel() {
deque.move_front_to_back();
}
}
/// Returns the bucket indices to locate the bucket that the timer event
/// should be added to.
fn bucket_indices(&self, time: Instant) -> (usize, usize) {
let duration_nanos = self.duration_nanos_since_last_advanced(time);
let time_nanos = self.time_nanos(time);
for level in 0..=NUM_LEVELS {
if duration_nanos < SPANS[level + 1] {
let ticks = time_nanos >> SHIFT[level];
let index = ticks & (BUCKET_COUNTS[level] - 1);
return (level, index as usize);
}
}
(OVERFLOW_QUEUE_INDEX, 0)
}
// Returns nano-seconds between the given `time` and the time when this timer
// wheel was advanced. If the `time` is earlier than other, returns zero.
fn duration_nanos_since_last_advanced(&self, time: Instant) -> u64 {
// If `time` is earlier than `self.current`, use zero. This could happen
// when a user provided `Expiry` method returned zero or a very short
// duration.
time.saturating_duration_since(self.current).as_nanos() as u64
}
// Returns nano-seconds between the given `time` and `self.origin`, the time when
// this timer wheel was created.
//
// - If the `time` is earlier than other, returns zero.
// - If the `time` is later than `self.origin + u64::MAX`, returns `u64::MAX`,
// which is ~584 years in nanoseconds.
//
fn time_nanos(&self, time: Instant) -> u64 {
let nanos_u128 = time
// If `time` is earlier than `self.origin`, use zero. This would never
// happen in practice as there should be some delay between the timer
// wheel was created and the first timer event is scheduled. But we will
// do this just in case.
.saturating_duration_since(self.origin)
.as_nanos();
// Convert an `u128` into an `u64`. If the value is too large, use `u64::MAX`
// (~584 years)
nanos_u128.try_into().unwrap_or(u64::MAX)
}
}
/// A timer event, which is either an expired/rescheduled cache entry, or a
/// descheduled timer. `TimerWheel::advance` method returns an iterator over timer
/// events.
#[derive(Debug)]
pub(crate) enum TimerEvent<K> {
/// This cache entry has expired.
Expired(Box<DeqNode<TimerNode<K>>>),
// This cache entry has been rescheduled. Rescheduling includes moving a timer
// from one wheel to another in a lower level of the hierarchy. (This variant
// is mainly used for testing)
#[cfg(test)]
Rescheduled(MiniArc<EntryInfo<K>>),
#[cfg(not(test))]
Rescheduled(()),
/// This timer node (containing a cache entry) has been removed from the timer.
/// (This variant is mainly used for testing)
Descheduled,
}
/// An iterator over expired cache entries.
pub(crate) struct TimerEventsIter<'iter, K> {
timer_wheel: &'iter mut TimerWheel<K>,
previous_time: Instant,
current_time: Instant,
is_done: bool,
level: usize,
index: u8,
end_index: u8,
index_mask: u64,
is_new_level: bool,
is_new_index: bool,
}
impl<'iter, K> TimerEventsIter<'iter, K> {
fn new(
timer_wheel: &'iter mut TimerWheel<K>,
previous_time: Instant,
current_time: Instant,
) -> Self {
Self {
timer_wheel,
previous_time,
current_time,
is_done: false,
level: 0,
index: 0,
end_index: 0,
index_mask: 0,
is_new_level: true,
is_new_index: true,
}
}
}
impl<K> Drop for TimerEventsIter<'_, K> {
fn drop(&mut self) {
if !self.is_done {
// This iterator was dropped before consuming all events. Reset the
// `current` to the time when the timer wheel was last successfully
// advanced.
self.timer_wheel.current = self.previous_time;
}
}
}
impl<K> Iterator for TimerEventsIter<'_, K> {
type Item = TimerEvent<K>;
/// NOTE: When necessary, this iterator will unset the timer node pointer in the
/// `ValueEntry`.
fn next(&mut self) -> Option<Self::Item> {
if self.is_done {
return None;
}
loop {
if self.is_new_level {
let previous_time_nanos = self.timer_wheel.time_nanos(self.previous_time);
let current_time_nanos = self.timer_wheel.time_nanos(self.current_time);
let previous_ticks = previous_time_nanos >> SHIFT[self.level];
let current_ticks = current_time_nanos >> SHIFT[self.level];
if current_ticks <= previous_ticks {
self.is_done = true;
return None;
}
self.index_mask = BUCKET_COUNTS[self.level] - 1;
self.index = (previous_ticks & self.index_mask) as u8;
let steps =
(current_ticks - previous_ticks + 1).min(BUCKET_COUNTS[self.level]) as u8;
self.end_index = self.index + steps;
self.is_new_level = false;
self.is_new_index = true;
// dbg!(self.level, self.index, self.end_index);
}
let i = self.index & self.index_mask as u8;
if self.is_new_index {
// Move the sentinel to the back of the queue.
self.timer_wheel
.reset_timer_node_positions(self.level, i as usize);
self.is_new_index = false;
}
// Pop the next timer event (cache entry) from the queue at the current
// level and index.
//
// We will repeat processing this level until we see the sentinel.
// (`pop_timer_node` will return `None` when it sees the sentinel)
if let Some(node) = self.timer_wheel.pop_timer_node(self.level, i as usize) {
if let Some(t) = node.as_ref().element.entry_info().expiration_state().0 {
if t <= self.current_time {
// The cache entry has expired. Unset the timer node from
// the ValueEntry and return the node.
node.as_ref().element.unset_timer_node_in_deq_nodes();
return Some(TimerEvent::Expired(node));
}
// The cache entry has not expired. Reschedule it.
let node_p = NonNull::new(Box::into_raw(node)).expect("Got a null ptr");
#[cfg(test)]
// Get the entry info before rescheduling (mutating) the node to
// avoid Stacked Borrows/Tree Borrows violations on `node_p`.
let entry_info =
MiniArc::clone(unsafe { node_p.as_ref() }.element.entry_info());
match self.timer_wheel.schedule_existing_node(node_p) {
ReschedulingResult::Rescheduled => {
#[cfg(test)]
return Some(TimerEvent::Rescheduled(entry_info));
#[cfg(not(test))]
return Some(TimerEvent::Rescheduled(()));
}
ReschedulingResult::Removed(node) => {
// The timer event has been removed from the timer
// wheel. Unset the timer node from the ValueEntry.
node.as_ref().element.unset_timer_node_in_deq_nodes();
return Some(TimerEvent::Descheduled);
}
}
} else {
node.as_ref().element.unset_timer_node_in_deq_nodes();
}
} else {
// Done with the current queue. Move to the next index
// and/or next level.
self.index += 1;
self.is_new_index = true;
if self.index >= self.end_index {
self.level += 1;
// No more levels to process. We are done.
if self.level >= BUCKET_COUNTS.len() {
self.is_done = true;
return None;
}
self.is_new_level = true;
}
}
}
}
}
#[cfg(test)]
mod tests {
use std::{sync::Arc, time::Duration};
use super::{TimerEvent, TimerWheel, SPANS};
use crate::common::{
concurrent::{arc::MiniArc, entry_info::EntryInfo, KeyHash},
time::{Clock, Instant, Mock},
};
#[test]
fn test_bucket_indices() {
fn bi(timer: &TimerWheel<()>, now: Instant, dur: Duration) -> (usize, usize) {
let t = now.saturating_add(dur);
timer.bucket_indices(t)
}
let (clock, mock) = Clock::mock();
let now = clock.now();
let mut timer = TimerWheel::<()>::new(now);
timer.enable();
assert_eq!(timer.bucket_indices(now), (0, 0));
// Level 0: 1.07s
assert_eq!(bi(&timer, now, n2d(SPANS[0] - 1)), (0, 0));
assert_eq!(bi(&timer, now, n2d(SPANS[0])), (0, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 63)), (0, 63));
// Level 1: 1.14m
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 64)), (1, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[1])), (1, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[1] * 63 + SPANS[0] * 63)), (1, 63));
// Level 2: 1.22h
assert_eq!(bi(&timer, now, n2d(SPANS[1] * 64)), (2, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[2])), (2, 1));
assert_eq!(
bi(
&timer,
now,
n2d(SPANS[2] * 31 + SPANS[1] * 63 + SPANS[0] * 63)
),
(2, 31)
);
// Level 3: 1.63dh
assert_eq!(bi(&timer, now, n2d(SPANS[2] * 32)), (3, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[3])), (3, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[3] * 3)), (3, 3));
// Overflow
assert_eq!(bi(&timer, now, n2d(SPANS[3] * 4)), (4, 0));
assert_eq!(bi(&timer, now, n2d(SPANS[4])), (4, 0));
assert_eq!(bi(&timer, now, n2d(SPANS[4] * 100)), (4, 0));
// Increment the clock by 5 ticks. (1 tick ~= 1.07s)
let now = advance_clock(&clock, &mock, n2d(SPANS[0] * 5));
timer.current = now;
// Level 0: 1.07s
assert_eq!(bi(&timer, now, n2d(SPANS[0] - 1)), (0, 5));
assert_eq!(bi(&timer, now, n2d(SPANS[0])), (0, 6));
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 63)), (0, 4));
// Level 1: 1.14m
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 64)), (1, 1));
assert_eq!(bi(&timer, now, n2d(SPANS[1])), (1, 1));
assert_eq!(
bi(&timer, now, n2d(SPANS[1] * 63 + SPANS[0] * (63 - 5))),
(1, 63)
);
// Increment the clock by 61 ticks. (total 66 ticks)
let now = advance_clock(&clock, &mock, n2d(SPANS[0] * 61));
timer.current = now;
// Level 0: 1.07s
assert_eq!(bi(&timer, now, n2d(SPANS[0] - 1)), (0, 2));
assert_eq!(bi(&timer, now, n2d(SPANS[0])), (0, 3));
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 63)), (0, 1));
// Level 1: 1.14m
assert_eq!(bi(&timer, now, n2d(SPANS[0] * 64)), (1, 2));
assert_eq!(bi(&timer, now, n2d(SPANS[1])), (1, 2));
assert_eq!(
bi(&timer, now, n2d(SPANS[1] * 63 + SPANS[0] * (63 - 2))),
(1, 0)
);
}
#[test]
fn test_advance() {
fn schedule_timer(timer: &mut TimerWheel<u32>, key: u32, now: Instant, ttl: Duration) {
let hash = key as u64;
let key_hash = KeyHash::new(Arc::new(key), hash);
let policy_weight = 0;
let entry_info = MiniArc::new(EntryInfo::new(key_hash, now, policy_weight));
let expiry_gen = entry_info.set_expiration_time(Some(now.saturating_add(ttl)));
let deq_nodes = Default::default();
let timer_node = timer.schedule(entry_info, MiniArc::clone(&deq_nodes), expiry_gen);
deq_nodes.lock().set_timer_node(timer_node, expiry_gen);
}
fn expired_key(maybe_entry: Option<TimerEvent<u32>>) -> u32 {
let entry = maybe_entry.expect("entry is none");
match entry {
TimerEvent::Expired(node) => *node.element.entry_info().key_hash().key,
_ => panic!("Expected an expired entry. Got {entry:?}"),
}
}
fn rescheduled_key(maybe_entry: Option<TimerEvent<u32>>) -> u32 {
let entry = maybe_entry.expect("entry is none");
match entry {
TimerEvent::Rescheduled(entry) => *entry.key_hash().key,
_ => panic!("Expected a rescheduled entry. Got {entry:?}"),
}
}
let (clock, mock) = Clock::mock();
let now = advance_clock(&clock, &mock, s2d(10));
let mut timer = TimerWheel::<u32>::new(now);
timer.enable();
// Add timers that will expire in some seconds.
schedule_timer(&mut timer, 1, now, s2d(5));
schedule_timer(&mut timer, 2, now, s2d(1));
schedule_timer(&mut timer, 3, now, s2d(63));
schedule_timer(&mut timer, 4, now, s2d(3));
let now = advance_clock(&clock, &mock, s2d(4));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 2);
assert_eq!(expired_key(expired_entries.next()), 4);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(4));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 1);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(64 - 8));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 3);
assert!(expired_entries.next().is_none());
drop(expired_entries);
// Add timers that will expire in some minutes.
const MINUTES: u64 = 60;
schedule_timer(&mut timer, 1, now, s2d(5 * MINUTES));
#[allow(clippy::identity_op)]
schedule_timer(&mut timer, 2, now, s2d(1 * MINUTES));
schedule_timer(&mut timer, 3, now, s2d(63 * MINUTES));
schedule_timer(&mut timer, 4, now, s2d(3 * MINUTES));
let now = advance_clock(&clock, &mock, s2d(4 * MINUTES));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 2);
assert_eq!(expired_key(expired_entries.next()), 4);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(4 * MINUTES));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 1);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d((64 - 8) * MINUTES));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 3);
assert!(expired_entries.next().is_none());
drop(expired_entries);
// Add timers that will expire in some hours.
const HOURS: u64 = 60 * 60;
schedule_timer(&mut timer, 1, now, s2d(5 * HOURS));
#[allow(clippy::identity_op)]
schedule_timer(&mut timer, 2, now, s2d(1 * HOURS));
schedule_timer(&mut timer, 3, now, s2d(31 * HOURS));
schedule_timer(&mut timer, 4, now, s2d(3 * HOURS));
let now = advance_clock(&clock, &mock, s2d(4 * HOURS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 2);
assert_eq!(expired_key(expired_entries.next()), 4);
assert_eq!(rescheduled_key(expired_entries.next()), 1);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(4 * HOURS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 1);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d((32 - 8) * HOURS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 3);
assert!(expired_entries.next().is_none());
drop(expired_entries);
// Add timers that will expire in a few days.
const DAYS: u64 = 24 * 60 * 60;
schedule_timer(&mut timer, 1, now, s2d(5 * DAYS));
#[allow(clippy::identity_op)]
schedule_timer(&mut timer, 2, now, s2d(1 * DAYS));
schedule_timer(&mut timer, 3, now, s2d(2 * DAYS));
// Longer than ~6.5 days, so this should be stored in the overflow area.
schedule_timer(&mut timer, 4, now, s2d(8 * DAYS));
let now = advance_clock(&clock, &mock, s2d(3 * DAYS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 2);
assert_eq!(expired_key(expired_entries.next()), 3);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(3 * DAYS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 1);
assert_eq!(rescheduled_key(expired_entries.next()), 4);
assert!(expired_entries.next().is_none());
drop(expired_entries);
let now = advance_clock(&clock, &mock, s2d(3 * DAYS));
let mut expired_entries = timer.advance(now);
assert_eq!(expired_key(expired_entries.next()), 4);
assert!(expired_entries.next().is_none());
drop(expired_entries);
}
//
// Utility functions
//
fn advance_clock(clock: &Clock, mock: &Arc<Mock>, duration: Duration) -> Instant {
mock.increment(duration);
clock.now()
}
/// Convert nano-seconds to duration.
fn n2d(nanos: u64) -> Duration {
Duration::from_nanos(nanos)
}
/// Convert seconds to duration.
fn s2d(secs: u64) -> Duration {
Duration::from_secs(secs)
}
}
+150
View File
@@ -0,0 +1,150 @@
//! Provides a thread-safe, concurrent asynchronous (futures aware) cache
//! implementation.
//!
//! To use this module, enable a crate feature called "future".
use crossbeam_channel::Sender;
use futures_util::future::{BoxFuture, Shared};
use std::{future::Future, hash::Hash, sync::Arc};
use crate::common::{concurrent::WriteOp, time::Instant};
mod base_cache;
mod builder;
mod cache;
mod entry_selector;
mod housekeeper;
mod invalidator;
mod key_lock;
mod notifier;
mod value_initializer;
pub use {
builder::CacheBuilder,
cache::Cache,
entry_selector::{OwnedKeyEntrySelector, RefKeyEntrySelector},
};
/// The type of the unique ID to identify a predicate used by
/// [`Cache::invalidate_entries_if`][invalidate-if] method.
///
/// A `PredicateId` is a `String` of UUID (version 4).
///
/// [invalidate-if]: ./struct.Cache.html#method.invalidate_entries_if
pub type PredicateId = String;
pub(crate) type PredicateIdStr<'a> = &'a str;
// Empty struct to be used in `InitResult::InitErr` to represent the Option None.
pub(crate) struct OptionallyNone;
// Empty struct to be used in `InitResult::InitErr` to represent the Compute None.
pub(crate) struct ComputeNone;
impl<T: ?Sized> FutureExt for T where T: Future {}
pub trait FutureExt: Future {
fn boxed<'a, T>(self) -> BoxFuture<'a, T>
where
Self: Future<Output = T> + Sized + Send + 'a,
{
Box::pin(self)
}
}
/// Iterator visiting all key-value pairs in a cache in arbitrary order.
///
/// Call [`Cache::iter`](./struct.Cache.html#method.iter) method to obtain an `Iter`.
pub struct Iter<'i, K, V>(crate::common::iter::Iter<'i, K, V>);
impl<'i, K, V> Iter<'i, K, V> {
pub(crate) fn new(inner: crate::common::iter::Iter<'i, K, V>) -> Self {
Self(inner)
}
}
impl<K, V> Iterator for Iter<'_, K, V>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
type Item = (Arc<K>, V);
fn next(&mut self) -> Option<Self::Item> {
self.0.next()
}
}
/// Operation that has been interrupted (stopped polling) by async cancellation.
pub(crate) enum InterruptedOp<K, V> {
CallEvictionListener {
ts: Instant,
// 'static means that the future can capture only owned value and/or static
// references. No non-static references are allowed.
future: Shared<BoxFuture<'static, ()>>,
op: WriteOp<K, V>,
},
SendWriteOp {
ts: Instant,
op: WriteOp<K, V>,
},
}
/// Drop guard for an async task being performed. If this guard is dropped while it
/// is still having the shared `future` or the write `op`, it will convert them to an
/// `InterruptedOp` and send it to the interrupted operations channel. Later, the
/// interrupted op will be retried by `retry_interrupted_ops` method of
/// `BaseCache`.
struct CancelGuard<'a, K, V> {
interrupted_op_ch: &'a Sender<InterruptedOp<K, V>>,
ts: Instant,
future: Option<Shared<BoxFuture<'static, ()>>>,
op: Option<WriteOp<K, V>>,
}
impl<'a, K, V> CancelGuard<'a, K, V> {
fn new(interrupted_op_ch: &'a Sender<InterruptedOp<K, V>>, ts: Instant) -> Self {
Self {
interrupted_op_ch,
ts,
future: None,
op: None,
}
}
fn set_future_and_op(&mut self, future: Shared<BoxFuture<'static, ()>>, op: WriteOp<K, V>) {
self.future = Some(future);
self.op = Some(op);
}
fn set_op(&mut self, op: WriteOp<K, V>) {
self.op = Some(op);
}
fn unset_future(&mut self) {
self.future = None;
}
fn clear(&mut self) {
self.future = None;
self.op = None;
}
}
impl<K, V> Drop for CancelGuard<'_, K, V> {
fn drop(&mut self) {
let interrupted_op = match (self.future.take(), self.op.take()) {
(Some(future), Some(op)) => InterruptedOp::CallEvictionListener {
ts: self.ts,
future,
op,
},
(None, Some(op)) => InterruptedOp::SendWriteOp { ts: self.ts, op },
_ => return,
};
self.interrupted_op_ch
.send(interrupted_op)
.expect("Failed to send a pending op");
}
}
File diff suppressed because it is too large Load Diff
+489
View File
@@ -0,0 +1,489 @@
use super::{Cache, FutureExt};
use crate::{
common::{builder_utils, concurrent::Weigher, time::Clock, HousekeeperConfig},
notification::{AsyncEvictionListener, ListenerFuture, RemovalCause},
policy::{EvictionPolicy, ExpirationPolicy},
Expiry,
};
use std::{
collections::hash_map::RandomState,
hash::{BuildHasher, Hash},
marker::PhantomData,
sync::Arc,
time::Duration,
};
/// Builds a [`Cache`][cache-struct] with various configuration knobs.
///
/// [cache-struct]: ./struct.Cache.html
///
/// # Example: Expirations
///
/// ```rust
/// // Cargo.toml
/// //
/// // [dependencies]
/// // moka = { version = "0.12", features = ["future"] }
/// // tokio = { version = "1", features = ["rt-multi-thread", "macros" ] }
/// // futures = "0.3"
///
/// use moka::future::Cache;
/// use std::time::Duration;
///
/// #[tokio::main]
/// async fn main() {
/// let cache = Cache::builder()
/// // Max 10,000 entries
/// .max_capacity(10_000)
/// // Time to live (TTL): 30 minutes
/// .time_to_live(Duration::from_secs(30 * 60))
/// // Time to idle (TTI): 5 minutes
/// .time_to_idle(Duration::from_secs( 5 * 60))
/// // Create the cache.
/// .build();
///
/// // This entry will expire after 5 minutes (TTI) if there is no get().
/// cache.insert(0, "zero").await;
///
/// // This get() will extend the entry life for another 5 minutes.
/// cache.get(&0);
///
/// // Even though we keep calling get(), the entry will expire
/// // after 30 minutes (TTL) from the insert().
/// }
/// ```
///
#[must_use]
pub struct CacheBuilder<K, V, C> {
name: Option<String>,
max_capacity: Option<u64>,
initial_capacity: Option<usize>,
weigher: Option<Weigher<K, V>>,
eviction_policy: EvictionPolicy,
eviction_listener: Option<AsyncEvictionListener<K, V>>,
expiration_policy: ExpirationPolicy<K, V>,
housekeeper_config: HousekeeperConfig,
invalidator_enabled: bool,
clock: Clock,
cache_type: PhantomData<C>,
}
impl<K, V> Default for CacheBuilder<K, V, Cache<K, V, RandomState>>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
fn default() -> Self {
Self {
name: None,
max_capacity: None,
initial_capacity: None,
weigher: None,
eviction_policy: EvictionPolicy::default(),
eviction_listener: None,
expiration_policy: ExpirationPolicy::default(),
housekeeper_config: HousekeeperConfig::default(),
invalidator_enabled: false,
clock: Clock::default(),
cache_type: PhantomData,
}
}
}
impl<K, V> CacheBuilder<K, V, Cache<K, V, RandomState>>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
/// Construct a new `CacheBuilder` that will be used to build a `Cache` holding
/// up to `max_capacity` entries.
pub fn new(max_capacity: u64) -> Self {
Self {
max_capacity: Some(max_capacity),
..Self::default()
}
}
/// Builds a `Cache<K, V>`.
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build(self) -> Cache<K, V, RandomState> {
let build_hasher = RandomState::default();
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
Cache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
build_hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
/// Builds a `Cache<K, V, S>` with the given `hasher` of type `S`.
///
/// # Examples
///
/// This example uses AHash hasher from [AHash][ahash-crate] crate.
///
/// [ahash-crate]: https://crates.io/crates/ahash
///
/// ```rust
/// // Cargo.toml
/// // [dependencies]
/// // ahash = "0.8"
/// // moka = { version = ..., features = ["future"] }
/// // tokio = { version = "1", features = ["rt-multi-thread", "macros" ] }
///
/// use moka::future::Cache;
///
/// #[tokio::main]
/// async fn main() {
/// // The type of this cache is: Cache<i32, String, ahash::RandomState>
/// let cache = Cache::builder()
/// .max_capacity(100)
/// .build_with_hasher(ahash::RandomState::default());
/// cache.insert(1, "one".to_string()).await;
/// }
/// ```
///
/// Note: If you need to add a type annotation to your cache, you must use the
/// form of `Cache<K, V, S>` instead of `Cache<K, V>`. That `S` is the type of
/// the build hasher, and its default is the `RandomState` from
/// `std::collections::hash_map` module . If you use a different build hasher,
/// you must specify `S` explicitly.
///
/// Here is a good example:
///
/// ```rust
/// # use moka::future::Cache;
/// # #[tokio::main]
/// # async fn main() {
/// # let cache = Cache::builder()
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Good {
/// // Specifying the type in Cache<K, V, S> format.
/// cache: Cache<i32, String, ahash::RandomState>,
/// }
///
/// // Storing the cache from above example. This should compile.
/// Good { cache };
/// # }
/// ```
///
/// Here is a bad example. This struct cannot store the above cache because it
/// does not specify `S`:
///
/// ```compile_fail
/// # use moka::future::Cache;
/// # #[tokio::main]
/// # async fn main() {
/// # let cache = Cache::builder()
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Bad {
/// // Specifying the type in Cache<K, V> format.
/// cache: Cache<i32, String>,
/// }
///
/// // This should not compile.
/// Bad { cache };
/// // => error[E0308]: mismatched types
/// // expected struct `std::collections::hash_map::RandomState`,
/// // found struct `ahash::RandomState`
/// # }
/// ```
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build_with_hasher<S>(self, hasher: S) -> Cache<K, V, S>
where
S: BuildHasher + Clone + Send + Sync + 'static,
{
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
Cache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
}
impl<K, V, C> CacheBuilder<K, V, C> {
/// Sets the name of the cache. Currently the name is used for identification
/// only in logging messages.
pub fn name(self, name: &str) -> Self {
Self {
name: Some(name.to_string()),
..self
}
}
/// Sets the max capacity of the cache.
pub fn max_capacity(self, max_capacity: u64) -> Self {
Self {
max_capacity: Some(max_capacity),
..self
}
}
/// Sets the initial capacity (number of entries) of the cache.
pub fn initial_capacity(self, number_of_entries: usize) -> Self {
Self {
initial_capacity: Some(number_of_entries),
..self
}
}
/// Sets the eviction (and admission) policy of the cache.
///
/// The default policy is TinyLFU. See [`EvictionPolicy`][eviction-policy] for
/// more details.
///
/// [eviction-policy]: ../policy/struct.EvictionPolicy.html
pub fn eviction_policy(self, policy: EvictionPolicy) -> Self {
Self {
eviction_policy: policy,
..self
}
}
/// Sets the weigher closure to the cache.
///
/// The closure should take `&K` and `&V` as the arguments and returns a `u32`
/// representing the relative size of the entry.
pub fn weigher(self, weigher: impl Fn(&K, &V) -> u32 + Send + Sync + 'static) -> Self {
Self {
weigher: Some(Arc::new(weigher)),
..self
}
}
/// Sets the eviction listener closure to the cache. The closure should take
/// `Arc<K>`, `V` and [`RemovalCause`][removal-cause] as the arguments.
///
/// See [this example][example] for a usage of eviction listener.
///
/// # Sync or Async Eviction Listener
///
/// The closure can be either synchronous or asynchronous, and `CacheBuilder`
/// provides two methods for setting the eviction listener closure:
///
/// - If you do not need to `.await` anything in the eviction listener, use this
/// `eviction_listener` method.
/// - If you need to `.await` something in the eviction listener, use
/// [`async_eviction_listener`](#method.async_eviction_listener) method
/// instead.
///
/// # Panics
///
/// It is very important to make the listener closure not to panic. Otherwise,
/// the cache will stop calling the listener after a panic. This is an intended
/// behavior because the cache cannot know whether it is memory safe or not to
/// call the panicked listener again.
///
/// [removal-cause]: ../notification/enum.RemovalCause.html
/// [example]: ./struct.Cache.html#per-entry-expiration-policy
pub fn eviction_listener<F>(self, listener: F) -> Self
where
F: Fn(Arc<K>, V, RemovalCause) + Send + Sync + 'static,
{
let async_listener = move |k, v, c| {
{
listener(k, v, c);
std::future::ready(())
}
.boxed()
};
self.async_eviction_listener(async_listener)
}
/// Sets the eviction listener closure to the cache. The closure should take
/// `Arc<K>`, `V` and [`RemovalCause`][removal-cause] as the arguments, and
/// return a [`ListenerFuture`][listener-future].
///
/// See [this example][example] for a usage of asynchronous eviction listener.
///
/// # Sync or Async Eviction Listener
///
/// The closure can be either synchronous or asynchronous, and `CacheBuilder`
/// provides two methods for setting the eviction listener closure:
///
/// - If you do not need to `.await` anything in the eviction listener, use
/// [`eviction_listener`](#method.eviction_listener) method instead.
/// - If you need to `.await` something in the eviction listener, use
/// this method.
///
/// # Panics
///
/// It is very important to make the listener closure not to panic. Otherwise,
/// the cache will stop calling the listener after a panic. This is an intended
/// behavior because the cache cannot know whether it is memory safe or not to
/// call the panicked listener again.
///
/// [removal-cause]: ../notification/enum.RemovalCause.html
/// [listener-future]: ../notification/type.ListenerFuture.html
/// [example]: ./struct.Cache.html#example-eviction-listener
pub fn async_eviction_listener<F>(self, listener: F) -> Self
where
F: Fn(Arc<K>, V, RemovalCause) -> ListenerFuture + Send + Sync + 'static,
{
Self {
eviction_listener: Some(Box::new(listener)),
..self
}
}
/// Sets the time to live of the cache.
///
/// A cached entry will be expired after the specified duration past from
/// `insert`.
///
/// # Panics
///
/// `CacheBuilder::build*` methods will panic if the given `duration` is longer
/// than 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn time_to_live(self, duration: Duration) -> Self {
let mut builder = self;
builder.expiration_policy.set_time_to_live(duration);
builder
}
/// Sets the time to idle of the cache.
///
/// A cached entry will be expired after the specified duration past from `get`
/// or `insert`.
///
/// # Panics
///
/// `CacheBuilder::build*` methods will panic if the given `duration` is longer
/// than 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn time_to_idle(self, duration: Duration) -> Self {
let mut builder = self;
builder.expiration_policy.set_time_to_idle(duration);
builder
}
/// Sets the given `expiry` to the cache.
///
/// See [the example][per-entry-expiration-example] for per-entry expiration
/// policy in the `Cache` documentation.
///
/// [per-entry-expiration-example]:
/// ./struct.Cache.html#per-entry-expiration-policy
pub fn expire_after(self, expiry: impl Expiry<K, V> + Send + Sync + 'static) -> Self {
let mut builder = self;
builder.expiration_policy.set_expiry(Arc::new(expiry));
builder
}
#[cfg(test)]
pub(crate) fn housekeeper_config(self, conf: HousekeeperConfig) -> Self {
Self {
housekeeper_config: conf,
..self
}
}
#[cfg(test)]
pub(crate) fn clock(self, clock: Clock) -> Self {
Self { clock, ..self }
}
/// Enables support for [`Cache::invalidate_entries_if`][cache-invalidate-if]
/// method.
///
/// The cache will maintain additional internal data structures to support
/// `invalidate_entries_if` method.
///
/// [cache-invalidate-if]: ./struct.Cache.html#method.invalidate_entries_if
pub fn support_invalidation_closures(self) -> Self {
Self {
invalidator_enabled: true,
..self
}
}
}
#[cfg(test)]
mod tests {
use super::CacheBuilder;
use std::time::Duration;
#[tokio::test]
async fn build_cache() {
// Cache<char, String>
let cache = CacheBuilder::new(100).build();
let policy = cache.policy();
assert_eq!(policy.max_capacity(), Some(100));
assert_eq!(policy.time_to_live(), None);
assert_eq!(policy.time_to_idle(), None);
assert_eq!(policy.num_segments(), 1);
cache.insert('a', "Alice").await;
assert_eq!(cache.get(&'a').await, Some("Alice"));
let cache = CacheBuilder::new(100)
.time_to_live(Duration::from_secs(45 * 60))
.time_to_idle(Duration::from_secs(15 * 60))
.build();
let policy = cache.policy();
assert_eq!(policy.max_capacity(), Some(100));
assert_eq!(policy.time_to_live(), Some(Duration::from_secs(45 * 60)));
assert_eq!(policy.time_to_idle(), Some(Duration::from_secs(15 * 60)));
assert_eq!(policy.num_segments(), 1);
cache.insert('a', "Alice").await;
assert_eq!(cache.get(&'a').await, Some("Alice"));
}
#[tokio::test]
#[should_panic(expected = "time_to_live is longer than 1000 years")]
async fn build_cache_too_long_ttl() {
let thousand_years_secs: u64 = 1000 * 365 * 24 * 3600;
let builder: CacheBuilder<char, String, _> = CacheBuilder::new(100);
let duration = Duration::from_secs(thousand_years_secs);
builder
.time_to_live(duration + Duration::from_secs(1))
.build();
}
#[tokio::test]
#[should_panic(expected = "time_to_idle is longer than 1000 years")]
async fn build_cache_too_long_tti() {
let thousand_years_secs: u64 = 1000 * 365 * 24 * 3600;
let builder: CacheBuilder<char, String, _> = CacheBuilder::new(100);
let duration = Duration::from_secs(thousand_years_secs);
builder
.time_to_idle(duration + Duration::from_secs(1))
.build();
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+215
View File
@@ -0,0 +1,215 @@
use crate::common::{
concurrent::constants::{
LOG_SYNC_INTERVAL_MILLIS, READ_LOG_FLUSH_POINT, WRITE_LOG_FLUSH_POINT,
},
time::{AtomicInstant, Instant},
HousekeeperConfig,
};
use std::{
hash::{BuildHasher, Hash},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
time::Duration,
};
#[cfg(test)]
use std::sync::atomic::AtomicUsize;
use async_lock::Mutex;
use futures_util::future::{BoxFuture, Shared};
use super::base_cache::Inner;
pub(crate) struct Housekeeper {
/// A shared `Future` of the maintenance task that is currently being resolved.
current_task: Mutex<Option<Shared<BoxFuture<'static, bool>>>>,
run_after: AtomicInstant,
/// A flag to indicate if the last call on `run_pending_tasks` method left some
/// entries to evict.
///
/// Used only when the eviction listener closure is set for this cache instance
/// because, if not, `run_pending_tasks` will never leave entries to evict.
more_entries_to_evict: Option<AtomicBool>,
/// The timeout duration for the `run_pending_tasks` method. This is a safe-guard
/// to prevent cache read/write operations (that may call `run_pending_tasks`
/// internally) from being blocked for a long time when the user wrote a slow
/// eviction listener closure.
///
/// Used only when the eviction listener closure is set for this cache instance.
maintenance_task_timeout: Option<Duration>,
/// The maximum repeat count for receiving operation logs from the read and write
/// log channels. Default: `MAX_LOG_SYNC_REPEATS`.
max_log_sync_repeats: u32,
/// The batch size of entries to be processed by each internal eviction method.
/// Default: `EVICTION_BATCH_SIZE`.
eviction_batch_size: u32,
auto_run_enabled: AtomicBool,
#[cfg(test)]
pub(crate) start_count: AtomicUsize,
#[cfg(test)]
pub(crate) complete_count: AtomicUsize,
}
impl Housekeeper {
pub(crate) fn new(
is_eviction_listener_enabled: bool,
config: HousekeeperConfig,
now: Instant,
) -> Self {
let (more_entries_to_evict, maintenance_task_timeout) = if is_eviction_listener_enabled {
(
Some(AtomicBool::new(false)),
Some(config.maintenance_task_timeout),
)
} else {
(None, None)
};
Self {
current_task: Mutex::default(),
run_after: AtomicInstant::new(Self::sync_after(now)),
more_entries_to_evict,
maintenance_task_timeout,
max_log_sync_repeats: config.max_log_sync_repeats,
eviction_batch_size: config.eviction_batch_size,
auto_run_enabled: AtomicBool::new(true),
#[cfg(test)]
start_count: Default::default(),
#[cfg(test)]
complete_count: Default::default(),
}
}
pub(crate) fn should_apply_reads(&self, ch_len: usize, now: Instant) -> bool {
self.more_entries_to_evict() || self.should_apply(ch_len, READ_LOG_FLUSH_POINT, now)
}
pub(crate) fn should_apply_writes(&self, ch_len: usize, now: Instant) -> bool {
self.more_entries_to_evict() || self.should_apply(ch_len, WRITE_LOG_FLUSH_POINT, now)
}
#[inline]
fn more_entries_to_evict(&self) -> bool {
self.more_entries_to_evict
.as_ref()
.map(|v| v.load(Ordering::Acquire))
.unwrap_or(false)
}
fn set_more_entries_to_evict(&self, v: bool) {
if let Some(flag) = &self.more_entries_to_evict {
flag.store(v, Ordering::Release);
}
}
#[inline]
fn should_apply(&self, ch_len: usize, ch_flush_point: usize, now: Instant) -> bool {
self.auto_run_enabled.load(Ordering::Relaxed)
&& (ch_len >= ch_flush_point || now >= self.run_after.instant().unwrap())
}
pub(crate) async fn run_pending_tasks<K, V, S>(&self, cache: Arc<Inner<K, V, S>>)
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Clone + Send + Sync + 'static,
{
let mut current_task = self.current_task.lock().await;
self.do_run_pending_tasks(Arc::clone(&cache), &mut current_task)
.await;
drop(current_task);
// If there are any async tasks waiting in `BaseCache::schedule_write_op`
// method for the write op channel, notify them.
cache.write_op_ch_ready_event.notify(usize::MAX);
}
/// Tries to run the pending tasks if the lock is free. Returns `true` if there
/// are more entries to evict in next run.
pub(crate) async fn try_run_pending_tasks<K, V, S>(&self, cache: &Arc<Inner<K, V, S>>) -> bool
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Clone + Send + Sync + 'static,
{
if let Some(mut current_task) = self.current_task.try_lock() {
self.do_run_pending_tasks(Arc::clone(cache), &mut current_task)
.await;
} else {
return false;
}
// The `current_task` lock should be free now.
// If there are any async tasks waiting in `BaseCache::schedule_write_op`
// method for the write op channel, notify them.
cache.write_op_ch_ready_event.notify(usize::MAX);
true
}
async fn do_run_pending_tasks<K, V, S>(
&self,
cache: Arc<Inner<K, V, S>>,
current_task: &mut Option<Shared<BoxFuture<'static, bool>>>,
) where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Clone + Send + Sync + 'static,
{
use futures_util::FutureExt;
let now = cache.current_time();
let more_to_evict;
// Async Cancellation Safety: Our maintenance task is cancellable as we save
// it in the lock. If it is canceled, we will resume it in the next run.
if let Some(task) = &*current_task {
// This task was cancelled in the previous run due to the enclosing
// Future was dropped. Resume the task now by awaiting.
more_to_evict = task.clone().await;
} else {
let timeout = self.maintenance_task_timeout;
let repeats = self.max_log_sync_repeats;
let batch_size = self.eviction_batch_size;
// Create a new maintenance task and await it.
let task = async move {
cache
.do_run_pending_tasks(timeout, repeats, batch_size)
.await
}
.boxed()
.shared();
*current_task = Some(task.clone());
#[cfg(test)]
self.start_count.fetch_add(1, Ordering::AcqRel);
more_to_evict = task.await;
}
// If we are here, it means that the maintenance task has been completed.
// We can remove it from the lock.
*current_task = None;
self.run_after.set_instant(Self::sync_after(now));
self.set_more_entries_to_evict(more_to_evict);
#[cfg(test)]
self.complete_count.fetch_add(1, Ordering::AcqRel);
}
fn sync_after(now: Instant) -> Instant {
let dur = Duration::from_millis(LOG_SYNC_INTERVAL_MILLIS);
now.saturating_add(dur)
}
}
#[cfg(test)]
impl Housekeeper {
pub(crate) fn disable_auto_run(&self) {
self.auto_run_enabled.store(false, Ordering::Relaxed);
}
}
+391
View File
@@ -0,0 +1,391 @@
use super::{base_cache::Inner, PredicateId, PredicateIdStr};
use crate::{
common::{
concurrent::{arc::MiniArc, AccessTime, KvEntry, ValueEntry},
time::Instant,
},
notification::RemovalCause,
PredicateError,
};
use async_lock::{Mutex, MutexGuard};
use std::{
hash::{BuildHasher, Hash},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use uuid::Uuid;
pub(crate) type PredicateFun<K, V> = Arc<dyn Fn(&K, &V) -> bool + Send + Sync + 'static>;
const PREDICATE_MAP_NUM_SEGMENTS: usize = 16;
pub(crate) struct KeyDateLite<K> {
key: Arc<K>,
hash: u64,
timestamp: Instant,
}
impl<K> Clone for KeyDateLite<K> {
fn clone(&self) -> Self {
Self {
key: Arc::clone(&self.key),
hash: self.hash,
timestamp: self.timestamp,
}
}
}
impl<K> KeyDateLite<K> {
pub(crate) fn new(key: &Arc<K>, hash: u64, timestamp: Instant) -> Self {
Self {
key: Arc::clone(key),
hash,
timestamp,
}
}
}
pub(crate) struct Invalidator<K, V, S> {
predicates: crate::cht::SegmentedHashMap<PredicateId, Predicate<K, V>, S>,
is_empty: AtomicBool,
scan_context: Arc<ScanContext<K, V>>,
}
//
// Crate public methods.
//
impl<K, V, S> Invalidator<K, V, S> {
pub(crate) fn new(hasher: S) -> Self
where
S: BuildHasher,
{
const CAPACITY: usize = 0;
let predicates = crate::cht::SegmentedHashMap::with_num_segments_capacity_and_hasher(
PREDICATE_MAP_NUM_SEGMENTS,
CAPACITY,
hasher,
);
Self {
predicates,
is_empty: AtomicBool::new(true),
scan_context: Arc::new(ScanContext::default()),
}
}
pub(crate) fn is_empty(&self) -> bool {
self.is_empty.load(Ordering::Acquire)
}
pub(crate) fn remove_predicates_registered_before(&self, ts: Instant)
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher,
{
let pred_map = &self.predicates;
let removing_ids = pred_map
.iter()
.filter(|(_, pred)| pred.registered_at <= ts)
.map(|(id, _)| id)
.collect::<Vec<_>>();
for id in removing_ids {
let hash = pred_map.hash(&id);
pred_map.remove(hash, |k| k == &id);
}
if pred_map.is_empty() {
self.is_empty.store(true, Ordering::Release);
}
}
pub(crate) fn register_predicate(
&self,
predicate: PredicateFun<K, V>,
registered_at: Instant,
) -> Result<PredicateId, PredicateError>
where
K: Hash + Eq,
S: BuildHasher,
{
const MAX_RETRY: usize = 1_000;
let mut tries = 0;
let preds = &self.predicates;
while tries < MAX_RETRY {
let id = Uuid::new_v4().as_hyphenated().to_string();
let hash = preds.hash(&id);
if preds.contains_key(hash, |k| k == &id) {
tries += 1;
continue; // Retry
}
let pred = Predicate::new(&id, predicate, registered_at);
preds.insert_entry_and(id.clone(), hash, pred, |_, _| ());
self.is_empty.store(false, Ordering::Release);
return Ok(id);
}
// Since we are using 128-bit UUID for the ID and we do retries for MAX_RETRY
// times, this panic should extremely unlikely occur (unless there is a bug in
// UUID generation).
panic!("Cannot assign a new PredicateId to a predicate");
}
// This method will be called by the get method of Cache.
#[inline]
pub(crate) fn apply_predicates(&self, key: &Arc<K>, entry: &MiniArc<ValueEntry<K, V>>) -> bool
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Send + Sync + 'static,
{
if self.is_empty() {
false
} else if let Some(ts) = entry.last_modified() {
Self::do_apply_predicates(
self.predicates.iter().map(|(_, v)| v),
key,
&entry.value,
ts,
)
} else {
false
}
}
pub(crate) async fn scan_and_invalidate(
&self,
cache: &Inner<K, V, S>,
candidates: Vec<KeyDateLite<K>>,
is_truncated: bool,
) -> (Vec<KvEntry<K, V>>, bool)
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Send + Sync + 'static,
{
let mut predicates = self.scan_context.predicates.lock().await;
if predicates.is_empty() {
*predicates = self.predicates.iter().map(|(_k, v)| v).collect();
}
let mut invalidated = Vec::default();
let mut newest_timestamp = None;
for candidate in &candidates {
let key = &candidate.key;
let hash = candidate.hash;
let ts = candidate.timestamp;
if self.apply(&predicates, cache, key, hash, ts) {
if let Some(entry) = Self::invalidate(cache, key, hash, ts).await {
invalidated.push(KvEntry {
key: Arc::clone(key),
entry,
});
}
}
newest_timestamp = Some(ts);
}
self.remove_finished_predicates(predicates, is_truncated, newest_timestamp);
(invalidated, self.predicates.is_empty())
}
}
//
// Private methods.
//
impl<K, V, S> Invalidator<K, V, S>
where
K: Hash + Eq,
S: BuildHasher + Send + Sync + 'static,
{
#[inline]
fn do_apply_predicates<I>(predicates: I, key: &K, value: &V, ts: Instant) -> bool
where
I: Iterator<Item = Predicate<K, V>>,
{
for predicate in predicates {
if predicate.is_applicable(ts) && predicate.apply(key, value) {
return true;
}
}
false
}
fn remove_finished_predicates(
&self,
mut predicates: MutexGuard<'_, Vec<Predicate<K, V>>>,
is_truncated: bool,
newest_timestamp: Option<Instant>,
) where
K: Hash + Eq,
S: BuildHasher,
{
let predicates = &mut *predicates;
if is_truncated {
if let Some(ts) = newest_timestamp {
let (active, finished): (Vec<_>, Vec<_>) =
predicates.drain(..).partition(|p| p.is_applicable(ts));
// Remove finished predicates from the predicate registry.
self.remove_predicates(&finished);
// Set the active predicates to the scan context.
*predicates = active;
} else {
unreachable!();
}
} else {
// Remove all the predicates from the predicate registry and scan context.
self.remove_predicates(predicates);
predicates.clear();
}
}
fn remove_predicates(&self, predicates: &[Predicate<K, V>])
where
K: Hash + Eq,
S: BuildHasher,
{
let pred_map = &self.predicates;
for p in predicates {
let hash = pred_map.hash(p.id());
pred_map.remove(hash, |k| k == p.id());
}
if pred_map.is_empty() {
self.is_empty.store(true, Ordering::Release);
}
}
fn apply(
&self,
predicates: &[Predicate<K, V>],
cache: &Inner<K, V, S>,
key: &Arc<K>,
hash: u64,
ts: Instant,
) -> bool {
if let Some(entry) = cache.cache.get(hash, |k| k == key) {
if let Some(lm) = entry.last_modified() {
if lm == ts {
return Self::do_apply_predicates(
predicates.iter().cloned(),
key,
&entry.value,
lm,
);
}
}
}
false
}
async fn invalidate(
cache: &Inner<K, V, S>,
key: &Arc<K>,
hash: u64,
ts: Instant,
) -> Option<MiniArc<ValueEntry<K, V>>>
where
K: Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
// Lock the key for removal if blocking removal notification is enabled.
let kl = cache.maybe_key_lock(key);
let _klg = if let Some(lock) = &kl {
Some(lock.lock().await)
} else {
None
};
let maybe_entry = cache.cache.remove_if(
hash,
|k| k == key,
|_, v| {
if let Some(lm) = v.last_modified() {
lm == ts
} else {
false
}
},
);
if let Some(entry) = &maybe_entry {
if cache.is_removal_notifier_enabled() {
cache
.notify_single_removal(Arc::clone(key), entry, RemovalCause::Explicit)
.await;
}
}
maybe_entry
}
}
//
// for testing
//
#[cfg(test)]
impl<K, V, S> Invalidator<K, V, S> {
pub(crate) fn predicate_count(&self) -> usize {
self.predicates.len()
}
}
struct ScanContext<K, V> {
predicates: Mutex<Vec<Predicate<K, V>>>,
}
impl<K, V> Default for ScanContext<K, V> {
fn default() -> Self {
Self {
predicates: Mutex::new(Vec::default()),
}
}
}
struct Predicate<K, V> {
id: PredicateId,
f: PredicateFun<K, V>,
registered_at: Instant,
}
impl<K, V> Clone for Predicate<K, V> {
fn clone(&self) -> Self {
Self {
id: self.id.clone(),
f: Arc::clone(&self.f),
registered_at: self.registered_at,
}
}
}
impl<K, V> Predicate<K, V> {
fn new(id: PredicateIdStr<'_>, f: PredicateFun<K, V>, registered_at: Instant) -> Self {
Self {
id: id.to_string(),
f,
registered_at,
}
}
fn id(&self) -> PredicateIdStr<'_> {
&self.id
}
fn is_applicable(&self, last_modified: Instant) -> bool {
last_modified <= self.registered_at
}
fn apply(&self, key: &K, value: &V) -> bool {
(self.f)(key, value)
}
}
+94
View File
@@ -0,0 +1,94 @@
use std::{
hash::{BuildHasher, Hash},
sync::Arc,
};
use crate::{cht::SegmentedHashMap, common::concurrent::arc::MiniArc};
use async_lock::{Mutex, MutexGuard};
const LOCK_MAP_NUM_SEGMENTS: usize = 64;
type LockMap<K, S> = SegmentedHashMap<Arc<K>, MiniArc<Mutex<()>>, S>;
// We need the `where` clause here because of the Drop impl.
pub(crate) struct KeyLock<'a, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
map: &'a LockMap<K, S>,
key: Arc<K>,
hash: u64,
lock: MiniArc<Mutex<()>>,
}
impl<K, S> Drop for KeyLock<'_, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
fn drop(&mut self) {
if MiniArc::count(&self.lock) <= 2 {
self.map.remove_if(
self.hash,
|k| k == &self.key,
|_k, v| MiniArc::count(v) <= 2,
);
}
}
}
impl<'a, K, S> KeyLock<'a, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
fn new(map: &'a LockMap<K, S>, key: &Arc<K>, hash: u64, lock: MiniArc<Mutex<()>>) -> Self {
Self {
map,
key: Arc::clone(key),
hash,
lock,
}
}
pub(crate) async fn lock(&self) -> MutexGuard<'_, ()> {
self.lock.lock().await
}
}
pub(crate) struct KeyLockMap<K, S> {
locks: LockMap<K, S>,
}
impl<K, S> KeyLockMap<K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
pub(crate) fn with_hasher(hasher: S) -> Self {
Self {
locks: SegmentedHashMap::with_num_segments_and_hasher(LOCK_MAP_NUM_SEGMENTS, hasher),
}
}
pub(crate) fn key_lock(&self, key: &Arc<K>) -> KeyLock<'_, K, S> {
let hash = self.locks.hash(key);
let kl = MiniArc::new(Mutex::new(()));
match self
.locks
.insert_if_not_present(Arc::clone(key), hash, kl.clone())
{
None => KeyLock::new(&self.locks, key, hash, kl),
Some(existing_kl) => KeyLock::new(&self.locks, key, hash, existing_kl),
}
}
}
#[cfg(test)]
impl<K, S> KeyLockMap<K, S> {
pub(crate) fn is_empty(&self) -> bool {
self.locks.len() == 0
}
}
+78
View File
@@ -0,0 +1,78 @@
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use futures_util::FutureExt;
use crate::notification::{AsyncEvictionListener, RemovalCause};
pub(crate) struct RemovalNotifier<K, V> {
listener: AsyncEvictionListener<K, V>,
is_enabled: AtomicBool,
#[cfg(feature = "logging")]
cache_name: Option<String>,
}
impl<K, V> RemovalNotifier<K, V> {
pub(crate) fn new(listener: AsyncEvictionListener<K, V>, _cache_name: Option<String>) -> Self {
Self {
listener,
is_enabled: AtomicBool::new(true),
#[cfg(feature = "logging")]
cache_name: _cache_name,
}
}
pub(crate) async fn notify(&self, key: Arc<K>, value: V, cause: RemovalCause) {
use std::panic::{catch_unwind, AssertUnwindSafe};
if !self.is_enabled.load(Ordering::Acquire) {
return;
}
// This macro unwraps the result of the catch_unwind call if it is Ok. And
// disable the notifier and do early return if the listener panicked.
macro_rules! try_or_disable {
($match_expr:expr) => {
match $match_expr {
Ok(v) => v,
Err(_payload) => {
self.is_enabled.store(false, Ordering::Release);
#[cfg(feature = "logging")]
log_panic(&*_payload, self.cache_name.as_deref());
return;
}
}
};
}
let listener_clo = || (self.listener)(key, value, cause);
// Safety: It is safe to assert unwind safety here because we will not
// call the listener again if it has been panicked.
let fut = try_or_disable!(catch_unwind(AssertUnwindSafe(listener_clo)));
try_or_disable!(AssertUnwindSafe(fut).catch_unwind().await);
}
}
#[cfg(feature = "logging")]
fn log_panic(payload: &(dyn std::any::Any + Send + 'static), cache_name: Option<&str>) {
// Try to downcast the payload into &str or String.
//
// NOTE: Clippy will complain if we use `if let Some(_)` here.
// https://rust-lang.github.io/rust-clippy/master/index.html#manual_map
let message: Option<std::borrow::Cow<'_, str>> =
(payload.downcast_ref::<&str>().map(|s| (*s).into()))
.or_else(|| payload.downcast_ref::<String>().map(Into::into));
let cn = cache_name
.map(|name| format!("[{name}] "))
.unwrap_or_default();
if let Some(m) = message {
log::error!("{cn}Disabled the eviction listener because it panicked at '{m}'");
} else {
log::error!("{cn}Disabled the eviction listener because it panicked");
}
}
@@ -0,0 +1,681 @@
use async_lock::{RwLock, RwLockWriteGuard};
use futures_util::FutureExt;
use std::{
any::{Any, TypeId},
fmt,
future::Future,
hash::{BuildHasher, Hash},
pin::Pin,
sync::Arc,
};
use crate::{
common::concurrent::arc::MiniArc,
ops::compute::{CompResult, Op},
Entry,
};
use super::{Cache, ComputeNone, OptionallyNone};
const WAITER_MAP_NUM_SEGMENTS: usize = 64;
type ErrorObject = Arc<dyn Any + Send + Sync + 'static>;
pub(crate) enum InitResult<V, E> {
Initialized(V),
ReadExisting(V),
InitErr(Arc<E>),
}
enum WaiterValue<V> {
Computing,
Ready(Result<V, ErrorObject>),
ReadyNone,
// https://github.com/moka-rs/moka/issues/43
InitFuturePanicked,
// https://github.com/moka-rs/moka/issues/59
EnclosingFutureAborted,
}
impl<V> fmt::Debug for WaiterValue<V> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
WaiterValue::Computing => write!(f, "Computing"),
WaiterValue::Ready(_) => write!(f, "Ready"),
WaiterValue::ReadyNone => write!(f, "ReadyNone"),
WaiterValue::InitFuturePanicked => write!(f, "InitFuturePanicked"),
WaiterValue::EnclosingFutureAborted => write!(f, "EnclosingFutureAborted"),
}
}
}
type Waiter<V> = MiniArc<RwLock<WaiterValue<V>>>;
type WaiterMap<K, V, S> = crate::cht::SegmentedHashMap<(Arc<K>, TypeId), Waiter<V>, S>;
struct WaiterGuard<'a, K, V, S>
// NOTE: We usually do not attach trait bounds to here at the struct definition, but
// the Drop trait requires these bounds here.
where
K: Eq + Hash,
V: Clone,
S: BuildHasher,
{
w_key: Option<(Arc<K>, TypeId)>,
w_hash: u64,
waiters: &'a WaiterMap<K, V, S>,
write_lock: RwLockWriteGuard<'a, WaiterValue<V>>,
}
impl<'a, K, V, S> WaiterGuard<'a, K, V, S>
where
K: Eq + Hash,
V: Clone,
S: BuildHasher,
{
fn new(
w_key: (Arc<K>, TypeId),
w_hash: u64,
waiters: &'a WaiterMap<K, V, S>,
write_lock: RwLockWriteGuard<'a, WaiterValue<V>>,
) -> Self {
Self {
w_key: Some(w_key),
w_hash,
waiters,
write_lock,
}
}
fn set_waiter_value(mut self, v: WaiterValue<V>) {
*self.write_lock = v;
if let Some(w_key) = self.w_key.take() {
remove_waiter(self.waiters, w_key, self.w_hash);
}
}
}
impl<K, V, S> Drop for WaiterGuard<'_, K, V, S>
where
K: Eq + Hash,
V: Clone,
S: BuildHasher,
{
fn drop(&mut self) {
if let Some(w_key) = self.w_key.take() {
// Value is not set. This means the future containing `*get_with` method
// has been aborted. Remove our waiter to prevent the issue described in
// https://github.com/moka-rs/moka/issues/59
*self.write_lock = WaiterValue::EnclosingFutureAborted;
remove_waiter(self.waiters, w_key, self.w_hash);
}
}
}
pub(crate) struct ValueInitializer<K, V, S> {
// TypeId is the type ID of the concrete error type of generic type E in the
// try_get_with method. We use the type ID as a part of the key to ensure that we
// can always downcast the trait object ErrorObject (in Waiter<V>) into its
// concrete type.
waiters: MiniArc<WaiterMap<K, V, S>>,
}
impl<K, V, S> ValueInitializer<K, V, S>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Clone + Send + Sync + 'static,
{
pub(crate) fn with_hasher(hasher: S) -> Self {
Self {
waiters: MiniArc::new(crate::cht::SegmentedHashMap::with_num_segments_and_hasher(
WAITER_MAP_NUM_SEGMENTS,
hasher,
)),
}
}
//
// NOTES: We use `Pin<&mut impl Future>` instead of `impl Future` here for the
// `init` argument. This is because we want to avoid the future size inflation
// caused by calling nested async functions. See the following links for more
// details:
//
// - https://github.com/moka-rs/moka/issues/212
// - https://swatinem.de/blog/future-size/
//
/// # Panics
/// Panics if the `init` future has been panicked.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn try_init_or_read<I, O, E>(
&self,
c_key: &Arc<K>,
c_hash: u64,
type_id: TypeId,
cache: &Cache<K, V, S>,
mut ignore_if: Option<I>,
// Future to initialize a new value.
init: Pin<&mut impl Future<Output = O>>,
// Function to convert a value O, returned from the init future, into
// Result<V, E>.
post_init: fn(O) -> Result<V, E>,
) -> InitResult<V, E>
where
I: FnMut(&V) -> bool + Send,
E: Send + Sync + 'static,
{
use std::panic::{resume_unwind, AssertUnwindSafe};
use InitResult::{InitErr, Initialized, ReadExisting};
const MAX_RETRIES: usize = 200;
let mut retries = 0;
let (w_key, w_hash) = waiter_key_hash(&self.waiters, c_key, type_id);
let waiter = MiniArc::new(RwLock::new(WaiterValue::Computing));
// NOTE: We have to acquire a write lock before `try_insert_waiter`,
// so that any concurrent attempt will get our lock and wait on it.
let lock = waiter.write().await;
loop {
let Some(existing_waiter) =
try_insert_waiter(&self.waiters, w_key.clone(), w_hash, &waiter)
else {
// Inserted.
break;
};
// Somebody else's waiter already exists, so wait for its result to become available.
let waiter_result = existing_waiter.read().await;
match &*waiter_result {
WaiterValue::Ready(Ok(value)) => return ReadExisting(value.clone()),
WaiterValue::Ready(Err(e)) => return InitErr(Arc::clone(e).downcast().unwrap()),
// Somebody else's init future has been panicked.
WaiterValue::InitFuturePanicked => {
retries += 1;
panic_if_retry_exhausted_for_panicking(retries, MAX_RETRIES);
// Retry from the beginning.
continue;
}
// Somebody else (a future containing `get_with`/`try_get_with`)
// has been aborted.
WaiterValue::EnclosingFutureAborted => {
retries += 1;
panic_if_retry_exhausted_for_aborting(retries, MAX_RETRIES);
// Retry from the beginning.
continue;
}
// Unexpected state.
s @ (WaiterValue::Computing | WaiterValue::ReadyNone) => panic!(
"Got unexpected state `{s:?}` after resolving `init` future. \
This might be a bug in Moka"
),
}
}
// Our waiter was inserted.
// Create a guard. This will ensure to remove our waiter when the
// enclosing future has been aborted:
// https://github.com/moka-rs/moka/issues/59
let waiter_guard = WaiterGuard::new(w_key, w_hash, &self.waiters, lock);
// Check if the value has already been inserted by other thread.
if let Some(value) = cache
.base
.get_with_hash(&**c_key, c_hash, ignore_if.as_mut(), false, false)
.await
.map(Entry::into_value)
{
// Yes. Set the waiter value, remove our waiter, and return
// the existing value.
waiter_guard.set_waiter_value(WaiterValue::Ready(Ok(value.clone())));
return ReadExisting(value);
}
// The value still does note exist. Let's resolve the init
// future. Catching panic is safe here as we do not try to
// resolve the future again.
match AssertUnwindSafe(init).catch_unwind().await {
// Resolved.
Ok(value) => match post_init(value) {
Ok(value) => {
cache
.insert_with_hash(Arc::clone(c_key), c_hash, value.clone())
.await;
waiter_guard.set_waiter_value(WaiterValue::Ready(Ok(value.clone())));
Initialized(value)
}
Err(e) => {
let err: ErrorObject = Arc::new(e);
waiter_guard.set_waiter_value(WaiterValue::Ready(Err(Arc::clone(&err))));
InitErr(err.downcast().unwrap())
}
},
// Panicked.
Err(payload) => {
waiter_guard.set_waiter_value(WaiterValue::InitFuturePanicked);
resume_unwind(payload);
}
}
// The lock will be unlocked here.
}
/// # Panics
/// Panics if the `init` future has been panicked.
pub(crate) async fn try_compute<'a, F, Fut, O, E>(
&'a self,
c_key: Arc<K>,
c_hash: u64,
cache: &Cache<K, V, S>,
f: F,
post_init: fn(O) -> Result<Op<V>, E>,
allow_nop: bool,
) -> Result<CompResult<K, V>, E>
where
F: FnOnce(Option<Entry<K, V>>) -> Fut,
Fut: Future<Output = O> + 'a,
E: Send + Sync + 'static,
{
use std::panic::{resume_unwind, AssertUnwindSafe};
let type_id = TypeId::of::<ComputeNone>();
let (w_key, w_hash) = waiter_key_hash(&self.waiters, &c_key, type_id);
let waiter = MiniArc::new(RwLock::new(WaiterValue::Computing));
// NOTE: We have to acquire a write lock before `try_insert_waiter`,
// so that any concurrent attempt will get our lock and wait on it.
let lock = waiter.write().await;
loop {
let Some(existing_waiter) =
try_insert_waiter(&self.waiters, w_key.clone(), w_hash, &waiter)
else {
// Inserted.
break;
};
// Somebody else's waiter already exists, so wait for it to finish
// (wait for it to release the write lock).
let waiter_result = existing_waiter.read().await;
match &*waiter_result {
// Unexpected state.
WaiterValue::Computing => panic!(
"Got unexpected state `Computing` after resolving `init` future. \
This might be a bug in Moka"
),
_ => {
// Try to insert our waiter again.
continue;
}
}
}
// Our waiter was inserted.
// Create a guard. This will ensure to remove our waiter when the
// enclosing future has been aborted:
// https://github.com/moka-rs/moka/issues/59
let waiter_guard = WaiterGuard::new(w_key, w_hash, &self.waiters, lock);
// Get the current value.
let ignore_if = None as Option<&mut fn(&V) -> bool>;
let maybe_entry = cache
.base
.get_with_hash(&*c_key, c_hash, ignore_if, true, true)
.await;
let maybe_value = if allow_nop {
maybe_entry.as_ref().map(|ent| ent.value().clone())
} else {
None
};
let entry_existed = maybe_entry.is_some();
// Evaluate the `f` closure and get a future. Catching panic is safe here as
// we will not evaluate the closure again.
let fut = match std::panic::catch_unwind(AssertUnwindSafe(|| f(maybe_entry))) {
// Evaluated.
Ok(fut) => fut,
// Panicked.
Err(payload) => {
waiter_guard.set_waiter_value(WaiterValue::InitFuturePanicked);
resume_unwind(payload);
}
};
// Resolve the `fut` future. Catching panic is safe here as we will not
// resolve the future again.
let output = match AssertUnwindSafe(fut).catch_unwind().await {
// Resolved.
Ok(output) => output,
// Panicked.
Err(payload) => {
waiter_guard.set_waiter_value(WaiterValue::InitFuturePanicked);
resume_unwind(payload);
}
};
// Defer waiter removal until after the cache mutation so that concurrent
// callers cannot insert their own waiter and observe stale cache state
// between the waiter removal and the cache write.
match post_init(output) {
Ok(Op::Nop) => {
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if let Some(value) = maybe_value {
Ok(CompResult::Unchanged(Entry::new(
Some(c_key),
value,
false,
false,
)))
} else {
Ok(CompResult::StillNone(c_key))
}
}
Ok(Op::Put(value)) => {
cache
.insert_with_hash(Arc::clone(&c_key), c_hash, value.clone())
.await;
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if entry_existed {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), value, true, true);
Ok(CompResult::ReplacedWith(entry))
} else {
let entry = Entry::new(Some(c_key), value, true, false);
Ok(CompResult::Inserted(entry))
}
}
Ok(Op::Remove) => {
let maybe_prev_v = cache.invalidate_with_hash(&*c_key, c_hash, true).await;
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if let Some(prev_v) = maybe_prev_v {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), prev_v, false, false);
Ok(CompResult::Removed(entry))
} else {
Ok(CompResult::StillNone(c_key))
}
}
Err(e) => {
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
Err(e)
}
}
// The lock will be unlocked here.
}
pub(crate) async fn try_compute_if_nobody_else<'a, F, Fut, O, E>(
&'a self,
c_key: Arc<K>,
c_hash: u64,
cache: &Cache<K, V, S>,
f: F,
post_init: fn(O) -> Result<Op<V>, E>,
allow_nop: bool,
) -> Result<CompResult<K, V>, E>
where
F: FnOnce(Option<Entry<K, V>>) -> Fut,
Fut: Future<Output = O> + 'a,
E: Send + Sync + 'static,
{
use std::panic::{resume_unwind, AssertUnwindSafe};
let type_id = TypeId::of::<ComputeNone>();
let (w_key, w_hash) = waiter_key_hash(&self.waiters, &c_key, type_id);
let waiter = MiniArc::new(RwLock::new(WaiterValue::Computing));
// NOTE: We have to acquire a write lock before `try_insert_waiter`,
// so that any concurrent attempt will get our lock and wait on it.
let lock = waiter.write().await;
if let Some(_existing_waiter) =
try_insert_waiter(&self.waiters, w_key.clone(), w_hash, &waiter)
{
// There's already a waiter computing for this entry, cancel this computation.
// Get the current value.
let ignore_if = None as Option<&mut fn(&V) -> bool>;
let maybe_entry = cache
.base
.get_with_hash(&*c_key, c_hash, ignore_if, true, true)
.await;
let maybe_value = maybe_entry.as_ref().map(|ent| ent.value().clone());
return if let Some(value) = maybe_value {
Ok(CompResult::Unchanged(Entry::new(
Some(c_key),
value,
false,
false,
)))
} else {
Ok(CompResult::StillNone(c_key))
};
// The lock will be unlocked here.
} else {
// Inserted.
}
// Our waiter was inserted.
// Create a guard. This will ensure to remove our waiter when the
// enclosing future has been aborted:
// https://github.com/moka-rs/moka/issues/59
let waiter_guard = WaiterGuard::new(w_key, w_hash, &self.waiters, lock);
// Get the current value.
let ignore_if = None as Option<&mut fn(&V) -> bool>;
let maybe_entry = cache
.base
.get_with_hash(&*c_key, c_hash, ignore_if, true, true)
.await;
let maybe_value = if allow_nop {
maybe_entry.as_ref().map(|ent| ent.value().clone())
} else {
None
};
let entry_existed = maybe_entry.is_some();
// Evaluate the `f` closure and get a future. Catching panic is safe here as
// we will not evaluate the closure again.
let fut = match std::panic::catch_unwind(AssertUnwindSafe(|| f(maybe_entry))) {
// Evaluated.
Ok(fut) => fut,
Err(payload) => {
waiter_guard.set_waiter_value(WaiterValue::InitFuturePanicked);
resume_unwind(payload);
}
};
// Resolve the `fut` future. Catching panic is safe here as we will not
// resolve the future again.
let output = match AssertUnwindSafe(fut).catch_unwind().await {
// Resolved.
Ok(output) => output,
// Panicked.
Err(payload) => {
waiter_guard.set_waiter_value(WaiterValue::InitFuturePanicked);
resume_unwind(payload);
}
};
// Defer waiter removal until after the cache mutation so that concurrent
// callers cannot insert their own waiter and observe stale cache state
// between the waiter removal and the cache write.
match post_init(output) {
Ok(Op::Nop) => {
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if let Some(value) = maybe_value {
Ok(CompResult::Unchanged(Entry::new(
Some(c_key),
value,
false,
false,
)))
} else {
Ok(CompResult::StillNone(c_key))
}
}
Ok(Op::Put(value)) => {
cache
.insert_with_hash(Arc::clone(&c_key), c_hash, value.clone())
.await;
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if entry_existed {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), value, true, true);
Ok(CompResult::ReplacedWith(entry))
} else {
let entry = Entry::new(Some(c_key), value, true, false);
Ok(CompResult::Inserted(entry))
}
}
Ok(Op::Remove) => {
let maybe_prev_v = cache.invalidate_with_hash(&*c_key, c_hash, true).await;
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
if let Some(prev_v) = maybe_prev_v {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), prev_v, false, false);
Ok(CompResult::Removed(entry))
} else {
Ok(CompResult::StillNone(c_key))
}
}
Err(e) => {
waiter_guard.set_waiter_value(WaiterValue::ReadyNone);
Err(e)
}
}
// The lock will be unlocked here.
}
/// The `post_init` function for the `get_with` method of cache.
pub(crate) fn post_init_for_get_with(value: V) -> Result<V, ()> {
Ok(value)
}
/// The `post_init` function for the `optionally_get_with` method of cache.
pub(crate) fn post_init_for_optionally_get_with(
value: Option<V>,
) -> Result<V, Arc<OptionallyNone>> {
// `value` can be either `Some` or `None`. For `None` case, without change
// the existing API too much, we will need to convert `None` to Arc<E> here.
// `Infallible` could not be instantiated. So it might be good to use an
// empty struct to indicate the error type.
value.ok_or(Arc::new(OptionallyNone))
}
/// The `post_init` function for `try_get_with` method of cache.
pub(crate) fn post_init_for_try_get_with<E>(result: Result<V, E>) -> Result<V, E> {
result
}
/// The `post_init` function for the `and_upsert_with` method of cache.
pub(crate) fn post_init_for_upsert_with(value: V) -> Result<Op<V>, ()> {
Ok(Op::Put(value))
}
/// The `post_init` function for the `and_compute_with` method of cache.
pub(crate) fn post_init_for_compute_with(op: Op<V>) -> Result<Op<V>, ()> {
Ok(op)
}
/// The `post_init` function for the `and_try_compute_with` method of cache.
pub(crate) fn post_init_for_try_compute_with<E>(op: Result<Op<V>, E>) -> Result<Op<V>, E>
where
E: Send + Sync + 'static,
{
op
}
/// The `post_init` function for the `and_try_compute_if_nobody_else` method of cache.
pub(crate) fn post_init_for_try_compute_with_if_nobody_else<E>(
op: Result<Op<V>, E>,
) -> Result<Op<V>, E>
where
E: Send + Sync + 'static,
{
op
}
/// Returns the `type_id` for `get_with` method of cache.
pub(crate) fn type_id_for_get_with() -> TypeId {
// NOTE: We use a regular function here instead of a const fn because TypeId
// is not stable as a const fn. (as of our MSRV)
TypeId::of::<()>()
}
/// Returns the `type_id` for `optionally_get_with` method of cache.
pub(crate) fn type_id_for_optionally_get_with() -> TypeId {
TypeId::of::<OptionallyNone>()
}
/// Returns the `type_id` for `try_get_with` method of cache.
pub(crate) fn type_id_for_try_get_with<E: 'static>() -> TypeId {
TypeId::of::<E>()
}
}
#[cfg(test)]
impl<K, V, S> ValueInitializer<K, V, S> {
pub(crate) fn waiter_count(&self) -> usize {
self.waiters.len()
}
}
#[inline]
fn remove_waiter<K, V, S>(waiter_map: &WaiterMap<K, V, S>, w_key: (Arc<K>, TypeId), w_hash: u64)
where
(Arc<K>, TypeId): Eq + Hash,
S: BuildHasher,
{
waiter_map.remove(w_hash, |k| k == &w_key);
}
#[inline]
fn try_insert_waiter<K, V, S>(
waiter_map: &WaiterMap<K, V, S>,
w_key: (Arc<K>, TypeId),
w_hash: u64,
waiter: &Waiter<V>,
) -> Option<Waiter<V>>
where
(Arc<K>, TypeId): Eq + Hash,
S: BuildHasher,
{
let waiter = MiniArc::clone(waiter);
waiter_map.insert_if_not_present(w_key, w_hash, waiter)
}
#[inline]
fn waiter_key_hash<K, V, S>(
waiter_map: &WaiterMap<K, V, S>,
c_key: &Arc<K>,
type_id: TypeId,
) -> ((Arc<K>, TypeId), u64)
where
(Arc<K>, TypeId): Eq + Hash,
S: BuildHasher,
{
let w_key = (Arc::clone(c_key), type_id);
let w_hash = waiter_map.hash(&w_key);
(w_key, w_hash)
}
fn panic_if_retry_exhausted_for_panicking(retries: usize, max: usize) {
assert!(
retries < max,
"Too many retries. Tried to read the return value from the `init` future \
but failed {retries} times. Maybe the `init` kept panicking?"
);
}
fn panic_if_retry_exhausted_for_aborting(retries: usize, max: usize) {
assert!(
retries < max,
"Too many retries. Tried to read the return value from the `init` future \
but failed {retries} times. Maybe the future containing `get_with`/`try_get_with` \
kept being aborted?"
);
}
+310
View File
@@ -0,0 +1,310 @@
#![warn(clippy::all)]
#![warn(rust_2018_idioms)]
// Temporary disable this lint as the MSRV (1.51) require an older lint name:
// #![deny(rustdoc::broken_intra_doc_links)]
#![cfg_attr(docsrs, feature(doc_cfg))]
//! Moka is a fast, concurrent cache library for Rust. Moka is inspired by the
//! [Caffeine][caffeine-git] library for Java.
//!
//! Moka provides in-memory concurrent cache implementations on top of hash maps.
//! They support full concurrency of retrievals and a high expected concurrency for
//! updates. They utilize a lock-free concurrent hash table as the central key-value
//! storage.
//!
//! All cache implementations perform a best-effort bounding of the map using an
//! entry replacement algorithm to determine which entries to evict when the capacity
//! is exceeded.
//!
//! [caffeine-git]: https://github.com/ben-manes/caffeine
//!
//! **NOTE**:
//! If you have any questions about Moka's APIs or internal design, you can ask the
//! AI chatbot at DeepWiki in a natural language:
//! <https://deepwiki.com/moka-rs/moka>
//!
//! # Features
//!
//! - Thread-safe, highly concurrent in-memory cache implementations:
//! - Synchronous caches that can be shared across OS threads.
//! - An asynchronous (futures aware) cache.
//! - A cache can be bounded by one of the followings:
//! - The maximum number of entries.
//! - The total weighted size of entries. (Size aware eviction)
//! - Maintains near optimal hit ratio by using an entry replacement algorithms
//! inspired by Caffeine:
//! - Admission to a cache is controlled by the Least Frequently Used (LFU)
//! policy.
//! - Eviction from a cache is controlled by the Least Recently Used (LRU)
//! policy.
//! - [More details and some benchmark results are available here][tiny-lfu].
//! - Supports expiration policies:
//! - Time to live.
//! - Time to idle.
//! - Per-entry variable expiration.
//! - Supports eviction listener, a callback function that will be called when an
//! entry is removed from the cache.
//!
//! [tiny-lfu]: https://github.com/moka-rs/moka/wiki#admission-and-eviction-policies
//!
//! ## Cache Policies
//!
//! When a cache is full, it has to select and evict existing entries to make some
//! room. A cache policy is a strategy to determine which entry to evict.
//!
//! The choice of the cache policy may have a significant impact on the performance
//! of the cache. Because the time for cache misses is usually much greater than the
//! time for cache hits, the miss rate (number of misses per second) has a
//! significant impact on the performance.
//!
//! Moka provides the following policies:
//!
//! - TinyLFU
//! - LRU
//!
//! ### TinyLFU
//!
//! TinyLFU is the default policy of the cache, and will be suitable for most
//! workloads.
//!
//! TinyLFU is a combination of the LRU eviction policy and the LFU admission policy.
//! LRU stands for Least Recently Used, which is very popular in many cache systems.
//! LFU stands for Least Frequently Used.
//!
//! ![The lifecycle of cached entries with TinyLFU][tiny-lfu-image]
//!
//! [tiny-lfu-image]:
//! https://github.com/moka-rs/moka/wiki/images/benchmarks/moka-tiny-lfu.png
//!
//! With TinyLFU policy, the cache will admit a new entry based on its popularity. If
//! the key of the entry is popular, it will be admitted to the cache. Otherwise, it
//! will be rejected.
//!
//! The popularity of the key is estimated by the historic popularity estimator
//! called LFU filter. It is a modified Count-Min Sketch, and it can estimate the
//! frequency of keys with a very low memory footprint (thus the name “tiny”). Note
//! that it tracks not only the keys currently in the cache, but all hit and missed
//! keys.
//!
//! Once the entry is admitted to the cache, it will be evicted based on the LRU
//! policy. It evicts the least recently used entry from the cache.
//!
//! TinyLFU will be suitable for most workloads, such as database, search, and
//! analytics.
//!
//! ### LRU
//!
//! LRU stands for Least Recently Used.
//!
//! With LRU policy, the cache will evict the least recently used entry. It is a
//! simple policy and has been used in many cache systems.
//!
//! LRU will be suitable for recency-biased workloads, such as job queues and event
//! streams.
//!
//! # Examples
//!
//! See the following document:
//!
//! - Thread-safe, synchronous caches:
//! - [`sync::Cache`][sync-cache-struct]
//! - [`sync::SegmentedCache`][sync-seg-cache-struct]
//! - An asynchronous (futures aware) cache:
//! - [`future::Cache`][future-cache-struct] (Requires "future" feature)
//!
//! [future-cache-struct]: ./future/struct.Cache.html
//! [sync-cache-struct]: ./sync/struct.Cache.html
//! [sync-seg-cache-struct]: ./sync/struct.SegmentedCache.html
//!
//! **NOTE:** The following caches have been moved to a separate crate called
//! "[mini-moka][mini-moka-crate]".
//!
//! - Non concurrent cache for single threaded applications:
//! - `moka::unsync::Cache` → [`mini_moka::unsync::Cache`][unsync-cache-struct]
//! - A simple, thread-safe, synchronous cache:
//! - `moka::dash::Cache` → [`mini_moka::sync::Cache`][dash-cache-struct]
//!
//! [mini-moka-crate]: https://crates.io/crates/mini-moka
//! [unsync-cache-struct]:
//! https://docs.rs/mini-moka/latest/mini_moka/unsync/struct.Cache.html
//! [dash-cache-struct]:
//! https://docs.rs/mini-moka/latest/mini_moka/sync/struct.Cache.html
//!
//! # Minimum Supported Rust Versions
//!
//! This crate's minimum supported Rust versions (MSRV) are the followings:
//!
//! | Feature | MSRV |
//! |:---------|:-----------------------------:|
//! | `future` | Rust 1.71.1 (August 3, 2023) |
//! | `sync` | Rust 1.71.1 (August 3, 2023) |
//!
//! It will keep a rolling MSRV policy of at least 6 months. If the default features
//! with a mandatory features (`future` or `sync`) are enabled, MSRV will be updated
//! conservatively. When using other features, MSRV might be updated more frequently,
//! up to the latest stable.
//!
//! In both cases, increasing MSRV is _not_ considered a semver-breaking change.
//!
//! # Implementation Details
//!
//! ## Concurrency
//!
//! The entry replacement algorithms are kept eventually consistent with the
//! concurrent hash table. While updates to the cache are immediately applied to the
//! hash table, recording of reads and writes may not be immediately reflected on the
//! cache policy's data structures.
//!
//! These cache policy structures are guarded by a lock and operations are applied in
//! batches to avoid lock contention.
//!
//! Recap:
//!
//! - The concurrent hash table in the cache is _strong consistent_:
//! - It is a lock-free data structure and immediately applies updates.
//! - It is guaranteed that the inserted entry will become visible immediately to
//! all threads.
//! - The cache policy's data structures are _eventually consistent_:
//! - They are guarded by a lock and operations are applied in batches.
//! - An example of eventual consistency: the `entry_count` method may return an
//! outdated value.
//!
//! ### Bounded Channels
//!
//! In order to hold the recordings of reads and writes until they are applied to the
//! cache policy's data structures, the cache uses two bounded channels, one for
//! reads and the other for writes. Bounded means that a channel have a maximum
//! number of elements that can be stored.
//!
//! These channels are drained when one of the following conditions is met:
//!
//! - The numbers of read or write recordings reach to the configured amounts.
//! - It is currently hard-coded to 64.
//! - Or, the certain time past from the last draining.
//! - It is currently hard-coded to 300 milliseconds.
//!
//! Cache does not have a dedicated thread for draining. Instead, it is done by a
//! user thread. When user code calls certain cache methods, such as `get`,
//! `get_with`, `insert`, and `run_pending_tasks`, the cache checks if the above
//! condition is met, and if so, it will start draining as a part of the method call
//! and apply the recordings to the cache policy's data structures. See [the
//! Maintenance Tasks section](#maintenance-tasks) for more details of applying the
//! recordings.
//!
//! ### When a Bounded Channels is Full
//!
//! Under heavy concurrent operations from clients, draining may not be able to catch
//! up and the bounded channels can become full. In this case, the cache will do one
//! of the followings:
//!
//! - For the read channel, recordings of new reads will be discarded, so that
//! retrievals will never be blocked. This behavior may have some impact to the hit
//! rate of the cache.
//! - For the write channel, updates from clients to the cache will be blocked until
//! the draining task catches up.
//!
//! ## Maintenance Tasks
//!
//! When draining the read and write recordings from the channels, the cache will do
//! the following maintenance tasks:
//!
//! 1. Determine whether to admit an entry to the cache or not, based on its
//! popularity.
//! - If not, the entry is removed from the internal concurrent hash table.
//! 2. Apply the recording of cache reads and writes to the internal data structures
//! for the cache policies, such as the LFU filter, LRU queues, and hierarchical
//! timer wheels.
//! - The hierarchical timer wheels are used for the per-entry expiration policy.
//! 3. When cache's max capacity is exceeded, remove least recently used (LRU)
//! entries.
//! 4. Remove expired entries.
//! 5. Find and remove the entries that have been invalidated by the `invalidate_all`
//! or `invalidate_entries_if` methods.
//! 6. Deliver removal notifications to the eviction listener. (Call the eviction
//! listener closure with the information about the evicted entry)
//!
//! The following cache method calls may trigger the maintenance tasks:
//!
//! - All cache write methods: `insert`, `get_with`, `invalidate`, etc., except for
//! `invalidate_all` and `invalidate_entries_if`.
//! - Some of the cache read methods: `get`
//! - `run_pending_tasks` method, which executes the pending maintenance tasks
//! explicitly.
//!
//! Except `run_pending_tasks` method, the maintenance tasks are executed lazily
//! when one of the conditions in the [Bounded Channels](#bounded-channels) section
//! is met.
#[cfg(not(any(feature = "sync", feature = "future")))]
compile_error!(
"At least one of the crate features `sync` or `future` must be enabled for \
`moka` crate. Please update your dependencies in Cargo.toml"
);
// Reexport(s)
pub use equivalent::Equivalent;
#[cfg(feature = "future")]
#[cfg_attr(docsrs, doc(cfg(feature = "future")))]
pub mod future;
#[cfg(feature = "sync")]
#[cfg_attr(docsrs, doc(cfg(feature = "sync")))]
pub mod sync;
#[cfg(any(feature = "sync", feature = "future"))]
#[cfg_attr(docsrs, doc(cfg(any(feature = "sync", feature = "future"))))]
pub mod notification;
#[cfg(any(feature = "sync", feature = "future"))]
pub(crate) mod cht;
#[cfg(any(feature = "sync", feature = "future"))]
pub(crate) mod common;
#[cfg(any(feature = "sync", feature = "future"))]
#[cfg_attr(docsrs, doc(cfg(any(feature = "sync", feature = "future"))))]
pub mod ops;
#[cfg(any(feature = "sync", feature = "future"))]
pub mod policy;
#[cfg(any(feature = "sync", feature = "future"))]
#[cfg_attr(docsrs, doc(cfg(any(feature = "sync", feature = "future"))))]
pub use common::error::PredicateError;
#[cfg(any(feature = "sync", feature = "future"))]
#[cfg_attr(docsrs, doc(cfg(any(feature = "sync", feature = "future"))))]
pub use common::entry::Entry;
#[cfg(any(feature = "sync", feature = "future"))]
#[cfg_attr(docsrs, doc(cfg(any(feature = "sync", feature = "future"))))]
pub use policy::{Expiry, Policy};
#[cfg(feature = "unstable-debug-counters")]
#[cfg_attr(docsrs, doc(cfg(feature = "unstable-debug-counters")))]
pub use common::concurrent::debug_counters::GlobalDebugCounters;
#[cfg(test)]
mod tests {
#[cfg(trybuild)]
#[test]
fn trybuild_default() {
let t = trybuild::TestCases::new();
t.compile_fail("tests/compile_tests/default/clone/*.rs");
}
#[cfg(all(trybuild, feature = "future"))]
#[test]
fn trybuild_future() {
let t = trybuild::TestCases::new();
t.compile_fail("tests/compile_tests/future/clone/*.rs");
}
}
#[cfg(all(doctest, feature = "sync"))]
mod doctests {
// https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#include-items-only-when-collecting-doctests
#[doc = include_str!("../README.md")]
struct ReadMeDoctests;
}
+47
View File
@@ -0,0 +1,47 @@
//! Common data types for notifications.
#[cfg(feature = "sync")]
pub(crate) mod notifier;
use std::{future::Future, pin::Pin, sync::Arc};
/// A future returned by an eviction listener.
///
/// You can use the [`boxed` method][boxed-method] of `FutureExt` trait to convert a
/// regular `Future` object into `ListenerFuture`.
///
/// [boxed-method]: ../future/trait.FutureExt.html#method.boxed
pub type ListenerFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
#[cfg(feature = "sync")]
pub(crate) type EvictionListener<K, V> =
Arc<dyn Fn(Arc<K>, V, RemovalCause) + Send + Sync + 'static>;
#[cfg(feature = "future")]
pub(crate) type AsyncEvictionListener<K, V> =
Box<dyn Fn(Arc<K>, V, RemovalCause) -> ListenerFuture + Send + Sync + 'static>;
// NOTE: Currently, dropping the cache will drop all entries without sending
// notifications. Calling `invalidate_all` method of the cache will trigger
// the notifications, but currently there is no way to know when all entries
// have been invalidated and their notifications have been sent.
/// Indicates the reason why a cached entry was removed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RemovalCause {
/// The entry's expiration timestamp has passed.
Expired,
/// The entry was manually removed by the user.
Explicit,
/// The entry itself was not actually removed, but its value was replaced by
/// the user.
Replaced,
/// The entry was evicted due to size constraints.
Size,
}
impl RemovalCause {
pub fn was_evicted(&self) -> bool {
matches!(self, Self::Expired | Self::Size)
}
}
+64
View File
@@ -0,0 +1,64 @@
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use crate::notification::{EvictionListener, RemovalCause};
pub(crate) struct RemovalNotifier<K, V> {
listener: EvictionListener<K, V>,
is_enabled: AtomicBool,
#[cfg(feature = "logging")]
cache_name: Option<String>,
}
impl<K, V> RemovalNotifier<K, V> {
pub(crate) fn new(listener: EvictionListener<K, V>, _cache_name: Option<String>) -> Self {
Self {
listener,
is_enabled: AtomicBool::new(true),
#[cfg(feature = "logging")]
cache_name: _cache_name,
}
}
pub(crate) fn notify(&self, key: Arc<K>, value: V, cause: RemovalCause) {
use std::panic::{catch_unwind, AssertUnwindSafe};
if !self.is_enabled.load(Ordering::Acquire) {
return;
}
let listener_clo = || (self.listener)(key, value, cause);
// Safety: It is safe to assert unwind safety here because we will not
// call the listener again if it has been panicked.
let result = catch_unwind(AssertUnwindSafe(listener_clo));
if let Err(_payload) = result {
self.is_enabled.store(false, Ordering::Release);
#[cfg(feature = "logging")]
log_panic(&*_payload, self.cache_name.as_deref());
}
}
}
#[cfg(feature = "logging")]
fn log_panic(payload: &(dyn std::any::Any + Send + 'static), cache_name: Option<&str>) {
// Try to downcast the payload into &str or String.
//
// NOTE: Clippy will complain if we use `if let Some(_)` here.
// https://rust-lang.github.io/rust-clippy/master/index.html#manual_map
let message: Option<std::borrow::Cow<'_, str>> =
(payload.downcast_ref::<&str>().map(|s| (*s).into()))
.or_else(|| payload.downcast_ref::<String>().map(Into::into));
let cn = cache_name
.map(|name| format!("[{name}] "))
.unwrap_or_default();
if let Some(m) = message {
log::error!("{cn}Disabled the eviction listener because it panicked at '{m}'");
} else {
log::error!("{cn}Disabled the eviction listener because it panicked");
}
}
+71
View File
@@ -0,0 +1,71 @@
//! Cache operations.
/// Operations used by the `and_compute_with` and similar methods.
pub mod compute {
use std::sync::Arc;
use crate::Entry;
/// Instructs the `and_compute_with` and similar methods how to modify the cached
/// entry.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Op<V> {
/// No-operation. Do not modify the cached entry.
Nop,
/// Insert or update the value of the cached entry.
Put(V),
/// Remove the cached entry.
Remove,
}
/// The result of the `and_compute_with` and similar methods.
#[derive(Debug)]
pub enum CompResult<K, V> {
/// The entry did not exist and still does not exist.
StillNone(Arc<K>),
/// The entry already existed and was not modified. The returned entry
/// contains the existing value.
Unchanged(Entry<K, V>),
/// The entry did not exist and was inserted. The returned entry contains
/// the inserted value.
Inserted(Entry<K, V>),
/// The entry already existed and its value was replaced with a new one. The
/// returned entry contains the new value (not the replaced value).
ReplacedWith(Entry<K, V>),
/// The entry already existed and was removed. The returned entry contains
/// the removed value.
///
/// Note: `StillNone` is returned instead of `Removed` if `Op::Remove` was
/// requested but the entry did not exist.
Removed(Entry<K, V>),
}
impl<K, V> CompResult<K, V> {
/// Returns the contained `Some(Entry)` if any. Otherwise returns `None`.
/// Consumes the `self` value.
pub fn into_entry(self) -> Option<Entry<K, V>> {
match self {
CompResult::StillNone(_) => None,
CompResult::Unchanged(entry) => Some(entry),
CompResult::Inserted(entry) => Some(entry),
CompResult::ReplacedWith(entry) => Some(entry),
CompResult::Removed(entry) => Some(entry),
}
}
/// Unwraps the contained `Entry`, consuming the `self` value.
///
/// # Panics
///
/// Panics if the `self` value is `StillNone`.
pub fn unwrap(self) -> Entry<K, V> {
match self {
CompResult::StillNone(_) => panic!("`CompResult::unwrap` called on `StillNone`"),
CompResult::Unchanged(entry) => entry,
CompResult::Inserted(entry) => entry,
CompResult::ReplacedWith(entry) => entry,
CompResult::Removed(entry) => entry,
}
}
}
}
+433
View File
@@ -0,0 +1,433 @@
use std::{
fmt,
ops::Deref,
sync::Arc,
time::{Duration, Instant},
};
#[derive(Clone, Debug)]
/// The policy of a cache.
pub struct Policy {
max_capacity: Option<u64>,
num_segments: usize,
time_to_live: Option<Duration>,
time_to_idle: Option<Duration>,
}
impl Policy {
pub(crate) fn new(
max_capacity: Option<u64>,
num_segments: usize,
time_to_live: Option<Duration>,
time_to_idle: Option<Duration>,
) -> Self {
Self {
max_capacity,
num_segments,
time_to_live,
time_to_idle,
}
}
/// Returns the `max_capacity` of the cache.
pub fn max_capacity(&self) -> Option<u64> {
self.max_capacity
}
#[cfg(feature = "sync")]
pub(crate) fn set_max_capacity(&mut self, capacity: Option<u64>) {
self.max_capacity = capacity;
}
/// Returns the number of internal segments of the cache.
pub fn num_segments(&self) -> usize {
self.num_segments
}
#[cfg(feature = "sync")]
pub(crate) fn set_num_segments(&mut self, num: usize) {
self.num_segments = num;
}
/// Returns the `time_to_live` of the cache.
pub fn time_to_live(&self) -> Option<Duration> {
self.time_to_live
}
/// Returns the `time_to_idle` of the cache.
pub fn time_to_idle(&self) -> Option<Duration> {
self.time_to_idle
}
}
/// The eviction (and admission) policy of a cache.
///
/// When the cache is full, the eviction/admission policy is used to determine which
/// items should be admitted to the cache and which cached items should be evicted.
/// The choice of a policy will directly affect the performance (hit rate) of the
/// cache.
///
/// The following policies are available:
///
/// - **TinyLFU** (default):
/// - Suitable for most workloads.
/// - TinyLFU combines the LRU eviction policy and an admission policy based on the
/// historical popularity of keys.
/// - Note that it tracks not only the keys currently in the cache, but all hit and
/// missed keys. The data structure used to _estimate_ the popularity of keys is
/// a modified Count-Min Sketch, which has a very low memory footprint (thus the
/// name "tiny").
/// - **LRU**:
/// - Suitable for some workloads with strong recency bias, such as streaming data
/// processing.
///
/// LFU stands for Least Frequently Used. LRU stands for Least Recently Used.
///
/// Use associate function [`EvictionPolicy::tiny_lfu`](#method.tiny_lfu) or
/// [`EvictionPolicy::lru`](#method.lru) to obtain an instance of `EvictionPolicy`.
#[derive(Clone, Default)]
pub struct EvictionPolicy {
pub(crate) config: EvictionPolicyConfig,
}
impl EvictionPolicy {
/// Returns the TinyLFU policy, which is suitable for most workloads.
///
/// TinyLFU is a combination of the LRU eviction policy and the admission policy
/// based on the historical popularity of keys.
///
/// Note that it tracks not only the keys currently in the cache, but all hit and
/// missed keys. The data structure used to _estimate_ the popularity of keys is
/// a modified Count-Min Sketch, which has a very low memory footprint (thus the
/// name "tiny").
pub fn tiny_lfu() -> Self {
Self {
config: EvictionPolicyConfig::TinyLfu,
}
}
/// Returns the LRU policy.
///
/// Suitable for some workloads with strong recency bias, such as streaming data
/// processing.
pub fn lru() -> Self {
Self {
config: EvictionPolicyConfig::Lru,
}
}
}
impl fmt::Debug for EvictionPolicy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.config {
EvictionPolicyConfig::TinyLfu => write!(f, "EvictionPolicy::TinyLfu"),
EvictionPolicyConfig::Lru => write!(f, "EvictionPolicy::Lru"),
}
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) enum EvictionPolicyConfig {
#[default]
TinyLfu,
Lru,
}
/// Calculates when cache entries expire. A single expiration time is retained on
/// each entry so that the lifetime of an entry may be extended or reduced by
/// subsequent evaluations.
///
/// `Expiry` trait provides three methods. They specify the expiration time of an
/// entry by returning a `Some(duration)` until the entry expires:
///
/// - [`expire_after_create`](#method.expire_after_create) &mdash; Returns the
/// duration (or none) after the entry's creation.
/// - [`expire_after_read`](#method.expire_after_read) &mdash; Returns the duration
/// (or none) after its last read.
/// - [`expire_after_update`](#method.expire_after_update) &mdash; Returns the
/// duration (or none) after its last update.
///
/// The default implementations are provided that return `None` (no expiration) or
/// `current_duration: Option<Instant>` (not modify the current expiration time).
/// Override some of them as you need.
///
pub trait Expiry<K, V> {
/// Specifies that the entry should be automatically removed from the cache once
/// the duration has elapsed after the entry's creation. This method is called
/// for cache write methods such as `insert` and `get_with` but only when the key
/// was not present in the cache.
///
/// # Parameters
///
/// - `key` &mdash; A reference to the key of the entry.
/// - `value` &mdash; A reference to the value of the entry.
/// - `created_at` &mdash; The time when this entry was inserted.
///
/// # Return value
///
/// The returned `Option<Duration>` is used to set the expiration time of the
/// entry.
///
/// - Returning `Some(duration)` &mdash; The expiration time is set to
/// `created_at + duration`.
/// - Returning `None` &mdash; The expiration time is cleared (no expiration).
/// - This is the value that the default implementation returns.
///
/// # Notes on `time_to_live` and `time_to_idle` policies
///
/// When the cache is configured with `time_to_live` and/or `time_to_idle`
/// policies, the entry will be evicted after the earliest of the expiration time
/// returned by this expiry, the `time_to_live` and `time_to_idle` policies.
#[allow(unused_variables)]
fn expire_after_create(&self, key: &K, value: &V, created_at: Instant) -> Option<Duration> {
None
}
/// Specifies that the entry should be automatically removed from the cache once
/// the duration has elapsed after its last read. This method is called for cache
/// read methods such as `get` and `get_with` but only when the key is present in
/// the cache.
///
/// # Parameters
///
/// - `key` &mdash; A reference to the key of the entry.
/// - `value` &mdash; A reference to the value of the entry.
/// - `read_at` &mdash; The time when this entry was read.
/// - `duration_until_expiry` &mdash; The remaining duration until the entry
/// expires. (Calculated by `expiration_time - read_at`)
/// - `last_modified_at` &mdash; The time when this entry was created or updated.
///
/// # Return value
///
/// The returned `Option<Duration>` is used to set the expiration time of the
/// entry.
///
/// - Returning `Some(duration)` &mdash; The expiration time is set to
/// `read_at + duration`.
/// - Returning `None` &mdash; The expiration time is cleared (no expiration).
/// - Returning `duration_until_expiry` will not modify the expiration time.
/// - This is the value that the default implementation returns.
///
/// # Notes on `time_to_live` and `time_to_idle` policies
///
/// When the cache is configured with `time_to_live` and/or `time_to_idle`
/// policies, then:
///
/// - The entry will be evicted after the earliest of the expiration time
/// returned by this expiry, the `time_to_live` and `time_to_idle` policies.
/// - The `duration_until_expiry` takes in account the `time_to_live` and
/// `time_to_idle` policies.
#[allow(unused_variables)]
fn expire_after_read(
&self,
key: &K,
value: &V,
read_at: Instant,
duration_until_expiry: Option<Duration>,
last_modified_at: Instant,
) -> Option<Duration> {
duration_until_expiry
}
/// Specifies that the entry should be automatically removed from the cache once
/// the duration has elapsed after the replacement of its value. This method is
/// called for cache write methods such as `insert` but only when the key is
/// already present in the cache.
///
/// # Parameters
///
/// - `key` &mdash; A reference to the key of the entry.
/// - `value` &mdash; A reference to the value of the entry.
/// - `updated_at` &mdash; The time when this entry was updated.
/// - `duration_until_expiry` &mdash; The remaining duration until the entry
/// expires. (Calculated by `expiration_time - updated_at`)
///
/// # Return value
///
/// The returned `Option<Duration>` is used to set the expiration time of the
/// entry.
///
/// - Returning `Some(duration)` &mdash; The expiration time is set to
/// `updated_at + duration`.
/// - Returning `None` &mdash; The expiration time is cleared (no expiration).
/// - Returning `duration_until_expiry` will not modify the expiration time.
/// - This is the value that the default implementation returns.
///
/// # Notes on `time_to_live` and `time_to_idle` policies
///
/// When the cache is configured with `time_to_live` and/or `time_to_idle`
/// policies, then:
///
/// - The entry will be evicted after the earliest of the expiration time
/// returned by this expiry, the `time_to_live` and `time_to_idle` policies.
/// - The `duration_until_expiry` takes in account the `time_to_live` and
/// `time_to_idle` policies.
#[allow(unused_variables)]
fn expire_after_update(
&self,
key: &K,
value: &V,
updated_at: Instant,
duration_until_expiry: Option<Duration>,
) -> Option<Duration> {
duration_until_expiry
}
}
impl<K, V, T> Expiry<K, V> for T
where
T: Deref<Target = dyn Expiry<K, V> + Send + Sync>,
{
fn expire_after_create(&self, key: &K, value: &V, created_at: Instant) -> Option<Duration> {
self.deref().expire_after_create(key, value, created_at)
}
fn expire_after_read(
&self,
key: &K,
value: &V,
read_at: Instant,
duration_until_expiry: Option<Duration>,
last_modified_at: Instant,
) -> Option<Duration> {
self.deref()
.expire_after_read(key, value, read_at, duration_until_expiry, last_modified_at)
}
fn expire_after_update(
&self,
key: &K,
value: &V,
updated_at: Instant,
duration_until_expiry: Option<Duration>,
) -> Option<Duration> {
self.deref()
.expire_after_update(key, value, updated_at, duration_until_expiry)
}
}
pub(crate) struct ExpirationPolicy<K, V> {
time_to_live: Option<Duration>,
time_to_idle: Option<Duration>,
expiry: Option<Arc<dyn Expiry<K, V> + Send + Sync + 'static>>,
}
impl<K, V> Default for ExpirationPolicy<K, V> {
fn default() -> Self {
Self {
time_to_live: None,
time_to_idle: None,
expiry: None,
}
}
}
impl<K, V> Clone for ExpirationPolicy<K, V> {
fn clone(&self) -> Self {
Self {
time_to_live: self.time_to_live,
time_to_idle: self.time_to_idle,
expiry: self.expiry.clone(),
}
}
}
impl<K, V> ExpirationPolicy<K, V> {
#[cfg(test)]
pub(crate) fn new(
time_to_live: Option<Duration>,
time_to_idle: Option<Duration>,
expiry: Option<Arc<dyn Expiry<K, V> + Send + Sync + 'static>>,
) -> Self {
Self {
time_to_live,
time_to_idle,
expiry,
}
}
/// Returns the `time_to_live` of the cache.
pub(crate) fn time_to_live(&self) -> Option<Duration> {
self.time_to_live
}
pub(crate) fn set_time_to_live(&mut self, duration: Duration) {
self.time_to_live = Some(duration);
}
/// Returns the `time_to_idle` of the cache.
pub(crate) fn time_to_idle(&self) -> Option<Duration> {
self.time_to_idle
}
pub(crate) fn set_time_to_idle(&mut self, duration: Duration) {
self.time_to_idle = Some(duration);
}
pub(crate) fn expiry(&self) -> Option<Arc<dyn Expiry<K, V> + Send + Sync + 'static>> {
self.expiry.clone()
}
pub(crate) fn set_expiry(&mut self, expiry: Arc<dyn Expiry<K, V> + Send + Sync + 'static>) {
self.expiry = Some(expiry);
}
}
#[cfg(test)]
pub(crate) mod test_utils {
use std::sync::atomic::{AtomicU8, Ordering};
#[derive(Default)]
pub(crate) struct ExpiryCallCounters {
expected_creations: AtomicU8,
expected_reads: AtomicU8,
expected_updates: AtomicU8,
actual_creations: AtomicU8,
actual_reads: AtomicU8,
actual_updates: AtomicU8,
}
impl ExpiryCallCounters {
pub(crate) fn incl_expected_creations(&self) {
self.expected_creations.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn incl_expected_reads(&self) {
self.expected_reads.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn incl_expected_updates(&self) {
self.expected_updates.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn incl_actual_creations(&self) {
self.actual_creations.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn incl_actual_reads(&self) {
self.actual_reads.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn incl_actual_updates(&self) {
self.actual_updates.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn verify(&self) {
assert_eq!(
self.expected_creations.load(Ordering::Relaxed),
self.actual_creations.load(Ordering::Relaxed),
"expected_creations != actual_creations"
);
assert_eq!(
self.expected_reads.load(Ordering::Relaxed),
self.actual_reads.load(Ordering::Relaxed),
"expected_reads != actual_reads"
);
assert_eq!(
self.expected_updates.load(Ordering::Relaxed),
self.actual_updates.load(Ordering::Relaxed),
"expected_updates != actual_updates"
);
}
}
}
+40
View File
@@ -0,0 +1,40 @@
//! Provides thread-safe, concurrent cache implementations.
mod base_cache;
mod builder;
mod cache;
mod entry_selector;
mod invalidator;
mod key_lock;
mod segment;
mod value_initializer;
/// The type of the unique ID to identify a predicate used by
/// [`Cache::invalidate_entries_if`][invalidate-if] method.
///
/// A `PredicateId` is a `String` of UUID (version 4).
///
/// [invalidate-if]: ./struct.Cache.html#method.invalidate_entries_if
pub type PredicateId = String;
pub(crate) type PredicateIdStr<'a> = &'a str;
pub use crate::common::iter::Iter;
pub use {
builder::CacheBuilder,
cache::Cache,
entry_selector::{OwnedKeyEntrySelector, RefKeyEntrySelector},
segment::SegmentedCache,
};
/// Provides extra methods that will be useful for testing.
pub trait ConcurrentCacheExt<K, V> {
/// Performs any pending maintenance operations needed by the cache.
fn sync(&self);
}
// Empty struct to be used in `InitResult::InitErr` to represent the Option None.
pub(crate) struct OptionallyNone;
// Empty struct to be used in `InitResult::InitErr`` to represent the Compute None.
pub(crate) struct ComputeNone;
File diff suppressed because it is too large Load Diff
+618
View File
@@ -0,0 +1,618 @@
use super::{Cache, SegmentedCache};
use crate::{
common::{builder_utils, concurrent::Weigher, time::Clock, HousekeeperConfig},
notification::{EvictionListener, RemovalCause},
policy::{EvictionPolicy, ExpirationPolicy},
Expiry,
};
use std::{
collections::hash_map::RandomState,
hash::{BuildHasher, Hash},
marker::PhantomData,
sync::Arc,
time::Duration,
};
/// Builds a [`Cache`][cache-struct] or [`SegmentedCache`][seg-cache-struct]
/// with various configuration knobs.
///
/// [cache-struct]: ./struct.Cache.html
/// [seg-cache-struct]: ./struct.SegmentedCache.html
///
/// # Example: Expirations
///
/// ```rust
/// use moka::sync::Cache;
/// use std::time::Duration;
///
/// let cache = Cache::builder()
/// // Max 10,000 entries
/// .max_capacity(10_000)
/// // Time to live (TTL): 30 minutes
/// .time_to_live(Duration::from_secs(30 * 60))
/// // Time to idle (TTI): 5 minutes
/// .time_to_idle(Duration::from_secs( 5 * 60))
/// // Create the cache.
/// .build();
///
/// // This entry will expire after 5 minutes (TTI) if there is no get().
/// cache.insert(0, "zero");
///
/// // This get() will extend the entry life for another 5 minutes.
/// cache.get(&0);
///
/// // Even though we keep calling get(), the entry will expire
/// // after 30 minutes (TTL) from the insert().
/// ```
///
#[must_use]
pub struct CacheBuilder<K, V, C> {
name: Option<String>,
max_capacity: Option<u64>,
initial_capacity: Option<usize>,
num_segments: Option<usize>,
weigher: Option<Weigher<K, V>>,
eviction_policy: EvictionPolicy,
eviction_listener: Option<EvictionListener<K, V>>,
expiration_policy: ExpirationPolicy<K, V>,
housekeeper_config: HousekeeperConfig,
invalidator_enabled: bool,
clock: Clock,
cache_type: PhantomData<C>,
}
impl<K, V> Default for CacheBuilder<K, V, Cache<K, V, RandomState>>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
fn default() -> Self {
Self {
name: None,
max_capacity: None,
initial_capacity: None,
num_segments: None,
weigher: None,
eviction_listener: None,
eviction_policy: EvictionPolicy::default(),
expiration_policy: ExpirationPolicy::default(),
housekeeper_config: HousekeeperConfig::default(),
invalidator_enabled: false,
clock: Clock::default(),
cache_type: PhantomData,
}
}
}
impl<K, V> CacheBuilder<K, V, Cache<K, V, RandomState>>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
/// Construct a new `CacheBuilder` that will be used to build a `Cache` or
/// `SegmentedCache` holding up to `max_capacity` entries.
pub fn new(max_capacity: u64) -> Self {
Self {
max_capacity: Some(max_capacity),
..Default::default()
}
}
/// Sets the number of segments of the cache.
///
/// # Panics
///
/// Panics if `num_segments` is zero.
pub fn segments(
self,
num_segments: usize,
) -> CacheBuilder<K, V, SegmentedCache<K, V, RandomState>> {
assert!(num_segments != 0);
CacheBuilder {
name: self.name,
max_capacity: self.max_capacity,
initial_capacity: self.initial_capacity,
num_segments: Some(num_segments),
weigher: self.weigher,
eviction_policy: self.eviction_policy,
eviction_listener: self.eviction_listener,
expiration_policy: self.expiration_policy,
housekeeper_config: self.housekeeper_config,
invalidator_enabled: self.invalidator_enabled,
clock: self.clock,
cache_type: PhantomData,
}
}
/// Builds a `Cache<K, V>`.
///
/// If you want to build a `SegmentedCache<K, V>`, call `segments` method before
/// calling this method.
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build(self) -> Cache<K, V, RandomState> {
let build_hasher = RandomState::default();
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
Cache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
build_hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
/// Builds a `Cache<K, V, S>` with the given `hasher` of type `S`.
///
/// # Examples
///
/// This example uses AHash hasher from [AHash][ahash-crate] crate.
///
/// [ahash-crate]: https://crates.io/crates/ahash
///
/// ```rust
/// // Cargo.toml
/// // [dependencies]
/// // ahash = "0.8"
/// // moka = ...
///
/// use moka::sync::Cache;
///
/// // The type of this cache is: Cache<i32, String, ahash::RandomState>
/// let cache = Cache::builder()
/// .max_capacity(100)
/// .build_with_hasher(ahash::RandomState::default());
/// cache.insert(1, "one".to_string());
/// ```
///
/// Note: If you need to add a type annotation to your cache, you must use the
/// form of `Cache<K, V, S>` instead of `Cache<K, V>`. That `S` is the type of
/// the build hasher, and its default is the `RandomState` from
/// `std::collections::hash_map` module . If you use a different build hasher,
/// you must specify `S` explicitly.
///
/// Here is a good example:
///
/// ```rust
/// # use moka::sync::Cache;
/// # let cache = Cache::builder()
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Good {
/// // Specifying the type in Cache<K, V, S> format.
/// cache: Cache<i32, String, ahash::RandomState>,
/// }
///
/// // Storing the cache from above example. This should compile.
/// Good { cache };
/// ```
///
/// Here is a bad example. This struct cannot store the above cache because it
/// does not specify `S`:
///
/// ```compile_fail
/// # use moka::sync::Cache;
/// # let cache = Cache::builder()
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Bad {
/// // Specifying the type in Cache<K, V> format.
/// cache: Cache<i32, String>,
/// }
///
/// // This should not compile.
/// Bad { cache };
/// // => error[E0308]: mismatched types
/// // expected struct `std::collections::hash_map::RandomState`,
/// // found struct `ahash::RandomState`
/// ```
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build_with_hasher<S>(self, hasher: S) -> Cache<K, V, S>
where
S: BuildHasher + Clone + Send + Sync + 'static,
{
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
Cache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
}
impl<K, V> CacheBuilder<K, V, SegmentedCache<K, V, RandomState>>
where
K: Eq + Hash + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
/// Builds a `SegmentedCache<K, V>`.
///
/// If you want to build a `Cache<K, V>`, do not call `segments` method before
/// calling this method.
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build(self) -> SegmentedCache<K, V, RandomState> {
let build_hasher = RandomState::default();
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
SegmentedCache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
self.num_segments.unwrap(),
build_hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
/// Builds a `SegmentedCache<K, V, S>` with the given `hasher`.
///
///
/// # Examples
///
/// This example uses AHash hasher from [AHash][ahash-crate] crate.
///
/// [ahash-crate]: https://crates.io/crates/ahash
///
/// ```rust
/// // Cargo.toml
/// // [dependencies]
/// // ahash = "0.8"
/// // moka = ...
///
/// use moka::sync::SegmentedCache;
///
/// // The type of this cache is: SegmentedCache<i32, String, ahash::RandomState>
/// let cache = SegmentedCache::builder(4)
/// .max_capacity(100)
/// .build_with_hasher(ahash::RandomState::default());
/// cache.insert(1, "one".to_string());
/// ```
///
/// Note: If you need to add a type annotation to your cache, you must use the
/// form of `SegmentedCache<K, V, S>` instead of `SegmentedCache<K, V>`. That `S`
/// is the type of the build hasher, whose default is the `RandomState` from
/// `std::collections::hash_map` module . If you use a different build hasher,
/// you must specify `S` explicitly.
///
/// Here is a good example:
///
/// ```rust
/// # use moka::sync::SegmentedCache;
/// # let cache = SegmentedCache::builder(4)
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Good {
/// // Specifying the type in SegmentedCache<K, V, S> format.
/// cache: SegmentedCache<i32, String, ahash::RandomState>,
/// }
///
/// // Storing the cache from above example. This should compile.
/// Good { cache };
/// ```
///
/// Here is a bad example. This struct cannot store the above cache because it
/// does not specify `S`:
///
/// ```compile_fail
/// # use moka::sync::SegmentedCache;
/// # let cache = SegmentedCache::builder(4)
/// # .build_with_hasher(ahash::RandomState::default());
/// struct Bad {
/// // Specifying the type in SegmentedCache<K, V> format.
/// cache: SegmentedCache<i32, String>,
/// }
///
/// // This should not compile.
/// Bad { cache };
/// // => error[E0308]: mismatched types
/// // expected struct `std::collections::hash_map::RandomState`,
/// // found struct `ahash::RandomState`
/// ```
///
/// # Panics
///
/// Panics if configured with either `time_to_live` or `time_to_idle` higher than
/// 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn build_with_hasher<S>(self, hasher: S) -> SegmentedCache<K, V, S>
where
S: BuildHasher + Clone + Send + Sync + 'static,
{
let exp = &self.expiration_policy;
builder_utils::ensure_expirations_or_panic(exp.time_to_live(), exp.time_to_idle());
SegmentedCache::with_everything(
self.name,
self.max_capacity,
self.initial_capacity,
self.num_segments.unwrap(),
hasher,
self.weigher,
self.eviction_policy,
self.eviction_listener,
self.expiration_policy,
self.housekeeper_config,
self.invalidator_enabled,
self.clock,
)
}
}
impl<K, V, C> CacheBuilder<K, V, C> {
/// Sets the name of the cache. Currently the name is used for identification
/// only in logging messages.
pub fn name(self, name: &str) -> Self {
Self {
name: Some(name.to_string()),
..self
}
}
/// Sets the max capacity of the cache.
pub fn max_capacity(self, max_capacity: u64) -> Self {
Self {
max_capacity: Some(max_capacity),
..self
}
}
/// Sets the initial capacity (number of entries) of the cache.
pub fn initial_capacity(self, number_of_entries: usize) -> Self {
Self {
initial_capacity: Some(number_of_entries),
..self
}
}
/// Sets the eviction (and admission) policy of the cache.
///
/// The default policy is TinyLFU. See [`EvictionPolicy`][eviction-policy] for
/// more details.
///
/// [eviction-policy]: ../policy/struct.EvictionPolicy.html
pub fn eviction_policy(self, policy: EvictionPolicy) -> Self {
Self {
eviction_policy: policy,
..self
}
}
/// Sets the weigher closure to the cache.
///
/// The closure should take `&K` and `&V` as the arguments and returns a `u32`
/// representing the relative size of the entry.
pub fn weigher(self, weigher: impl Fn(&K, &V) -> u32 + Send + Sync + 'static) -> Self {
Self {
weigher: Some(Arc::new(weigher)),
..self
}
}
/// Sets the eviction listener closure to the cache.
///
/// The closure should take `Arc<K>`, `V` and [`RemovalCause`][removal-cause] as
/// the arguments.
///
/// # Panics
///
/// It is very important to make the listener closure not to panic. Otherwise,
/// the cache will stop calling the listener after a panic. This is an intended
/// behavior because the cache cannot know whether it is memory safe or not to
/// call the panicked listener again.
///
/// [removal-cause]: ../notification/enum.RemovalCause.html
pub fn eviction_listener(
self,
listener: impl Fn(Arc<K>, V, RemovalCause) + Send + Sync + 'static,
) -> Self {
Self {
eviction_listener: Some(Arc::new(listener)),
..self
}
}
/// Sets the time to live of the cache.
///
/// A cached entry will be expired after the specified duration past from
/// `insert`.
///
/// # Panics
///
/// `CacheBuilder::build*` methods will panic if the given `duration` is longer
/// than 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn time_to_live(self, duration: Duration) -> Self {
let mut builder = self;
builder.expiration_policy.set_time_to_live(duration);
builder
}
/// Sets the time to idle of the cache.
///
/// A cached entry will be expired after the specified duration past from `get`
/// or `insert`.
///
/// # Panics
///
/// `CacheBuilder::build*` methods will panic if the given `duration` is longer
/// than 1000 years. This is done to protect against overflow when computing key
/// expiration.
pub fn time_to_idle(self, duration: Duration) -> Self {
let mut builder = self;
builder.expiration_policy.set_time_to_idle(duration);
builder
}
/// Sets the given `expiry` to the cache.
///
/// See [the example][per-entry-expiration-example] for per-entry expiration
/// policy in the `Cache` documentation.
///
/// [per-entry-expiration-example]:
/// ./struct.Cache.html#per-entry-expiration-policy
pub fn expire_after(self, expiry: impl Expiry<K, V> + Send + Sync + 'static) -> Self {
let mut builder = self;
builder.expiration_policy.set_expiry(Arc::new(expiry));
builder
}
#[cfg(test)]
pub(crate) fn housekeeper_config(self, conf: HousekeeperConfig) -> Self {
Self {
housekeeper_config: conf,
..self
}
}
#[cfg(test)]
pub(crate) fn clock(self, clock: Clock) -> Self {
Self { clock, ..self }
}
/// Enables support for [`Cache::invalidate_entries_if`][cache-invalidate-if]
/// method.
///
/// The cache will maintain additional internal data structures to support
/// `invalidate_entries_if` method.
///
/// [cache-invalidate-if]: ./struct.Cache.html#method.invalidate_entries_if
pub fn support_invalidation_closures(self) -> Self {
Self {
invalidator_enabled: true,
..self
}
}
}
#[cfg(test)]
mod tests {
use super::CacheBuilder;
use std::time::Duration;
#[test]
fn build_cache() {
// Cache<char, String>
let cache = CacheBuilder::new(100).build();
let policy = cache.policy();
assert_eq!(policy.max_capacity(), Some(100));
assert_eq!(policy.time_to_live(), None);
assert_eq!(policy.time_to_idle(), None);
assert_eq!(policy.num_segments(), 1);
cache.insert('a', "Alice");
assert_eq!(cache.get(&'a'), Some("Alice"));
let cache = CacheBuilder::new(100)
.time_to_live(Duration::from_secs(45 * 60))
.time_to_idle(Duration::from_secs(15 * 60))
.build();
let config = cache.policy();
assert_eq!(config.max_capacity(), Some(100));
assert_eq!(config.time_to_live(), Some(Duration::from_secs(45 * 60)));
assert_eq!(config.time_to_idle(), Some(Duration::from_secs(15 * 60)));
assert_eq!(config.num_segments(), 1);
cache.insert('a', "Alice");
assert_eq!(cache.get(&'a'), Some("Alice"));
}
#[test]
fn build_segmented_cache() {
// SegmentCache<char, String>
let cache = CacheBuilder::new(100).segments(15).build();
let policy = cache.policy();
assert_eq!(policy.max_capacity(), Some(100));
assert!(policy.time_to_live().is_none());
assert!(policy.time_to_idle().is_none());
assert_eq!(policy.num_segments(), 16_usize.next_power_of_two());
cache.insert('b', "Bob");
assert_eq!(cache.get(&'b'), Some("Bob"));
let listener = move |_key, _value, _cause| ();
let builder = CacheBuilder::new(400)
.time_to_live(Duration::from_secs(45 * 60))
.time_to_idle(Duration::from_secs(15 * 60))
.eviction_listener(listener)
.name("tracked_sessions")
// Call segments() at the end to check all field values in the current
// builder struct are copied to the new builder:
// https://github.com/moka-rs/moka/issues/207
.segments(24);
assert!(builder.eviction_listener.is_some());
let cache = builder.build();
let policy = cache.policy();
assert_eq!(policy.max_capacity(), Some(400));
assert_eq!(policy.time_to_live(), Some(Duration::from_secs(45 * 60)));
assert_eq!(policy.time_to_idle(), Some(Duration::from_secs(15 * 60)));
assert_eq!(policy.num_segments(), 24_usize.next_power_of_two());
assert_eq!(cache.name(), Some("tracked_sessions"));
cache.insert('b', "Bob");
assert_eq!(cache.get(&'b'), Some("Bob"));
}
#[test]
#[should_panic(expected = "time_to_live is longer than 1000 years")]
fn build_cache_too_long_ttl() {
let thousand_years_secs: u64 = 1000 * 365 * 24 * 3600;
let builder: CacheBuilder<char, String, _> = CacheBuilder::new(100);
let duration = Duration::from_secs(thousand_years_secs);
builder
.time_to_live(duration + Duration::from_secs(1))
.build();
}
#[test]
#[should_panic(expected = "time_to_idle is longer than 1000 years")]
fn build_cache_too_long_tti() {
let thousand_years_secs: u64 = 1000 * 365 * 24 * 3600;
let builder: CacheBuilder<char, String, _> = CacheBuilder::new(100);
let duration = Duration::from_secs(thousand_years_secs);
builder
.time_to_idle(duration + Duration::from_secs(1))
.build();
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+385
View File
@@ -0,0 +1,385 @@
use super::{base_cache::Inner, PredicateId, PredicateIdStr};
use crate::{
common::{
concurrent::{arc::MiniArc, AccessTime, KvEntry, ValueEntry},
time::Instant,
},
notification::RemovalCause,
PredicateError,
};
use parking_lot::{Mutex, MutexGuard};
use std::{
hash::{BuildHasher, Hash},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use uuid::Uuid;
pub(crate) type PredicateFun<K, V> = Arc<dyn Fn(&K, &V) -> bool + Send + Sync + 'static>;
const PREDICATE_MAP_NUM_SEGMENTS: usize = 16;
pub(crate) struct KeyDateLite<K> {
key: Arc<K>,
hash: u64,
timestamp: Instant,
}
impl<K> Clone for KeyDateLite<K> {
fn clone(&self) -> Self {
Self {
key: Arc::clone(&self.key),
hash: self.hash,
timestamp: self.timestamp,
}
}
}
impl<K> KeyDateLite<K> {
pub(crate) fn new(key: &Arc<K>, hash: u64, timestamp: Instant) -> Self {
Self {
key: Arc::clone(key),
hash,
timestamp,
}
}
}
pub(crate) struct Invalidator<K, V, S> {
predicates: crate::cht::SegmentedHashMap<PredicateId, Predicate<K, V>, S>,
is_empty: AtomicBool,
scan_context: Arc<ScanContext<K, V>>,
}
//
// Crate public methods.
//
impl<K, V, S> Invalidator<K, V, S> {
pub(crate) fn new(hasher: S) -> Self
where
S: BuildHasher,
{
const CAPACITY: usize = 0;
let predicates = crate::cht::SegmentedHashMap::with_num_segments_capacity_and_hasher(
PREDICATE_MAP_NUM_SEGMENTS,
CAPACITY,
hasher,
);
Self {
predicates,
is_empty: AtomicBool::new(true),
scan_context: Arc::new(ScanContext::default()),
}
}
pub(crate) fn is_empty(&self) -> bool {
self.is_empty.load(Ordering::Acquire)
}
pub(crate) fn remove_predicates_registered_before(&self, ts: Instant)
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher,
{
let pred_map = &self.predicates;
let removing_ids = pred_map
.iter()
.filter(|(_, pred)| pred.registered_at <= ts)
.map(|(id, _)| id)
.collect::<Vec<_>>();
for id in removing_ids {
let hash = pred_map.hash(&id);
pred_map.remove(hash, |k| k == &id);
}
if pred_map.is_empty() {
self.is_empty.store(true, Ordering::Release);
}
}
pub(crate) fn register_predicate(
&self,
predicate: PredicateFun<K, V>,
registered_at: Instant,
) -> Result<PredicateId, PredicateError>
where
K: Hash + Eq,
S: BuildHasher,
{
const MAX_RETRY: usize = 1_000;
let mut tries = 0;
let preds = &self.predicates;
while tries < MAX_RETRY {
let id = Uuid::new_v4().as_hyphenated().to_string();
let hash = preds.hash(&id);
if preds.contains_key(hash, |k| k == &id) {
tries += 1;
continue; // Retry
}
let pred = Predicate::new(&id, predicate, registered_at);
preds.insert_entry_and(id.clone(), hash, pred, |_, _| ());
self.is_empty.store(false, Ordering::Release);
return Ok(id);
}
// Since we are using 128-bit UUID for the ID and we do retries for MAX_RETRY
// times, this panic should extremely unlikely occur (unless there is a bug in
// UUID generation).
panic!("Cannot assign a new PredicateId to a predicate");
}
// This method will be called by the get method of Cache.
#[inline]
pub(crate) fn apply_predicates(&self, key: &Arc<K>, entry: &MiniArc<ValueEntry<K, V>>) -> bool
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher,
{
if self.is_empty() {
false
} else if let Some(ts) = entry.last_modified() {
Self::do_apply_predicates(
self.predicates.iter().map(|(_, v)| v),
key,
&entry.value,
ts,
)
} else {
false
}
}
pub(crate) fn scan_and_invalidate(
&self,
cache: &Inner<K, V, S>,
candidates: Vec<KeyDateLite<K>>,
is_truncated: bool,
) -> (Vec<KvEntry<K, V>>, bool)
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher,
{
let mut predicates = self.scan_context.predicates.lock();
if predicates.is_empty() {
*predicates = self.predicates.iter().map(|(_k, v)| v).collect();
}
let mut invalidated = Vec::default();
let mut newest_timestamp = None;
for candidate in &candidates {
let key = &candidate.key;
let hash = candidate.hash;
let ts = candidate.timestamp;
if self.apply(&predicates, cache, key, hash, ts) {
if let Some(entry) = Self::invalidate(cache, key, hash, ts) {
invalidated.push(KvEntry {
key: Arc::clone(key),
entry,
});
}
}
newest_timestamp = Some(ts);
}
self.remove_finished_predicates(predicates, is_truncated, newest_timestamp);
(invalidated, self.predicates.is_empty())
}
}
//
// Private methods.
//
impl<K, V, S> Invalidator<K, V, S>
where
K: Hash + Eq,
S: BuildHasher,
{
#[inline]
fn do_apply_predicates<I>(predicates: I, key: &K, value: &V, ts: Instant) -> bool
where
I: Iterator<Item = Predicate<K, V>>,
{
for predicate in predicates {
if predicate.is_applicable(ts) && predicate.apply(key, value) {
return true;
}
}
false
}
fn remove_finished_predicates(
&self,
mut predicates: MutexGuard<'_, Vec<Predicate<K, V>>>,
is_truncated: bool,
newest_timestamp: Option<Instant>,
) where
K: Hash + Eq,
S: BuildHasher,
{
let predicates = &mut *predicates;
if is_truncated {
if let Some(ts) = newest_timestamp {
let (active, finished): (Vec<_>, Vec<_>) =
predicates.drain(..).partition(|p| p.is_applicable(ts));
// Remove finished predicates from the predicate registry.
self.remove_predicates(&finished);
// Set the active predicates to the scan context.
*predicates = active;
} else {
unreachable!();
}
} else {
// Remove all the predicates from the predicate registry and scan context.
self.remove_predicates(predicates);
predicates.clear();
}
}
fn remove_predicates(&self, predicates: &[Predicate<K, V>])
where
K: Hash + Eq,
S: BuildHasher,
{
let pred_map = &self.predicates;
for p in predicates.iter() {
let hash = pred_map.hash(p.id());
pred_map.remove(hash, |k| k == p.id());
}
if pred_map.is_empty() {
self.is_empty.store(true, Ordering::Release);
}
}
fn apply(
&self,
predicates: &[Predicate<K, V>],
cache: &Inner<K, V, S>,
key: &Arc<K>,
hash: u64,
ts: Instant,
) -> bool {
if let Some(entry) = cache.cache.get(hash, |k| k == key) {
if let Some(lm) = entry.last_modified() {
if lm == ts {
return Invalidator::<_, _, S>::do_apply_predicates(
predicates.iter().cloned(),
key,
&entry.value,
lm,
);
}
}
}
false
}
fn invalidate(
cache: &Inner<K, V, S>,
key: &Arc<K>,
hash: u64,
ts: Instant,
) -> Option<MiniArc<ValueEntry<K, V>>>
where
K: Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
{
// Lock the key for removal if blocking removal notification is enabled.
let kl = cache.maybe_key_lock(key);
let _klg = &kl.as_ref().map(|kl| kl.lock());
let maybe_entry = cache.cache.remove_if(
hash,
|k| k == key,
|_, v| {
if let Some(lm) = v.last_modified() {
lm == ts
} else {
false
}
},
);
if let Some(entry) = &maybe_entry {
if cache.is_removal_notifier_enabled() {
cache.notify_single_removal(Arc::clone(key), entry, RemovalCause::Explicit);
}
}
maybe_entry
}
}
//
// for testing
//
#[cfg(test)]
impl<K, V, S> Invalidator<K, V, S> {
pub(crate) fn predicate_count(&self) -> usize {
self.predicates.len()
}
}
struct ScanContext<K, V> {
predicates: Mutex<Vec<Predicate<K, V>>>,
}
impl<K, V> Default for ScanContext<K, V> {
fn default() -> Self {
Self {
predicates: Mutex::new(Vec::default()),
}
}
}
struct Predicate<K, V> {
id: PredicateId,
f: PredicateFun<K, V>,
registered_at: Instant,
}
impl<K, V> Clone for Predicate<K, V> {
fn clone(&self) -> Self {
Self {
id: self.id.clone(),
f: Arc::clone(&self.f),
registered_at: self.registered_at,
}
}
}
impl<K, V> Predicate<K, V> {
fn new(id: PredicateIdStr<'_>, f: PredicateFun<K, V>, registered_at: Instant) -> Self {
Self {
id: id.to_string(),
f,
registered_at,
}
}
fn id(&self) -> PredicateIdStr<'_> {
&self.id
}
fn is_applicable(&self, last_modified: Instant) -> bool {
last_modified <= self.registered_at
}
fn apply(&self, key: &K, value: &V) -> bool {
(self.f)(key, value)
}
}
+94
View File
@@ -0,0 +1,94 @@
use std::{
hash::{BuildHasher, Hash},
sync::Arc,
};
use crate::{cht::SegmentedHashMap, common::concurrent::arc::MiniArc};
use parking_lot::{Mutex, MutexGuard};
const LOCK_MAP_NUM_SEGMENTS: usize = 64;
type LockMap<K, S> = SegmentedHashMap<Arc<K>, MiniArc<Mutex<()>>, S>;
// We need the `where` clause here because of the Drop impl.
pub(crate) struct KeyLock<'a, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
map: &'a LockMap<K, S>,
key: Arc<K>,
hash: u64,
lock: MiniArc<Mutex<()>>,
}
impl<K, S> Drop for KeyLock<'_, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
fn drop(&mut self) {
if MiniArc::count(&self.lock) <= 2 {
self.map.remove_if(
self.hash,
|k| k == &self.key,
|_k, v| MiniArc::count(v) <= 2,
);
}
}
}
impl<'a, K, S> KeyLock<'a, K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
fn new(map: &'a LockMap<K, S>, key: &Arc<K>, hash: u64, lock: MiniArc<Mutex<()>>) -> Self {
Self {
map,
key: Arc::clone(key),
hash,
lock,
}
}
pub(crate) fn lock(&self) -> MutexGuard<'_, ()> {
self.lock.lock()
}
}
pub(crate) struct KeyLockMap<K, S> {
locks: LockMap<K, S>,
}
impl<K, S> KeyLockMap<K, S>
where
K: Eq + Hash,
S: BuildHasher,
{
pub(crate) fn with_hasher(hasher: S) -> Self {
Self {
locks: SegmentedHashMap::with_num_segments_and_hasher(LOCK_MAP_NUM_SEGMENTS, hasher),
}
}
pub(crate) fn key_lock(&self, key: &Arc<K>) -> KeyLock<'_, K, S> {
let hash = self.locks.hash(key);
let kl = MiniArc::new(Mutex::new(()));
match self
.locks
.insert_if_not_present(Arc::clone(key), hash, kl.clone())
{
None => KeyLock::new(&self.locks, key, hash, kl),
Some(existing_kl) => KeyLock::new(&self.locks, key, hash, existing_kl),
}
}
}
#[cfg(test)]
impl<K, S> KeyLockMap<K, S> {
pub(crate) fn is_empty(&self) -> bool {
self.locks.len() == 0
}
}
File diff suppressed because it is too large Load Diff
+390
View File
@@ -0,0 +1,390 @@
use parking_lot::RwLock;
use std::{
any::{Any, TypeId},
fmt,
hash::{BuildHasher, Hash},
sync::Arc,
};
use crate::{
common::concurrent::arc::MiniArc,
ops::compute::{CompResult, Op},
Entry,
};
use super::{Cache, ComputeNone, OptionallyNone};
const WAITER_MAP_NUM_SEGMENTS: usize = 64;
type ErrorObject = Arc<dyn Any + Send + Sync + 'static>;
// type WaiterValue<V> = Option<Result<V, ErrorObject>>;
enum WaiterValue<V> {
Computing,
Ready(Result<V, ErrorObject>),
ReadyNone,
// https://github.com/moka-rs/moka/issues/43
InitClosurePanicked,
}
impl<V> fmt::Debug for WaiterValue<V> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
WaiterValue::Computing => write!(f, "Computing"),
WaiterValue::Ready(_) => write!(f, "Ready"),
WaiterValue::ReadyNone => write!(f, "ReadyNone"),
WaiterValue::InitClosurePanicked => write!(f, "InitFuturePanicked"),
}
}
}
type Waiter<V> = MiniArc<RwLock<WaiterValue<V>>>;
pub(crate) enum InitResult<V, E> {
Initialized(V),
ReadExisting(V),
InitErr(Arc<E>),
}
pub(crate) struct ValueInitializer<K, V, S> {
// TypeId is the type ID of the concrete error type of generic type E in the
// try_get_with method. We use the type ID as a part of the key to ensure that
// we can always downcast the trait object ErrorObject (in Waiter<V>) into
// its concrete type.
waiters: crate::cht::SegmentedHashMap<(Arc<K>, TypeId), Waiter<V>, S>,
}
impl<K, V, S> ValueInitializer<K, V, S>
where
K: Hash + Eq + Send + Sync + 'static,
V: Clone + Send + Sync + 'static,
S: BuildHasher + Clone + Send + Sync + 'static,
{
pub(crate) fn with_hasher(hasher: S) -> Self {
Self {
waiters: crate::cht::SegmentedHashMap::with_num_segments_and_hasher(
WAITER_MAP_NUM_SEGMENTS,
hasher,
),
}
}
/// # Panics
/// Panics if the `init` closure has been panicked.
pub(crate) fn try_init_or_read<O, E>(
&self,
key: &Arc<K>,
type_id: TypeId,
// Closure to get an existing value from cache.
mut get: impl FnMut() -> Option<V>,
// Closure to initialize a new value.
init: impl FnOnce() -> O,
// Closure to insert a new value into cache.
mut insert: impl FnMut(V),
// Function to convert a value O, returned from the init future, into
// Result<V, E>.
post_init: fn(O) -> Result<V, E>,
) -> InitResult<V, E>
where
E: Send + Sync + 'static,
{
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
use InitResult::{InitErr, ReadExisting};
const MAX_RETRIES: usize = 200;
let mut retries = 0;
let (w_key, w_hash) = self.waiter_key_hash(key, type_id);
let waiter = MiniArc::new(RwLock::new(WaiterValue::Computing));
let mut lock = waiter.write();
loop {
let Some(existing_waiter) = self.try_insert_waiter(w_key.clone(), w_hash, &waiter)
else {
// Inserted.
break;
};
// Somebody else's waiter already exists, so wait for its result to become available.
let waiter_result = existing_waiter.read();
match &*waiter_result {
WaiterValue::Ready(Ok(value)) => return ReadExisting(value.clone()),
WaiterValue::Ready(Err(e)) => return InitErr(Arc::clone(e).downcast().unwrap()),
// Somebody else's init closure has been panicked.
WaiterValue::InitClosurePanicked => {
retries += 1;
assert!(
retries < MAX_RETRIES,
"Too many retries. Tried to read the return value from the `init` \
closure but failed {retries} times. Maybe the `init` kept panicking?"
);
// Retry from the beginning.
continue;
}
// Unexpected state.
s @ (WaiterValue::Computing | WaiterValue::ReadyNone) => panic!(
"Got unexpected state `{s:?}` after resolving `init` future. \
This might be a bug in Moka"
),
}
}
// Our waiter was inserted.
// Check if the value has already been inserted by other thread.
if let Some(value) = get() {
// Yes. Set the waiter value, remove our waiter, and return
// the existing value.
*lock = WaiterValue::Ready(Ok(value.clone()));
self.remove_waiter(w_key, w_hash);
return InitResult::ReadExisting(value);
}
// The value still does note exist. Let's evaluate the init
// closure. Catching panic is safe here as we do not try to
// evaluate the closure again.
match catch_unwind(AssertUnwindSafe(init)) {
// Evaluated.
Ok(value) => {
let init_res = match post_init(value) {
Ok(value) => {
insert(value.clone());
*lock = WaiterValue::Ready(Ok(value.clone()));
InitResult::Initialized(value)
}
Err(e) => {
let err: ErrorObject = Arc::new(e);
*lock = WaiterValue::Ready(Err(Arc::clone(&err)));
InitResult::InitErr(err.downcast().unwrap())
}
};
self.remove_waiter(w_key, w_hash);
init_res
}
// Panicked.
Err(payload) => {
*lock = WaiterValue::InitClosurePanicked;
// Remove the waiter so that others can retry.
self.remove_waiter(w_key, w_hash);
resume_unwind(payload);
}
}
// The write lock will be unlocked here.
}
/// # Panics
/// Panics if the `init` closure has been panicked.
pub(crate) fn try_compute<F, O, E>(
&self,
c_key: Arc<K>,
c_hash: u64,
cache: &Cache<K, V, S>,
f: F,
post_init: fn(O) -> Result<Op<V>, E>,
allow_nop: bool,
) -> Result<CompResult<K, V>, E>
where
V: 'static,
F: FnOnce(Option<Entry<K, V>>) -> O,
E: Send + Sync + 'static,
{
use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
let type_id = TypeId::of::<ComputeNone>();
let (w_key, w_hash) = self.waiter_key_hash(&c_key, type_id);
let waiter = MiniArc::new(RwLock::new(WaiterValue::Computing));
// NOTE: We have to acquire a write lock before `try_insert_waiter`,
// so that any concurrent attempt will get our lock and wait on it.
let mut lock = waiter.write();
loop {
let Some(existing_waiter) = self.try_insert_waiter(w_key.clone(), w_hash, &waiter)
else {
// Inserted.
break;
};
// Somebody else's waiter already exists, so wait for it to finish
// (wait for it to release the write lock).
let waiter_result = existing_waiter.read();
match &*waiter_result {
// Unexpected state.
WaiterValue::Computing => panic!(
"Got unexpected state `Computing` after resolving `init` future. \
This might be a bug in Moka"
),
_ => {
// Try to insert our waiter again.
continue;
}
}
}
// Our waiter was inserted.
// Get the current value.
let ignore_if = None as Option<&mut fn(&V) -> bool>;
let maybe_entry = cache
.base
.get_with_hash_and_ignore_if(&*c_key, c_hash, ignore_if, true);
let maybe_value = if allow_nop {
maybe_entry.as_ref().map(|ent| ent.value().clone())
} else {
None
};
let entry_existed = maybe_entry.is_some();
// Evaluate the `f` closure. Catching panic is safe here as we will not
// evaluate the closure again.
let output = match catch_unwind(AssertUnwindSafe(|| f(maybe_entry))) {
// Evaluated.
Ok(output) => {
*lock = WaiterValue::ReadyNone;
output
}
// Panicked.
Err(payload) => {
*lock = WaiterValue::InitClosurePanicked;
// Remove the waiter so that others can retry.
self.remove_waiter(w_key, w_hash);
resume_unwind(payload);
}
};
let op = match post_init(output) {
Ok(op) => op,
Err(e) => {
self.remove_waiter(w_key, w_hash);
return Err(e);
}
};
let result = match op {
Op::Nop => {
if let Some(value) = maybe_value {
Ok(CompResult::Unchanged(Entry::new(
Some(c_key),
value,
false,
false,
)))
} else {
Ok(CompResult::StillNone(c_key))
}
}
Op::Put(value) => {
cache.insert_with_hash(Arc::clone(&c_key), c_hash, value.clone());
if entry_existed {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), value, true, true);
Ok(CompResult::ReplacedWith(entry))
} else {
let entry = Entry::new(Some(c_key), value, true, false);
Ok(CompResult::Inserted(entry))
}
}
Op::Remove => {
let maybe_prev_v = cache.invalidate_with_hash(&*c_key, c_hash, true);
if let Some(prev_v) = maybe_prev_v {
crossbeam_epoch::pin().flush();
let entry = Entry::new(Some(c_key), prev_v, false, false);
Ok(CompResult::Removed(entry))
} else {
Ok(CompResult::StillNone(c_key))
}
}
};
self.remove_waiter(w_key, w_hash);
result
// The lock will be unlocked here.
}
/// The `post_init` function for the `get_with` method of cache.
pub(crate) fn post_init_for_get_with(value: V) -> Result<V, ()> {
Ok(value)
}
/// The `post_init` function for the `optionally_get_with` method of cache.
pub(crate) fn post_init_for_optionally_get_with(
value: Option<V>,
) -> Result<V, Arc<OptionallyNone>> {
// `value` can be either `Some` or `None`. For `None` case, without change
// the existing API too much, we will need to convert `None` to Arc<E> here.
// `Infallible` could not be instantiated. So it might be good to use an
// empty struct to indicate the error type.
value.ok_or(Arc::new(OptionallyNone))
}
/// The `post_init` function for `try_get_with` method of cache.
pub(crate) fn post_init_for_try_get_with<E>(result: Result<V, E>) -> Result<V, E> {
result
}
/// The `post_init` function for the `and_upsert_with` method of cache.
pub(crate) fn post_init_for_upsert_with(value: V) -> Result<Op<V>, ()> {
Ok(Op::Put(value))
}
/// The `post_init` function for the `and_compute_with` method of cache.
pub(crate) fn post_init_for_compute_with(op: Op<V>) -> Result<Op<V>, ()> {
Ok(op)
}
/// The `post_init` function for the `and_try_compute_with` method of cache.
pub(crate) fn post_init_for_try_compute_with<E>(op: Result<Op<V>, E>) -> Result<Op<V>, E>
where
E: Send + Sync + 'static,
{
op
}
/// Returns the `type_id` for `get_with` method of cache.
pub(crate) fn type_id_for_get_with() -> TypeId {
// NOTE: We use a regular function here instead of a const fn because TypeId
// is not stable as a const fn. (as of our MSRV)
TypeId::of::<()>()
}
/// Returns the `type_id` for `optionally_get_with` method of cache.
pub(crate) fn type_id_for_optionally_get_with() -> TypeId {
TypeId::of::<OptionallyNone>()
}
/// Returns the `type_id` for `try_get_with` method of cache.
pub(crate) fn type_id_for_try_get_with<E: 'static>() -> TypeId {
TypeId::of::<E>()
}
#[inline]
fn remove_waiter(&self, w_key: (Arc<K>, TypeId), w_hash: u64) {
self.waiters.remove(w_hash, |k| k == &w_key);
}
#[inline]
fn try_insert_waiter(
&self,
w_key: (Arc<K>, TypeId),
w_hash: u64,
waiter: &Waiter<V>,
) -> Option<Waiter<V>> {
let waiter = MiniArc::clone(waiter);
self.waiters.insert_if_not_present(w_key, w_hash, waiter)
}
#[inline]
fn waiter_key_hash(&self, c_key: &Arc<K>, type_id: TypeId) -> ((Arc<K>, TypeId), u64) {
let w_key = (Arc::clone(c_key), type_id);
let w_hash = self.waiters.hash(&w_key);
(w_key, w_hash)
}
}
#[cfg(test)]
impl<K, V, S> ValueInitializer<K, V, S> {
pub(crate) fn waiter_count(&self) -> usize {
self.waiters.len()
}
}
@@ -0,0 +1,67 @@
#![cfg(feature = "future")]
/// Regression test for TOCTOU race in `and_compute_with`.
///
/// In `value_initializer.rs::try_compute`, the waiter was removed from the waiter
/// map (via `set_waiter_value(ReadyNone)`) before the actual cache mutation
/// (`cache.insert_with_hash`). This allowed a concurrent `and_compute_with` caller
/// to insert its own waiter, read stale cache state, and both callers to execute
/// `Op::Put` based on the same old value — losing one update.
use moka::future::Cache;
use moka::ops::compute;
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn test_and_compute_with_concurrent_atomicity() {
let cache: Cache<String, u64> = Cache::builder().max_capacity(100).build();
let key = "counter".to_string();
// Seed the cache with an initial value.
cache.insert(key.clone(), 0).await;
let n_writers: u64 = 8;
let writes_per_writer: u64 = 100;
let mut handles = Vec::new();
for _ in 0..n_writers {
let cache = cache.clone();
let key = key.clone();
handles.push(tokio::spawn(async move {
for _ in 0..writes_per_writer {
loop {
let result = cache
.entry_by_ref(&key)
.and_compute_with(|entry| async move {
match entry {
Some(entry) => {
let val: u64 = entry.into_value();
compute::Op::Put(val + 1)
}
None => compute::Op::Nop,
}
})
.await;
if matches!(result, compute::CompResult::ReplacedWith(_)) {
break;
}
}
}
}));
}
for h in handles {
h.await.unwrap();
}
let expected: u64 = n_writers * writes_per_writer;
let actual: u64 = cache.get(&key).await.unwrap();
assert_eq!(
actual,
expected,
"Lost {} increments out of {expected}. \
and_compute_with did not properly serialize concurrent calls on the same key.",
expected.saturating_sub(actual)
);
}
+87
View File
@@ -0,0 +1,87 @@
#![cfg(all(test, feature = "future"))]
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
use actix_rt::Runtime;
use async_lock::Barrier;
use moka::future::Cache;
const NUM_THREADS: u8 = 16;
#[test]
fn test_get_with() -> Result<(), Box<dyn std::error::Error>> {
const TEN_MIB: usize = 10 * 1024 * 1024; // 10MiB
let cache = Cache::new(100);
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
let rt = Runtime::new()?;
let tasks: Vec<_> = (0..NUM_THREADS)
.map(|task_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
rt.spawn(async move {
my_barrier.wait().await;
println!("Task {task_id} started.");
let key = "key1".to_string();
let value = match task_id % 4 {
0 => {
my_cache
.get_with(key.clone(), async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
}
1 => {
my_cache
.get_with_by_ref(key.as_str(), async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
}
2 => my_cache
.entry(key.clone())
.or_insert_with(async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
.into_value(),
3 => my_cache
.entry_by_ref(key.as_str())
.or_insert_with(async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
.into_value(),
_ => unreachable!(),
};
assert_eq!(value.len(), TEN_MIB);
assert!(my_cache.get(key.as_str()).await.is_some());
println!("Task {task_id} got the value. (len: {})", value.len());
})
})
.collect();
rt.block_on(futures_util::future::join_all(tasks));
assert_eq!(call_counter.load(Ordering::Acquire), 1);
Ok(())
}
+248
View File
@@ -0,0 +1,248 @@
#![cfg(all(test, feature = "sync"))]
use std::{
path::Path,
sync::atomic::{AtomicUsize, Ordering},
sync::{Arc, Barrier},
thread,
};
use moka::{
sync::{Cache, SegmentedCache},
Entry,
};
const NUM_THREADS: u8 = 16;
const FILE: &str = "./Cargo.toml";
macro_rules! generate_test_get_with {
($test_fn_name:ident, $cache_init:expr) => {
#[test]
fn $test_fn_name() {
const TEN_MIB: usize = 10 * 1024 * 1024; // 10MiB
let cache = $cache_init;
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
let threads: Vec<_> = (0..NUM_THREADS)
.map(|thread_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
thread::spawn(move || {
my_barrier.wait();
println!("Thread {thread_id} started.");
let key = "key1".to_string();
let value = match thread_id % 4 {
0 => my_cache.get_with(key.clone(), || {
println!("Thread {thread_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
}),
1 => my_cache.get_with_by_ref(key.as_str(), || {
println!("Thread {thread_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
}),
2 => my_cache
.entry(key.clone())
.or_insert_with(|| {
println!("Thread {thread_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.into_value(),
3 => my_cache
.entry_by_ref(key.as_str())
.or_insert_with(|| {
println!("Thread {thread_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.into_value(),
_ => unreachable!(),
};
assert_eq!(value.len(), TEN_MIB);
assert!(my_cache.get(key.as_str()).is_some());
println!("Thread {thread_id} got the value. (len: {})", value.len());
})
})
.collect();
threads
.into_iter()
.for_each(|t| t.join().expect("Thread failed"));
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
};
}
macro_rules! generate_test_optionally_get_with {
($test_fn_name:ident, $cache_init:expr) => {
#[test]
fn $test_fn_name() {
let cache = $cache_init;
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
fn get_file_size(
thread_id: u8,
path: impl AsRef<Path>,
call_counter: &AtomicUsize,
) -> Option<u64> {
println!("get_file_size() called by thread {thread_id}.");
call_counter.fetch_add(1, Ordering::AcqRel);
std::fs::metadata(path).ok().map(|m| m.len())
}
let threads: Vec<_> = (0..NUM_THREADS)
.map(|thread_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
thread::spawn(move || {
my_barrier.wait();
println!("Thread {thread_id} started.");
let key = "key1".to_string();
let value = match thread_id % 4 {
0 => my_cache.optionally_get_with(key.clone(), || {
get_file_size(thread_id, FILE, &my_call_counter)
}),
1 => my_cache.optionally_get_with_by_ref(key.as_str(), || {
get_file_size(thread_id, FILE, &my_call_counter)
}),
2 => my_cache
.entry(key.clone())
.or_optionally_insert_with(|| {
get_file_size(thread_id, FILE, &my_call_counter)
})
.map(Entry::into_value),
3 => my_cache
.entry_by_ref(key.as_str())
.or_optionally_insert_with(|| {
get_file_size(thread_id, FILE, &my_call_counter)
})
.map(Entry::into_value),
_ => unreachable!(),
};
assert!(value.is_some());
assert!(my_cache.get(key.as_str()).is_some());
println!(
"Thread {thread_id} got the value. (len: {})",
value.unwrap()
);
})
})
.collect();
threads
.into_iter()
.for_each(|t| t.join().expect("Thread failed"));
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
};
}
macro_rules! generate_test_try_get_with {
($test_fn_name:ident, $cache_init:expr) => {
#[test]
fn $test_fn_name() {
let cache = $cache_init;
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
fn get_file_size(
thread_id: u8,
path: impl AsRef<Path>,
call_counter: &AtomicUsize,
) -> Result<u64, std::io::Error> {
println!("get_file_size() called by thread {thread_id}.");
call_counter.fetch_add(1, Ordering::AcqRel);
Ok(std::fs::metadata(path)?.len())
}
let threads: Vec<_> = (0..NUM_THREADS)
.map(|thread_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
thread::spawn(move || {
my_barrier.wait();
println!("Thread {thread_id} started.");
let key = "key1".to_string();
let value = match thread_id % 4 {
0 => my_cache.try_get_with(key.clone(), || {
get_file_size(thread_id, FILE, &my_call_counter)
}),
1 => my_cache.try_get_with_by_ref(key.as_str(), || {
get_file_size(thread_id, FILE, &my_call_counter)
}),
2 => my_cache
.entry(key.clone())
.or_try_insert_with(|| {
get_file_size(thread_id, FILE, &my_call_counter)
})
.map(Entry::into_value),
3 => my_cache
.entry_by_ref(key.as_str())
.or_try_insert_with(|| {
get_file_size(thread_id, FILE, &my_call_counter)
})
.map(Entry::into_value),
_ => unreachable!(),
};
assert!(value.is_ok());
assert!(my_cache.get(key.as_str()).is_some());
println!(
"Thread {thread_id} got the value. (len: {})",
value.unwrap()
);
})
})
.collect();
threads
.into_iter()
.for_each(|t| t.join().expect("Thread failed"));
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
};
}
generate_test_get_with!(test_cache_get_with, Cache::<String, Arc<Vec<u8>>>::new(100));
generate_test_get_with!(
test_seg_cache_get_with,
SegmentedCache::<String, Arc<Vec<u8>>>::new(100, 4)
);
generate_test_optionally_get_with!(
test_cache_optionally_get_with,
Cache::<String, u64>::new(100)
);
generate_test_optionally_get_with!(
test_seg_cache_optionally_get_with,
SegmentedCache::<String, u64>::new(100, 4)
);
generate_test_try_get_with!(test_cache_try_get_with, Cache::<String, u64>::new(100));
generate_test_try_get_with!(
test_seg_cache_try_get_with,
SegmentedCache::<String, u64>::new(100, 4)
);
+222
View File
@@ -0,0 +1,222 @@
#![cfg(all(test, feature = "future"))]
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
use async_lock::Barrier;
use moka::{future::Cache, Entry};
const NUM_THREADS: u8 = 16;
const SITE: &str = "https://www.rust-lang.org/";
#[tokio::test]
async fn test_get_with() {
const TEN_MIB: usize = 10 * 1024 * 1024; // 10MiB
let cache = Cache::new(100);
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
let tasks: Vec<_> = (0..NUM_THREADS)
.map(|task_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
tokio::spawn(async move {
my_barrier.wait().await;
println!("Task {task_id} started.");
let key = "key1".to_string();
let value = match task_id % 4 {
0 => {
my_cache
.get_with(key.clone(), async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
}
1 => {
my_cache
.get_with_by_ref(key.as_str(), async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
}
2 => my_cache
.entry(key.clone())
.or_insert_with(async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
.into_value(),
3 => my_cache
.entry_by_ref(key.as_str())
.or_insert_with(async move {
println!("Task {task_id} inserting a value.");
my_call_counter.fetch_add(1, Ordering::AcqRel);
Arc::new(vec![0u8; TEN_MIB])
})
.await
.into_value(),
_ => unreachable!(),
};
assert_eq!(value.len(), TEN_MIB);
assert!(my_cache.get(key.as_str()).await.is_some());
println!("Task {task_id} got the value. (len: {})", value.len());
})
})
.collect();
futures_util::future::join_all(tasks).await;
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
#[tokio::test]
async fn test_optionally_get_with() {
let cache = Cache::new(100);
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
async fn get_html(task_id: u8, uri: &str, call_counter: &AtomicUsize) -> Option<String> {
println!("get_html() called by task {task_id}.");
call_counter.fetch_add(1, Ordering::AcqRel);
reqwest::get(uri).await.ok()?.text().await.ok()
}
let tasks: Vec<_> = (0..NUM_THREADS)
.map(|task_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
tokio::spawn(async move {
my_barrier.wait().await;
println!("Task {task_id} started.");
let key = "key1".to_string();
let value = match task_id % 4 {
0 => {
my_cache
.optionally_get_with(
key.clone(),
get_html(task_id, SITE, &my_call_counter),
)
.await
}
1 => {
my_cache
.optionally_get_with_by_ref(
key.as_str(),
get_html(task_id, SITE, &my_call_counter),
)
.await
}
2 => my_cache
.entry(key.clone())
.or_optionally_insert_with(get_html(task_id, SITE, &my_call_counter))
.await
.map(Entry::into_value),
3 => my_cache
.entry_by_ref(key.as_str())
.or_optionally_insert_with(get_html(task_id, SITE, &my_call_counter))
.await
.map(Entry::into_value),
_ => unreachable!(),
};
assert!(value.is_some());
assert!(my_cache.get(key.as_str()).await.is_some());
println!(
"Task {task_id} got the value. (len: {})",
value.unwrap().len()
);
})
})
.collect();
futures_util::future::join_all(tasks).await;
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
#[tokio::test]
async fn test_try_get_with() {
let cache = Cache::new(100);
let call_counter = Arc::new(AtomicUsize::default());
let barrier = Arc::new(Barrier::new(NUM_THREADS as usize));
async fn get_html(
task_id: u8,
uri: &str,
call_counter: &AtomicUsize,
) -> Result<String, reqwest::Error> {
println!("get_html() called by task {task_id}.");
call_counter.fetch_add(1, Ordering::AcqRel);
reqwest::get(uri).await?.text().await
}
let tasks: Vec<_> = (0..NUM_THREADS)
.map(|task_id| {
let my_cache = cache.clone();
let my_call_counter = Arc::clone(&call_counter);
let my_barrier = Arc::clone(&barrier);
tokio::spawn(async move {
my_barrier.wait().await;
println!("Task {task_id} started.");
let key = "key1".to_string();
let value = match task_id % 4 {
0 => {
my_cache
.try_get_with(key.clone(), get_html(task_id, SITE, &my_call_counter))
.await
}
1 => {
my_cache
.try_get_with_by_ref(
key.as_str(),
get_html(task_id, SITE, &my_call_counter),
)
.await
}
2 => my_cache
.entry(key.clone())
.or_try_insert_with(get_html(task_id, SITE, &my_call_counter))
.await
.map(Entry::into_value),
3 => my_cache
.entry_by_ref(key.as_str())
.or_try_insert_with(get_html(task_id, SITE, &my_call_counter))
.await
.map(Entry::into_value),
_ => unreachable!(),
};
assert!(value.is_ok());
assert!(my_cache.get(key.as_str()).await.is_some());
println!(
"Task {task_id} got the value. (len: {})",
value.unwrap().len()
);
})
})
.collect();
futures_util::future::join_all(tasks).await;
assert_eq!(call_counter.load(Ordering::Acquire), 1);
}
+101
View File
@@ -0,0 +1,101 @@
#![cfg(all(test, feature = "future"))]
use std::sync::Arc;
use actix_rt::System;
use moka::future::Cache;
use tokio::sync::Barrier;
#[actix_rt::test]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
const NUM_TASKS: usize = 12;
const NUM_THREADS: usize = 4;
const NUM_KEYS_PER_TASK: usize = 64;
fn value(n: usize) -> String {
format!("value {n}")
}
// Create a cache that can store up to 10,000 entries.
let cache = Cache::new(10_000);
let barrier = Arc::new(Barrier::new(NUM_THREADS + NUM_TASKS));
// Spawn async tasks and write to and read from the cache.
// NOTE: Actix Runtime is single threaded.
let tasks: Vec<_> = (0..NUM_TASKS)
.map(|i| {
// To share the same cache across the async tasks and OS threads, clone
// it. This is a cheap operation.
let my_cache = cache.clone();
let my_barrier = Arc::clone(&barrier);
let start = i * NUM_KEYS_PER_TASK;
let end = (i + 1) * NUM_KEYS_PER_TASK;
actix_rt::spawn(async move {
// Wait for the all async tasks and threads to be spawned.
my_barrier.wait().await;
// Insert 64 entries. (NUM_KEYS_PER_TASK = 64)
for key in start..end {
my_cache.insert(key, value(key)).await;
assert_eq!(my_cache.get(&key).await, Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
my_cache.invalidate(&key).await;
}
})
})
.collect();
// Spawn OS threads and write to and read from the cache.
let threads: Vec<_> = (0..NUM_THREADS)
.map(|i| i + NUM_TASKS)
.map(|i| {
let my_cache = cache.clone();
let my_barrier = Arc::clone(&barrier);
let start = i * NUM_KEYS_PER_TASK;
let end = (i + 1) * NUM_KEYS_PER_TASK;
std::thread::spawn(move || {
// It seems there is no way to get a SystemRunner from the current
// System (`System::current()`). So, create a new System.
let runner = System::new(); // Returns a SystemRunner.
// Wait for the all async tasks and threads to be spawned.
runner.block_on(my_barrier.wait());
// Insert 64 entries. (NUM_KEYS_PER_TASK = 64)
for key in start..end {
runner.block_on(my_cache.insert(key, value(key)));
assert_eq!(runner.block_on(my_cache.get(&key)), Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
runner.block_on(my_cache.invalidate(&key));
}
})
})
.collect();
futures_util::future::join_all(tasks).await;
for t in threads {
t.join().unwrap();
}
// Verify the result.
for key in 0..(NUM_TASKS * NUM_KEYS_PER_TASK) {
if key % 4 == 0 {
assert_eq!(cache.get(&key).await, None);
} else {
assert_eq!(cache.get(&key).await, Some(value(key)));
}
}
System::current().stop();
Ok(())
}
+93
View File
@@ -0,0 +1,93 @@
#![cfg(all(test, feature = "future"))]
use std::sync::Arc;
use moka::future::Cache;
use tokio::sync::Barrier;
#[tokio::test]
async fn main() {
const NUM_TASKS: usize = 12;
const NUM_THREADS: usize = 4;
const NUM_KEYS_PER_TASK: usize = 64;
fn value(n: usize) -> String {
format!("value {n}")
}
// Create a cache that can store up to 10,000 entries.
let cache = Cache::new(10_000);
let barrier = Arc::new(Barrier::new(NUM_THREADS + NUM_TASKS));
// Spawn async tasks and write to and read from the cache.
let tasks: Vec<_> = (0..NUM_TASKS)
.map(|i| {
// To share the same cache across the async tasks and OS threads, clone
// it. This is a cheap operation.
let my_cache = cache.clone();
let my_barrier = Arc::clone(&barrier);
let start = i * NUM_KEYS_PER_TASK;
let end = (i + 1) * NUM_KEYS_PER_TASK;
tokio::spawn(async move {
// Wait for the all async tasks and threads to be spawned.
my_barrier.wait().await;
// Insert 64 entries. (NUM_KEYS_PER_TASK = 64)
for key in start..end {
my_cache.insert(key, value(key)).await;
assert_eq!(my_cache.get(&key).await, Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
my_cache.invalidate(&key).await;
}
})
})
.collect();
// Spawn OS threads and write to and read from the cache.
let threads: Vec<_> = (0..NUM_THREADS)
.map(|i| i + NUM_TASKS)
.map(|i| {
let my_cache = cache.clone();
let my_barrier = Arc::clone(&barrier);
let start = i * NUM_KEYS_PER_TASK;
let end = (i + 1) * NUM_KEYS_PER_TASK;
let rt = tokio::runtime::Handle::current();
std::thread::spawn(move || {
// Wait for the all async tasks and threads to be spawned.
rt.block_on(my_barrier.wait());
// Insert 64 entries. (NUM_KEYS_PER_TASK = 64)
for key in start..end {
rt.block_on(my_cache.insert(key, value(key)));
assert_eq!(rt.block_on(my_cache.get(&key)), Some(value(key)));
}
// Invalidate every 4 element of the inserted entries.
for key in (start..end).step_by(4) {
rt.block_on(my_cache.invalidate(&key));
}
})
})
.collect();
// Wait for all tasks and threads to complete.
futures_util::future::join_all(tasks).await;
for t in threads {
t.join().unwrap();
}
// Verify the result.
for key in 0..(NUM_TASKS * NUM_KEYS_PER_TASK) {
if key % 4 == 0 {
assert_eq!(cache.get(&key).await, None);
} else {
assert_eq!(cache.get(&key).await, Some(value(key)));
}
}
}
@@ -0,0 +1,346 @@
//! This test reproduces a race condition where returning `None` from `Expiry`
//! methods (to unset expiration) can cause a use-after-free panic in the timer wheel.
//!
//! The bug occurs because:
//! 1. `expiration_time` is set atomically (immediately) when `Expiry` returns `None`
//! 2. `timer_node` is only updated during housekeeping (later)
//! 3. This creates a window where `expiration_time` is `None` but `timer_node` still
//! points to a freed/invalid timer node
//!
//! Pattern that triggers the bug:
//! - Insert with `Err` value → gets TTL (timer node created)
//! - Update to `Ok` value → `Expiry` returns `None` (should remove timer node)
//! - Concurrent `get` or housekeeping reads stale `timer_node` pointer
#![cfg(feature = "sync")]
use moka::sync::Cache;
use moka::Expiry;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::{Duration, Instant};
type HandleResult<T> = Result<T, String>;
struct CustomExpiry {
error_ttl: Duration,
}
impl Default for CustomExpiry {
fn default() -> Self {
Self {
error_ttl: Duration::from_millis(100),
}
}
}
impl<K, V> Expiry<K, HandleResult<V>> for CustomExpiry {
fn expire_after_create(&self, _: &K, value: &HandleResult<V>, _: Instant) -> Option<Duration> {
match value {
Ok(_) => None,
Err(_) => Some(self.error_ttl),
}
}
fn expire_after_update(
&self,
_: &K,
value: &HandleResult<V>,
_: Instant,
_: Option<Duration>,
) -> Option<Duration> {
match value {
Ok(_) => None,
Err(_) => Some(self.error_ttl),
}
}
}
/// This test runs for a short duration (5 seconds by default) to catch the race condition.
/// In production, the bug was observed within 2 months of operation, but with aggressive
/// concurrent operations, it can be triggered much faster.
///
/// The test pattern:
/// 1. Insert threads: repeatedly insert `Err` then `Ok` values (triggers timer node create/remove)
/// 2. Get threads: continuously read entries (triggers `expire_after_read` and housekeeping)
/// 3. Housekeeping thread: explicitly runs `run_pending_tasks()` to process timer wheel operations
#[test]
fn test_timer_wheel_panic() {
// Use shorter duration for CI, increase for more thorough testing
let test_duration = Duration::from_secs(5);
const NUM_KEYS: u64 = if cfg!(miri) { 25 } else { 100 };
let panics = Arc::new(AtomicUsize::new(0));
let cache: Cache<u64, HandleResult<u64>> = Cache::builder()
.name("test_cache")
.expire_after(CustomExpiry::default())
.time_to_idle(Duration::from_secs(240))
.max_capacity(10000)
.build();
let cache = Arc::new(cache);
let start = Instant::now();
// Insert threads: Err -> Ok pattern
let insert_handles: Vec<_> = (0..4)
.map(|tid| {
let cache = Arc::clone(&cache);
let panics = Arc::clone(&panics);
let duration = test_duration;
thread::spawn(move || {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut ops = 0u64;
while start.elapsed() < duration {
for key in 0..NUM_KEYS {
let key = key + (tid as u64 * 10000);
// Insert error first (creates timer node with TTL)
cache.insert(key, Err("error".into()));
// Update to success (Expiry returns None - removes TTL)
cache.insert(key, Ok(ops));
// Sometimes go back to error
if ops % 3 == 0 {
cache.insert(key, Err("retry".into()));
}
ops += 1;
}
}
println!("[Insert {}] done: {} ops", tid, ops);
}))
.unwrap_or_else(|_| {
panics.fetch_add(1, Ordering::Relaxed);
eprintln!("[Insert {}] PANICKED!", tid);
});
})
})
.collect();
// Get threads - trigger expire_after_read and housekeeping
let get_handles: Vec<_> = (0..4)
.map(|tid| {
let cache = Arc::clone(&cache);
let panics = Arc::clone(&panics);
let duration = test_duration;
thread::spawn(move || {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut ops = 0u64;
while start.elapsed() < duration {
for key in 0..NUM_KEYS {
// Read from different thread's key space to increase contention
let key = key + ((ops % 4) * 10000);
let _ = cache.get(&key);
ops += 1;
}
}
println!("[Get {}] done: {} ops", tid, ops);
}))
.unwrap_or_else(|_| {
panics.fetch_add(1, Ordering::Relaxed);
eprintln!("[Get {}] PANICKED!", tid);
});
})
})
.collect();
let cache_hk = Arc::clone(&cache);
let panics_hk = Arc::clone(&panics);
let hk = thread::spawn(move || {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
while start.elapsed() < test_duration {
cache_hk.run_pending_tasks();
thread::sleep(Duration::from_millis(50));
}
}))
.unwrap_or_else(|_| {
panics_hk.fetch_add(1, Ordering::Relaxed);
eprintln!("[HK] PANICKED!");
});
});
for h in insert_handles {
let _ = h.join();
}
for h in get_handles {
let _ = h.join();
}
let _ = hk.join();
let total_panics = panics.load(Ordering::Relaxed);
assert_eq!(
total_panics, 0,
"Timer wheel panic detected! {} threads panicked",
total_panics
);
}
/// Extended stress test - runs longer for more thorough testing.
///
/// To run this test, do one of the following:
///
/// ```console
/// ## Normal cargo test command for release build
/// $ cargo test --release -p moka -F sync --test timer_wheel_panic_test stress -- \
/// --ignored --no-capture
///
/// ## Miri command with appropriate flags
/// ##
/// ## - `miri-ignore-leaks` for ignoring `crossbeam-epoch`'s not yet reclaimed memory.
/// ## - `miri-permissive-provenance` to avoid provenance-related warnings for `tagptr`
/// ## crate.
/// ## - `miri-disable-isolation` to use the wall-clock time for the test duration,
/// ## instead of Miri's emulated virtual clock, which may advance significantly
/// ## slower.
/// $ MIRIFLAGS='-Zmiri-tree-borrows -Zmiri-ignore-leaks -Zmiri-permissive-provenance -Zmiri-disable-isolation' \
/// cargo +nightly miri test stress_test_timer_wheel_panic -F sync -- \
/// --ignored --no-capture
/// ```
#[test]
#[ignore]
fn stress_test_timer_wheel_panic() {
// If Miri is used, extend the duration significantly.
const DURATION_MINUTES: u64 = if cfg!(miri) { 20 } else { 1 };
let test_duration = Duration::from_secs(DURATION_MINUTES * 60);
let panics = Arc::new(AtomicUsize::new(0));
let cache: Cache<u64, HandleResult<u64>> = Cache::builder()
.name("stress_test_cache")
.expire_after(CustomExpiry::default())
.time_to_idle(Duration::from_secs(240))
.max_capacity(10000)
.build();
let cache = Arc::new(cache);
let start = Instant::now();
// More aggressive thread count for stress testing
const NUM_INSERT_THREADS: i32 = if cfg!(miri) { 6 } else { 8 };
const NUM_GET_THREADS: i32 = if cfg!(miri) { 4 } else { 8 };
// Insert threads
let insert_handles: Vec<_> = (0..NUM_INSERT_THREADS)
.map(|tid| {
let cache = Arc::clone(&cache);
let panics = Arc::clone(&panics);
let duration = test_duration;
thread::spawn(move || {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut ops = 0u64;
const REPORT_INTERVAL: u64 = if cfg!(miri) { 5 } else { 100_000 };
'l: loop {
for key in 0..200u64 {
let key = key + (tid as u64 * 10000);
cache.insert(key, Err("error".into()));
cache.insert(key, Ok(ops));
if ops % 3 == 0 {
cache.insert(key, Err("retry".into()));
}
if ops % 7 == 0 {
cache.invalidate(&key);
}
ops += 1;
if ops % REPORT_INTERVAL == 0 {
println!("[Insert {}] running: {} ops", tid, ops);
}
if start.elapsed() >= duration {
break 'l;
}
}
}
println!("[Insert {}] done: {} ops", tid, ops);
}))
.unwrap_or_else(|e| {
panics.fetch_add(1, Ordering::Relaxed);
eprintln!("[Insert {}] PANICKED: {:?}", tid, e);
});
})
})
.collect();
// Get threads
let get_handles: Vec<_> = (0..NUM_GET_THREADS)
.map(|tid| {
let cache = Arc::clone(&cache);
let panics = Arc::clone(&panics);
let duration = test_duration;
thread::spawn(move || {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let mut ops = 0u64;
const REPORT_INTERVAL: u64 = if cfg!(miri) { 50 } else { 100_000 };
'l: loop {
for key in 0..200u64 {
let key = key + ((ops % NUM_GET_THREADS as u64) * 10000);
let _ = cache.get(&key);
ops += 1;
if ops % REPORT_INTERVAL == 0 {
println!("[Get {}] running: {} ops", tid, ops);
}
if start.elapsed() >= duration {
break 'l;
}
}
}
println!("[Get {}] done: {} ops", tid, ops);
}))
.unwrap_or_else(|e| {
panics.fetch_add(1, Ordering::Relaxed);
eprintln!("[Get {}] PANICKED: {:?}", tid, e);
});
})
})
.collect();
let hk_handles = if cfg!(miri) {
None
} else {
let handles = (0..2)
.map(|tid| {
let cache = Arc::clone(&cache);
let panics = Arc::clone(&panics);
let duration = test_duration;
thread::spawn(move || {
let mut ops = 0u64;
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
while start.elapsed() < duration {
cache.run_pending_tasks();
thread::sleep(Duration::from_millis(25));
ops += 1;
if ops % 50 == 0 {
println!("[HK {}] running: {} ops", tid, ops);
}
}
}))
.unwrap_or_else(|e| {
panics.fetch_add(1, Ordering::Relaxed);
eprintln!("[HK {}] PANICKED: {:?}", tid, e);
});
})
})
.collect::<Vec<_>>();
Some(handles)
};
// Wait for all threads
for h in insert_handles {
let _ = h.join();
}
for h in get_handles {
let _ = h.join();
}
if let Some(hk_handles) = hk_handles {
for h in hk_handles {
let _ = h.join();
}
}
let total_panics = panics.load(Ordering::Relaxed);
assert_eq!(
total_panics, 0,
"Timer wheel panic detected! {} threads panicked during stress test.",
total_panics
);
}