diff options
Diffstat (limited to 'sys-fs/bees')
| -rw-r--r-- | sys-fs/bees/Manifest | 5 | ||||
| -rw-r--r-- | sys-fs/bees/bees-0.10-r1.ebuild | 112 | ||||
| -rw-r--r-- | sys-fs/bees/bees-9999.ebuild | 6 | ||||
| -rw-r--r-- | sys-fs/bees/files/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch | 36 | ||||
| -rw-r--r-- | sys-fs/bees/files/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch | 43 |
5 files changed, 198 insertions, 4 deletions
diff --git a/sys-fs/bees/Manifest b/sys-fs/bees/Manifest index cab62ad6564b..8f062277fcd9 100644 --- a/sys-fs/bees/Manifest +++ b/sys-fs/bees/Manifest @@ -1,5 +1,8 @@ AUX 0001-HACK-crucible-Work-around-kernel-memory-fragmentatio.patch 1183 BLAKE2B 784ad6e8150a5d791c839c04060aa1116f72a29c0e2f4d1d790d6e1fb1d56b5a6ced843f38ac56930a63426ed7e9c5506f81c1f7189728937e0ee49f9f5af079 SHA512 29ac086a2c84d7eb89090498849b2e01e99da62c3ce49b5c4d36f071a3a8ff506e15b227a2303c01da25bdc4572e5dc1129767df496897de8ca2bed579dd3017 +AUX 0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch 1500 BLAKE2B c6f6895b92caae4ac8de83931574f87274d7798889c0589859204d73cd41072ee6967ae908c8509e7f5c929f3a4f04c73932b72a27ab3d244cef254ebc15030e SHA512 82e3e8872e4b9943afff883c3ee420a8850b337914fc0b388171f1331c4cd900778c2263b98ff1705f50f7cd3ffe5868d7f88ec48ab68217471a21573a559479 +AUX 0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch 1445 BLAKE2B e1ebe9c50deeaf7eed82c738d172ab332c5048ab192150cc967843ea16f387631ee44c201c43f49c3c0a032a735219f3a08881e5f92fbbf5b85d050eebb1c7f8 SHA512 75655e40084a0f8660773293693d0a214d34c6bb2f861c2e502a4cc5a1576d5c8084b8af87f790155f419affa56886c23eed09532f82c29b2623f553a5c82999 DIST bees-0.10.tar.gz 177509 BLAKE2B 269cf65d5250a07297dfaee6c0fd6bc25199d63fe423d63ce39df8bcdb64d6f8110a0e03d0dc7dc1b105a8886f2a9364da8878481546eaa8dbf3dc46abc569ed SHA512 77a4ae3d66430c2d9dc0351c16c60bad95f21e238fc00f5f5787f55929ba75ee25212ac4bb4a3e4a9419cec27bbcfd2987aa925a54bc2872609b38a698dbedc3 +EBUILD bees-0.10-r1.ebuild 4151 BLAKE2B d0b2ddea9523905f3a5620aa6e0da3c595a405afd34a9ab5488a20e7b1f5c77130d9b56b3856a236004982d6069f213ddacf28a9234f3498cc805b4f9a980f56 SHA512 35d51de04f606c5fe35bd0dd071c58e435398dc304f0520e307974a2436bc358856580f35f1611a8e45b6e1d16b4b67d050308ac61dfb8ea8c60872c616f47cd EBUILD bees-0.10.ebuild 4076 BLAKE2B dad615a8ad7fada7c1cd31a5cffa6fad2cdcbc1cf241cc6e602d927cd9c182289d944663365cbeaa27a2f14f89cd8150e7c7196c19b5bbd4cb2580dae80bc19c SHA512 53a60088a2e078e1dec4899d77961c51976f941ca3952de32ddacdb6c03b4e6350d977f159b7a845ec45901bb4cb77b19ca9aebeec2ef1cc7867740df74adbbb -EBUILD bees-9999.ebuild 4077 BLAKE2B 8208983c244fd03a1c615b485011bb1fefbdffe91b0d1846aec7a13775c9c1da06dd895092f847a9da0ce2e1a88dac10611f0f2a35df1589042e58a70f11b1a4 SHA512 41bc775e23be08cc87fd06da485b1101e3924dca1ab00150e4abbc50bcfc2b892a08bce0560403e9830a70fac9be0ea669f8017c71a78a414ab5cf1f580644e7 +EBUILD bees-9999.ebuild 4151 BLAKE2B d0b2ddea9523905f3a5620aa6e0da3c595a405afd34a9ab5488a20e7b1f5c77130d9b56b3856a236004982d6069f213ddacf28a9234f3498cc805b4f9a980f56 SHA512 35d51de04f606c5fe35bd0dd071c58e435398dc304f0520e307974a2436bc358856580f35f1611a8e45b6e1d16b4b67d050308ac61dfb8ea8c60872c616f47cd MISC metadata.xml 529 BLAKE2B 69ea069e33d1af7c2c38aa7f8fc61516f23c586ad162d9581859829b084c8da3ef76278173e1ac3afa1ef432355ab14f2ecc01d07f039f8447c20cc27b22d633 SHA512 cb9ba7a32c6a8269f7303ebd41bf0e21f4325560b40bba10aff08a05d07170f9f0e35e687e688f6793c0abf994c9b23f90a47b9d32544636f2d6b5ab51140bd3 diff --git a/sys-fs/bees/bees-0.10-r1.ebuild b/sys-fs/bees/bees-0.10-r1.ebuild new file mode 100644 index 000000000000..e1d5da90f548 --- /dev/null +++ b/sys-fs/bees/bees-0.10-r1.ebuild @@ -0,0 +1,112 @@ +# Copyright 1999-2024 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +inherit linux-info systemd toolchain-funcs + +DESCRIPTION="Best-Effort Extent-Same, a btrfs dedup agent" +HOMEPAGE="https://github.com/Zygo/bees" + +if [[ ${PV} == 9999 ]] ; then + inherit git-r3 + EGIT_REPO_URI="https://github.com/Zygo/bees.git" +else + SRC_URI="https://github.com/Zygo/bees/archive/v${PV}.tar.gz -> ${P}.tar.gz" + KEYWORDS="~amd64 ~arm64" +fi + +LICENSE="GPL-3" +SLOT="0" + +DEPEND=" + >=sys-apps/util-linux-2.30.2 + >=sys-fs/btrfs-progs-4.20.2 +" +RDEPEND="${DEPEND}" + +CONFIG_CHECK="~BTRFS_FS" +ERROR_BTRFS_FS="CONFIG_BTRFS_FS: bees does currently only work with btrfs" + +PATCHES=( + "${FILESDIR}/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch" + "${FILESDIR}/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch" +) + +pkg_pretend() { + if [[ ${MERGE_TYPE} != buildonly ]]; then + if kernel_is -lt 4 11; then + ewarn "With kernel versions below 4.11, bees may severely degrade system performance" + ewarn "and responsiveness. Especially, the kernel may deadlock while bees is" + ewarn "running, it's recommended to run at least kernel 4.11." + ewarn + elif kernel_is -lt 4 14 29; then + ewarn "With kernel versions below 4.14.29, bees may generate a lot of bogus WARN_ON()" + ewarn "messages in the kernel log. These messages can be ignored and this is fixed" + ewarn "with more recent kernels:" + ewarn "# WARNING: CPU: 3 PID: 18172 at fs/btrfs/backref.c:1391 find_parent_nodes+0xc41/0x14e0" + ewarn + elif kernel_is -lt 5 7 0; then + ewarn "With kernel versions below 5.4.96 and 5.7, the kernel may hold file system" + ewarn "locks for a long time while at the same time CPU usage increases when bees is" + ewarn "operating. bees tries to avoid this behavior by excluding very common extents" + ewarn "from deduplication. This has only a minimal impact on dedupe effectiveness." + ewarn + fi + if kernel_is -lt 5 1 0; then + ewarn "IMPORTANT: With kernel versions below 5.1.0, you may experience data corruption" + ewarn "due to bees using compression in btrfs. You are adviced to use a chronologically" + ewarn "later kernel, that includes older LTS versions released after 5.0.4:" + ewarn "Fixed in: 5.1+, 5.0.4+, 4.19.31+, 4.14.108+, 4.9.165+, 4.4.177+, 3.18.137+" + ewarn "# commit 8e92821 btrfs: fix corruption reading shared and compressed extents after hole punching" + ewarn + fi + if kernel_is -lt 5 4 19; then + ewarn "With kernel versions below 5.4.19, bees may trigger a btrfs bug when running" + ewarn "btrfs-balance in parallel. This may lead to meta-data corruption in the worst" + ewarn "case. Especially, kernels 5.1.21 and 5.2.21 should be avoided. Kernels 5.0.x" + ewarn "after 5.0.21 should be safe. In the best case, affected kernels may force" + ewarn "the device RO without writing corrupted meta-data. More details:" + ewarn "https://github.com/Zygo/bees/blob/master/docs/btrfs-kernel.md" + ewarn + fi + if kernel_is -gt 5 15 106; then + if kernel_is -lt 6 3 10; then + ewarn "With kernel versions 5.15.107 or later, there is a memory fragmentation" + ewarn "issue with LOGICAL_INO which can lead to cache thrashing and cause IO" + ewarn "latency spikes. This version ships with a work-around at the cost of not" + ewarn "handling highly duplicated filesystems that well. More details:" + ewarn "https://github.com/Zygo/bees/issues/260" + ewarn + fi + fi + + elog "Bees recommends running the latest current kernel for performance and" + elog "reliability reasons, see README.md." + fi +} + +src_prepare() { + default + sed -i 's/ -Werror//' makeflags || die +} + +src_configure() { + tc-export CC CXX AR + cat >localconf <<-EOF || die + ETC_PREFIX="${EPREFIX}/etc" + LIBEXEC_PREFIX="${EPREFIX}/usr/libexec" + PREFIX="${EPREFIX}/usr" + SYSTEMD_SYSTEM_UNIT_DIR="$(systemd_get_systemunitdir)" + DEFAULT_MAKE_TARGET=all + EOF + if [[ ${PV} != "9999" ]] ; then + echo BEES_VERSION=v${PV} >>localconf || die + fi +} + +src_compile() { + default + # localconf quotes leak in the systemd unit but are still needed for spaces + sed -i 's/"//g' scripts/beesd@.service || die +} diff --git a/sys-fs/bees/bees-9999.ebuild b/sys-fs/bees/bees-9999.ebuild index 1b164eb56b08..e1d5da90f548 100644 --- a/sys-fs/bees/bees-9999.ebuild +++ b/sys-fs/bees/bees-9999.ebuild @@ -1,4 +1,4 @@ -# Copyright 1999-2023 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 EAPI=8 @@ -18,7 +18,6 @@ fi LICENSE="GPL-3" SLOT="0" -IUSE="" DEPEND=" >=sys-apps/util-linux-2.30.2 @@ -30,7 +29,8 @@ CONFIG_CHECK="~BTRFS_FS" ERROR_BTRFS_FS="CONFIG_BTRFS_FS: bees does currently only work with btrfs" PATCHES=( - "${FILESDIR}/0001-HACK-crucible-Work-around-kernel-memory-fragmentatio.patch" + "${FILESDIR}/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch" + "${FILESDIR}/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch" ) pkg_pretend() { diff --git a/sys-fs/bees/files/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch b/sys-fs/bees/files/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch new file mode 100644 index 000000000000..387ae7f1100c --- /dev/null +++ b/sys-fs/bees/files/0001-context-demote-abandoned-toxic-match-to-debug-log-le.patch @@ -0,0 +1,36 @@ +From fad9230cea8a9970980c36b442b71b864e69ded3 Mon Sep 17 00:00:00 2001 +From: Kai Krakow <kai@kaishome.de> +Date: Sun, 30 Jun 2024 16:27:20 +0200 +Subject: [PATCH 1/2] context: demote "abandoned toxic match" to debug log + level + +This log message creates a overwhelmingly lot of messages in the system +journal, leading to write-back flushing storms under high activity. As +it is a work-around message, it is probably only useful to developers, +thus demote to debug level. + +This fixes latency spikes in desktop usage after adding a lot of new +files, especially since systemd-journal starts to flush caches if it +sees memory pressure. + +Signed-off-by: Kai Krakow <kai@kaishome.de> +--- + src/bees-context.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/bees-context.cc b/src/bees-context.cc +index 09ec152..2251783 100644 +--- a/src/bees-context.cc ++++ b/src/bees-context.cc +@@ -471,7 +471,7 @@ BeesContext::scan_one_extent(const BeesFileRange &bfr, const Extent &e) + + // Hash is toxic + if (found_addr.is_toxic()) { +- BEESLOGWARN("WORKAROUND: abandoned toxic match for hash " << hash << " addr " << found_addr << " matching bbd " << bbd); ++ BEESLOGDEBUG("WORKAROUND: abandoned toxic match for hash " << hash << " addr " << found_addr << " matching bbd " << bbd); + // Don't push these back in because we'll never delete them. + // Extents may become non-toxic so give them a chance to expire. + // hash_table->push_front_hash_addr(hash, found_addr); +-- +2.44.2 + diff --git a/sys-fs/bees/files/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch b/sys-fs/bees/files/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch new file mode 100644 index 000000000000..b62957c05291 --- /dev/null +++ b/sys-fs/bees/files/0002-HACK-crucible-Work-around-kernel-memory-fragmentatio_v2.patch @@ -0,0 +1,43 @@ +From 291574986ccfaff8b620f6baa185a567f2f0cfb5 Mon Sep 17 00:00:00 2001 +From: Kai Krakow <kai@kaishome.de> +Date: Sun, 9 Jul 2023 12:24:40 +0200 +Subject: [PATCH 2/2] HACK: crucible: Work around kernel memory fragmentation + +According to @Zygo: + +> This will drastically reduce the number of reflinks bees can handle +> to a single extent, but that size is beyond more than enough for most +> filesystems. Making the buffer smaller may also reduce the size of +> vmallocs which might be aggravating the kernel's memory manager. + +v2: + +According to @Zygo: + +> Lately I've been running with vm.swappiness=0 and +> BEES_MAX_EXTENT_REF_COUNT set to 9999. With those settings there's no +> swapping at all. + +Link: https://github.com/Zygo/bees/issues/260#issuecomment-1627586574 +Link: https://github.com/Zygo/bees/issues/260#issuecomment-2068287231 +Signed-off-by: Kai Krakow <kai@kaishome.de> +--- + src/bees.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/bees.h b/src/bees.h +index 4dbc005..a3c3872 100644 +--- a/src/bees.h ++++ b/src/bees.h +@@ -96,7 +96,7 @@ const double BEES_TOO_LONG = 5.0; + const double BEES_TOXIC_SYS_DURATION = 0.1; + + // Maximum number of refs to a single extent +-const size_t BEES_MAX_EXTENT_REF_COUNT = (16 * 1024 * 1024 / 24) - 1; ++const size_t BEES_MAX_EXTENT_REF_COUNT = (10000) - 1; + + // How long between hash table histograms + const double BEES_HASH_TABLE_ANALYZE_INTERVAL = BEES_STATS_INTERVAL; +-- +2.44.2 + |