5 files changed, 119 insertions, 0 deletions
diff --git a/sci-libs/Manifest.gz b/sci-libs/Manifest.gz
index fa774d0359ff..0cbbcc3c614c 100644
--- a/sci-libs/Manifest.gz
+++ b/sci-libs/Manifest.gz
diff --git a/sci-libs/evaluate/Manifest b/sci-libs/evaluate/Manifest
new file mode 100644
index 000000000000..fc58f197984e
--- /dev/null
+++ b/sci-libs/evaluate/Manifest
@@ -0,0 +1,4 @@
+AUX evaluate-0.4.0-tests.patch 2471 BLAKE2B 9744c3f24f1e95a88f9a06d065234ba37b094723261884681c021888c794c9a8c1d6253390105379c898f59d8f18b947db9a3a1229a85fdf3f341909e5eb4eb3 SHA512 03a4b335f856da64423bce7fd10a21558d4ced499794868ac5c34f68885f39627ae75bd01634370bd7c175be330e7abb8adb1317d0e5a3adb938f140c9216387
+DIST evaluate-0.4.0.gh.tar.gz 292250 BLAKE2B f88428b263820c1af43d02ae676625257251476092efe624490f29e63a045d698db01e4a7a802c2330027d01bc6ccf16986f28ecf8202ecbfd943c5d7c40f6ec SHA512 f2136196fc4e5717859e36e173cd49d049fc5ef50c89f466e13edd0142830574dec0b5485a4a1097eec9cb9df756a617216ff48c141db008cb0c2b85288d7fc9
+EBUILD evaluate-0.4.0.ebuild 1189 BLAKE2B 75bf8bb8f0c8bd9e2994b196f85315d6d2677d70ee07ac076aca379a725d30d7920f476994b299c71fe033196460a5c6a642128d7c0c35d9bf5d7407b33d0b8f SHA512 6a3ed06a927967a4e6395f0493776953aad00e12a7732afab7578edc9b4eaf6a43fa6c877e5f0bec88a545e74373ae8388d3f7b17f53a5c9fee1ef3f511f08b7
+MISC metadata.xml 379 BLAKE2B a717b46962e59358925c866c64b2d0bc1dcd5d55d73e814686a09f703e339d2c0de52f6c214c8f795518d6d9dbb46443be11374643d415ff681dedca1511732b SHA512 03d6b58cad73cad46f1101fedf88ec94dc6d7a2028399a20b39939bead4fa402d00224085206a175a33d92417176cc45853060b18faa13769b80527fac9254e1
diff --git a/sci-libs/evaluate/evaluate-0.4.0.ebuild b/sci-libs/evaluate/evaluate-0.4.0.ebuild
new file mode 100644
index 000000000000..60382685f160
--- /dev/null
+++ b/sci-libs/evaluate/evaluate-0.4.0.ebuild
@@ -0,0 +1,43 @@
+# Copyright 2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+DISTUTILS_USE_PEP517=setuptools
+PYTHON_COMPAT=( python3_11 )
+inherit distutils-r1
+
+DESCRIPTION="makes evaluating, comparing models and reporting their performance easier"
+HOMEPAGE="
+	https://pypi.org/project/evaluate/
+	https://github.com/huggingface/evaluate
+"
+SRC_URI="https://github.com/huggingface/${PN}/archive/refs/tags/v${PV}.tar.gz
+	-> ${P}.gh.tar.gz"
+
+LICENSE="Apache-2.0"
+SLOT="0"
+KEYWORDS="~amd64"
+
+RDEPEND="
+	dev-python/pyarrow[${PYTHON_USEDEP},parquet]
+	dev-python/unidecode[${PYTHON_USEDEP}]
+"
+BDEPEND="test? (
+	sci-libs/jiwer[${PYTHON_USEDEP}]
+	sci-libs/seqeval[${PYTHON_USEDEP}]
+)"
+
+PATCHES=( "${FILESDIR}"/${P}-tests.patch )
+
+distutils_enable_tests pytest
+
+src_prepare() {
+	# These require packages not available on gentoo
+	rm -r metrics/{bertscore,bleurt,character,charcut_mt,chrf,code_eval} || die
+	rm -r metrics/{competition_math,coval,google_bleu,mauve,meteor} || die
+	rm -r metrics/{nist_mt,rl_reliability,rouge,sacrebleu,sari} || die
+	rm -r metrics/{ter,trec_eval,wiki_split,xtreme_s} || die
+	rm -r measurements/word_length || die
+	distutils-r1_src_prepare
+}
diff --git a/sci-libs/evaluate/files/evaluate-0.4.0-tests.patch b/sci-libs/evaluate/files/evaluate-0.4.0-tests.patch
new file mode 100644
index 000000000000..1e7e808576e3
--- /dev/null
+++ b/sci-libs/evaluate/files/evaluate-0.4.0-tests.patch
@@ -0,0 +1,60 @@
+--- a/tests/test_evaluator.py	2023-05-14 11:01:54.449768849 +0200
++++ b/tests/test_evaluator.py	2023-05-14 11:06:15.182738125 +0200
+@@ -16,6 +16,7 @@
+ 
+ from time import sleep
+ from unittest import TestCase, mock
++from unittest import skip
+ 
+ from datasets import ClassLabel, Dataset, Features, Sequence, Value
+ from PIL import Image
+@@ -335,6 +335,7 @@
+         )
+         self.assertEqual(results["accuracy"], 1.0)
+ 
++    @skip("not working")
+     def test_bootstrap(self):
+         data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]})
+ 
+@@ -368,6 +369,7 @@
+         self.assertAlmostEqual(results["samples_per_second"], len(self.data) / results["total_time_in_seconds"], 5)
+         self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(self.data), 5)
+ 
++    @skip("not working")
+     def test_bootstrap_and_perf(self):
+         data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]})
+ 
+@@ -877,6 +877,7 @@
+         results = self.evaluator.compute(data=self.data)
+         self.assertIsInstance(results["unique_words"], int)
+ 
++    @skip("require nltk")
+     def test_overwrite_default_metric(self):
+         word_length = load("word_length")
+         results = self.evaluator.compute(
+@@ -939,6 +940,7 @@
+         results = self.evaluator.compute(data=self.data)
+         self.assertEqual(results["bleu"], 0)
+ 
++    @skip("require rouge_score")
+     def test_overwrite_default_metric(self):
+         rouge = load("rouge")
+         results = self.evaluator.compute(
+@@ -949,6 +952,7 @@
+         )
+         self.assertEqual(results["rouge1"], 1.0)
+ 
++    @skip("require rouge_score")
+     def test_summarization(self):
+         pipe = DummyText2TextGenerationPipeline(task="summarization", prefix="summary")
+         e = evaluator("summarization")
+--- a/tests/test_trainer_evaluator_parity.py	2023-05-14 17:50:29.224525549 +0200
++++ b/tests/test_trainer_evaluator_parity.py	2023-05-14 17:37:40.947501195 +0200
+@@ -269,6 +269,7 @@
+         self.assertEqual(transformers_results["eval_HasAns_f1"], evaluator_results["HasAns_f1"])
+         self.assertEqual(transformers_results["eval_NoAns_f1"], evaluator_results["NoAns_f1"])
+ 
++    @unittest.skip('require eval_results.json')
+     def test_token_classification_parity(self):
+         model_name = "hf-internal-testing/tiny-bert-for-token-classification"
+         n_samples = 500
diff --git a/sci-libs/evaluate/metadata.xml b/sci-libs/evaluate/metadata.xml
new file mode 100644
index 000000000000..f1e8571190f9
--- /dev/null
+++ b/sci-libs/evaluate/metadata.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "https://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+	<maintainer type="person">
+		<email>tupone@gentoo.org</email>
+		<name>Tupone Alfredo</name>
+	</maintainer>
+	<upstream>
+		<remote-id type="pypi">evaluate</remote-id>
+		<remote-id type="github">huggingface/evaluate</remote-id>
+	</upstream>
+</pkgmetadata>