diff options
Diffstat (limited to 'sci-libs/tokenizers')
| -rw-r--r-- | sci-libs/tokenizers/Manifest | 2 | ||||
| -rw-r--r-- | sci-libs/tokenizers/tokenizers-0.20.1.ebuild | 23 |
2 files changed, 14 insertions, 11 deletions
diff --git a/sci-libs/tokenizers/Manifest b/sci-libs/tokenizers/Manifest index f608a0c464f7..2d38f56d3ef9 100644 --- a/sci-libs/tokenizers/Manifest +++ b/sci-libs/tokenizers/Manifest @@ -250,5 +250,5 @@ DIST windows_x86_64_msvc-0.52.6.crate 832564 BLAKE2B 8e9a3044654c6de99a9153e7cac DIST zerocopy-0.7.35.crate 152645 BLAKE2B 8f13123c9d9257ac5a5c6954d38c3510fa658624442f7e03cdcc6db5a0977d9f26bb4e277be172b7872ec365cf6e58ac742e5578636f7698f9f37093e9249d9a SHA512 17fcb31c029ae89c01e5bae5fb2bb46bd434120199a3dc2c7fe0012dbbcfe2a0bb38934c4a7d3a4920e3fe47c097403beee554fefa54b66cb390f8b1de638d3c DIST zerocopy-derive-0.7.35.crate 37829 BLAKE2B badeb7fa5e0bfe93a6788d93fd297604ed31de526b121549300ead3c49d450f49265f499e2e7ce606dcce2b59dd01f7fa817b8fbb3f237475185c5b42f5299c4 SHA512 dbe23573b62a6267d7bc8c744320b75b2fbda03b908c1d175211f7394374fe182bce58021e25485c10671d726b2007f250565dfe53134d51c89293bb607e9feb DIST zeroize-1.8.1.crate 20029 BLAKE2B 092eba034cd35ec47290020e0c2b213177ff5dbe14ab9e7f0b4ef3cb1ecbc42fbec2b951414e26ab00bc65aaddc2c93eddd5a1963b27c6cd613ac71c65d5cc24 SHA512 dd40ebe98b98fd742608d4066b5ab66caba94b2e679428fcaff9fe547d8cd6ff2360dc85d671ee9183e32fb79cb554d00d6aef9eb8f3d8ad0ec92d0435aa4ebe -EBUILD tokenizers-0.20.1.ebuild 9431 BLAKE2B 9790660c6fefa3b7a893e4d84cb2bca561672774ca04df521ca75fdbbac70af166b5a6ffc6b506e11b8d440a6a4bcfbe59bc806311c0cf3c9a03daf4bd5f496f SHA512 29bb9cfa5b2e488be2c59262a91f34d5da7396f693a37324a01b3302e2c517e9b3a7419adc68e3ac5f2a36b0109ea69a1c22e37bbabe7f48e85cd58bc7ac4f79 +EBUILD tokenizers-0.20.1.ebuild 9686 BLAKE2B fefa9a1e8dd8206808832741fab110aad7a5e04d9175251b99728134a34beb2de0c1ed808c3e1f9d3e18141908bdc27c5dca0811f9937488888c9baa65ecc94e SHA512 df45b722d376a3688de25c4c6e19aa409a5a43fa68da3835d9b803eeaefcb170ab47ad5fa599b182b69371381915502ffbbddadafd5c211c7d70cf7d5dd24c89 MISC metadata.xml 335 BLAKE2B 55e4d7b301e4315210bf24a4056741cc923c6a72ae323158c5715fade20db86de8a03464bf8f12fe7116689b41aa0519a0070eebff5f63801e23a9c4a9698a92 SHA512 77f537f495b90ad031842f704c5a629cc995fcd848f11defeb597cc16d2d7bab07d262f05a50b07788d2f58fb60eacbefdfa8328215b89b0c661a05b18c4555d diff --git a/sci-libs/tokenizers/tokenizers-0.20.1.ebuild b/sci-libs/tokenizers/tokenizers-0.20.1.ebuild index f2172b1cfd5a..f5e839834cd8 100644 --- a/sci-libs/tokenizers/tokenizers-0.20.1.ebuild +++ b/sci-libs/tokenizers/tokenizers-0.20.1.ebuild @@ -325,7 +325,6 @@ src_test() { cd ../bindings/python local EPYTEST_DESELECT=( "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_formats" - "tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets" "tests/bindings/test_encoding.py::TestEncoding::test_sequence_ids" "tests/bindings/test_encoding.py::TestEncoding::test_n_sequences" "tests/bindings/test_encoding.py::TestEncoding::test_word_to_tokens" @@ -339,28 +338,32 @@ src_test() { "tests/bindings/test_encoding.py::TestEncoding::test_invalid_truncate_direction" "tests/bindings/test_models.py::TestBPE::test_instantiate" "tests/bindings/test_models.py::TestWordLevel::test_instantiate" + "tests/bindings/test_models.py::TestWordPiece::test_instantiate" "tests/bindings/test_processors.py::TestByteLevelProcessing::test_processing" - "tests/bindings/test_trainers.py::TestUnigram::test_train" - "tests/documentation/test_pipeline.py::TestPipeline::test_pipeline" - "tests/documentation/test_pipeline.py::TestPipeline::test_bert_example" - "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode" - "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase" - "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding" - "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism" - "tests/test_serialization.py::TestSerialization::test_full_serialization_albert" - "tests/test_serialization.py::TestSerialization::test_str_big" "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_add_special_tokens" "tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained" "tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained_revision" "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_special_tokens" "tests/bindings/test_tokenizer.py::TestTokenizer::test_splitting" + "tests/bindings/test_trainers.py::TestUnigram::test_continuing_prefix_trainer_mistmatch" + "tests/bindings/test_trainers.py::TestUnigram::test_train" + "tests/documentation/test_pipeline.py::TestPipeline::test_pipeline" + "tests/documentation/test_pipeline.py::TestPipeline::test_bert_example" "tests/documentation/test_quicktour.py::TestQuicktour::test_quicktour" + "tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets" + "tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_gzip" "tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_basic_encode" "tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_multiprocessing_with_parallelism" "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_basic_encode" "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_add_prefix_space" "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_lowerspace" "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_multiprocessing_with_parallelism" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism" + "tests/test_serialization.py::TestSerialization::test_full_serialization_albert" + "tests/test_serialization.py::TestSerialization::test_str_big" ) local -x EPYTEST_IGNORE=(benches/) distutils-r1_src_test |