diff options
| author | V3n3RiX <venerix@koprulu.sector> | 2024-10-27 03:04:35 +0000 |
|---|---|---|
| committer | V3n3RiX <venerix@koprulu.sector> | 2024-10-27 03:04:35 +0000 |
| commit | 83d9fce566bb89f6976f36f5314bfc88b6d261fa (patch) | |
| tree | 8d8a18f75ff0a6a98313046fcd757ee0135db98c /sci-libs/tokenizers | |
| parent | d0349ca0f4903608beca459a9db2068dab9fa6ea (diff) | |
gentoo auto-resync : 27:10:2024 - 03:04:35
Diffstat (limited to 'sci-libs/tokenizers')
| -rw-r--r-- | sci-libs/tokenizers/Manifest | 2 | ||||
| -rw-r--r-- | sci-libs/tokenizers/tokenizers-0.20.1.ebuild | 35 |
2 files changed, 36 insertions, 1 deletions
diff --git a/sci-libs/tokenizers/Manifest b/sci-libs/tokenizers/Manifest index b729e9d0a38f..f608a0c464f7 100644 --- a/sci-libs/tokenizers/Manifest +++ b/sci-libs/tokenizers/Manifest @@ -250,5 +250,5 @@ DIST windows_x86_64_msvc-0.52.6.crate 832564 BLAKE2B 8e9a3044654c6de99a9153e7cac DIST zerocopy-0.7.35.crate 152645 BLAKE2B 8f13123c9d9257ac5a5c6954d38c3510fa658624442f7e03cdcc6db5a0977d9f26bb4e277be172b7872ec365cf6e58ac742e5578636f7698f9f37093e9249d9a SHA512 17fcb31c029ae89c01e5bae5fb2bb46bd434120199a3dc2c7fe0012dbbcfe2a0bb38934c4a7d3a4920e3fe47c097403beee554fefa54b66cb390f8b1de638d3c DIST zerocopy-derive-0.7.35.crate 37829 BLAKE2B badeb7fa5e0bfe93a6788d93fd297604ed31de526b121549300ead3c49d450f49265f499e2e7ce606dcce2b59dd01f7fa817b8fbb3f237475185c5b42f5299c4 SHA512 dbe23573b62a6267d7bc8c744320b75b2fbda03b908c1d175211f7394374fe182bce58021e25485c10671d726b2007f250565dfe53134d51c89293bb607e9feb DIST zeroize-1.8.1.crate 20029 BLAKE2B 092eba034cd35ec47290020e0c2b213177ff5dbe14ab9e7f0b4ef3cb1ecbc42fbec2b951414e26ab00bc65aaddc2c93eddd5a1963b27c6cd613ac71c65d5cc24 SHA512 dd40ebe98b98fd742608d4066b5ab66caba94b2e679428fcaff9fe547d8cd6ff2360dc85d671ee9183e32fb79cb554d00d6aef9eb8f3d8ad0ec92d0435aa4ebe -EBUILD tokenizers-0.20.1.ebuild 6699 BLAKE2B 4eaec03645f933123600976b892319da75930e2ac128222e2948a35fc7e28300e21345931a467e645eb9e2cb664fe88c4d8a965121914dadedf19994d86acc06 SHA512 42daaa960b98bdcfa9d0fa23d202075306db0d8663bcf165b2d402a47b74911e8c2a7dd2ada1e0c76165b9efe7f8685a5480e5cdeeaa534c56b8eba92d604828 +EBUILD tokenizers-0.20.1.ebuild 9431 BLAKE2B 9790660c6fefa3b7a893e4d84cb2bca561672774ca04df521ca75fdbbac70af166b5a6ffc6b506e11b8d440a6a4bcfbe59bc806311c0cf3c9a03daf4bd5f496f SHA512 29bb9cfa5b2e488be2c59262a91f34d5da7396f693a37324a01b3302e2c517e9b3a7419adc68e3ac5f2a36b0109ea69a1c22e37bbabe7f48e85cd58bc7ac4f79 MISC metadata.xml 335 BLAKE2B 55e4d7b301e4315210bf24a4056741cc923c6a72ae323158c5715fade20db86de8a03464bf8f12fe7116689b41aa0519a0070eebff5f63801e23a9c4a9698a92 SHA512 77f537f495b90ad031842f704c5a629cc995fcd848f11defeb597cc16d2d7bab07d262f05a50b07788d2f58fb60eacbefdfa8328215b89b0c661a05b18c4555d diff --git a/sci-libs/tokenizers/tokenizers-0.20.1.ebuild b/sci-libs/tokenizers/tokenizers-0.20.1.ebuild index 1feae074105a..f2172b1cfd5a 100644 --- a/sci-libs/tokenizers/tokenizers-0.20.1.ebuild +++ b/sci-libs/tokenizers/tokenizers-0.20.1.ebuild @@ -326,6 +326,41 @@ src_test() { local EPYTEST_DESELECT=( "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_formats" "tests/documentation/test_tutorial_train_from_iterators.py::TestTrainFromIterators::test_datasets" + "tests/bindings/test_encoding.py::TestEncoding::test_sequence_ids" + "tests/bindings/test_encoding.py::TestEncoding::test_n_sequences" + "tests/bindings/test_encoding.py::TestEncoding::test_word_to_tokens" + "tests/bindings/test_encoding.py::TestEncoding::test_word_to_chars" + "tests/bindings/test_encoding.py::TestEncoding::test_token_to_sequence" + "tests/bindings/test_encoding.py::TestEncoding::test_token_to_chars" + "tests/bindings/test_encoding.py::TestEncoding::test_token_to_word" + "tests/bindings/test_encoding.py::TestEncoding::test_char_to_token" + "tests/bindings/test_encoding.py::TestEncoding::test_char_to_word" + "tests/bindings/test_encoding.py::TestEncoding::test_truncation" + "tests/bindings/test_encoding.py::TestEncoding::test_invalid_truncate_direction" + "tests/bindings/test_models.py::TestBPE::test_instantiate" + "tests/bindings/test_models.py::TestWordLevel::test_instantiate" + "tests/bindings/test_processors.py::TestByteLevelProcessing::test_processing" + "tests/bindings/test_trainers.py::TestUnigram::test_train" + "tests/documentation/test_pipeline.py::TestPipeline::test_pipeline" + "tests/documentation/test_pipeline.py::TestPipeline::test_bert_example" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_basic_encode" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_lowercase" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_decoding" + "tests/implementations/test_char_bpe.py::TestCharBPETokenizer::test_multiprocessing_with_parallelism" + "tests/test_serialization.py::TestSerialization::test_full_serialization_albert" + "tests/test_serialization.py::TestSerialization::test_str_big" + "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_add_special_tokens" + "tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained" + "tests/bindings/test_tokenizer.py::TestTokenizer::test_from_pretrained_revision" + "tests/bindings/test_tokenizer.py::TestTokenizer::test_encode_special_tokens" + "tests/bindings/test_tokenizer.py::TestTokenizer::test_splitting" + "tests/documentation/test_quicktour.py::TestQuicktour::test_quicktour" + "tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_basic_encode" + "tests/implementations/test_bert_wordpiece.py::TestBertWordPieceTokenizer::test_multiprocessing_with_parallelism" + "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_basic_encode" + "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_add_prefix_space" + "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_lowerspace" + "tests/implementations/test_byte_level_bpe.py::TestByteLevelBPE::test_multiprocessing_with_parallelism" ) local -x EPYTEST_IGNORE=(benches/) distutils-r1_src_test |