sOi$ddlZddlmZddlZddlZddlmZddlm Z m Z ddl m Z dZ dZeGdd ZGd d Zy) N) dataclass)SnowballStemmer)get_all_punctuationremove_non_alphanumeric)SparseEmbeddingi}icLeZdZUeed<eeed<eed<eed<eeed<y) WordEmbeddingwordformscountword_id embeddingN)__name__ __module__ __qualname__str__annotations__listintfloatv/volume1/homes/robertsu/coba/app/.venv/lib/python3.12/site-packages/fastembed/sparse/utils/sparse_vectors_converter.pyr r s% I 9 J LE{rr c eZdZ ddeededededef dZedede d e fd Z d e d e d efd Z ede ed e efdZ ddeeefde d eeeffdZdeeefde de d efdZdeeefde de d efdZy)SparseVectorConverter stopwordsstemmerkbavg_lenctt}hd}||_||z|z|_||_||_||_y)N>[CLS][PAD][SEP][UNK][MASK])setrrunwanted_tokensrrr )selfrrrrr punctuationspecial_tokenss r__init__zSparseVectorConverter.__init__sE-/0 G *^;iG rr shiftreturncdttj|}t|z }|||zz}|S)N)absmmh3hash INT32_MAX)clsr r- token_hash range_size remapped_hashs runkn_word_token_idz(SparseVectorConverter.unkn_word_token_id+s54) & j!89 rnum_occurrences sentence_lenc||jdzz}|||jd|jz |j|z|jz zzzz}|S)N)rrr )r)r9r:ress rbm25_tfzSparseVectorConverter.bm25_tf6sS!, 1tvv:8MPTP\P\8\+\!]]] rvectorct|Dcgc]}|dz c}dz}|dkr|S|Dcgc]}||z  c}Scc}wcc}w)Ng?g:0yE>)sum)r4r?xnorms rnormalize_vectorz&SparseVectorConverter.normalize_vector;sS&)&QAqD&)*c1 $;M"()&QD&))**s : ?sentence_embeddingtoken_max_lengthcHi}|jD] \}}|jdkDr|||<||jvr+t|j }t |dkDsS|j D]}|jj|}t ||ks-||jvs<||vr#tj|||<|||_ c||xj|jz c_ ||xj|jz c_ |S)u Clean miniCOIL-produced sentence_embedding, as unknown to the miniCOIL's stemmer tokens should fully resemble our BM25 token representation. sentence_embedding = {"9°": {"word": "9°", "word_id": -1, "count": 2, "embedding": [1], "forms": ["9°"]}, "9": {"word": "9", "word_id": -1, "count": 2, "embedding": [1], "forms": ["9"]}, "bat": {"word": "bat", "word_id": 2, "count": 3, "embedding": [0.2, 0.1, -0.2, -0.2], "forms": ["bats", "bat"]}, "9°9": {"word": "9°9", "word_id": -1, "count": 1, "embedding": [1], "forms": ["9°9"]}, "screech": {"word": "screech", "word_id": -1, "count": 1, "embedding": [1], "forms": ["screech"]}, "screeched": {"word": "screeched", "word_id": -1, "count": 1, "embedding": [1], "forms": ["screeched"]} } cleaned_embedding_ground_truth = { "9": {"word": "9", "word_id": -1, "count": 6, "embedding": [1], "forms": ["9°", "9", "9°9", "9°9"]}, "bat": {"word": "bat", "word_id": 2, "count": 3, "embedding": [0.2, 0.1, -0.2, -0.2], "forms": ["bats", "bat"]}, "screech": {"word": "screech", "word_id": -1, "count": 2, "embedding": [1], "forms": ["screech", "screeched"]} } r)itemsr r(rstriplensplitr stem_wordcopydeepcopyr r r ) r)rFrGnew_sentence_embeddingr r word_cleanedsubwordstemmed_subwords r clean_wordsz!SparseVectorConverter.clean_wordsBs%*<>1779OD)  1$/8&t,4/// 7t<BBD |$q(#/#5#5#7/3||/E/Eg/N04DD /t7K7K K.6LLJN--XaJb 6 GO^ 6 G L 6 G M MQZQ`Q` ` M 6 G M MQZQ`Q` ` M$8/ :H&%rembedding_size vocab_sizecg}g}||ztzdztz}|j|}d}|jD]} || jz }|jD]} | j} | j} |j | |} | dkDr\| j } |j| }t|D]0\}}|j| |z|z|j|| z2|j|j| j||j| ttj|tjtj|tj S)a Convert miniCOIL sentence embedding to Qdrant sparse vector Example input: ``` { "vector": WordEmbedding({ // Vocabulary word, encoded with miniCOIL normally "word": "vector", "forms": ["vector", "vectors"], "count": 2, "word_id": 1231, "embedding": [0.1, 0.2, 0.3, 0.4] }), "axiotic": WordEmbedding({ // Out-of-vocabulary word, fallback to BM25 "word": "axiotic", "forms": ["axiotics"], "count": 1, "word_id": -1, }) } ``` rArdtypeindicesvalues)GAPrTr\r r r>rrE enumerateappendr8r rnparrayint32float32)r)rFrUrVr[r\unknown_words_shiftsentence_embedding_cleanedr:rr r9tfembedding_valuesnormalized_embeddingval_idvalues rembedding_to_vectorz)SparseVectorConverter.embedding_to_vectorsc>  !+^ ;CaG3N%)%5%56H%I" 3::rrEdictr rTrrkrmrrrrrs_  s8!      $" s#% *d5k*d5k**UW;&"&sM'9":;&NQ;& c= !;&zM  m!34M M  M  M ^&  m!34& &  &  & rr)rN dataclassesrr1numpyr`py_rust_stemmersrfastembed.common.utilsrr&fastembed.sparse.sparse_embedding_baserr]r3r rrrrrvsI ! ,OB     \ \ r