
    sOi*                     (   d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZmZ d dlmZmZ d dlmZmZmZmZ d d	lmZmZ d d
lmZmZ d dl m!Z! d dl"m#Z# d dl$m%Z%  G d dee         Z& G d dee         Z' G d dee         Z(y)    N)get_all_start_methods)Path)AnyIterableSequenceType)Image)Encoding	Tokenizer)OnnxProvider
ImageInput)EmbeddingWorker	OnnxModelOnnxOutputContextT)load_tokenizerload_preprocessor)
NumpyArrayDevice)
iter_batch)Compose)ParallelWorkerPoolc                        e Zd ZU dZee   dz  ed<   d) fdZdeee	f   de
deee	f   fdZdeee	f   de
deee	f   fdZeded	   fd
       Zeded   fd       Zdedee   fdZdedee   fdZdej.                  ddfdedededz  dee   dz  deez  dedz  deee
f   dz  ddf fdZd)dZdee   de
dee   fdZ dee   de
defdZ!dddej.                  ddddfdededeee   z  d ed!edz  dee   dz  deez  d"ee   dz  d#ed$edz  deee
f   dz  de
dee   fd%Z"d&ee#   de
defd'Z$dddej.                  ddddfdeded&ee#   e#z  d ed!edz  dee   dz  deez  d"ee   dz  d#ed$edz  deee
f   dz  de
dee   fd(Z% xZ&S )*OnnxMultimodalModelNONNX_OUTPUT_NAMESreturnc                 L    t         |           d | _        d | _        i | _        y N)super__init__	tokenizer	processorspecial_token_to_id)self	__class__s    /volume1/homes/robertsu/coba/app/.venv/lib/python3.12/site-packages/fastembed/late_interaction_multimodal/onnx_multimodal_model.pyr    zOnnxMultimodalModel.__init__   s$    +/)-35     
onnx_inputkwargsc                     |S z,
        Preprocess the onnx input.
         r$   r(   r)   s      r&   _preprocess_onnx_text_inputz/OnnxMultimodalModel._preprocess_onnx_text_input   
     r'   c                     |S r+   r,   r-   s      r&   _preprocess_onnx_image_inputz0OnnxMultimodalModel._preprocess_onnx_image_input%   r/   r'   zTextEmbeddingWorker[T]c                     t        d      Nz%Subclasses must implement this methodNotImplementedErrorclss    r&   _get_text_worker_classz*OnnxMultimodalModel._get_text_worker_class-       !"IJJr'   zImageEmbeddingWorker[T]c                     t        d      r3   r4   r6   s    r&   _get_image_worker_classz+OnnxMultimodalModel._get_image_worker_class1   r9   r'   outputc                     t        d      r3   r4   r$   r<   s     r&   _post_process_onnx_image_outputz3OnnxMultimodalModel._post_process_onnx_image_output5       !"IJJr'   c                     t        d      r3   r4   r>   s     r&   _post_process_onnx_text_outputz2OnnxMultimodalModel._post_process_onnx_text_output8   r@   r'   	model_dir
model_filethreads	providerscuda	device_idextra_session_optionsc           	          t         |   |||||||       t        |      \  | _        | _        | j                  J t        |      | _        y )N)rC   rD   rE   rF   rG   rH   rI   )rC   )r   _load_onnx_modelr   r!   r#   r   r"   )	r$   rC   rD   rE   rF   rG   rH   rI   r%   s	           r&   rK   z$OnnxMultimodalModel._load_onnx_model;   s_     	 !"7 	! 	
 4BI3V00~~)))*Y?r'   c                     t        d      r3   r4   )r$   s    r&   load_onnx_modelz#OnnxMultimodalModel.load_onnx_modelR   r@   r'   	documentsc                 8    | j                   j                  |      S r   )r!   encode_batch)r$   rN   r)   s      r&   tokenizezOnnxMultimodalModel.tokenizeU   s    ~~**955r'   c           
          | j                   |fi |}t        j                  |D cg c]  }|j                   c}      }t        j                  |D cg c]  }|j                   c}      }| j
                  j                         D ch c]  }|j                   }}dt        j                  |t        j                        i}	d|v r(t        j                  |t        j                        |	d<   d|v rbt        j                  |D cg c]0  }t        j                  t        |      t        j                        2 c}t        j                        |	d<    | j                  |	fi |}	| j
                  j                  | j                  |	      }
t        |
d   |	j                  d|      |	j                  d|            S c c}w c c}w c c}w c c}w )N	input_ids)dtypeattention_masktoken_type_idsr   )model_outputrU   rS   )rQ   nparrayidsrU   model
get_inputsnameint64zeroslenr.   runr   r   get)r$   rN   r)   encodederS   rU   nodeinput_namesr(   rW   s              r&   onnx_embed_textz#OnnxMultimodalModel.onnx_embed_textX   s   
  $--	4V4HHW5WaeeW56	W"EW1#3#3W"EF-1ZZ-B-B-DE-DTtyy-DE)288<-

 {*+-88N"((+SJ'({*+-88;DE9a#a&19ERXX,J'( 6T55jKFK
zz~~d&<&<jI %a%>>*:NK nn[)<
 	
 6"EE Fs   GG
G	5G   F
model_name	cache_dir
batch_sizeparallel
device_idslocal_files_onlyspecific_model_pathc              +     K   d}t        |t              r|g}d}t        |t              rt        |      |k  rd}||rbt	        | d      r| j
                  | j                          t        ||      D ]*  }| j                  | j                  |            E d {    , y |dk(  rt        j                         }dt               v rdnd}||||	|
d|}||j                  |       t        |xs d| j                         |||	      } |j                   t        ||      fi |D ]  }| j                  |      E d {     y 7 7 	w
NFTr[   r   
forkserverspawn)ri   rj   rF   rn   ro      )num_workersworkerrG   rm   start_method)
isinstancestrlistr`   hasattrr[   rM   r   rB   rg   os	cpu_countr   updater   r8   ordered_map)r$   ri   rj   rN   rk   rl   rF   rG   rm   rn   ro   rI   r)   is_smallbatchrw   paramspools                     r&   _embed_documentsz$OnnxMultimodalModel._embed_documentss   s_     i%"IHi&9~
*x4)TZZ-?$$&#Iz:>>t?S?STY?Z[[[ ; 1}<<>+7;P;R+R<X_L(&&$4': F %034%$M224%)D *))*Y
*KVvV>>uEEE W3 \4 Fs%   BEEB#E<E=EEimagesc           	          t        j                         5 }|D cg c]B  }t        |t        j                        s$|j	                  t        j
                  |            n|D }}| j                  J d       t        j                  | j                  |            }d d d        di} | j                  |fi |}| j                  j                  d |      }|d   j                  t        |      d      }	t        |	      S c c}w # 1 sw Y   kxY w)NzProcessor is not initializedpixel_valuesr   )rW   )
contextlib	ExitStackrx   r	   enter_contextopenr"   rX   rY   r1   r[   ra   reshaper`   r   )
r$   r   r)   stackimageimage_filesrc   r(   rW   
embeddingss
             r&   onnx_embed_imagez$OnnxMultimodalModel.onnx_embed_image   s    !!#u
 $	 $E "%5 ##EJJu$56 $	   >>-M/MM-hht~~k:;G $ %g.
6T66zLVL
zz~~dJ7!!_,,S["=
 j99 $#s   DAC?!9D?DDc              +     K   d}t        |t        t        t        j                  f      r|g}d}t        |t              rt        |      |k  rd}||rbt        | d      r| j                  | j                          t        ||      D ]*  }| j                  | j                  |            E d {    , y |dk(  rt        j                         }dt               v rdnd}||||	|
d|}||j                  |       t!        |xs d| j#                         |||	      } |j$                  t        ||      fi |D ]  }| j                  |      E d {     y 7 7 	wrq   )rx   ry   r   r	   rz   r`   r{   r[   rM   r   r?   r   r|   r}   r   r~   r   r;   r   )r$   ri   rj   r   rk   rl   rF   rG   rm   rn   ro   rI   r)   r   r   rw   r   r   s                     r&   _embed_imagesz!OnnxMultimodalModel._embed_images   si     fsD%++67XFHfd#Fj(@Hx4)TZZ-?$$&#FJ7??@U@UV[@\]]] 8 1}<<>+7;P;R+R<X_L(&&$4': F %034%$M335%)D *))*VZ*HSFS??FFF T3 ^4 Gs%   B+E-E.B#EEEE)r   N)'__name__
__module____qualname__r   rz   ry   __annotations__r    dictr   r   r.   r1   classmethodr   r8   r;   r   r   r   r?   rB   r   AUTOr   intr   r   boolrK   rM   r
   rQ   rg   r   r   r   r   __classcell__r%   s   @r&   r   r      s   *.tCy4'.6sJ/;>	c:o	sJ/;>	c:o	 Kt,D'E K K K-F(G K KK6G KHUVK KK5F K8TU; K 48$kk $7;@@ @ t	@
 L)D0@ Vm@ :@  $CH~4@ 
@.K6$s) 6s 6tH~ 6
9
 
 
	
@ #37$kk'+!&*.7;7F7F 7F #&	7F
 7F *7F L)D07F Vm7F I$7F 7F !4Z7F  $CH~47F 7F 
!7Fr:tJ'7 :3 :K\ :* #37$kk'+!&*.7;7G7G 7G $z1	7G
 7G *7G L)D07G Vm7G I$7G 7G !4Z7G  $CH~47G 7G 
!7Gr'   r   c                   t     e Zd Zdededef fdZdedededefdZdee	e
ef      dee	e
ef      fdZ xZS )	TextEmbeddingWorkerri   rj   r)   c                 ,    |  t        |   ||fi | y r   r   r    r$   ri   rj   r)   r%   s       r&   r    zTextEmbeddingWorker.__init__        	Y9&9r'   r   c                     t               r   r4   r$   ri   rj   r)   s       r&   init_embeddingz"TextEmbeddingWorker.init_embedding        "##r'   itemsc              #   b   K   |D ]&  \  }}| j                   j                  |      }||f ( y wr   )r[   rg   )r$   r   idxr   onnx_outputs        r&   processzTextEmbeddingWorker.process  s3     JC**44U;K{""     -/r   r   r   ry   r   r    r   r   r   tupler   r   r   r   s   @r&   r   r      s|    :: : 	:$$ $ 	$
 
$#XeCHo6 #8E#s(O;T #r'   r   c                   t     e Zd Zdededef fdZdedededefdZdee	e
ef      dee	e
ef      fdZ xZS )	ImageEmbeddingWorkerri   rj   r)   c                 ,    |  t        |   ||fi | y r   r   r   s       r&   r    zImageEmbeddingWorker.__init__  r   r'   r   c                     t               r   r4   r   s       r&   r   z#ImageEmbeddingWorker.init_embedding  r   r'   r   c              #   b   K   |D ]&  \  }}| j                   j                  |      }||f ( y wr   )r[   r   )r$   r   r   r   r   s        r&   r   zImageEmbeddingWorker.process   s2     JC44U;Jz/!  r   r   r   s   @r&   r   r     s|    :: : 	:$$ $ 	$
 
$"XeCHo6 "8E#s(O;T "r'   r   ))r   r|   multiprocessingr   pathlibr   typingr   r   r   r   numpyrX   PILr	   
tokenizersr
   r   fastembed.commonr   r   fastembed.common.onnx_modelr   r   r   r   #fastembed.common.preprocessor_utilsr   r   fastembed.common.typesr   r   fastembed.common.utilsr   #fastembed.image.transform.operatorsr   fastembed.parallel_processorr   r   r   r   r,   r'   r&   <module>r      so     	 1  0 0   * 5 X X Q 5 - 7 ;_G)A, _GD#/!, #0"?1- "r'   