+
    Ԙj                       ^ RI Ht ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIt^ RI	t	^ RI
Ht ]! ]4      P                  4       P                  P                  t]P                   P#                  ^ ]! ]4      4       ]! ]R,          4       ^ RIHt ^ RIHtHtHt ^ RIHt ^ R	IHt ]P:                  ! R
4      t^tR R lt R R lt!R R lt"R R lt#R R lt$]%R8X  d
   ]$! 4        R# R# )    )annotationsN)Path)load_dotenvz.env)get_settings)
close_poolget_connection	init_pool)embed_texts)init_vertexingestc                    V ^8  d   QhRRRR/# )   pdf_pathr   returnstr )formats   "scripts/ingest.py__annotate__r      s      t      c           	        . p\         P                  ! V 4      ;_uu_ 4       p\        VP                  ^R7       F~  w  r4VP	                  4       ;'       g    Rp\
        P                  ! RRV4      p\
        P                  ! RRV4      P                  4       pV'       g   Kg  VP                  RV RV 24       K  	  R	R	R	4       V'       g   \        R
V  24      hRP                  V4      #   + '       g   i     L7; i)   start z[ \t]+ z\n{3,}z

z[Page z]
NzNo extractable text in )
pdfplumberopen	enumeratepagesextract_textresubstripappendRuntimeErrorjoin)r   r    pdfipagerawcleaneds   &      r   extract_pdf_textr-      s    E		"	"c !4GA##%++CffYS1GffY8>>@GwvaSG956 5 
# 4XJ?@@;;u 
#	"s   3C*AC* C**C:	c               (    V ^8  d   QhRRRRRRRR/# )r   textr   chunk_tokensintoverlapr   	list[str]r   )r   s   "r   r   r   )   s(      S  c i r   c                l   \         P                  ! R 4      pVP                  V 4      pW8:  d   \        R4      h. p^ p\	        V4      pWg8  dg   \        Wa,           V4      pVP                  WFV 4      P                  4       p	V	'       d   VP                  V	4       W8X  d    V# WaV,
          ,          pKl  V# )cl100k_basez0CHUNK_TOKENS must be greater than CHUNK_OVERLAP.)	tiktokenget_encodingencode
ValueErrorlenmindecoder$   r%   )
r/   r0   r2   enc	token_idschunksr   nendpieces
   &&&       r   
chunk_textrC   )   s    



.C

4 IKLLFEIA
)%&*

93/0668MM% 8M 	''Mr   c               (    V ^8  d   QhRRRRRRRR/# )	r   source_filer   r?   r3   
embeddingszlist[list[float]]r   Noner   )r   s   "r   r   r   =   s*       Y DU Z^ r   c                ~   \        4       ;_uu_ 4       pVP                  4       ;_uu_ 4       pVP                  R V 34       \        \	        WRR7      4       F  w  pw  rgVP                  RWgW34       K  	  RRR4       VP                  4        RRR4       R#   + '       g   i     L*; i  + '       g   i     R# ; i)z2DELETE FROM document_chunks WHERE source_file = %sT)strictz
                    INSERT INTO document_chunks (text_chunk, embedding, source_file, chunk_index)
                    VALUES (%s, %s, %s, %s)
                    N)r   cursorexecuter   zipcommit)rE   r?   rF   conncurindexr/   vectors   &&&     r   replace_chunksrR   =   s    			T[[]]cKKD *33vRV3W)X%~ ;6 *Y  	 
	] 
		s#   B+A
B	6B+B(#B++B<	c               $    V ^8  d   QhRRRRRR/# )r   itemsr3   sizer1   r   zlist[list[str]]r   )r   s   "r   r   r   O   s&     E E9 EC EO Er   c                h    \        ^ \        V 4      V4       Uu. uF  q W"V,            NK  	  up# u upi )r   )ranger:   )rT   rU   r)   s   && r   batchedrX   O   s0    ).q#e*d)CD)CA!$h)CDDDs   /c                   V ^8  d   QhRR/# )r   r   rG   r   )r   s   "r   r   r   S   s        d  r   c                 ,   \         P                  ! \         P                  R R7       \        P                  ! RR7      p V P                  RRRR7       V P                  4       p\        4       p\        VP                  ;'       g    VP                  4      P                  4       P                  4       pVP                  4       '       g   \        RV R	24      h\        P!                  R
V4       \#        V4      p\%        WBP&                  VP(                  4      p\        P!                  R\+        V4      VP&                  VP(                  4       \-        4        \/        4         . p\1        \3        V\4        4      ^R7       FA  w  rx\        P!                  RV\+        V4      4       VP7                  \9        VRR7      4       KC  	  VP:                  p	\=        WV4       \        P!                  R\+        V4      V	4       \?        4        R#   \?        4        i ; i)z%(levelname)s %(message)s)levelr   z%Ingest a local PDF into PartnerLogic.)descriptionz--pdfNzPath to the source PDF.)defaulthelpzPDF not found: ze
Place the document at data/shareholder_agreement.pdf or run:
  python scripts/generate_sample_pdf.pyzExtracting %sz2Split into %s chunks (%s-token window, %s overlap)r   zEmbedding batch %s (%s chunks)RETRIEVAL_DOCUMENT)	task_typez3Wrote %s chunks to document_chunks (source_file=%s)) loggingbasicConfigINFOargparseArgumentParseradd_argument
parse_argsr   r   r(   r   
expanduserresolveexists
SystemExitloginfor-   rC   r0   chunk_overlapr:   r   r	   r   rX   EMBED_BATCHextendr
   namerR   r   )
parserargssettingsr   r/   r?   rF   batch_nobatchrE   s
             r   mainrw   S   s   gll3NO$$1XYF
4MND~HDHH11 1 12==?GGIH??hZ (6 6
 	
 HH_h'H%D33X5K5KLFHHA3v;PXPePegog}g}~MK
(*
()EQOOHHH5xULk%;OPQ  P mm{J7FFU`a
s   !BH H__main__)&
__future__r   rd   ra   r"   syspathlibr   r   r6   dotenvr   __file__ri   parentROOTpathinsertr   
app.configr   app.dbr   r   r	   app.embeddingsr
   
app.vertexr   	getLoggerrl   ro   r-   rC   rR   rX   rw   __name__r   r   r   <module>r      s    "   	 
    H~&&-- 3t9  D6M  # 8 8 & "!($E F zF r   