o
    *2iӋ                     @   sZ  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlZd dlmZm	Z	 d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z* d dl+m,Z, d dlm-Z-m.Z.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4 d dl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z< d dl=m>Z>m?Z? d dl@mAZA d dlBmCZC d dlDmEZE d dlFmGZGmHZHmIZI d dlJmKZLmMZN d dlOmPZPmQZQmRZRmSZSmTZT d dlUmVZV d dlWmXZXmYZY d dlZZZd dl+m,Z, d dl[m\Z\ d dl Z d dl]Z]d dlZd dlZd d l5m8Z8m6Z6 d dlBmCZC d dlDmEZE zd d
lmZ^ W n   dZ^Y d dl]Z]d!d" Z_d#d$ Z`d%d& ZaeCeEd'd( Zbd)d* ZceCeEd+d, ZdeCeEd-d. Zed dlfZfeCeEd/d0 Zgd1d2 ZheCeEd3d4 ZYeCeEd5d6 Zid7d8 ZjdFd:d;ZkdGd<d=Zld>d? Zmd@dA ZndBdC ZodDdE ZpdS )H    N)BytesIO)CaseWhen)Image)	PdfReader)	Converter)convert_from_bytesconvert_from_path)Document)Inchesconvert)canvas)letter)settings)messages)ContentFile)default_storage)	Paginator	EmptyPagePageNotAnInteger)
connection)QValueF	CharField	TextFieldIntegerField)Cast)HttpResponseHttpResponseRedirectJsonResponseHttp404FileResponseHttpResponseBadRequestHttpResponseServerError)renderget_object_or_404)slugify)csrf_exempt)require_POST)dumpcurrencytgl_indo)koleksimaster)ContentVersionContentVersionAttachmentContents
ContentTagTag)Users)upload_to_driveconvert_to_images)embed_model)r!   r   c           
      C   s   | j dkr]| jdr]| jd }t|}tj|dd}tdd}d|j d|d	< t	j
|td
}t\}}|d }| D ]}	|d||	 |d8 }|dk rV|  |d }q=|  |S tdddS )NPOSTfileindlangapplication/pdfcontent_typeattachment; filename="z.pdf"Content-Disposition)pagesize2      Invalid request  status)methodFILESgetr   openpytesseractimage_to_stringr   namer   Canvasr   
splitlines
drawStringshowPagesave)
requestimg_fileimgtextresponsecwidthheightyline r`   </var/www/html/mfile_papteng_new/webapp/views/fileman/main.pyconvert_image_to_pdf]   s&   


rb   c                 C   s   | j dkrE| jdrE| jd }t|}tj|dd}t }|| t	
 }|| |d t| dd}d|j d	|d
< |S tdddS )Nr9   r:   engr<   r   Gapplication/vnd.openxmlformats-officedocument.wordprocessingml.documentr?   rA   .docx"rB   rF   rG   rH   )rJ   rK   rL   r   rM   rN   rO   r
   add_paragraphior   rU   seekr   getvaluerP   )rV   rW   rX   rY   docbufferrZ   r`   r`   ra   convert_image_to_docx|   s    




rl   c              
   C   s8  t   }d|v r:zdd l}ddlm} |  || | |  |W S  ty9 } z	tdt	| d }~ww zLt
j|}t
j|d}t
j|dd t
j }||d< d	|d
< d| |d< ddddddd|| g	}	tj|	tjtjd|d}
|
jdkrt|
j|W S  ty } z	tdt	| d }~ww )Nwindowsr   r   zdocx2pdf Windows gagal: 
lo_profileTexist_okHOMEwww-dataUSERfile://UserInstallationsoffice
--headless--nologo--nofirststartwizard--convert-topdf--outdirstdoutstderrrY   envu)   LibreOffice gagal konversi DOCX → PDF: )platformsystemlower	pythoncomdocx2pdfr   CoInitializeCoUninitialize	Exceptionstrospathdirnamejoinmakedirsenvironcopy
subprocessrunPIPE
returncoder   )
input_pathoutput_pathr   r   docx2pdf_convertetmpdirrn   r   cmdresultr`   r`   ra   safe_docx2pdf   sJ   




r   c                 C   sj  dd l }z|| jd}W n   tddidd Y S |d}|dd	}|r1| d
s9tddiddS t|}|j	dkrKtddiddS zt
 }ttj|d }t jd d }tj|| d| d
}	tj|| d| d}
t|	d}||j W d    n1 sw   Y  t|	|
 tj|
stddiddW  d    W S t|
dd}tjtjd| d| }tj|dd g }t|ddD ](\}}d| d}tj||}||d |tj  d| d| d|  qW d    n	1 sw   Y  td|d W S  t!y4 } ztdt"|iddW  Y d }~S d }~ww )!Nr   utf-8errorPayload tidak validrG   rH   urlrP   zdocument.docx.docxzURL tidak valid atau bukan DOCX   zTidak bisa mengunduh DOCX   -.pdfwbPDF gagal dibuat  dpiconverted_imagesTro      startpage_.jpgJPEGconverted_images//successrI   images)#jsonloadsbodydecoder!   rL   r   endswithrequestsstatus_codetempfileTemporaryDirectoryr(   r   r   splitextuuiduuid4hexr   rM   writecontentr   existsr	   r   
MEDIA_ROOTr   	enumeraterU   append	MEDIA_URLr   r   )rV   r   datar   rP   rr   baseuniq	docx_pathpdf_pathfr   
out_folderurlsirX   filename	save_pathr   r`   r`   ra   convert_docx_url_to_jpg   sR   




$' r   c           
   
   C   s  t   }d|v r4td u rtdtj|tj| 	dd}dd l
}|  t| | |  |S tj|d}tj|dd tj }||d	< d
|d< d| |d< dddddddd|| g
}tj|tjtjd|d}|jdkrztd|j tj| 	dd}	tj||	S )Nrm   z#docx2pdf tidak tersedia di Windows.r   r   r   rn   Tro   rq   rr   rs   rt   ru   rv   rw   rx   ry   z--invisiblerz   zpdf:writer_pdf_Exportr|   r}   zLibreOffice error:
)r   r   r   r   r   r   r   r   basenamereplacer   r   r   r   r   r   r   r   r   r   r   )
r   r   r   r   r   rn   r   r   r   r   r`   r`   ra   convert_docx_universal  sL   


r   c              
   C   s~  z| j d }t }tj||j}t|d}| D ]}|	| qW d    n1 s/w   Y  |j
dd}tj||}t|| t||}tj|sctddiddW  d    W S t|d	}	|	 }
W d    n1 sww   Y  t|
d
d}dtj| d|d< |W  d    W S 1 sw   Y  W d S  ty } ztdt|iddW  Y d }~S d }~ww )Nr:   wb+ _r   r   r   rH   rbr>   r?   rA   "rB   )rK   r   r   r   r   r   rP   rM   chunksr   r   renamer   r   r!   readr   r   r   r   )rV   uploaded_filer   original_pathr   chunk	safe_namer   r   pdf_filepdf_datarZ   r   r`   r`   ra   convert_word_to_pdfS  s6   



( r   c                 C   sl  | j d}zt|}W n   tddidd Y S |d}|dd}|r/| d	s7tdd
iddS t|}|j	dkrItddiddS t
jdd	d}||j |j}W d   n1 sdw   Y  |d	d}t|}	|	j|ddd |	  t|d}
t|
 dd}dt|d	d d|d< W d   n1 sw   Y  t| t| |S )zP
    Konversi PDF berdasarkan URL lampiran menjadi DOCX dan kirim langsung.
    r   r   r   rG   rH   r   rP   document.pdfr   URL tidak valid atau bukan PDFr   !Tidak bisa mengunduh PDF dari URLFdeletesuffixNr   r   r   endr   rd   r?   rA    re   rB   )r   r   r   r   r!   rL   r   r   r   r   r   NamedTemporaryFiler   r   rP   r   r   r   closerM   r   r   r(   r   remove)rV   r   payloadr   rP   r   tmpftmp_pathr   cvfprespr`   r`   ra   convert_pdf_url_to_word|  s6   




r   c              
   C   s  | j d}zt|}W n   tddidd Y S |d}|dd}|r/| d	s7tdd
iddS t|}|j	dkrItddiddS z	t
|jdd}W n tyo } ztdd| iddW  Y d}~S d}~ww ttj|d }t jdd }	tjtjd| d|	 }
tj|
dd g }t|ddD ](\}}d| d}tj|
|}||d |tj d| d|	 d|  qtd|dS ) zo
    Konversi PDF berdasarkan URL menjadi satu atau beberapa gambar JPG, kembalikan JSON array URL gambar.
    r   r   r   rG   rH   r   rP   r   r   r   r   r   r   zKonversi gagal: r   Nr   r   r   r   Tro   r   r   r   r   r   r   r   r   r   )r   r   r   r   r!   rL   r   r   r   r   r   r   r   r(   r   r   r   r   r   r   r   r   r   r   r   rU   r   r   )rV   r   r   r   rP   r   r   r   r   uniquefolderr   idxrX   fnr   r`   r`   ra   convert_pdf_url_to_jpg  s:   


"$r  c              
   C   s   | j dkr@| jd}|r9zt||j}td|jddW S  ty8 } ztdt|dW  Y d}~S d}~ww tdd	dS tdd
dS )z9
    Upload PDF ke Google Drive dan kembalikan link.
    r9   r:   r   zHhttps://drive.google.com/drive/folders/19pH19iCJ8hNr9OF0TwyWwMiKeR-_iecS)rI   	file_nameweb_linkr   rI   messageNzFile tidak ditemukan.zMetode request harus POST)rJ   rK   rL   r6   rP   r!   r   r   )rV   r:   linkr   r`   r`   ra   upload_pdf_to_drive  s    

r  c                 C   s  | j d}|stdddddS t jdd }ttj	|j
d	 }| d
| d}tjtjd|}tjtj|dd t|d}| D ]}|| qLW d   n1 s^w   Y  t|}g }	tjtjd| d
| }
tj|
dd tt|D ]8}||}|jtddd}d|d  d}tj|
|}|| tj d| d
| d| }|	| q|  td|	dS )z0
    Konversi PDF ke beberapa gambar (PNG).
    r:   r   zNo PDF file providedr  rG   rH   Nr   r   r   r   converted_pdfsTro   r   r      )matrixr   r   z.pngr   r   r   r   )rK   rL   r!   r   r   r   r(   r   r   r   rP   r   r   r   r   r   r   rM   r   r   fitzrangelen	load_page
get_pixmapMatrixrU   r   r   r   )rV   r   	unique_id	base_namer   r   destinationr   rj   
image_urlsimage_folderr   pagepiximage_filename
image_path	image_urlr`   r`   ra   r7     s6   


r7   c                 C   s   | j d}|r|jdstdddS tjddd}||  |j}W d   n1 s0w   Y  |	dd	}t
|}|j|d
dd |  t|d}t| dd}dtj| d|d< W d   n1 sow   Y  t| t| |S )z,
    Konversi PDF ke file Word (.docx).
    r:   r   z(File tidak valid. Harap unggah file PDF.rG   rH   Fr   Nr   r   r   r   rd   r?   rA   r   rB   )rK   rL   rP   r   r   r   r   r   r   r   r   r   r   rM   r   r   r   r   )rV   r:   tmp_pdftmp_pdf_pathtmp_docx_pathr   	docx_filerZ   r`   r`   ra   convert_to_word  s*   

r  c                 C   sP   t | trz| jdddW S  ty   | jddd Y S w | d ur&t| S dS )Nr   ignore)errorszlatin-1r   )
isinstancebytesr   r   r   )valr`   r`   ra   safe_str8  s   
r%  d   c                 C   s  | sg S |d u rg d}t j| gddd }dddd | D  d	 }d
}d|  d}d| d}|| || ||g}||  || t }	|	|| |	 }
W d    n1 s_w   Y  i }|
D ]}|d }|d }||vrzg ||< ||| vr|| | qh|S )Nr>   
image/jpeg	image/pngrd   T)convert_to_numpyr   [,c                 s   s    | ]}t |V  qd S N)r   ).0xr`   r`   ra   	<genexpr>R  s    z.semantic_search_content_ids.<locals>.<genexpr>]zac.chunk_content ILIKE %s%a  
        SELECT
            atch.pk_contents,
            ac.pk_attachment,
            (ae.embedding <-> %s::vector) AS semantic_dist,
            COALESCE(ts_rank_cd(ac.tsv, plainto_tsquery(%s)), 0) AS keyword_score,

            (
                0.6 * (ae.embedding <-> %s::vector)
                +
                0.4 * (1.0 - COALESCE(ts_rank_cd(ac.tsv, plainto_tsquery(%s)), 0))
            ) AS hybrid_score
        FROM attachments_chunks ac
        JOIN attachments_embeddings ae ON ae.pk_chunk = ac.pk
        JOIN attachments atch ON atch.pk = ac.pk_attachment
        WHERE atch.type = ANY(%s)
            AND (
                z
                OR plainto_tsquery('indonesian', %s) @@ ac.tsv
                OR TRUE    -- supaya semantic tetap jalan
            )
        ORDER BY hybrid_score ASC
        LIMIT %s;
    r   )	r8   encoder   tolistr   r   cursorexecutefetchall)
query_textallowed_typestop_kq_embq_vecilike_clausesearch_patternsqlparamscurrowsmatched_mapr   c_pka_pkr`   r`   ra   semantic_search_content_idsD  s8    



rF  c           '      C   s^  i }t  }| jdd}|dkrdg}n|dkrddg}ng d}| js0d	D ]	}| j|d
 q&d| jvr:d| jd< | jdd }| jdd }| jdd}	td|	 td| |re|| jd< n| jdd
 | jd}
| jd}| jd}| jd}| jd}| jd}| jdd}t| jddpd}| jdd }|r|| jd< n| jdd
 |r|dkrt|	 rt|| jd< n|rt|	 rt|| jd< n|d
u r|s| jdd
 |
r|
| jd< |r|
 rt|| jd< |r|d kr	d nd!| jd"< |r|
 rt|| jd#< |r%|| jd< |d$kr7d%D ]
}| j|d
 q,| jdrGt| jd |d< | jdrWt| jd |d< | jdrgt| jd |d< | jdrz| jd }|t |d&M }| D ]\}}|t dDi ||iM }q~g }d'}|	d(kr|rd)}td*| t||d+}t| }td,| |r|t |d-M }n|t g d-M }| jdr| jd }d.d/ tjj D }|rt  }|D ]}|t dDi | d0|iO }q||M }|r|rtd1d/ t|D  }tjj||}n| jddkr!d2nd3}tjj||}|jtd4t d5d4}| jd#d6} | d7vrHd6} t|| }!|!|}"g }#|"D ]]}$tj|$}$tjj d|$d8 d'd9}%|r|$d8 |v r||$d8  }&|%j|&d-}%|%! |$d:< d;d/ |%D |$d<< d=d/ tjj d|$d8 d>D |$d?< t"j#j d|$d d@! |$d< |#$|$ qVt%| dA|rt|	 rt|ndtj&j'dddBt"j#j( |#|"| jddt| jd	 rt| jdndt| jdd	 rt| jddndt| jd	 rt| jdnd| jd"d | | jdd| jdddCS )EzI
    Tampilkan daftar konten dengan filter, sorting, dan pagination.
    typeallr{   r>   imager(  r)  r'  )termsearch_skpdsearch_kategoritahun
search_tagdisplayshowsortingNrQ  descrJ  r   
searchTerm
searchModezDEBUG search_mode:zDEBUG search_text:
pk_skpd_dgpk_categoriesrM  dispitem_jmlresetr   r  r   tagrN  rL  rK  listcolrO  rP  1)rJ  rK  rL  rM  rN  ) contenttag__tag__name__icontainsFsemanticTz	SEMANTIC:)r9  zDEBUG content_ids:)pk__inc                 S   s&   g | ]}t |ttfr|js|jqS r`   )r"  r   r   is_relationrP   )r.  fieldr`   r`   ra   
<listcomp>  s    zpage_list.<locals>.<listcomp>__icontainsc                 S   s   g | ]
\}}t ||d qS ))pkthen)r   )r.  posre  r`   r`   ra   rc  #  s    -createdcreated
attributes)attributessr   )   r            re  asispk_contents
is_deletedattachmentsc                 S   s"   g | ]}t |jt |jd qS ))rP   r   )r%  rP   r   r.  ar`   r`   ra   rc  L  s   " attachments_listc                 S   s   g | ]	}d t |jiqS )r   )r%  r   ru  r`   r`   ra   rc  M      )rq  rr  	thumbnailrq  re  zfileman/list.html)	pk_parentdeep)rV  categories_list	skpd_listr[  pagingterm_currentsearch_skpd_dgrL  search_tahunrO  rP  rQ  rN  r`   ))r   GETrL   sessionpopstripprintintr   isdigit	isnumericitemsrF  r[  keys	m_koleksir2   _meta
get_fieldsr   r   objectsfilterorder_byannotater   r   deferr   get_page_sanitize_resultAttachments
_read_manycountm_masterSkpddgr   r&   
Categories
_recursiverH  )'rV   re  	my_filterr   	file_typer9  keyrJ  search_textsearch_moderK  search_kategori_getrM  rW  rX  rQ  rY  r  tag_term
tag_filtervaluesemantic_idsis_semanticsemantic_mapcontent_ids
term_valuefieldsqrb  	preservedqueryorderitem_per_page	paginatorr  r   itemattachments_qsrelevant_file_idsr`   r`   ra   	page_list  s   
	











 
	


"
(,(r  c                 C   s  t jjd|d}t jjd|dd}|j}| jd}| jd}|j|ks)|dv r+dnd	}d
\}}	}
|j	r}t j
j|j	jd}t j
j|j	jd}|rM|d nd}	|jD ]*}|	rh|	| d dkrht|j| |j|< |	r||	| d dkr|t|j| |j|< qR|jrtjj|jjd}
tjjd|jdj }tjj|dd}| }d}|rtjj|jdd }|r|jnd|_t| d||||	|
||||d	S )z
    Detail satu konten.
    r   rz  Frp  re  role)ADMINKOORDINATORyesno)NNNre  rj  NrG  moneydate)rr  rh  
pk_versionzfileman/item.html)	r  rt  
categoriesrj  skpdallowuser_yg_inputstatus_kontenversions)r  r2   	_read_oner  r  r  r  rL   
user_inputrV  r  _recursive_reversere  rj  r,   r-   rU  r  r  r5   usrupperr0   r  r  r  firstr1   r   
latest_pdfr&   )rV   re  r   rt  r  user_pk	user_roler  r  rj  r  attr_objr  r  r  latest_versionlatest_attachmentr`   r`   ra   	page_itemb  sP   

r  c           	         s   t jj|d}t "}|d|g dd |jD   fdd| D }W d   n1 s0w   Y  |D ]}tj	j
|d d|d	< q7t|d
}| jd}||}t| d||dS )z)
    Tampilkan riwayat versi konten.
    r  z
            SELECT cv.*, u.usr AS user_nama
            FROM content_versions cv
            LEFT JOIN users u ON u.pk = cv.user_input
            WHERE cv.pk_contents = %s
            ORDER BY cv.created DESC
        c                 S   s   g | ]}|d  qS )r   r`   )r.  r\  r`   r`   ra   rc    s    z page_history.<locals>.<listcomp>c                    s   g | ]	}t t |qS r`   )dictzip)r.  rowcolumnsr`   ra   rc    rx  Nre  r  rt     r  zfileman/history.html)r  r  )r  r2   r  r   r5  r6  descriptionr7  r1   r  r  r   r  rL   r  r&   )	rV   re  r   r5  versions_listvr  page_numberversions_pager`   r  ra   page_history  s"   


r  c                 C   s   t t| d}tjj|dd}t|d}| j	d}z|
|}W n ty2   |
d}Y n ty@   |
|j}Y nw t| d||dS )	z;
    Detail tag dan daftar konten dengan tag tersebut.
    )rP   )contenttag__tagrh  r   r  r   zfileman/tags_detail.html)rZ  contents)r'   r4   r   r2   r  r  r  r   r  rL   r  r   r   	num_pagesr&   )rV   tag_namerZ  contents_listr  r  r  r`   r`   ra   page_tag_detail  s   
r  c                 C   sP   t j d}| jd}|r|j|d}| jd}|r&|jt|d}|S )zO
    Helper untuk API & view biasa
    Supaya query list berkas konsisten.
    rh  search)title__icontainskategori)rV  )r2   r  rH  r  r  rL   r  r  )rV   r  r  r  r`   r`   ra   _get_queryset  s   r  )Nr&  r-  )qr   r   shutilr   r   rg   r   django.db.modelsr   r   r   r  PILr   PyPDF2r   pdf2docxr   	pdf2imager   r	   docxr
   docx.sharedr   r   r   reportlab.pdfgenr   reportlab.lib.pagesizesr   django.confr   django.contribr   django.core.files.baser   django.core.files.storager   django.core.paginatorr   r   r   	django.dbr   r   r   r   r   r   r   django.db.models.functionsr   django.httpr   r    r!   r"   r#   r$   r%   django.shortcutsr&   r'   django.utils.textr(   django.views.decorators.csrfr)   django.views.decorators.httpr*   webapp.commonr+   r,   r-   webapp.modelsr.   r  r/   r  webapp.models.koleksir0   r1   r2   r3   r4   webapp.models.masterr5   webapp.utilsr6   r7   rN   webapp.ml.e5larger8   r   r   rb   rl   r   r   r   r   r   r   r  r  r  r%  rF  r  r  r  r  r  r`   r`   r`   ra   <module>   s    $	8D<'(*+

V IV4