U
     R'he                     @   s   d dl mZmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlZdd	 Zd
d Zdd Zdd Zdd Zedgeefdd ZdS )    )api_viewpermission_classes)AllowAny)KeywordGroups)JsonResponseN)ThreadPoolExecutoras_completed)datetimec                 C   s    t | }|j d|j }|S )N.)
tldextractextractdomainsuffix)site_urlextdomain_name r   M/home/richestsoft/Projects/NewRankTracker/api/serp/keyword_insight_tracker.pyparse_domain   s    
r   c              
   C   sH   zt dd | D d}|W S  tk
rB } z
W Y dS d }~X Y nX d S )Nc                 s   s$   | ]}| d dk	r|d  V  qdS )search_volumeNget.0itemr   r   r   	<genexpr>   s      z)get_last_search_volume.<locals>.<genexpr>r   NA)next	Exception)search_volume_dataZlast_search_volumeer   r   r   get_last_search_volume   s
    r"   c           $      C   s  dd | D }dd | D }dd | D }d}d}| d| }t |dd}d}d	| d
d}	|d d|ddg}
tj||	t|
d}| }|ddkrt	d|d d|d  g S |d d |d ddg}d}tj||	t|d}| }|ddkr:t	d|d d|d  i }|dg D ]R}|dg D ]>}|dg D ]*}|d}|dd}|rj|||< qjqZqJ|d d dd g}d!}tj||	t|d}| }i }|dg D ]Z}|dg D ]F}|dg D ]2}|d}|d"i }|r|d#d||< qqqg }|dg D ]}|dg D ]}|dprd}t
|d$}|d%d k	r|d%nd&}|d$pg } ||d}!||d}"| d k	rd'd | D }#|||||!|"|#d( q`qP|S ))Nc                 S   s   g | ]
}|j qS r   keywordr   kr   r   r   
<listcomp>   s     z#ranked_keywords.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r   )location_coder%   r   r   r   r'      s     c                 S   s   g | ]
}|j qS r   )r   r%   r   r   r   r'      s      :zutf-8zIhttps://api.dataforseo.com/v3/keywords_data/google_ads/search_volume/livezBasic zapplication/json)AuthorizationzContent-Typer   enr   )r(   language_codekeywordsZsort_by)headersdatastatus_codei N  zError:  - Zstatus_messagei  )r.   r(   r-   zQhttps://api.dataforseo.com/v3/dataforseo_labs/google/bulk_keyword_difficulty/livezDifficulty API Error: tasksresultitemsr$   keyword_difficultyr   )r.   r-   zGhttps://api.dataforseo.com/v3/dataforseo_labs/google/search_intent/liveZkeyword_intentlabelmonthly_searchescpcg        c                 S   s6   g | ].}| d pd| dp d| dp,ddqS )yearr   monthr   )r:   r;   r   r   )r   mr   r   r   r'   d   s     )r$   r   r9   r6   search_intentr8   )base64	b64encodeencodedecoderequestspostjsondumpsr   printr"   append)$keyword_objsZkeyword_listZlocation_listZsite_url_listZdataseo_usernameZdataseo_passwordZauth_stringZauth_base64urlr/   payloadresponseZ	json_dataZdifficulty_payloadZdifficulty_urlZdifficulty_responseZdifficulty_dataZdifficulty_maptaskr4   r   r$   Z
difficultyZsearch_intent_payloadZsearch_intent_urlZsearch_intent_responseZsearch_intent_dataZsearch_intent_mapZintentZextracted_datar   r9   r8   r6   r=   Zcleaned_monthly_searchesr   r   r   ranked_keywords   sl      


 rM   c              
   C   s   zt | }W n8 tk
rD } ztd|  g  W Y S d }~X Y nX dd |D }| D ]j}||j}|r|d|_|d|_|d|_|d|_|d|_	qXd	|_d	|_d	|_d	|_	qX| S )
Nz[Search Volume Metrics Error] c                 S   s    i | ]}| d r|d  |qS r#   r   r   r   r   r   
<dictcomp>s   s     
  z)process_keyword_batch.<locals>.<dictcomp>r=   r   r8   r9   r6   r   )
rM   r   rF   r   r$   r=   r   r    r9   r6   )rH   resultsr!   Z
result_mapr$   r0   r   r   r   process_keyword_batchk   s&    rP   c              	      sV  | j }tjj|djdd t|d dkr0dnd}ttjj|dd	|sltjj|djd
d d S d  fddt	dt
 D }g }tdd<fdd|D }t|D ]}| }|r|| qW 5 Q R X |r<|D ]P}	|	j}
|
d k	r|
d d d |	_tjj|	j dj|	j|	j|	j|	j|	jd
d qtjj|djd
d d S )NidSCHDmetric_status      created_datez-created_dateINIT)fk_group_idrU   COMPd   c                    s   g | ]}||   qS r   r   )r   i)
batch_sizer.   r   r   r'      s     z!process_group.<locals>.<listcomp>r      )max_workersc                    s   g | ]}  t|qS r   )submitrP   )r   batch)executorr   r   r'      s     )r9   r6   r=   r   r    rU   )rR   r   objectsfilterupdateintlistr   order_byrangelenr   r	   r4   extendr    r9   r6   r=   r   )group
sort_ordergroup_idZ
sort_fieldbatchesZupdated_keywordsfuturesfuturer4   r$   r    r   )r^   rc   r.   r   process_group   s.     
.rt   GETc              
   C   s   t jjdd }|sDt jjdd }|rDt jj|jdjdd |sVtdddS zt|| W nT tk
r } z6t	d|j d	t
|  t jj|jdjd
d W 5 d}~X Y nX tdddS )z<Cron job to process one group at a time using metric_status.rS   rT   rY   rQ   truezNo groups to process)statusmessagez[Group Error] r2   FAILNzGroup processing completed)r   re   rf   firstrR   rg   r   rt   r   rF   str)requestro   rn   r!   r   r   r   insightTracker   s    *r}   )rest_framework.decoratorsr   r   Zrest_framework.permissionsr   Zserp.modelsr   r   django.httpr   rB   rD   r>   concurrent.futuresr   r	   r
   r   r   r"   rM   rP   rt   r}   r   r   r   r   <module>   s"   P#