
    Sj(                         d Z ddlZddlZddlZddlZddlmZmZmZm	Z	m
Z
 ddlZddlmZmZmZ ddlmZmZmZ ddlmZ e	d   Z G d	 d
      Zy)z
Async WebSocket watcher with async iterator interface for v2 jobs.

Usage:
    async for snapshot in AsyncWatcher(client, job_id, kind="crawl"):
        print(snapshot.status)
    N)AsyncIteratorDictListLiteralOptional)ConnectionClosedConnectionClosedOKConnectionClosedError   )BatchScrapeJobCrawlJobDocument)normalize_document_input)crawlbatchc                       e Zd Zdddddedededed	ee   d
dfdZd
e	e   fdZ
d
efdZd
e	e   fdZd ZdefdZd Zdddededeee      fdZy)AsyncWatcherr      N)kindpoll_intervaltimeoutclientjob_idr   r   r   returnc                <   || _         || _        || _        || _        t	        dt        |            | _        t        |dd       }|%t        |dd       | _        t        |dd       | _	        n$t        |dd       | _        t        |dd       | _	        d| _
        g | _        y )N        http_clientapi_urlapi_keyscraping)_client_job_id_kind_timeoutmaxfloat_poll_intervalgetattr_api_url_api_key_status_data)selfr   r   r   r   r   r   s          M/root/.hermes/venv/lib/python3.12/site-packages/firecrawl/v2/watcher_async.py__init__zAsyncWatcher.__init__   s     
%(eM.B%CfmT:"#KDADM#KDADM $FIt<DM#FIt<DM&!#
    c                 "    | j                         S N)_iterater-   s    r.   	__aiter__zAsyncWatcher.__aiter__4   s    }}r0   c                     | j                   st        d      | j                   j                  dd      j                  ddd      }| j                  dk(  r| d| j                   S | d	| j                   S )
Nz)API URL is required for WebSocket watcherzhttps://zwss://zhttp://zws://r   r   z
/v2/crawl/z/v2/batch/scrape/)r)   
ValueErrorreplacer#   r"   )r-   ws_bases     r.   _build_ws_urlzAsyncWatcher._build_ws_url7   ss    }}HII--''
H=EEiQXZ[\:: Yj77+DLL>::r0   c           
     D  K   | j                         }g }| j                  r |j                  dd| j                   f       	 t        j                  |d |      4 d {   }| j
                  r/t        j                         j                         | j
                  z   nd }	 | j                          d {   }| |j                  dv r	 d d d       d {    y 	 	 	 |Mt        d|t        j                         j                         z
        }|dkD  rt        | j                  |      nd}n| j                  }t        j                  |j!                         |       d {   }	 t/        j0                  |      }
|
j3                  d
      }|dk(  rSd| _        | j6                  dk(  rt9        ddddd d g        nt;        ddddd d g        	 d d d       d {    y |dk(  re|
j3                  di       }|j3                  d| j4                        | _        |j3                  dg       xs g }| j<                  j?                  |       n|dk(  r=|
j3                  d      }tA        |tB              r| j<                  j                  |       n|dk(  rd| _        |
j3                  di       xs i }|j3                  dg       xs g }tA        |tD              r5|r3|D ].  }tA        |tB              s| j<                  j                  |       0 | jG                  d|| j<                         	 d d d       d {    y |
j3                  d|
      }|j3                  d|
j3                  d| j4                              }| jG                  ||      }| |dv r	 d d d       d {    y 7 T7 7 # t        $ r Y w xY w7 g# t        j"                  $ r | j%                          d {  7  }	|	&|	 |	j                  dv rY d d d       d {  7   y |8t        j                         j                         |k\  rY d d d       d {  7   y Y t&        t(        t*        f$ r t        j                         | j
                  xs dz   }	 	 | j                          d {  7  }	|	 |	j                  dv rY d d d       d {  7   y n!# t        $ r Y Y d d d       d {  7   y w xY wt        j                         |k\  rY d d d       d {  7   y t        j,                  d	       d {  7   w xY w# t        $ r Y tw xY w7 u7 7 # 1 d {  7  sw Y   y xY w# t        $ r t        j                         | j
                  xs dz   }	 	 | j                          d {  7  }	|	 |	j                  dv rY y n# t        $ r Y Y y w xY wt        j                         |k\  rY y t        j,                  d	       d {  7   xw xY ww)NAuthorizationzBearer )max_sizeadditional_headers)	completedfailed	cancelledr   r   )r      r   typeerrorr@   r   statusr?   totalcredits_used
expires_atnextdatacatchuprK   rF   documentdoner?   )rF   payloaddocs_override)rF   rO   )$r:   r*   append
websocketsconnectr$   asyncioget_event_looptime_fetch_job_statusrF   	Exceptionr%   minr'   wait_forrecvTimeoutError_safe_fetchr	   r   r
   sleepjsonloadsgetr+   r#   r   r   r,   extend
isinstancedictlist_make_snapshot)r-   uriheaders_list	websocketdeadlinepre	remainingr   msgjobbodymsg_typeddocs_indocraw_payloadrO   
status_strsnapshots                      r.   r3   zAsyncWatcher._iterate?   s      "==GDMM?2K LMe	'!))#Q]^ V VbkNRmm711388:T]]Jae $ 6 6 88CIzz%IIV V V J
 3#/(+CG<R<R<T<Y<Y<[1[(\IMVYZ]c$*=*=y&I`cG&*&9&9G$+$4$4Y^^5Ew$WW2!#zz#  $xx/H7*'/::0"*(aq_`mqx|  DF  #G  G"0AUVefsw  C  JL  #M  MqV V Vr "Y. HHVR0'(uuXt||'D"#%%"3"9r

))'2!Z/"hhv.%c40 JJ--c2!V+'2&*hhvr&:&@b"-//&""="C%gt4'. ;#-c4#8$(JJ$5$5c$:; #11kimisis1tt]V V Vb #hhvt4G!(Xtxx$,,7W!XJ#22*g2VH"N!%IImV V V V 9	V !  X"// 	!$($4$4$666?"%I"zz-QQ &3V V V4 $/G4J4J4L4Q4Q4SW_4_"7V V V8 !.0@BWX 3#'99;$--2E2#F"',0,B,B,D&D&D&)	#&::1U#U$*KV V VH $V#, ' &OV V VL'#yy{h6 &SV V VT #*--"222 #3  % ! ![V V V V V Vn  	'yy{dmm&9r:H $ 6 6 888CIzz%II J  99;(*mmA&&& 	's  ?X U5 N2U5 ">U !N;4N55N;U5 N8U5 X U !BO$O%O*U?AU U5 "U#U5 'X (DU 4?U 3U5 >U?U5 X AU U5 *U+U5 /X 0U 2U5 5N;8U5 ;	OU OU O&U4O75UU U5 PU5 !X "'U	U 
U5 QU5 X U 9US,R/-SU U5 SU5 X U	S7"U#U $U5 /S20U5 5X 6S77UU U5 T U5 #X $U<T?=UU 	UU UU U5 U5 U5  U2&U)'U2.U5 1X 2U5 5/X%W8V;9WX X	W!XX  W!!X;X =XXXX c                    K   | j                   dk(  r| j                  d       d {   S | j                  d       d {   S 7 7 w)Nr   get_crawl_statusget_batch_scrape_status)r#   _call_status_methodr4   s    r.   rW   zAsyncWatcher._fetch_job_status   sF     :: 112DEEE--.GHHH FHs    $AAAA AAmethod_namec                 t  K   t        | j                  |d       }|^	  || j                        }|!t	        j
                  |      r
| d {   S |S t        j                  || j                         d {   S t        | j                  dd       }|mt        ||d       }|^	  || j                        }|!t	        j
                  |      r
| d {   S |S t        j                  || j                         d {   S t        d|       # t        $ r d }Y w xY w7 7 # t        $ r d }Y {w xY w7 b7 :w)Nv2zClient does not expose )	r(   r!   r"   	TypeErrorinspectisawaitablerT   	to_threadRuntimeError)r-   r{   methresultr}   s        r.   rz   z AsyncWatcher._call_status_method   s>    t||[$7dll+ !&&v.!'<' **4>>> T\\4.>2{D1D"!$,,/F %**62%+|+!M$..tT\\BBB4[MBCC1   ( ? ! "!F"  ,Bs   D8D D8D)D85D!6,D8#D# 5D8D4)D8;D6<D8DD8DD8!D8#D1.D80D11D86D8c                 ^   K   	 | j                          d {   S 7 # t        $ r Y y w xY wwr2   )rW   rX   r4   s    r.   r]   zAsyncWatcher._safe_fetch   s1     	//1111 		s(   -  - 	*-*-)rP   rF   rO   rP   c          
      P   g }||n|j                  dg       xs g }|D ]8  }t        |t              st        |      }|j	                  t        d
i |       : | j                  dk(  r`t        ||j                  dd      |j                  dd      |j                  dd      |j                  d      |j                  d      |	      S t        ||j                  dd      |j                  dd      |j                  d      |j                  d      |j                  d      |	      S )NrK   r   r?   r   rG   creditsUsed	expiresAtrJ   rE    )	ra   rc   rd   r   rQ   r   r#   r   r   )r-   rF   rO   rP   docssource_docsrs   rq   s           r.   rf   zAsyncWatcher._make_snapshot   s   '4'@mgkkRXZ\F]Fcac 	+C#t$,S1HMqM*	+
 :: !++k15kk'1-$[[:";;{3[[(  kk+q1++gq) ]3{{;/V$
 	
r0   )__name__
__module____qualname__objectstrJobKindintr   r/   r   r5   r:   r3   rW   rz   r]   r   r   rf   r   r0   r.   r   r      s      !%$$ $
 $ $ #$ 
$6=0 ;s ;l'f 5 l'\I
DS D@ cg 
 
d 
8TXY]T^K_ 
r0   r   )__doc__rT   r   r_   rV   typingr   r   r   r   r   rR   websockets.exceptionsr   r	   r
   typesr   r   r   utils.normalizer   r   r   r   r0   r.   <module>r      sG        ? ?  ] ] 5 5 5
"
#Z
 Z
r0   