
    Sj7                         d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZ ddlZddlmZmZmZ ddlmZ ed   Ze	eef   Z G d d	      Zy)
z
WebSocket-based watcher for v2 jobs (crawl and batch), mirroring v1 behavior.

Usage:
    watcher = client.watcher(job_id, kind="crawl")
    watcher.add_listener(lambda status: print(status.status))
    watcher.start()
    N)CallableListOptionalLiteralUnionDictAny   )CrawlJobBatchScrapeJobDocument)normalize_document_input)crawlbatchc                       e Zd Z	 	 	 ddededededee   ddfdZd	e	e
gdf   ddfd
Zde
ddfdZdede	eeef   gdf   ddfdZdedeeef   ddfdZdefdZddZdefdZddZddZddZy)WatcherNclientjob_idkindpoll_intervaltimeoutreturnc                 H   || _         || _        || _        || _        || _        g | _        d | _        t        j                         | _	        t        |dd       }t        |dd       | _        t        |dd       | _        d| _        g | _        g g g d| _        d| _        d| _        y )Nhttp_clientapi_urlapi_keyscraping)doneerrordocumentF)_client_job_id_kind_timeout_poll_interval
_listeners_thread	threadingEvent_stopgetattr_api_url_api_keystatusdata_event_handlers
_sent_done_sent_error)selfr   r   r   r   r   r   s          G/root/.hermes/venv/lib/python3.12/site-packages/firecrawl/v2/watcher.py__init__zWatcher.__init__   s     
+;=37__&
fmT:'.{It'L'.{It'L &*,	S

 !&!&    callbackc                 :    | j                   j                  |       y N)r&   append)r3   r7   s     r4   add_listenerzWatcher.add_listener:   s    x(r6   r.   c                 h    t        | j                        D ]  }	  ||        y # t        $ r Y w xY wr9   )listr&   	Exception)r3   r.   cbs      r4   _emitzWatcher._emit=   s:    t' 	B6
	  s   %	11
event_typehandlerc                 ^    || j                   v r| j                   |   j                  |       y y r9   )r0   r:   )r3   rA   rB   s      r4   add_event_listenerzWatcher.add_event_listenerE   s.    ---  ,33G< .r6   detailc                 z    || j                   v r| j                   |   D ]  }	  ||        y y # t        $ r Y w xY wr9   )r0   r>   )r3   rA   rE   rB   s       r4   dispatch_eventzWatcher.dispatch_eventI   sO    ---//
; FO . ! s   .	::c                     | j                   st        d      | j                   j                  dd      j                  ddd      }| j                  dk(  r| d| j                   S | d	| j                   S )
Nz)API URL is required for WebSocket watcherzhttps://zwss://zhttp://zws://r
   r   z
/v2/crawl/z/v2/batch/scrape/)r,   
ValueErrorreplacer#   r"   )r3   ws_bases     r4   _build_ws_urlzWatcher._build_ws_urlQ   ss    }}HII--''
H=EEiQXZ[\:: Yj77+DLL>::r6   c                 r  K   | j                         }g }| j                  r |j                  dd| j                   f       	 t        j                  |d |      4 d {   }| j
                  r/t        j                         j                         | j
                  z   nd }| j                  j                         sA|Jt        d|t        j                         j                         z
        }t        | j                  xs ||      }n| j                  xs d}	 t        j                  |j                         |       d {   }	 t5        j6                  |      }|j9                  d      }	|	dk(  rd| _        | j-                  d| j(                  | j.                  |j9                  d      | j0                  d       d
| _        | j<                  dk(  rt?        ddddd d g       }
ntA        ddddd d g       }
| jC                  |
       n|	dk(  r|j9                  di       }|j9                  d| j(                        | _        |j9                  dg       }| j.                  jE                  |       |D ]!  }| j-                  d|| j0                  d       # nH|	dk(  r^|j9                  d      }tG        |tH              r!| j.                  j                  |       | j-                  d|| j0                  d       n|	dk(  rd| _        |j9                  di       xs i }|j9                  dg       xs g }tG        |tJ              r5|r3|D ].  }tG        |tH              s| j.                  j                  |       0 | j-                  d| j(                  | j.                  | j0                  d	       d
| _        g }| j.                  D ]8  }tG        |tH              stM        |      }|j                  tO        di |       : | j<                  dk(  rat?        d|j9                  dd      |j9                  dd      |j9                  dd      |j9                  d      |j9                  d      |      }
n`tA        d|j9                  dd      |j9                  dd      |j9                  dd      |j9                  d      |j9                  d      |      }
| jC                  |
       n|j9                  d|      }tG        |tH              xr d|v xs d|v }|sq|j9                  d|j9                  d| j(                              }| j<                  dk(  rKg }|j9                  dg       D ]8  }tG        |tH              stM        |      }|j                  tO        di |       : t?        ||j9                  dd      |j9                  dd      |j9                  dd      |j9                  d      |j9                  d      |      }
| jC                  |
       |dv r|dk(  r=| j*                  s1| j-                  d|| j.                  | j0                  d	       d
| _        |dk(  r=| j:                  s1| j-                  d|| j.                  | j0                  d	       d
| _        ndg }|j9                  dg       D ]8  }tG        |tH              stM        |      }|j                  tO        di |       : tA        ||j9                  dd      |j9                  dd      |j9                  d      |j9                  d      |j9                  d      |      }
| jC                  |
       |dv r|dk(  r=| j*                  s1| j-                  d|| j.                  | j0                  d	       d
| _        |dk(  r=| j:                  s1| j-                  d|| j.                  | j0                  d	       d
| _        n| j                  j                         sAd d d       d {    | j(                  dk(  rI| j*                  s<| j-                  d| j(                  | j.                  | j0                  d	       d
| _        y y y 7 7 (# t        j                   $ r | j#                          d {  7  rY Y t        j$                  $ r Y t&        $ r | j                  j                         s^| j#                          d {  7  rkY d d d       d {  7   | j(                  dk(  rI| j*                  s<| j-                  d| j(                  | j.                  | j0                  d	       d
| _        y y y |t        j                         j                         |k\  rkY d d d       d {  7   | j(                  dk(  rI| j*                  s<| j-                  d| j(                  | j.                  | j0                  d	       d
| _        y y y t        j2                  | j                  xs d       d {  7   | j                  j                         s^Y d d d       d {  7   | j(                  dk(  rI| j*                  s<| j-                  d| j(                  | j.                  | j0                  d	       d
| _        y y y w xY w# t&        $ r Y 
w xY w7 # 1 d {  7  sw Y   xY w# t&        $ r Y w xY w# | j(                  dk(  rI| j*                  s<| j-                  d| j(                  | j.                  | j0                  d	       d
| _        w w w xY ww)NAuthorizationzBearer )max_sizeadditional_headersg           r   	completedr   r.   r/   idT   typer   failed)r.   r/   r   rU   r   r   )r.   rS   totalcredits_used
expires_atnextr/   catchupr/   r.   r    )r/   rU   rY   creditsUsed	expiresAtr\   rS   rX   	cancelled )(rL   r-   r:   
websocketsconnectr$   asyncioget_event_looptimer*   is_setmaxminr%   wait_forrecvTimeoutError_poll_status_onceCancelledErrorr>   r.   r1   rG   r/   r"   sleepjsonloadsgetr2   r#   r   r   r@   extend
isinstancedictr=   r   r   )r3   uriheaders_list	websocketdeadline	remainingr   msgbodymsg_typejobddocs_indocraw_payloaddocspayloadhas_status_field
status_strs                      r4   _run_wszWatcher._run_wsY   s	      "==GDMM?2K LMh	'!))#Q]^ `" `"bkNRmm711388:T]]Jae**++-+$'X8N8N8P8U8U8W-W$X	"%d&9&9&FY	"R"&"5"5":$+$4$4Y^^5Ew$WW&!#zz#
  $xx/H7*&.++G&*kk$(II%)XXg%6"&,,	6  ,0(::0"*(aq_`mqx|  DF  #GC"0AUVefsw  C  JL  #MC

3!Y. HHVR0&'eeHdkk&B"#%%"3		((1#* _C //
SPTP\P\<]^_!Z/"hhv.%c40 II,,S1 //
SPTP\P\<]^!V+&1&*hhvr&:&@b"-//&""="C%gt4'. :#-c4#8$(II$4$4S$9: ++Ft{{TXT]T]eieqeq4rs*./1#'99 ;C)#t4$<S$A $HMqM :;  ::0"*'2*5//+q*I&1oogq&A-8__]A-N+6??;+G%0__V%<%)#C #1'2*5//+q*I&1oogq&A-8__]A-N+6??;+G%0__V%<%)#C 

3"hhvt4G(27D(A(YhRYFY'p_gko_o$+ !(Xtxx$++7V!WJzzW,!#*;;vr#: ;C)#t4$<S$A $HMqM :; '#-&-kk+q&A")++gq"9)0]A)F'.{{;'?!(V!4!% 

3%)MM)[8 $ 3 3Fz[_[d[dlplxlx<y z26)X5d>N>N $ 3 3G
\`\e\emqmymy=z {37 0!!#*;;vr#: ;C)#t4$<S$A $HMqM :; -#-&-kk+q&A")++gq"9)0])C'.{{;'?!(V!4!% 

3%)MM)[8 $ 3 3Fz[_[d[dlplxlx<y z26)X5d>N>N $ 3 3G
\`\e\emqmymy=z {37 0!} **++-`" `"J {{k)$//##Ft{{DII]a]i]i,jk"& 3B)K`" X"// %!%!7!7!999!$"11 $ "&**"3"3"5%)%;%;%=== &/`" `" `"J {{k)$//##Ft{{DII]a]i]i,jk"& 3B)[  (38N8N8P8U8U8W[c8c &3`" `" `"J {{k)$//##Ft{{DII]a]i]i,jk"& 3B)W #*--0C0C0Hq"III #'**"3"3"5 7`" `" `"J {{k)$//##Ft{{DII]a]i]i,jk"& 3B)e % ! !?`" `" `" `"B  		 {{k)$//##Ft{{DII]a]i]i,jk"& 3B)s  ?l7k
 a%k
 "B5j4(a+ a(a+j!G?j4A9j4Fj45E
j4 D:j4<k
 j1k
 Al7%k
 (a++&jbjj4j.j407j'c*(j-j4.k
 9c<:k
 ?Al7'j?j4 k
 fk
 Al7*&jhj2j43k
 >i?k
 Al7jj4!	j.*j4-j..j41k
 4k:j=;kk
 
	kk kk Al44l7c           	        K   	 | j                   dk(  r=t        j                  | j                  j                  | j
                         d{   }n<t        j                  | j                  j                  | j
                         d{   }|j                  | _        | j                  |       |j                  dv r|j                  dk(  rc| j                  sW| j                  d|j                  |j                  D cg c]  }|j                          c}| j
                  d       d| _
        |j                  d	k(  rc| j                  sW| j                  d
|j                  |j                  D cg c]  }|j                          c}| j
                  d       d| _        yy7 X7 # t        $ r Y yw xY wc c}w c c}w w)z9Poll job status over HTTP once. Returns True if terminal.r   NFr`   rS   r   rT   TrX   r   )r#   re   	to_threadr!   get_crawl_statusr"   get_batch_scrape_statusr>   r.   r@   r1   rG   r/   
model_dumpr2   )r3   r   r   s      r4   rn   zWatcher._poll_status_once	  s~    	zzW$&-&7&78U8UW[WcWc&d d,3,=,=dll>b>bdhdpdp,q&q jj

3::==zz[(##FszzdgdldlKm_`ALLNKmuy  vB  vB  -C  D"&zzX%d.>.>##G

ehememLn`aQ\\^Lnvz  wC  wC  .D  E#' ! !e&q 		 Ln Mosg   GAF( 
F"<F( F%F( A0G<F7AG.F<G"F( %F( (	F41G3F44Gc                 J    t        j                  | j                                y r9   )re   runr   r3   s    r4   _loopzWatcher._loop  s    DLLN#r6   c                    | j                   r| j                   j                         ry | j                  j                          t	        j
                  | j                  d      | _         | j                   j                          y )NT)targetdaemon)r'   is_aliver*   clearr(   Threadr   startr   s    r4   r   zWatcher.start"  sS    <<DLL113

 ''tzz$Gr6   c                     | j                   j                          | j                  r| j                  j                  d       y y )Nr
   rR   )r*   setr'   joinr   s    r4   stopzWatcher.stop)  s1    

<<LLa( r6   )r   rV   N)r   N)__name__
__module____qualname__objectstrJobKindintr   r5   r   JobTyper;   r@   r   r	   rD   rG   rL   r   boolrn   r   r   r   rb   r6   r4   r   r      s   
  !%'' ' 	'
 ' #' 
'@)Xwio%> )4 )G  =S =8T#s(^DTVZDZ;[ =`d = d38n  ;s ;n'` ,$)r6   r   )__doc__re   rq   r(   typingr   r   r   r   r   r   r	   rc   typesr   r   r   utils.normalizer   r   r   r   rb   r6   r4   <module>r      sR       F F F  5 5 5 "
#
.(
)S) S)r6   