
    Sjk                    	   U d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlZddlZddlZddlZddlZddlZ ej,                  d      Zej0                  ed<   d Z e       Z ed      Z G d	 d
ej:                        Z G d dej:                        Z G d dej:                        Z  G d dej:                        Z! G d dej:                  ee         Z" G d dej:                        Z# G d dej:                        Z$ G d dej:                        Z% G d dej:                        Z& G d dej:                        Z' G d dej:                        Z( G d d ej:                        Z) G d! d"ej:                        Z* G d# d$ej:                        Z+ G d% d&ej:                        Z, G d' d(ej:                        Z- G d) d*ej:                        Z. G d+ d,ej:                        Z/ G d- d.ej:                        Z0 G d/ d0ej:                        Z1 G d1 d2e&      Z2 G d3 d4e"e   ee         Z3 G d5 d6ej:                        Z4 G d7 d8ej:                        Z5 G d9 d:ej:                        Z6 G d; d<ej:                        Z7 G d= d>ej:                        Z8 G d? d@ej:                        Z9 G dA dBej:                        Z: G dC dDej:                        Z; G dE dFej:                        Z< G dG dHej:                        Z= G dI dJej:                  ee         Z> G dK dLej:                        Z? G dM dNej:                        Z@ G dO dPej:                        ZA G dQ dRej:                        ZB G dS dTej:                        ZC G dU dVej:                        ZD G dW dXej:                        ZE G dY dZej:                        ZF G d[ d\ej:                        ZG G d] d^ej:                        ZH G d_ d`ej:                        ZI G da dbej:                        ZJ G dc ddej:                        ZK G de dfej:                        ZL G dg dhej:                        ZM G di djej:                        ZN G dk dlej:                        ZO G dm dNej:                        Z@ G dn dHej:                        Z= G do dp      ZP G dq dr      ZQ G ds dteP      ZR G du dveQ      ZSy)wa"  
Firecrawl v1 API Client - Legacy Implementation

This module provides the legacy v1 implementation of the Firecrawl SDK.
It contains the complete `V1FirecrawlApp` class with all v1 API methods and types
for backward compatibility. This is used by the unified client to provide
version-specific access patterns like app.v1.scrape_url().

Classes:
    - V1FirecrawlApp: Legacy v1 client for interacting with the Firecrawl API.
    - AsyncV1FirecrawlApp: Async version of the v1 client.
    - CrawlWatcher: WebSocket-based crawl monitoring for v1.
    N)	AnyDictOptionalListUnionCallableLiteralTypeVarGeneric)datetime	firecrawlloggerc                     	 ddl m}  t        j                  j	                  t
              } | t        j                  j                  |d            j                         }t        j                  d|t        j                        }|r|j                  d      j                         S y # t        $ r t        d       Y y w xY w)Nr   )Pathz__init__.pyz"^__version__ = ['\"]([^'\"]*)['\"]   z&Failed to get version from __init__.py)pathlibr   ospathdirname__file__join	read_textresearchMgroupstrip	Exceptionprint)r   package_pathversion_fileversion_matchs       F/root/.hermes/venv/lib/python3.12/site-packages/firecrawl/v1/client.pyget_versionr$      s    	WW__X.l"'',,|]CDNNPlii E|UWUYUYZm	$$Q'--/
/ 
	 45s   B"B& &B=<B=Tc                   :    e Zd ZU dZdZed   ed<   dZee	   ed<   y)V1AgentOptionszConfiguration for the agent.FIRE-1modelNprompt)
__name__
__module____qualname____doc__r)   r	   __annotations__r*   r   str     r#   r'   r'   P   s"    &'E78' FHSM r2   r'   c                   &    e Zd ZU dZdZed   ed<   y)V1AgentOptionsExtract2Configuration for the agent in extract operations.r(   r)   Nr+   r,   r-   r.   r)   r	   r/   r1   r2   r#   r4   r4   U       <'E78'r2   r4   c                   2    e Zd ZU dZee   ed<   ee   ed<   y)V1ActionsResultz,Result of actions performed during scraping.screenshotspdfsN)r+   r,   r-   r.   r   r0   r/   r1   r2   r#   r9   r9   Y   s    6c
s)Or2   r9   c                       e Zd ZU dZdZee   ed<   eed<   eed<   dZee	ee
f      ed<    ej                  dd      Zee
   ed	<   y)
V1ChangeTrackingDataz.
    Data for the change tracking format.
    NpreviousScrapeAtchangeStatus
visibilitydiffjsonalias
json_field)r+   r,   r-   r.   r>   r   r0   r/   rA   r   r   pydanticFieldrE   r1   r2   r#   r=   r=   ^   sU     '+hsm*O%)D(4S>
") .t6 BJBr2   r=   c                   @   e Zd ZU dZdZee   ed<   dZee   ed<   dZ	ee   ed<   dZ
ee   ed<   dZeee      ed<   dZee   ed<    ej                   dd	
      Zee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   y)V1FirecrawlDocumentz-Document retrieved or processed by Firecrawl.NurlmarkdownhtmlrawHtmllinksextractrB   rC   rE   
screenshotmetadataactionstitledescriptionchangeTracking)r+   r,   r-   r.   rJ   r   r0   r/   rK   rL   rM   rN   r   rO   r%   rF   rG   rE   rP   rQ   r   rR   r9   rS   rT   rU   r=   r1   r2   r#   rI   rI   h   s    7C#"Hhsm"D(3-!GXc]!!%E8DI%GXa[,hnnT@J@ $J$"Hhsm")-GXo&-E8C=!%K#%59NH129r2   rI   c                   @    e Zd ZU dZdZee   ed<   dZee	e      ed<   y)V1LocationConfigz$Location configuration for scraping.Ncountry	languages)
r+   r,   r-   r.   rX   r   r0   r/   rY   r   r1   r2   r#   rW   rW   x   s&    .!GXc]!%)IxS	")r2   rW   c                   x    e Zd ZU dZeed<   dZeeeef      ed<   dZ	eeeef      ed<   dZ
eeed         ed<   y)V1WebhookConfigzConfiguration for webhooks.rJ   NheadersrQ   )	completedfailedpagestartedevents)r+   r,   r-   r.   r0   r/   r\   r   r   rQ   ra   r   r	   r1   r2   r#   r[   r[   }   sR    %	H(,GXd38n%,)-HhtCH~&-PTFHT'"JKLMTr2   r[   c                       e Zd ZU dZdZeeed         ed<    e	j                  dd      Zee   ed<   dZee   ed<   dZee   ed	<   y)
V1ChangeTrackingOptionsz"Configuration for change tracking.N)zgit-diffrB   modesschemarC   schema_fieldr*   tag)r+   r,   r-   r.   rd   r   r   r	   r/   rF   rG   rf   r   r*   r0   rg   r1   r2   r#   rc   rc      sV    ,9=E8D!3456="0(..X"FL(3-F FHSM C#r2   rc   c                      e Zd ZU dZdZeeed         ed<   dZ	ee
eef      ed<   dZeee      ed<   dZeee      ed<   dZee   ed<   dZee   ed	<   d
Zee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZeed      ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   y)V1ScrapeOptions#Parameters for scraping operations.N
rK   rL   rM   contentrN   rP   screenshot@fullPagerO   rB   rU   formatsr\   includeTagsexcludeTagsonlyMainContentwaitFor0u  timeoutlocationmobileskipTlsVerificationremoveBase64ImagesblockAdsbasicstealthenhancedautoproxychangeTrackingOptionsmaxAgestoreInCacheparsePDF)r+   r,   r-   r.   rn   r   r   r	   r/   r\   r   r0   ro   rp   rq   boolrr   intrt   ru   rW   rv   rw   rx   ry   r   r   rc   r   r   r   r1   r2   r#   ri   ri      s3   - eiGXd7  $_  `  a  b  i(,GXd38n%,'+K$s)$+'+K$s)$+&*OXd^*!GXc]!"GXc]"+/Hh'(/!FHTN!*.$.)--#Hhtn#GKE8GBCDK?C8$;<C FHSM #'L(4.'#Hhtn#r2   ri   c                   J    e Zd ZU dZed   ed<   dZee   ed<   dZ	ee
   ed<   y)V1WaitActionz'Wait action to perform during scraping.waittypeNmillisecondsselector)r+   r,   r-   r.   r	   r/   r   r   r   r   r0   r1   r2   r#   r   r      s+    1
&/"&L(3-&"Hhsm"r2   r   c                   J    e Zd ZU dZed   ed<   dZee   ed<   dZ	ee
   ed<   y)V1ScreenshotActionz-Screenshot action to perform during scraping.rP   r   NfullPagequality)r+   r,   r-   r.   r	   r/   r   r   r   r   r   r1   r2   r#   r   r      s,    7
,
#Hhtn#!GXc]!r2   r   c                   ,    e Zd ZU dZed   ed<   eed<   y)V1ClickActionz(Click action to perform during scraping.clickr   r   Nr+   r,   r-   r.   r	   r/   r0   r1   r2   r#   r   r      s    2
'
Mr2   r   c                   ,    e Zd ZU dZed   ed<   eed<   y)V1WriteActionz(Write action to perform during scraping.writer   textNr   r1   r2   r#   r   r      s    2
'

Ir2   r   c                   ,    e Zd ZU dZed   ed<   eed<   y)V1PressActionz(Press action to perform during scraping.pressr   keyNr   r1   r2   r#   r   r      s    2
'
	Hr2   r   c                   F    e Zd ZU dZed   ed<   ed   ed<   dZee   ed<   y)V1ScrollActionz)Scroll action to perform during scraping.scrollr   )updown	directionNr   )	r+   r,   r-   r.   r	   r/   r   r   r0   r1   r2   r#   r   r      s)    3
(
|$$"Hhsm"r2   r   c                   "    e Zd ZU dZed   ed<   y)V1ScrapeActionz)Scrape action to perform during scraping.scraper   N)r+   r,   r-   r.   r	   r/   r1   r2   r#   r   r      s    3
(
r2   r   c                   ,    e Zd ZU dZed   ed<   eed<   y)V1ExecuteJavascriptActionz5Execute javascript action to perform during scraping.executeJavascriptr   scriptNr   r1   r2   r#   r   r      s    ?
%
&&Kr2   r   c                   d    e Zd ZU dZed   ed<   dZeed      ed<   dZee	   ed<   dZ
ee   ed<   y)	V1PDFActionz&PDF action to perform during scraping.pdfr   N)A0A1A2A3A4A5A6LetterLegalTabloidLedgerformat	landscapescale)r+   r,   r-   r.   r	   r/   r   r   r   r   r   floatr1   r2   r#   r   r      s@    0
%.rvFHWmnov $Ix~$!E8E?!r2   r   c                   &    e Zd ZU dZdZed   ed<   y)V1ExtractAgentr5   r(   r)   Nr6   r1   r2   r#   r   r      r7   r2   r   c                       e Zd ZU dZdZee   ed<    ej                  dd      Z
ee   ed<   dZee   ed<   dZee   ed<   y)	V1JsonConfigzConfiguration for extraction.Nr*   re   rC   rf   systemPromptagent)r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   r   r   r   r1   r2   r#   r   r      sK    ' FHSM "0(..X"FL(3-F"&L(3-&&*E8N#*r2   r   c                       e Zd ZU dZdZee   ed<   dZee   ed<   dZ	ee
eeeeeeeeeef	         ed<   dZee   ed<   dZee   ed<   y)V1ScrapeParamsrj   NrO   jsonOptionsrR   r   webhook)r+   r,   r-   r.   rO   r   r   r/   r   rR   r   r   r   r   r   r   r   r   r   r   r   r   r'   r   r[   r1   r2   r#   r   r      s    -&*GXl#**.K,'. }AGXd5/A=R_anp~  AO  Qj  lw  "w  x  y  z  A&*E8N#*)-GXo&-r2   r   c                   H    e Zd ZU dZdZeed<   dZee	   ed<   dZ
ee	   ed<   y)V1ScrapeResponsez"Response from scraping operations.TsuccessNwarningerror)r+   r,   r-   r.   r   r   r/   r   r   r0   r   r1   r2   r#   r   r      s+    ,GT!GXc]!E8C=r2   r   c                   v    e Zd ZU dZdZee   ed<   dZee   ed<   dZ	e
ed<   dZee   ed<   dZeee      ed<   y)	V1BatchScrapeResponsez&Response from batch scrape operations.NidrJ   Tr   r   invalidURLs)r+   r,   r-   r.   r   r   r0   r/   rJ   r   r   r   r   r   r1   r2   r#   r   r      sL    0BC#GTE8C='+K$s)$+r2   r   c                   |    e Zd ZU dZdZeed<   ed   ed<   eed<   eed<   eed<   e	ed	<   d
Z
ee   ed<   ee   ed<   y
)V1BatchScrapeStatusResponsez)Response from batch scrape status checks.Tr   scrapingr]   r^   	cancelledstatusr]   totalcreditsUsed	expiresAtNnextdatar+   r,   r-   r.   r   r   r/   r	   r   r   r   r   r0   r   rI   r1   r2   r#   r   r      sK    3GTBCCNJD(3-
"
##r2   r   c                   |   e Zd ZU dZdZeee      ed<   dZ	eee      ed<   dZ
ee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed	<   dZee   ed
<   dZee   ed<   dZee   ed<   dZeeeef      ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   y)V1CrawlParamsz#Parameters for crawling operations.NincludePathsexcludePathsmaxDepthmaxDiscoveryDepthlimitallowBackwardLinkscrawlEntireDomainallowExternalLinksignoreSitemapscrapeOptionsr   deduplicateSimilarURLsignoreQueryParametersregexOnFullURLdelaymaxConcurrencyallowSubdomains)r+   r,   r-   r.   r   r   r   r0   r/   r   r   r   r   r   r   r   r   r   r   r   ri   r   r   r[   r   r   r   r   r   r   r1   r2   r#   r   r      s   -(,L(49%,(,L(49%,"Hhsm"'+x}+E8C=)--(,x~,)--$(M8D>(/3M8O,359GXeC0129-1HTN1,08D>0%)NHTN)E8C=$(NHSM(&*OXd^*r2   r   c                   \    e Zd ZU dZdZee   ed<   dZee   ed<   dZ	e
ed<   dZee   ed<   y)V1CrawlResponsez"Response from crawling operations.Nr   rJ   Tr   r   )r+   r,   r-   r.   r   r   r0   r/   rJ   r   r   r   r1   r2   r#   r   r     s9    ,BC#GTE8C=r2   r   c                   |    e Zd ZU dZdZeed<   ed   ed<   eed<   eed<   eed<   e	ed	<   d
Z
ee   ed<   ee   ed<   y
)V1CrawlStatusResponsez"Response from crawl status checks.Tr   r   r   r]   r   r   r   Nr   r   r   r1   r2   r#   r   r     sK    ,GTBCCNJD(3-
"
##r2   r   c                   X    e Zd ZU dZeed<   dZee   ed<   eed<   dZ	ee   ed<   eed<   y)V1CrawlErrorzA crawl error.r   N	timestamprJ   coder   )
r+   r,   r-   r.   r0   r/   r   r   r   r   r1   r2   r#   r   r   "  s1    G$(Ix!(	HD(3-Jr2   r   c                   2    e Zd ZU dZee   ed<   ee   ed<   y)V1CrawlErrorsResponsez2Response from crawl/batch scrape error monitoring.errorsrobotsBlockedN)r+   r,   r-   r.   r   r   r/   r0   r1   r2   r#   r   r   *  s    <9r2   r   c                       e Zd ZU dZdZee   ed<   dZee	   ed<   dZ
ee	   ed<   dZee	   ed<   dZee   ed<   dZee   ed	<   dZee	   ed
<   dZee   ed<   y)V1MapParamsz"Parameters for mapping operations.Nr   r   includeSubdomainssitemapOnlyr   rs   rt   useIndexru   )r+   r,   r-   r.   r   r   r0   r/   r   r   r   r   r   r   rt   r   ru   rW   r1   r2   r#   r   r   /  sx    , FHSM $(M8D>((,x~,"&K$&E8C="GXc]"#Hhtn#+/Hh'(/r2   r   c                   N    e Zd ZU dZdZeed<   dZee	e
      ed<   dZee
   ed<   y)V1MapResponsez!Response from mapping operations.Tr   NrN   r   )r+   r,   r-   r.   r   r   r/   rN   r   r   r0   r   r1   r2   r#   r   r   :  s0    +GT!%E8DI%E8C=r2   r   c                       e Zd ZU dZdZee   ed<    ej                  dd      Z
ee   ed<   dZee   ed<   dZee   ed<   dZee   ed	<   dZee   ed
<   dZee   ed<   dZee   ed<   dZee   ed<   y)V1ExtractParamsz0Parameters for extracting information from URLs.Nr*   re   rC   rf   r   r   enableWebSearchr   originshowSourcesr   )r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   r   r   r   r  r   r  r  r   ri   r1   r2   r#   r   r   @  s    : FHSM "0(..X"FL(3-F"&L(3-&)--&*OXd^*(,x~, FHSM "&K$&/3M8O,3r2   r   c                       e Zd ZU dZdZee   ed<   dZee	d      ed<   dZ
ee   ed<   dZeed<   dZee   ed	<   dZee   ed
<   dZee   ed<   dZeeeef      ed<   dZee   ed<   y)V1ExtractResponsez!Response from extract operations.Nr   
processingr]   r^   r   r   Tr   r   r   r   sourcesr   )r+   r,   r-   r.   r   r   r0   r/   r   r	   r   r   r   r   r   r%   r   r   r  r   r   r   r   r1   r2   r#   r  r  L  s    +BEIFHW@ABI$(Ix!(GTD(1+E8C=!GXc]!(,GXd38n%,!%K#%r2   r  c                       e Zd ZU eed<   dZee   ed<   dZee   ed<   dZ	ee   ed<   dZ
ee   ed<   d	Zee   ed
<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   y)V1SearchParamsquery   r   NtbsfilterenlangusrX   ru   apir  i`  rt   r   )r+   r,   r-   r0   r/   r   r   r   r  r  r  rX   ru   r  rt   r   ri   r1   r2   r#   r
  r
  X  s    JE8C=C# FHSM D(3-!GXc]!"Hhsm"!FHSM!"GXc]"/3M8O,3r2   r
  c                   X    e Zd ZU dZdZeed<   ee   ed<   dZ	e
e   ed<   dZe
e   ed<   y)V1SearchResponsez Response from search operations.Tr   r   Nr   r   )r+   r,   r-   r.   r   r   r/   r   rI   r   r   r0   r   r1   r2   r#   r  r  d  s7    *GT
"
##!GXc]!E8C=r2   r  c                   T    e Zd ZU eed<   dZee   ed<   dZee   ed<   dZ	ee   ed<   y)V1CreditUsageDataremaining_creditsNplan_creditsbilling_period_startbilling_period_end)
r+   r,   r-   r   r/   r  r   r  r0   r  r1   r2   r#   r  r  k  s4    "&L(3-&*.(3-.(,,r2   r  c                   "    e Zd ZU eed<   eed<   y)V1CreditUsageResponser   r   N)r+   r,   r-   r   r/   r  r1   r2   r#   r  r  q  s    M
r2   r  c                   T    e Zd ZU eed<   dZee   ed<   dZee   ed<   dZ	ee   ed<   y)V1TokenUsageDataremaining_tokensNplan_tokensr  r  )
r+   r,   r-   r   r/   r   r   r  r0   r  r1   r2   r#   r  r  u  s4    !%K#%*.(3-.(,,r2   r  c                   "    e Zd ZU eed<   eed<   y)V1TokenUsageResponser   r   N)r+   r,   r-   r   r/   r  r1   r2   r#   r"  r"  {  s    M
r2   r"  c                   T    e Zd ZU dZee   ed<   dZee   ed<   dZee   ed<   e	ed<   y)V1CreditUsageHistoricalPeriodN	startDateendDateapiKeyr   
r+   r,   r-   r%  r   r0   r/   r&  r'  r   r1   r2   r#   r$  r$    s2    #Ix}#!GXc]! FHSM r2   r$  c                   (    e Zd ZU eed<   ee   ed<   y)V1CreditUsageHistoricalResponser   periodsN)r+   r,   r-   r   r/   r   r$  r1   r2   r#   r*  r*    s    M/00r2   r*  c                   T    e Zd ZU dZee   ed<   dZee   ed<   dZee   ed<   e	ed<   y)V1TokenUsageHistoricalPeriodNr%  r&  r'  
tokensUsedr(  r1   r2   r#   r-  r-    s1    #Ix}#!GXc]! FHSM Or2   r-  c                   (    e Zd ZU eed<   ee   ed<   y)V1TokenUsageHistoricalResponser   r+  N)r+   r,   r-   r   r/   r   r-  r1   r2   r#   r0  r0    s    M.//r2   r0  c                   b    e Zd ZU dZdZee   ed<   dZee	   ed<   dZ
ee	   ed<   dZee	   ed	<   y)
V1GenerateLLMsTextParamsz;
    Parameters for the LLMs.txt generation operation.
    
   maxUrlsFshowFullTextTcacheN._V1GenerateLLMsTextParams__experimental_stream)r+   r,   r-   r.   r4  r   r   r/   r5  r   r6  r7  r1   r2   r#   r2  r2    sB      GXc]#(L(4.( E8D> ,08D>0r2   r2  c                       e Zd ZU dZdZee   ed<   dZee   ed<   dZ	ee   ed<   dZ
ee   ed	<   dZee   ed
<   dZee   ed<   y)V1DeepResearchParamsz5
    Parameters for the deep research operation.
       r   i  	timeLimit   r4  NanalysisPromptr   /_V1DeepResearchParams__experimental_streamSteps)r+   r,   r-   r.   r   r   r   r/   r;  r4  r=  r0   r   r>  r   r1   r2   r#   r9  r9    s^      Hhsm"Ix}"GXc]$(NHSM("&L(3-&155r2   r9  c                   :    e Zd ZU dZeed<   eed<   dZee   ed<   y)V1DeepResearchResponsez4
    Response from the deep research operation.
    r   r   Nr   )	r+   r,   r-   r.   r   r/   r0   r   r   r1   r2   r#   r@  r@    s!     MGE8C=r2   r@  c                       e Zd ZU dZeed<   dZeee	e
f      ed<   e	ed<   dZee	   ed<   e	ed<   eed<   eed	<   eee	e
f      ed
<   eee	e
f      ed<   ee	   ed<   y)V1DeepResearchStatusResponsez;
    Status response from the deep research operation.
    r   Nr   r   r   r   currentDepthr   
activitiesr  	summaries)r+   r,   r-   r.   r   r/   r   r   r   r0   r   r   r   r   r1   r2   r#   rB  rB    sx     M%)D(4S>
")KE8C=NMT#s(^$$$sCx.!!Cyr2   rB  c                   >    e Zd ZU dZdZeed<   eed<   dZe	e   ed<   y)V1GenerateLLMsTextResponsez-Response from LLMs.txt generation operations.Tr   r   Nr   )
r+   r,   r-   r.   r   r   r/   r0   r   r   r1   r2   r#   rG  rG    s"    7GTGE8C=r2   rG  c                   ,    e Zd ZU eed<   dZee   ed<   y)$V1GenerateLLMsTextStatusResponseDatallmstxtNllmsfulltxt)r+   r,   r-   r0   r/   rK  r   r1   r2   r#   rI  rI    s    L!%K#%r2   rI  c                   b    e Zd ZU dZdZeed<   dZee	   ed<   e
d   ed<   dZee   ed<   eed	<   y)
 V1GenerateLLMsTextStatusResponsez4Status response from LLMs.txt generation operations.Tr   Nr   r  r   r   r   )r+   r,   r-   r.   r   r   r/   r   r   rI  r	   r   r0   r1   r2   r#   rM  rM    s>    >GT;?D(7
8?788E8C=Nr2   rM  c                   ^    e Zd ZU dZeed<   eeee	f      ed<   dZ
ee   ed<   dZee   ed<   y)r  z-
    Response from the search operation.
    r   r   Nr   r   )r+   r,   r-   r.   r   r/   r   r   r0   r   r   r   r   r1   r2   r#   r  r    s=     M
tCH~
!GXc]!E8C=r2   c                       e Zd ZU dZdZee   ed<    ej                  dd      Z
ee   ed<   dZee   ed<   dZee   ed	<   dZee   ed
<   dZee   ed<   dZee   ed<   dZeeeef      ed<   y)r   z/
    Parameters for the extract operation.
    Nr*   re   rC   rf   system_promptFallow_external_linksenable_web_searchr  show_sourcesr   )r+   r,   r-   r.   r*   r   r0   r/   rF   rG   rf   r   rP  rQ  r   rR  r  rS  r   r   r1   r2   r#   r   r     s     !FHSM "0(..X"FL(3-F#'M8C='+0(4.0(-x~-&+OXd^+#(L(4.(&*E8DcN#*r2   c            0          e Zd ZdZdZddee   dee   ddfdZdddddddddddddddddddddd	d
edeee	d         dee
eef      deee      deee      dee   dee   dee   dee   dee   dee   dee   dee   dee	d      dee   dee   dee   deeeeeeeeeeeef	         dee   dee   d ee   d!ee   dee   f.d"Zddddddddd#d$ed%ee   d&ee   d'ee   d(ee   d)ee   dee   dee   d*ee   defd+Zde fd,Z!de"fd-Z#dd/ede$fd0Z%dd/ede&fd1Z'ddddddddddddddddddd2dd3d
ed4eee      d5eee      d6ee   d7ee   d%ee   d8ee   d9ee   d:ee   d;ee   d*ee   d<eeee(f      d=ee   d>ee   d?ee   d@ee   dAee   dBee   d!ee   dCee   dDee   de)f,dEZ*ddddddddddddddddddddFd
ed4eee      d5eee      d6ee   d7ee   d%ee   d8ee   d9ee   d:ee   d;ee   d*ee   d<eeee(f      d=ee   d>ee   d?ee   d@ee   dAee   dBee   d!ee   dDee   de+f*dGZ,dHede)fdIZ-dHede.fdJZ/dHede
eef   fdKZ0ddddddddddddddddddddFd
ed4eee      d5eee      d6ee   d7ee   d%ee   d8ee   d9ee   d:ee   d;ee   d*ee   d<eeee(f      d=ee   d>ee   d?ee   d@ee   dAee   dBee   d!ee   dDee   ddLf*dMZ1dddddddddNd
edOee   d;ee   dPee   dQee   d%ee   dee   dRee   dee   de2fdSZ3dddddddddddddddddd2ddddTdUee   deee	dV         dee
eef      deee      deee      dee   dee   dee   dee   dee   dee   dee   dee   dee	d      dee   dee   deeeeeeeeeeeef	         dWee4   dCee   dBee   d!ee   dDee   de5f.dXZ6dddddddddddddddddddddYdUee   deee	dV         dee
eef      deee      deee      dee   dee   dee   dee   dee   dee   dee   dee   dee	d      dee   dee   deeeeeeeeeeeef	         dWee4   dBee   dDee   d!ee   de7f,dZZ8ddddddddddddddddddddd[dUee   deee	dV         dee
eef      deee      deee      dee   dee   dee   dee   dee   dee   dee   dee   dee	d      dee   dee   deeeeeeeeeeeef	         dWee4   dBee   d!ee   dDee   ddLf,d\Z9dHede5fd]Z:dHede.fd^Z;	 ddddd.d.d.dd_dUeee      d`ee   daee   dbee   d:ee   dcee   ddee   dWee
eef      de<e   fdeZ=dfede<e   fdgZ>	 ddddd.d.d.dd_dUeee      d`ee   daee   dbee   d:ee   dcee   ddee   dWee
eef      de<e   fdhZ?dddddid
edjee   dkee   dlee   dmee   de@fdnZAdddddid
edjee   dkee   dlee   dmee   deBfdoZCdHede@fdpZD	 ddDee   de
eef   fdqZE	 	 dd
edre
eef   de
eef   dsedteFdeGj                  fduZI	 	 dd
ede
eef   dsedteFdeGj                  f
dvZJ	 	 dd
ede
eef   dsedteFdeGj                  f
dwZKdHede
eef   dCede)fdxZLdyeGj                  dzeddfd{ZMd|edzed}ed~edef
dZNdddddddddd$ed6ee   dee   djee   dee   dbee   dee   deeOe
eef   gdf      deeOe
eef   gdf      dePfdZQdddddddd$ed6ee   dee   djee   dee   dbee   dee   de
eef   fdZRdHedePfdZSde
eef   deddfdZTd ZUddZVd ZWddZXd ZYd ZZd Z[ddZ\ddZ]y)V1FirecrawlAppa  
    Legacy v1 Firecrawl client for backward compatibility.
    
    This class provides the complete v1 API implementation including:
    - URL scraping with various formats and options
    - Website crawling with monitoring capabilities
    - Batch scraping operations
    - Search functionality
    - Data extraction with LLM integration
    - Deep research capabilities
    - LLMs.txt generation
    
    This is used by the unified client to provide version-specific access
    through app.v1.method_name() patterns.
    a  Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.Napi_keyapi_urlreturnc                 6   |xs t        j                  d      | _        |xs t        j                  dd      | _        d| j                  v r,| j                   t        j                  d       t        d      t        j                  d| j                          y)	z
        Initialize the V1FirecrawlApp instance with API key, API URL.

        Args:
            api_key (Optional[str]): API key for authenticating with the Firecrawl API.
            api_url (Optional[str]): Base URL for the Firecrawl API.
        FIRECRAWL_API_KEYFIRECRAWL_API_URLhttps://api.firecrawl.devzapi.firecrawl.devNz%No API key provided for cloud servicezNo API key providedz)Initialized V1FirecrawlApp with API URL: )r   getenvrV  rW  r   r   
ValueErrordebug)selfrV  rW  s      r#   __init__zV1FirecrawlApp.__init__  s{     @")),?"@]")),?A\"] $,,.4<<3GNNBC233@OPr2   rs   )rn   r\   include_tagsexclude_tagsonly_main_contentwait_forrt   ru   rv   skip_tls_verificationremove_base64_images	block_adsr   	parse_pdfrO   json_optionsrR   change_tracking_optionsmax_agestore_in_cachezero_data_retentionrJ   rn   rk   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rz   ri  rO   rj  rR   rk  rl  rm  rn  c                ,   | j                  |d       | j                         }|dt         d}|r||d<   |r||d<   |r||d<   |r||d<   |||d	<   |r||d
<   |r||d<   |	r|	j                  dd      |d<   |
|
|d<   |||d<   |||d<   |||d<   |r||d<   |||d<   |d| j	                  |      }t        |t              rd|v r| j	                  |d         |d<   t        |t              r|n|j                  dd      |d<   |d| j	                  |      }t        |t              rd|v r| j	                  |d         |d<   t        |t              r|n|j                  dd      |d<   |r6|D cg c]'  }t        |t              r|n|j                  dd      ) c}|d<   |r(t        |t              r|n|j                  dd      |d<   |||d<   |||d<   |||d<   |j                  |       d|v r)|d   r$d|d   v r| j	                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j	                  |d   d         |d   d<   t        j                  | j                   d||||dz  dz   nd       }|j                  d!k(  rW	 |j                         }|j                  d"      rd#|v rt        d(i |d#   S d$|v rt        d%|d$          t        d%|       | j!                  |d'       yc c}w # t        $ r t        d&      w xY w))aW  
        Scrape and extract content from a URL.

        Args:
          url (str): Target URL to scrape
          formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
          headers (Optional[Dict[str, str]]): Custom HTTP headers
          include_tags (Optional[List[str]]): HTML tags to include
          exclude_tags (Optional[List[str]]): HTML tags to exclude
          only_main_content (Optional[bool]): Extract main content only
          wait_for (Optional[int]): Wait for a specific element to appear
          timeout (Optional[int]): Request timeout (ms)
          location (Optional[LocationConfig]): Location configuration
          mobile (Optional[bool]): Use mobile user agent
          skip_tls_verification (Optional[bool]): Skip TLS verification
          remove_base64_images (Optional[bool]): Remove base64 images
          block_ads (Optional[bool]): Block ads
          proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
          extract (Optional[JsonConfig]): Content extraction settings
          json_options (Optional[JsonConfig]): JSON extraction settings
          actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform
          change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings
          zero_data_retention (Optional[bool]): Whether to delete data after scrape is done


        Returns:
          ScrapeResponse with:
          * Requested content formats
          * Page metadata
          * Extraction results
          * Success/error status

        Raises:
          Exception: If scraping fails
        
scrape_urlpython-sdk@rJ   r  rn   r\   ro   rp   Nrq   rr   rt   Tby_aliasexclude_noneru   rv   rw   rx   ry   r   r   re   rO   r   rR   r   r   r   zeroDataRetention
/v1/scrape     @@r  r\   rB   rt      r   r   r   Failed to scrape URL. Error: +Failed to parse Firecrawl response as JSON.z
scrape URLr1   )_validate_kwargs_prepare_headersversiondict_ensure_schema_dict
isinstanceupdaterequestspostrW  status_coderB   getr   r   r^  _handle_error)r`  rJ   rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   rk  rl  rm  rn  kwargs_headersscrape_paramsactionresponseresponse_jsons                                r#   rp  zV1FirecrawlApp.scrape_url  s6   | 	fl3((* #G9-
 '.M)$'.M)$+7M-(+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'" (1M*%..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$"PZ[rtxPy5L  @W  @\  @\  fj  y}  @\  @~M12&-M(#%,:M.)*1DM-.V$%-	*BxS`ajSkGk151I1I-XaJbckJl1mM)$X.M)mM.Jx[hiv[wOw595M5Mm\iNjksNt5uM-(2 ==||nJ'-4-@Wv%)d	
 3&	O ( $$Y/Fm4K+DmF.CDD-#&CMRYDZC[$\]]#&CM?$STT x6K (UD  O MNNOs   4,K92K> #K> >L)r   r  r  r  rX   ru   rt   scrape_optionsr  r   r  r  r  rX   r  c                   | j                  |
d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |j                  |
       |j                  d      }t	        dd|i|}|j                  d
d
      }dt
         |d<   |r||d<   t        j                  | j                   ddd| j                   i|      }|j                  dk(  rT	 |j                         }|j                  d      rd|v rt        di |S d|v rt        d|d          t        d|       | j                  |d       y# t        $ r t        d      w xY w)aS  
        Search for content using Firecrawl.

        Args:
            query (str): Search query string
            limit (Optional[int]): Max results (default: 5)
            tbs (Optional[str]): Time filter (e.g. "qdr:d")
            filter (Optional[str]): Custom result filter
            lang (Optional[str]): Language code (default: "en")
            country (Optional[str]): Country code (default: "us") 
            location (Optional[str]): Geo-targeting
            timeout (Optional[int]): Request timeout in milliseconds
            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
            **kwargs: Additional keyword arguments for future compatibility

        Returns:
            SearchResponse: Response containing:
                * success (bool): Whether request succeeded
                * data (List[FirecrawlDocument]): Search results
                * warning (Optional[str]): Warning message if any
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If search fails or response cannot be parsed
        r   Nr   r  r  r  rX   ru   rt   Trs  r   integrationr  rq  r  
/v1/searchAuthorizationBearer r\   rB   rz  r   r   r   zSearch failed. Error: r|  r1   )r}  r  r  r  r
  r  r  r  rW  rV  r  rB   r  r   r^  r  )r`  r  r   r  r  r  rX   ru   rt   r  r  search_params_integrationfinal_paramsparams_dictr  r  s                    r#   r   zV1FirecrawlApp.search  s   N 	fh/  %*M'"?#&M% &,M(#$(M&!'.M)$(0M*%'.M)$%-;-@-@$]a-@-bM/* 	V$$((7 &CEC]C"''D'I"-gY 7H)5K& ==||nJ'$~&>?
 3&	O ( $$Y/Fm4K+<m<<-#&<]7=S<T$UVV#&<]O$LMM x2  O MNNOs   7/E '#E E2c                 |   | j                         }| j                  | j                   d|      }|j                  dk(  rT	 |j	                         }|j                  d      rd|v rt        d
i |S d|v rt        d|d          t        d|       | j                  |d       y	# t        $ r t        d      w xY w)z1Get current credit usage and billing period (v1)./v1/team/credit-usagerz  r   r   r   z#Failed to get credit usage. Error: r|  zget credit usageNr1   )
r~  _get_requestrW  r  rB   r  r  r   r^  r  r`  r  r  r  s       r#   get_credit_usagezV1FirecrawlApp.get_credit_usage  s    ((*$$||n12

 3&	O ( $$Y/Fm4K0A=AA-#&I-X_J`Ia$bcc#&I-$YZZ x);<  O MNNO    /B& 0#B& &B;c                 |   | j                         }| j                  | j                   d|      }|j                  dk(  rT	 |j	                         }|j                  d      rd|v rt        d
i |S d|v rt        d|d          t        d|       | j                  |d       y	# t        $ r t        d      w xY w)z0Get current token usage and billing period (v1)./v1/team/token-usagerz  r   r   r   z"Failed to get token usage. Error: r|  zget token usageNr1   )
r~  r  rW  r  rB   r  r"  r   r^  r  r  s       r#   get_token_usagezV1FirecrawlApp.get_token_usage  s    ((*$$||n01

 3&	O ( $$Y/Fm4K/@-@@-#&HW^I_H`$abb#&H$XYY x):;  O MNNOr  F
by_api_keyc                    | j                         }| j                   d|rdndz   }| j                  ||      }|j                  dk(  rT	 |j	                         }|j                  d      rd|v rt        di |S d|v rt        d|d          t        d|       | j                  |d
       y# t        $ r t        d	      w xY w)z!Get historical credit usage (v1).z /v1/team/credit-usage/historical?byApiKey=true rz  r   r+  r   z.Failed to get historical credit usage. Error: r|  zget credit usage historicalNr1   )
r~  rW  r  r  rB   r  r*  r   r^  r  r`  r  r  rJ   r  r  s         r#   get_credit_usage_historicalz*V1FirecrawlApp.get_credit_usage_historical3  s    ((*>?WaCSgij$$S(33&	O ( $$Y/I4N:K]KK-#&TUbcjUkTl$mnn#&TUbTc$dee x)FG  O MNNO   	/B/ 9#B/ /Cc                    | j                         }| j                   d|rdndz   }| j                  ||      }|j                  dk(  rT	 |j	                         }|j                  d      rd|v rt        di |S d|v rt        d|d          t        d|       | j                  |d
       y# t        $ r t        d	      w xY w)z Get historical token usage (v1).z/v1/team/token-usage/historicalr  r  rz  r   r+  r   z-Failed to get historical token usage. Error: r|  zget token usage historicalNr1   )
r~  rW  r  r  rB   r  r0  r   r^  r  r  s         r#   get_token_usage_historicalz)V1FirecrawlApp.get_token_usage_historicalG  s    ((*=>V`BRfhi$$S(33&	O ( $$Y/I4N9JMJJ-#&STabiTjSk$lmm#&STaSb$cdd x)EF  O MNNOr     )include_pathsexclude_paths	max_depthmax_discovery_depthr   allow_backward_linkscrawl_entire_domainrQ  ignore_sitemapr  r   deduplicate_similar_urlsignore_query_parametersregex_on_full_urlr   allow_subdomainsmax_concurrencyrn  poll_intervalidempotency_keyr  r  r  r  r  r  rQ  r  r   r  r  r  r   r  r  r  r  c                &   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||j                  dd      |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |j                  |       |j                  d      }t	        d i |}|j                  dd      }||d<   dt
         |d<   |r||d<   | j                  |      }| j                  | j                   d||      }|j                  dk(  r3	 |j                         j                  d      }| j                  |||      S | j                  |d       y#  t        d      xY w)!a  
        Crawl a website starting from a URL.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            CrawlStatusResponse with:
            * Crawling status and progress
            * Crawled page contents
            * Success/error information

        Raises:
            Exception: If crawl fails
        	crawl_urlNr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   r   rv  r  rJ   rq  r  	/v1/crawlrz  r   r|  start crawl jobr1   )r}  r  r  r  r   r  r~  _post_requestrW  r  rB   r   _monitor_job_statusr  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  r  crawl_paramsr  r  r  r\   r  r   s                                 r#   r  zV1FirecrawlApp.crawl_url[  sX   ~ 	fk2 $+8L($+8L( '0L$*0CL,-$)L!*0CL,-!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+&-<L)**0CL,-F##''6 %4|4"''D'I E"-gY 7H)5K& ''8%%i&@+wW3&P]]_((. ++BGGx):;	P"MOOs   >F F)r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  c                   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||j                  dd      |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |j                  |       t        di |}|j                  dd      }||d<   dt         |d<   | j                  |      }| j                  | j                   d||      }|j                  dk(  r	 t        di |j                         S | j                  |d       y#  t        d      xY w)a  
        Start an asynchronous crawl job.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlResponse with:
            * success - Whether crawl started successfully
            * id - Unique identifier for the crawl job
            * url - Status check URL for the crawl
            * error - Error message if start failed

        Raises:
            Exception: If crawl initiation fails
        async_crawl_urlNr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   r   rv  rJ   rq  r  r  rz  r|  r  r1   )r}  r  r  r   r  r~  r  rW  r  r   rB   r   r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  r  r  r  r\   r  s                              r#   r  zV1FirecrawlApp.async_crawl_url  s%   | 	f&78 $+8L($+8L( '0L$*0CL,-$)L!*0CL,-!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+&-<L)**0CL,-F# %4|4"''D'I E"-gY 7H ''8%%i&@+wW3&P&999 x):;P"MOOs   &E Er   c                 2   d| }| j                         }| j                  | j                   | |      }|j                  dk(  rr	 |j	                         }|d   dk(  rd|v r|d   }d|v rt        |d         dk(  rn|j                  d      }|st        j                  d	       n~	 | j                  ||      }|j                  dk7  r#t        j                  d
|j                          n9	 |j	                         }	|j                  |	j                  dg              |	}d|v r||d<   |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      d}d|v r|d   |d<   d|v r|d   |d<   t        ddd|v rdndi|S | j                  |d       y#  t        d      xY w#  t        d      xY w# t
        $ r"}
t        j                  d|
        Y d}
~
d}
~
ww xY w)a$  
        Check the status and results of a crawl job.

        Args:
            id: Unique identifier for the crawl job

        Returns:
            V1CrawlStatusResponse containing:

            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of pages crawled
            * total - Total pages to crawl
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of crawled documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        
/v1/crawl/rz  r|  r   r]   r   r   r   Expected 'next' URL is missing.Failed to fetch next page: !Error during pagination request: Nr   r   r   r   r   r]   r   r   r   r   r   FTcheck crawl statusr1   )r~  r  rW  r  rB   r   lenr  r   r   r   extendr   r  r`  r   endpointr\   r  status_datar   next_urlstatus_response	next_dataes              r#   check_crawl_statusz!V1FirecrawlApp.check_crawl_statusY  sP   4  t$'')$$~hZ%@'J3&P&mmo 8$3[(&v.D K/{623q8!#.??6#:'"NN+LM!".2.?.?'.RO.::cA &/J?KfKfJg-h i %`,;,@,@,B	 !KK	fb(AB*3K# !K/* +/K' &//(3$1(__[9*}=(__[9#/H +%$/$8!$#.v#6 ( !(K!7T 
 x)=>aP"MOO$`&/2]&_ _  ) ""LL+LQC)PQ!"s=   G (AG+ -G =#G+ GG((G+ +	H4HHc                     | j                         }| j                  | j                   d| d|      }|j                  dk(  r	 t	        di |j                         S | j                  |d       y#  t        d      xY w)aL  
        Returns information about crawl errors.

        Args:
            id (str): The ID of the crawl job

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
                - id (str): Error ID
                - timestamp (str): When the error occurred
                - url (str): URL that caused the error
                - error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If error check fails
        r  /errorsrz  r|  zcheck crawl errorsNr1   r~  r  rW  r  r   rB   r   r  r`  r   r\   r  s       r#   check_crawl_errorsz!V1FirecrawlApp.check_crawl_errors  s    & '')$$~Zt7%KWU3&P,?x}}?? x)=>P"MOO   A/ /A<c                     | j                         }| j                  | j                   d| |      }|j                  dk(  r	 |j	                         S | j                  |d       y#  t        d      xY w)}  
        Cancel an asynchronous crawl job.

        Args:
            id (str): The ID of the crawl job to cancel

        Returns:
            Dict[str, Any] containing:
            * success (bool): Whether cancellation was successful
            * error (str, optional): Error message if cancellation failed

        Raises:
            Exception: If cancellation fails
        r  rz  r|  zcancel crawl jobN)r~  _delete_requestrW  r  rB   r   r  r  s       r#   cancel_crawlzV1FirecrawlApp.cancel_crawl  sz     '')''4<<.
2$(GQ3&P}}& x);<P"MOOs   A% %A2V1CrawlWatcherc                    | j                   |fi d|d|d|d|d|d|d|d|	d	|
d
|d|d|d|d|d|d|d|d|d||}|j                  r"|j                  rt        |j                  |       S t	        d      )aM  
        Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlWatcher: An instance to monitor the crawl job via WebSocket

        Raises:
            Exception: If crawl job fails to start
        r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  Crawl job failed to start)r  r   r   r  r   )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  rn  r  r  crawl_responses                          r#   crawl_url_and_watchz"V1FirecrawlApp.crawl_url_and_watch  s   r .--
'
 (
  	

 !4
 
 "6
 !4
 "6
 *
 *
 
 &>
 %<
 0
  !
" .#
$ ,%
& !4'
( ,+
. !!n&7&7!."3"3T::788r2   )r   r  include_subdomainssitemap_onlyr   rt   	use_indexru   r   r  r  r  c                   | j                  |
d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |j                  |
       |j                  d      }t	        di |}|j                  d
d
      }||d<   dt
         |d<   |r||d<   t        j                  | j                   ddd| j                   i|      }|j                  dk(  rT	 |j                         }|j                  d      rd|v rt        di |S d|v rt        d|d          t        d|       | j                  |d       y# t        $ r t        d      w xY w)a  
        Map and discover links from a URL.

        Args:
            url (str): Target URL to map
            search (Optional[str]): Filter pattern for URLs
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            include_subdomains (Optional[bool]): Include subdomain links
            sitemap_only (Optional[bool]): Only use sitemap.xml
            limit (Optional[int]): Maximum URLs to return
            timeout (Optional[int]): Request timeout in milliseconds
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1MapResponse: Response containing:
                * success (bool): Whether request succeeded
                * links (List[str]): Discovered URLs
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If mapping fails or response cannot be parsed
        map_urlNr   r   r   r   r   rt   r   Trs  ru   r  rJ   rq  r  /v1/mapr  r  r  rz  r   rN   r   zMap failed. Error: r|  mapr1   )r}  r  r  r  r   r  r  r  rW  rV  r  rB   r   r   r^  r  )r`  rJ   r   r  r  r  r   rt   r  ru   r  
map_paramsr  r  r  r  r  s                    r#   r  zV1FirecrawlApp.map_url7  s   H 	fi0 
 #)Jx %*8J').@J*+#(4J}%"'Jw$+Jy! %.Jz"%-]]Dt]%TJz" 	&!!~~m4 #0Z0"''D'I E"-gY 7H)5K& ==||nG$$~&>?
 3&	O ( $$Y/G}4L(9=99-#&9-:P9Q$RSS#&9-$IJJ x/  O MNNOs   :/E  *#E   E5)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  urls	rK   rL   rM   rl   rN   rP   rm   rO   rB   r   c                   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |r6|D cg c]'  }t        |t              r|n|j                  d
d
      ) c}|d<   ||j                  d
d
      |d<   |||d<   |||d<   |j	                  |       t        d!i |}|j                  d
d
      }||d<   dt         |d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   | j                  |      }| j                  | j                   d||      }|j                  dk(  r3	 |j                         j                  d      }| j                  |||      S | j                  |d        yc c}w #  t        d      xY w)"aH  
        Batch scrape multiple URLs and monitor until completion.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeStatusResponse with:
            * Scraping status and progress
            * Scraped content for each URL
            * Success/error information

        Raises:
            Exception: If batch scrape fails
        batch_scrape_urlsNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   rR   r   r   rv  r  rq  r  /v1/batch/scraperz  r   r|  start batch scrape jobr1   )r}  r  r  r  r  r   r  r~  r  rW  r  rB   r  r   r  r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  r  r  r  r  r  r  r   s                                 r#   r  z V1FirecrawlApp.batch_scrape_urls  s   @ 	f&9: '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$%*ZZDZ%QM'"&.=M*+*1DM-. 	V$ &66"''D'I"F"-gY 7H#I(>8{[dOeCe/3/G/GT]H^_gHh/iK	"8,K'K,F8WbcpWqKq373K3KKXeLfgoLp3qK&x0 ''8%%6F&GV]^3&P]]_((. ++BGGx)ABC (U:P"MOOs   ,KK K)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  c                   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |r6|D cg c]'  }t        |t              r|n|j                  d
d
      ) c}|d<   ||j                  d
d
      |d<   |||d<   |||d<   |j	                  |       t        d i |}|j                  d
d
      }||d<   dt         |d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   | j                  |      }| j                  | j                   d||      }|j                  dk(  r	 t        d i |j                         S | j                  |d       yc c}w #  t        d      xY w)!a~  
        Initiate a batch scrape job asynchronously.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeResponse with:
            * success - Whether job started successfully
            * id - Unique identifier for the job
            * url - Status check URL
            * error - Error message if start failed

        Raises:
            Exception: If job initiation fails
        async_batch_scrape_urlsNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   rR   r   r   rv  r  rq  r  r  rz  r|  r  r1   )r}  r  r  r  r  r   r  r~  r  rW  r  r   rB   r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  rn  r  r  r  r  r  r  s                               r#   r  z&V1FirecrawlApp.async_batch_scrape_urls!  s   @ 	f&?@ '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$%*ZZDZ%QM'"&.=M*+*1DM-. 	V$ &66"''D'I"F"-gY 7H#I(>8{[dOeCe/3/G/GT]H^_gHh/iK	"8,K'K,F8WbcpWqKq373K3KKXeLfgoLp3qK&x0 ''8%%6F&GV]^3&P,?x}}?? x)ABA (U:P"MOOs   ,J2J7 7K)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  rn  r  c                    | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |r6|D cg c]'  }t        |t              r|n|j                  d
d
      ) c}|d<   ||j                  d
d
      |d<   |||d<   |||d<   |j	                  |       t        d!i |}|j                  d
d
      }||d<   dt         |d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   | j                  |      }| j                  | j                   d||      }|j                  dk(  rS	 t        d!i |j                         }|j                  r"|j                  rt        |j                  |       S t!        d      | j#                  |d        yc c}w #  t!        d      xY w)"a   
        Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            max_concurrency (Optional[int]): Maximum number of concurrent scrapes
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket

        Raises:
            Exception: If batch scrape job fails to start
        batch_scrape_urls_and_watchNrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   rR   r   r   rv  r  rq  r  r  rz   Batch scrape job failed to startr|  r  r1   )r}  r  r  r  r  r   r  r~  r  rW  r  r   rB   r   r   r  r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  rn  r  r  r  r  r  r  r  r  s                                r#   r  z*V1FirecrawlApp.batch_scrape_urls_and_watch  s   x 	f&CD '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$%*ZZDZ%QM'"&.=M*+*1DM-. 	V$ &66"''D'I"F"-gY 7H#I(>8{[dOeCe/3/G/GT]H^_gHh/iK	"8,K'K,F8WbcpWqKq373K3KKXeLfgoLp3qK&x0 ''8%%6F&GV]^3&P!6!I!I!))n.?.?).*;*;TBB#$FGG x)ABI (UBP"MOOs   ,K+AK0 K0 0K=c                 <   d| }| j                         }| j                  | j                   | |      }|j                  dk(  rw	 |j	                         }|d   dk(  rd|v r|d   }d|v rt        |d         dk(  rn|j                  d      }|st        j                  d	       n~	 | j                  ||      }|j                  dk7  r#t        j                  d
|j                          n9	 |j	                         }	|j                  |	j                  dg              |	}d|v r||d<   t        di d|v rdnd|j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      |j                  d      d	S | j                  |d       y#  t        d      xY w#  t        d      xY w# t
        $ r"}
t        j                  d|
        Y d}
~
d}
~
ww xY w)a@  
        Check the status of a batch scrape job using the Firecrawl API.

        Args:
            id (str): The ID of the batch scrape job.

        Returns:
            V1BatchScrapeStatusResponse: The status of the batch scrape job.

        Raises:
            Exception: If the status check request fails.
        /v1/batch/scrape/rz  r|  r   r]   r   r   r   r  r  r  Nr   FTr   r   r   )	r   r   r   r]   r   r   r   r   r   zcheck batch scrape statusr1   )r~  r  rW  r  rB   r   r  r  r   r   r   r  r   r  r  s              r#   check_batch_scrape_statusz(V1FirecrawlApp.check_batch_scrape_status9  s,    'rd+'')$$~hZ%@'J3&P&mmo 8$3[(&v.D K/{623q8!#.??6#:'"NN+LM!".2.?.?'.RO.::cA &/J?KfKfJg-h i %`,;,@,@,B	 !KK	fb(AB*3K# !K/* +/K'. 
$+{$:5%//(3$1(__[9*}=(__[9#/#/$1
2 
 
 x)DEQP"MOO$`&/2]&_ _  ) ""LL+LQC)PQ!"s=   G (AG0 -G  =#G0 G G--G0 0	H9HHc                     | j                         }| j                  | j                   d| d|      }|j                  dk(  r	 t	        di |j                         S | j                  |d       y#  t        d      xY w)aX  
        Returns information about batch scrape errors.

        Args:
            id (str): The ID of the crawl job.

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
              * id (str): Error ID
              * timestamp (str): When the error occurred
              * url (str): URL that caused the error
              * error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If the error check request fails
        r  r  rz  r|  zcheck batch scrape errorsNr1   r  r  s       r#   check_batch_scrape_errorsz(V1FirecrawlApp.check_batch_scrape_errorsw  s    & '')$$~5Frd'%RT[\3&P,?x}}?? x)DEP"MOOr  r*   re   rP  rQ  rR  rS  r   r*   re   rP  rR  rS  c                H   ddl }
|
j                  dt        d       | j                  |	d       | j	                         }|s|st        d      |s|st        d      |r| j                  |      }|xs g ||||d	t                d
}|r||d<   |r||d<   |r||d<   |j                  |	       	 | j                  | j                   d||      }|j                  dk(  r	 |j                         }|d   r|j                  d      }|st        d      	 | j                  | j                   d| |      }|j                  dk(  rB	 |j                         }|d   dk(  rt!        d!i |S |d   dv r)t        d|d    d|d          | j#                  |d       t%        j&                  d       t        d|d          | j#                  |d       	 t!        dd       S #  t        d      xY w#  t        d      xY w# t        $ r}t        t)        |      d      d}~ww xY w)"a  
        Extract structured information from URLs.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If prompt/schema missing or extraction fails
        r   NThe extract endpoint is in maintenance mode and its use is discouraged. Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor to find a replacement.r  
stacklevelrO   #Either prompt or schema is required!Either urls or prompt is requiredrq  r  r   r  r  re   r  r*   r   r   /v1/extractrz  r|  r   r   )Job ID not returned from extract request./v1/extract/r   r]   r^   r   Extract job 	. Error: r   zextract-statusFailed to extract. Error:   FzInternal server error.r   r   r1   )warningswarnDeprecationWarningr}  r~  r^  r  r$   r  r  rW  r  rB   r   r  r  r  r  timesleepr0   )r`  r  r*   re   rP  rQ  rR  rS  r   r  r  r\   request_datar  r   job_idr  r  r  s                      r#   rO   zV1FirecrawlApp.extract  s   P 	%  	 	
 	fi0'')fBCCF@AA--f5F JB"60'#KM?3
 %+L"+8L($)L! 	F#)	*))<<.,H
 ##s*T#==?D 	?!XXd^F!'(STT *.*;*;#||nLA#+ +66#=`.=.B.B.D  +84C'8'G;'G G!,X!6:Q!Q&/,{8?T>UU^_jkr_s^t0u&v v ..@PQ

1# & $&@g$PQQ""8Y7 !6NOOCT#&QSS`&/2]&_ _  	*SVS))	*sP   //G> G /AG> G. G> 'A)G> G++G> .G;;G> >	H!HH!r  c                    ddl }|j                  dt        d       | j                         }	 | j	                  | j
                   d| |      }|j                  dk(  r	 t        di |j                         S | j                  |d	       y#  t        d      xY w# t        $ r}t        t        |      d
      d}~ww xY w)a  
        Retrieve the status of an extract job.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            job_id (str): The ID of the extract job.

        Returns:
            ExtractResponse[Any]: The status of the extract job.

        Raises:
            ValueError: If there is an error retrieving the status.
        r   Nr  r  r  r  rz  r|  zget extract statusr  r1   )r  r  r  r~  r  rW  r  r  rB   r   r  r^  r0   )r`  r  r  r\   r  r  s         r#   get_extract_statusz!V1FirecrawlApp.get_extract_status  s    $ 	%  	 	
 '')
	*((DLL>fX)NPWXH##s*T,?x}}?? ""8-ABT#&QSS  	*SVS))	*s/   0B B 8B BB 	B>$B99B>c                   ddl }	|	j                  dt        d       | j                         }
|}|r| j	                  |      }|||||dt
         d}|r||d<   |r||d	<   |r||d
<   	 | j                  | j                   d||
      }|j                  dk(  r	 t        di |j                         S | j                  |d       y#  t        d      xY w# t        $ r}t        t        |      d      d}~ww xY w)a  
        Initiate an asynchronous extract job.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (List[str]): URLs to extract information from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If job initiation fails
        r   Nr  r  r  rq  r  r*   r   r   r  rz  r|  zasync extractr  r1   )r  r  r  r~  r  r  r  rW  r  r  rB   r   r  r^  r0   )r`  r  r*   re   rP  rQ  rR  rS  r   r  r\   r  r  r  s                 r#   async_extractzV1FirecrawlApp.async_extract;  s.   N 	%  	 	
 '')--f5F "60'#G9-
 %+L"+8L($)L!
	*))T\\N+*FV]^H##s*T,?x}}?? ""8_=T#&QSS  	*SVS))	*s0   (/C C 1C CC 	C7C22C7max_urlsshow_full_textr6  experimental_streamr  r  r6  r  c                   t        ||||      }| j                  |||||      }|j                  r|j                  st	        dddd      S |j                  }	 | j                  |      }	|	j                  dk(  r|	S |	j                  dk(  r|	S |	j                  d	k7  rt	        dd
dd      S t        j                  d       g)a  
        Generate LLMs.txt for a given URL and poll until completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            GenerateLLMsTextStatusResponse with:
            * Generated LLMs.txt content
            * Full version if requested
            * Generation status
            * Success/error information

        Raises:
            Exception: If generation fails
        r4  r5  r6  __experimental_streamr  Fz#Failed to start LLMs.txt generationr^   r  r   r   r   r   r]   r  /LLMs.txt generation job terminated unexpectedlyr  )	r2  async_generate_llms_textr   r   rM  check_generate_llms_text_statusr   r  r  )
r`  rJ   r  r  r6  r  paramsr  r  r   s
             r#   generate_llms_textz!V1FirecrawlApp.generate_llms_text  s    8 *'"5	
 00) 3 1 
 x{{3;	  99&AF}}+(*,.7!K# 	  JJqM r2   c                   t        ||||      }| j                         }d|i|j                  dd      }dt         |d<   	 | j	                  | j
                   d||      }	|	j                         }
t        d|       t        d	|
       |
j                  d
      r	 t        di |
S | j                  |
d       	 t        dd      S #  t        d      xY w# t        $ r}t        t        |            d}~ww xY w)a  
        Initiate an asynchronous LLMs.txt generation operation.

        Args:
            url (str): The target URL to generate LLMs.txt from. Must be a valid HTTP/HTTPS URL.
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            GenerateLLMsTextResponse: A response containing:
            * success (bool): Whether the generation initiation was successful
            * id (str): The unique identifier for the generation job
            * error (str, optional): Error message if initiation failed

        Raises:
            Exception: If the generation job initiation fails.
        r  rJ   Trs  rq  r  /v1/llmstxt	json_datar  r   r|  zstart LLMs.txt generationNFInternal server errorr  r1   )r2  r~  r  r  r  rW  rB   r   r  rG  r   r  r^  r0   )r`  rJ   r  r  r6  r  r#  r\   r'  reqr  r  s               r#   r!  z'V1FirecrawlApp.async_generate_llms_text  s   6 *'"5	
 '')CQ6;;4;#PQ	 +G95	(	%$$~[%A9gVCxxzH+y)*h'||I&S5AAA ""8-HI *)
 	
S#$QRR  	%SV$$	%s1   AC 
C	 )C 	CC 	C;"C66C;c                    | j                         }	 | j                  | j                   d| |      }|j                  dk(  r	 |j	                         }t        di |S |j                  dk(  rt        d      | j                  |d       	 t        dd	d
d      S # t        $ r}t        dt        |             d}~ww xY w# t        $ r}t        t        |            d}~ww xY w)a;  
        Check the status of a LLMs.txt generation operation.

        Args:
            id (str): The unique identifier of the LLMs.txt generation job to check status for.

        Returns:
            GenerateLLMsTextStatusResponse: A response containing:
            * success (bool): Whether the generation was successful
            * status (str): Status of generation ("processing", "completed", "failed")
            * data (Dict[str, str], optional): Generated text with fields:
              * llmstxt (str): Generated LLMs.txt content
              * llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            Exception: If the status check fails.
        /v1/llmstxt/rz  zFFailed to parse Firecrawl response as GenerateLLMsTextStatusResponse: N  z!LLMs.txt generation job not foundz check LLMs.txt generation statusFr(  r^   r  r  r1   )
r~  r  rW  r  rB   rM  r   r0   r  r^  )r`  r   r\   r  r'  r  s         r#   r"  z.V1FirecrawlApp.check_generate_llms_text_status	  s    ( '')	%((DLL>bT)JGTH##s*w (I;HiHH %%, CDD""8-OP 0E\emy{|| ! w#&lmpqrmslt$uvvw  	%SV$$	%s;   0C B ,C 	B?#B::B??C 	C$CC$c                 P    |rdd| j                    |dS dd| j                    dS )a$  
        Prepare the headers for API requests.

        Args:
            idempotency_key (Optional[str]): A unique key to ensure idempotency of requests.

        Returns:
            Dict[str, str]: The headers including content type, authorization, and optionally idempotency key.
        zapplication/jsonr  )Content-Typer  zx-idempotency-key)r.  r  )rV  )r`  r  s     r#   r~  zV1FirecrawlApp._prepare_headers1	  sB      2#*4<<.!9%4  /&t||n5
 	
r2   r   retriesbackoff_factorc                     t        |      D ]]  }t        j                  |||d|v r|d   |d   dz  dz   nd      }|j                  dk(  rt	        j
                  |d|z  z         [|c S  S )a^  
        Make a POST request with retries.

        Args:
            url (str): The URL to send the POST request to.
            data (Dict[str, Any]): The JSON data to include in the POST request.
            headers (Dict[str, str]): The headers to include in the POST request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the POST request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        rt   Nrx  r  ry    r  )ranger  r  r  r  r  )r`  rJ   r   r\   r/  r0  attemptr  s           r#   r  zV1FirecrawlApp._post_requestI	  s    . W~ 	 G}}S'qz  C  rC  HL  MV  HW  HcPTU^P_bhPhklPl  im  oH##s*

>Q'\:;	  r2   c                     t        |      D ]G  }t        j                  ||      }|j                  dk(  rt	        j
                  |d|z  z         E|c S  S )a	  
        Make a GET request with retries.

        Args:
            url (str): The URL to send the GET request to.
            headers (Dict[str, str]): The headers to include in the GET request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the GET request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        r\   r2  r  )r3  r  r  r  r  r  r`  rJ   r\   r/  r0  r4  r  s          r#   r  zV1FirecrawlApp._get_requesth	  sV    * W~ 	 G||C9H##s*

>Q'\:;	  r2   c                     t        |      D ]G  }t        j                  ||      }|j                  dk(  rt	        j
                  |d|z  z         E|c S  S )a  
        Make a DELETE request with retries.

        Args:
            url (str): The URL to send the DELETE request to.
            headers (Dict[str, str]): The headers to include in the DELETE request.
            retries (int): Number of retries for the request.
            backoff_factor (float): Backoff factor for retries.

        Returns:
            requests.Response: The response from the DELETE request.

        Raises:
            requests.RequestException: If the request fails after the specified retries.
        r6  r2  r  )r3  r  deleter  r  r  r7  s          r#   r  zV1FirecrawlApp._delete_request	  sV    * W~ 	 GsG<H##s*

>Q'\:;	  r2   c                    	 | j                    d| }| j                  ||      }|j                  dk(  r	 |j                         }|d   dk(  rd|v rw|d   }d|v r^t        |d         dk(  rnL| j                  |d   |      }	 |j                         }|j                  |j                  dg              d|v r^||d<   t        di |S t	        d	      |d   d
v r"t        |d      }t        j                  |       n#t	        d|d          | j                  |d       #  t	        d      xY w#  t	        d      xY w)a  
        Monitor the status of a crawl job until completion.

        Args:
            id (str): The ID of the crawl job.
            headers (Dict[str, str]): The headers to include in the status check requests.
            poll_interval (int): Seconds between status checks.

        Returns:
            CrawlStatusResponse: The crawl results if the job is completed successfully.

        Raises:
            Exception: If the job fails or an error occurs during status checks.
        r  rz  r|  r   r]   r   r   r   z,Crawl job completed but no data was returnedactivepausedpendingqueuedwaitingr   r  z)Crawl job failed or was stopped. Status: r  r1   )rW  r  r  rB   r   r  r  r  r   maxr  r  r  )r`  r   r\   r  rW  r  r  r   s           r#   r  z"V1FirecrawlApp._monitor_job_status	  s   & j5G"//AO**c1T"1"6"6"8K x(K7,*62$3";v#671< %.2.?.?F@SU\.]O`.=.B.B.D !KK(CD %3 /3F+4C{CC'(VWW *.nn"%mA"6MJJ}-#&OP[\dPeOf$ghh""?4HI? T#&QSS`&/2]&_ _s   D! D1 !D.1D>r  r  c                    	 |j                         }|j                  dd      }|j                  dd      }| j                  |j                  |||      }t        j                  j                  ||      #  	 |j                  dd }|j                         rd| }d|j                   }nd	|j                   }d
}n # t
        $ r d|j                   }d
}Y nw xY wY xY w)ah  
        Handle errors from API responses.

        Args:
            response (requests.Response): The response object from the API request.
            action (str): Description of the action that was being performed.

        Raises:
            Exception: An exception with a message containing the status code and error details from the response.
        r   No error message provided.details%No additional error details provided.Nr  z#Server returned non-JSON response: zFull response status: z+Server returned empty response with status zNo additional details availablez0Server returned unreadable response with status )r  )
rB   r  r   r   r  r^  _get_error_messager  
exceptions	HTTPError)r`  r  r  r  error_messageerror_detailsresponse_textmessages           r#   r  zV1FirecrawlApp._handle_error	  s   	B$MMOM)--g7STM)--i9`aM ))(*>*>Wde !!++Gh+GG#	B
B (ds 3 &&(&I-$YM&<X=Q=Q<R$SM&QRZRfRfQg$hM$EM B"RS[SgSgRh i ABs0   4A5 5C 8AB>=C >CC CC r  rI  rJ  c                     |dk(  rd| d| d| S |dk(  rd| d| d| S |dk(  rd| d	| d| S |d
k(  rd| d| d| S |dk(  rd| d| d| S d| d| d| d| S )a  
        Generate a standardized error message based on HTTP status code.
        
        Args:
            status_code (int): The HTTP status code from the response
            action (str): Description of the action that was being performed
            error_message (str): The error message from the API response
            error_details (str): Additional error details from the API response
            
        Returns:
            str: A formatted error message
        i  zPayment Required: Failed to z. z - i  z!Website Not Supported: Failed to i  zRequest Timeout: Failed to z as the request timed out. i  zConflict: Failed to z due to a conflict. r  z!Internal Server Error: Failed to zUnexpected error during z: Status code r1   r`  r  r  rI  rJ  s        r#   rF  z!V1FirecrawlApp._get_error_message	  s     #1&M?#m_]]C6vhbsS`RabbC08STaSbbefsetuuC)&1Em_TWXeWfggC6vhbsS`Rabb-fX^K=PRS`Raaderdsttr2   )r  
time_limitr  analysis_promptrP  *_V1FirecrawlApp__experimental_stream_stepson_activity	on_sourcerO  rP  rQ  rR  rS  c                T   i }
|||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   t        di |
}
| j                  ||||||      }|j                  d	      rd
|vr|S |d
   }d}d}	 | j                  |      }|r)d|v r%|d   |d }|D ]
  } ||        t	        |d         }|	r)d|v r%|d   |d }|D ]
  } |	|        t	        |d         }|d   dk(  r|S |d   dk(  rt        d|j                  d             |d   dk7  rnt        j                  d       dddS a  
        Initiates a deep research operation on a given query and polls until completion.

        Args:
            query (str): Research query or topic to investigate
            max_depth (Optional[int]): Maximum depth of research exploration
            time_limit (Optional[int]): Time limit in seconds for research
            max_urls (Optional[int]): Maximum number of URLs to process
            analysis_prompt (Optional[str]): Custom prompt for analysis
            system_prompt (Optional[str]): Custom system prompt
            __experimental_stream_steps (Optional[bool]): Enable experimental streaming
            on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}
            on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}

        Returns:
            DeepResearchStatusResponse containing:
            * success (bool): Whether research completed successfully
            * status (str): Current state (processing/completed/failed)
            * error (Optional[str]): Error message if failed
            * id (str): Unique identifier for the research job
            * data (Any): Research findings and analysis
            * sources (List[Dict]): List of discovered sources
            * activities (List[Dict]): Research progress log
            * summaries (List[str]): Generated research summaries

        Raises:
            Exception: If research fails
        Nr   r;  r4  r=  r   __experimental_streamSteps)r  rO  r  rP  rP  r   r   r   rD  r  r   r]   r^   zDeep research failed. Error: r   r  r  Fz)Deep research job terminated unexpectedlyr  r1   )r9  async_deep_researchr  check_deep_research_statusr  r   r  r  )r`  r  r  rO  r  rP  rP  rQ  rR  rS  research_paramsr  r  last_activity_countlast_source_countr   new_activitiesactivitynew_sourcessources                       r#   deep_researchzV1FirecrawlApp.deep_research
  s   P  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AA++!+' , 
 ||I&$h*>O$44V<F|v5!'!56I6J!K . *H)*&)&*>&?#Y&0$Y/0A0BC) &Ff%&$'y(9$:!h;.!X-"?

7@S?T UVV!\1JJqM- 0 !+VWWr2   )r  rO  r  rP  rP  rQ  c                `   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   t        di |}| j                         }	d|i|j                  d	d	
      }
dt         |
d<   d|
v r3|
d   }|r,d|v r(t	        |d   d      r|d   j                         |
d   d<   	 | j                  | j                   d|
|	      }|j                  dk(  r	 |j                         S | j                  |d       	 dddS #  t        d      xY w# t        $ r}t        t        |            d}~ww xY w)  
        Initiates an asynchronous deep research operation.

        Args:
            query (str): Research query or topic to investigate
            max_depth (Optional[int]): Maximum depth of research exploration
            time_limit (Optional[int]): Time limit in seconds for research
            max_urls (Optional[int]): Maximum number of URLs to process
            analysis_prompt (Optional[str]): Custom prompt for analysis
            system_prompt (Optional[str]): Custom system prompt
            __experimental_stream_steps (Optional[bool]): Enable experimental streaming

        Returns:
            Dict[str, Any]: A response containing:
            * success (bool): Whether the research initiation was successful
            * id (str): The unique identifier for the research job
            * error (str, optional): Error message if initiation failed

        Raises:
            Exception: If the research initiation fails.
        Nr   r;  r4  r=  r   rV  r  Trs  rq  r  r   re   /v1/deep-researchrz  r|  zstart deep researchFr(  r  r1   )r9  r~  r  r  hasattrre   r  rW  r  rB   r   r  r^  r0   )r`  r  r  rO  r  rP  rP  rQ  rY  r\   r'  	json_optsr  r  s                 r#   rW  z"V1FirecrawlApp.async_deep_researchu
  s   >  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AA'')e^';';TX\';']^	 +G95	( I%!-0IX2wy?RT\7]5>x5H5O5O5Q	-(2
	%))T\\N:K*LiY`aH##s*S#==?* ""8-BC !+BCCS#$QRR  	%SV$$	%s0   #/D C; #D ;DD 	D-D((D-c                 t   | j                         }	 | j                  | j                   d| |      }|j                  dk(  r	 |j	                         S |j                  dk(  rt        d      | j                  |d       	 dd	d
S #  t        d      xY w# t
        $ r}t        t        |            d}~ww xY w)  
        Check the status of a deep research operation.

        Args:
            id (str): The ID of the deep research operation.

        Returns:
            DeepResearchResponse containing:

            Status:
            * success - Whether research completed successfully
            * status - Current state (processing/completed/failed)
            * error - Error message if failed
            
            Results:
            * id - Unique identifier for the research job
            * data - Research findings and analysis
            * sources - List of discovered sources
            * activities - Research progress log
            * summaries - Generated research summaries

        Raises:
            Exception: If the status check fails.
        /v1/deep-research/rz  r|  r,  zDeep research job not foundzcheck deep research statusNFr(  r  )	r~  r  rW  r  rB   r   r  r^  r0   )r`  r   r\   r  r  s        r#   rX  z)V1FirecrawlApp.check_deep_research_status
  s    2 '')	%((DLL>9KB4)PRYZH##s*S#==?* %%, =>>""8-IJ !+BCCS#$QRR
  	%SV$$	%s/   0B B ,B BB 	B7B22B7r  method_namec           	          |syh dh dh dh dh dh dh dh dd}|j                  |t                     }t        |j                               |z
  }|r!t        d	| d
dj	                  |       d      y)a  
        Validate additional keyword arguments before they are passed to the API.
        This provides early validation before the Pydantic model validation.

        Args:
            kwargs (Dict[str, Any]): Additional keyword arguments to validate
            method_name (str): Name of the method these kwargs are for

        Raises:
            ValueError: If kwargs contain invalid or unsupported parameters
        N>   r   rv   rR   rO   rn   rl  rt   ru   re  rh  r  rc  rb  rj  rd  rg  rf  rk  >	   r  r  r   r  rX   rt   ru   r  r  >   r   r   r  r  r  r  r  r  r  r  r  rQ  r  r  >   r   r   rt   r  r  r  r  >   r   r*   re   r  rS  rP  rR  rQ  >   r   r   rv   rR   rO   rn   r\   rt   r   ru   re  rh  rc  rb  rj  rd  rg  rf  )rp  r   r  r  rO   r  r  r  zUnsupported parameter(s) for z: z, zC. Please refer to the API documentation for the correct parameters.)r  setkeysr^  r   )r`  r  ri  method_paramsallowed_paramsunknown_paramss         r#   r}  zV1FirecrawlApp._validate_kwargs
  s     K | } R"@(F,J%
2 '**;> V[[]+n<<[MDIIVdLeKf  gj  k  l  l r2   c                    ||S t        |t              r8t        |d      r|j                         S t        |d      r|j	                         S t        |t
              r3|j                         D ci c]  \  }}|| j                  |       c}}S t        |t        t        f      r|D cg c]  }| j                  |       c}S |S c c}}w c c}w )zw
        Utility to ensure a schema is a dict, not a Pydantic model class. Recursively checks dicts and lists.
        model_json_schemare   )
r  r   rd  rq  re   r  itemsr  listtuple)r`  re   kvs       r#   r  z"V1FirecrawlApp._ensure_schema_dict  s     >Mfd#v23//11*}}&fd#?E||~Ntq!At//22NNftUm,9?@AD,,Q/@@ O@s   0C*Cc                 >   |rt        |t        t        f      sy|
t               }ddl} |j
                  |dt              }||v ry|j                  |       	 t        |t              rd|v rt        |d   t              r{|d   j                  d      }t        |      dk\  rY|d   d	k(  rQ|d
   dk(  rI|d   }||k(  r	 |j                  |       y||v r(| j                  ||   |||      |j                  |       S |j                         D ])  }	| j                  |	|||      s |j                  |       y n>t        |t              r.|D ])  }
| j                  |
|||      s |j                  |       y |j                  |       y# |j                  |       w xY w)a  
        Check if an object contains a recursive reference to a specific definition.
        
        Args:
            obj: Object to check
            target_def_name: Name of the definition to check for recursion
            defs: Dictionary of definitions
            visited: Set of visited object keys to detect cycles
            
        Returns:
            True if recursive reference is found, False otherwise
        FNr   T)	sort_keysdefault$ref/   #r   $defs)r  r  rs  rk  rB   dumpsr0   addsplitr  discard_contains_recursive_refvalues)r`  objtarget_def_namedefsvisitedrB   obj_keyref_pathdef_namevalueitems              r#   r  z&V1FirecrawlApp._contains_recursive_ref+  s    *S4,7?eG$**SD#>gG	%#t$S=ZFS%A"6{005H8})hqkS.@Xa[T[E[#+B<#6#' OOG$ $t+#'#?#?XP_aegn#o OOG$ !ZZ\ $E33E?DRYZ# OOG$$ C& $D33D/4QXY# OOG$$
 OOG$ OOG$s1   A#F	 F	 <'F	 $F	 7+F	 #F	 6F	 	Fc                 d    |sy|j                         D ]  \  }}| j                  |||      s y y)z
        Check if $defs contain circular references.
        
        Args:
            defs: Dictionary of definitions to check
            
        Returns:
            True if circular references are found, False otherwise
        FT)rr  r  )r`  r  r  	def_values       r#   _check_for_circular_defsz'V1FirecrawlApp._check_for_circular_defs]  s>     #'::< 	Hi++IxF	 r2   c           
      t   |rt        |t        t        f      r|dkD  r|S |
t               }t	        |      }||v r|S |j                  |       	 t        |t              rd|v rt        |d   t              r|d   j                  d      }t        |      dk\  rM|d   dk(  rE|d   dk(  r=|d	   }||v r4| j                  t        ||         |||dz         |j                  |       S ||j                  |       S i }|j                         D ]%  \  }	}
|	dk(  r| j                  |
|||dz         ||	<   ' ||j                  |       S t        |t              r6|D cg c]  }| j                  ||||dz          c}|j                  |       S 	 |j                  |       |S c c}w # |j                  |       w xY w)
a|  
        Resolve $ref references in a JSON schema object.
        
        Args:
            obj: Object to resolve references in
            defs: Dictionary of definitions
            visited: Set to track visited objects and prevent infinite recursion
            depth: Current recursion depth
            
        Returns:
            Object with resolved references
        r3  rz  r{  r|  r   r}  r   r~  r  )r  r  rs  rk  r   r  r0   r  r  _resolve_refsr  rr  )r`  r  r  r  depthobj_idr  r  resolvedr   r  r  s               r#   r  zV1FirecrawlApp._resolve_refsp  s    *S4,752:J?eGCWJF	$#t$S=ZFS%A"6{005H8})hqkS.@Xa[T[E[#+B<#t+#'#5#5d4>6JDRY[`cd[d#e OOF#  OOF# "%))+ XJCg~ $($6$6udGUUVY$WHSMX   OOF#	 C&WZ[t**4w	J[ OOF#	 ' OOF#
 \ OOF#s1   BF$ %F$ 8;F$ F$ F7F$ F$ $F7c                 ^    |rt        |t              s|S t               fd |      S )z
        Normalize a schema for OpenAI compatibility by handling recursive references.
        
        Args:
            schema: Schema to normalize
            
        Returns:
            Normalized schema
        c           
         t        | t              s(t        | t              r| D cg c]
  } |       c}S | S t        |       }|v r| S j	                  |       t        |       }d|v rj                  |       |S d|v r|j                  d      }i }|j                         D ]*  \  }}t        |t              rd|vr |      ||<   &|||<   , i |d|j                         D 	ci c]  \  }}	| |	       c}	}i}
j                  |       |
S |j                  d      dk(  rd|v r|j                  d      du r|d= |j                  d      dk(  rVd|v rRd|v rNt        |d   t              r8t        |d   t              r%|d   D cg c]  }||d   v r| }}|r||d<   n|d= n|d= t        |j                               D ]a  \  }}t        |t              rd|vr |      ||<   &t        |t              s7|D cg c]  }t        |t              r |      n| c}||<   c j                  |       |S c c}w c c}	}w c c}w c c}w )	Nrz  r~  r   object
propertiesadditionalPropertiesTrequired)	r  r  rs  r   r  r  poprr  r  )r  r  r  
normalizedr  processed_restr   r  ru  rv  resultfieldvalid_requirednormalize_objectr  s                r#   r  zEV1FirecrawlApp._normalize_schema_for_openai.<locals>.normalize_object  s   c4(c4(?BCt,T2CC
 WF 
KKcJ #'!!*$!~~g.!#","2"2"4 4JC"5$/F%4G.>u.Es+.3s+	4 hNgGY]YcYcYe5fQUQRTUa9I!9L6L5fg' v&(2
*56$>56v&(2j(
*z*5t<z,7>9CJ9O &I&+z,/G&G ', &IN &I%1?
:.&z2":. #:#3#3#56 u
UeT*vU/B&6u&=JsOt,ns&tfjDRVAW'7'=]a'a&tJsOu OOF#w D4 6g&I 'us   H=)I
I!Ir  r  rk  )r`  re   r  r  s     @@r#   _normalize_schema_for_openaiz+V1FirecrawlApp._normalize_schema_for_openai  s0     Z5M%>	@  ''r2   c                 ^    |rt        |t              syt               fd |       S )z
        Validate schema for OpenAI compatibility.
        
        Args:
            schema: Schema to validate
            
        Returns:
            True if schema is valid, False otherwise
        Tc                 X   t        | t              syt        |       }|v ryj                  |       d| v rj	                  |       y| j                  d      dk(  r-d| vr)d| vr%| j                  d      du rj	                  |       y| j                         D ]|  }t        |t              r d|vr |      s j	                  |        yt        |t              sD|D ]4  }t        |t              sd|vs |      s"j	                  |         y ~ j	                  |       y)	NFrz  r   r  r  patternPropertiesr  T)r  r  r   r  r  r  r  rs  )r  r  r  r  has_invalid_structurer  s       r#   r  zIV1FirecrawlApp._validate_schema_for_openai.<locals>.has_invalid_structure  s   c4(WF KK}'8+C'#3../47'  ,eT*vU/B,U3/#t, % ,%dD1fD6H4T: ' 7'+	,, OOF#r2   r  )r`  re   r  r  s     @@r#   _validate_schema_for_openaiz*V1FirecrawlApp._validate_schema_for_openai  s1     Z5%#	J )000r2   c                     |rt        |t              syddl} |j                  |      }d|v xs
 d|v xs d|v }t	        |j                  d      xs |j                  d            }|xs |S )	z
        Detect if a schema contains recursive references.
        
        Args:
            schema: Schema to analyze
            
        Returns:
            True if schema has recursive patterns, False otherwise
        Fr   N"$ref"#/$defs/z#/definitions/r~  definitions)r  r  rB   r  r   r  )r`  re   rB   schema_stringhas_refshas_defss         r#   _detect_recursive_schemaz'V1FirecrawlApp._detect_recursive_schema)  s{     Z5"

6*% .-'.- 	
 

7+Hvzz-/HI#8#r2   c                 F    |sdddS | j                  |      rdddS dddS )z
        Select appropriate model based on schema complexity.
        
        Args:
            schema: Schema to analyze
            
        Returns:
            Dict with modelName and reason
        zgpt-4o-mini	no_schema)	modelNamereasonzgpt-4orecursive_schema_detectedsimple_schema)r  )r`  re   s     r#   _select_model_for_schemaz'V1FirecrawlApp._select_model_for_schemaA  s7     !.+FF((0!)5PQQ*oFFr2   c                    t        |t              r#||v r| j                  ||         }|rt        |t              r|j                  di       }ddl} |j
                  |      }|j                  d      xs
 d|v xs d|v }|rK	 | j                  ||      } |j
                  |      }	d|	v xs d|	v }
|
s|}t        |t              rd|v r|d= n*	 | j                  ||      }t        |t              rd|v r|d= | j                  |      }| j                  |      st        | j                        |||<   |S # t        $ r Y Iw xY w# t        $ r Y Xw xY w)a1  
        Process and validate a schema container (like extract or json_options).
        
        Args:
            schema_container: Dict containing schema
            schema_key: Key where schema is stored (default: "schema")
            
        Returns:
            Processed schema container
        r~  r   Nr  r  )r  r  r  r  rB   r  r  r   r  r  r^  OPENAI_SCHEMA_ERROR_MESSAGE)r`  schema_container
schema_keyre   r  rB   r  has_any_refsresolved_schemaresolved_stringhas_remaining_refss              r#   _process_schema_with_validationz.V1FirecrawlApp._process_schema_with_validationS  s    &-*@P2P--.>z.JKF *VT2zz'2. *

6 2JJw' 0-0-/   *.*<*<VT*J*4$**_*E-5-H-iJZiLi*1%4F)&$7Gv<M$*7O!%!3!3FD!A%fd368I &w 66v>F33F; !A!ABB+1Z(# %  % s%   A	D7 )E 7	EE	EENN)FNr|        ?)Nr   )re   )^r+   r,   r-   r.   r  r   r0   ra  r   r	   r   r   r   rW   r   r   r   r   r   r   r   r   r   r   r   rc   r   r   rp  ri   r  r   r  r  r"  r  r*  r  r0  r  r[   r   r  r   r  r  r   r  r  r  r   r  r'   r   r  r   r  r  r  r  r  rO   r  r  rM  r$  rG  r!  r"  r~  r   r  Responser  r  r  r  r  rF  r   rB  r`  rW  rX  r}  r  r  r  r  r  r  r  r  r  r1   r2   r#   rU  rU    s   $	q  Q Qx} QX\ Q, mq04040404&*%*37%)4837(,OS(,.237 EIIM%)-1261U7U7 d7  ,g  $h  i  j	U7
 d38n-U7 #49-U7 #49-U7  (~U7 smU7 c]U7 /0U7 TNU7 $,D>U7 #+4.U7  ~U7  G$JKL!U7"  ~#U7$ l+%U7& #<0'U7( d57I=Zgiv  yG  IW  Yr  t  *  $@  A  B)U7* &..E%F+U7, c]-U7. %TN/U70 "*$1U72 *#.3U7v $(!%$("&%)&*%*8<]3]3 C=	]3
 #]3 SM]3 3-]3 c]]3 sm]3 c]]3 %_5]3 *]3~="7 =,<!5 <,Hd HGf H(GT GFd G0 .2-1#'-1#/3.2/3)-489=3726,0#+/)-.2'()-/@<@<  S	*	@<
  S	*@< C=@< &c]@< }@< 'tn@< &d^@< 'tn@< !@< !1@< %_ 456@< #+4.@<  "*$!@<" $D>#@<$ }%@<& #4.'@<( "#)@<* &d^+@<,  }-@<. "#/@<2 
3@<L .2-1#'-1#/3.2/3)-489=3726,0#+/)-.2)--z<z<  S	*	z<
  S	*z< C=z< &c]z< }z< 'tnz< &d^z< 'tnz< !z< !1z< %_ 456z< #+4.z<  "*$!z<" $D>#z<$ }%z<& #4.'z<( "#)z<* &d^+z<, "#-z<0 
1z<xQ?S Q?-B Q?f?S ?-B ?:=s =tCH~ =: 2615'+15#'372637-18<=A7;6:04#'/3-126-1-S9S9 $DI.	S9
 $DI.S9  }S9 "*#S9 C=S9 #+4.S9 "*$S9 #+4.S9 %TNS9 %_5S9 eC$89:S9 '/tnS9  &.d^!S9"  (~#S9$ C=%S9& 'tn'S9( &c])S9* "*$+S9, &c]-S90 
1S9r %)-115+/#'%*(,37[0[0 SM	[0
 %TN[0 !)[0 #4.[0 C=[0 c][0  ~[0 /0[0 '[0B W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.'()-.2)-1KC3iKC $w  (Q   R  S  T	KC
 $sCx.)KC tCy)KC tCy)KC $D>KC 3-KC #KC +,KC KC  (~KC 'tnKC D>KC   FGH!KC" ,'#KC$ |,%KC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'KC( ')KC*  }+KC, "#-KC. &d^/KC0 "#1KC4 
%5KCb W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.)-)-.2/JC3iJC $w  (Q   R  S  T	JC
 $sCx.)JC tCy)JC tCy)JC $D>JC 3-JC #JC +,JC JC  (~JC 'tnJC D>JC   FGH!JC" ,'#JC$ |,%JC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JC( ')JC* "#+JC, "#-JC. &d^/JC2 
3JC` W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.)-.2)-/JC3iJC $w  (Q   R  S  T	JC
 $sCx.)JC tCy)JC tCy)JC $D>JC 3-JC #JC +,JC JC  (~JC 'tnJC D>JC   FGH!JC" ,'#JC$ |,%JC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JC( ')JC* "#+JC, &d^-JC. "#/JC2 
3JCX<FC <F4O <F|FC F4I F> )-~P %)$(+/3805+0.2~P49%~P SM	~P
 SM~P $C=~P #+4.~P  (~~P #4.~P DcN+~P +3/~P@%* %*1B31G %*R )-O* %)$(+/3805+0.2O*49%O* SM	O*
 SMO* $C=O* #+4.O*  (~O* #4.O* DcN+O* 8I7MO*j '+-1$(26CC sm	C
 %TNC D>C "*$C <\CR '+-1$(268
8
 sm	8

 %TN8
 D>8
 "*$8
 <V8
t$}# $}:Z $}P .2
%c]
6:38n
: $' sCx. #s(^	
  " -5,=,=F $' #s(^ 	
 "
 -5,=,=B $' #s(^ 	
 "
 -5,=,=:2J2J #s(^2J 	2J $9	2Jh#H''#H #H !#HJuc u3 us ucf ukn u< (,(,&*-1+/:>FJDH^X^X  }	^X
 !^X sm^X &c]^X $C=^X *2$^X "(DcN+;T+A"BC^X  $sCx.)94)? @A^X Nj^XH (,(,&*-1+/:>EDED  }	ED
 !ED smED &c]ED $C=ED *2$ED DHS>EDN(DS (D5Q (DT.ltCH~ .lC .lD .lf$0d&0dO(b41l$0G$6 r2   rU  c                       e Zd ZdZdedefdZddZddZd	ed
e	e
eef   gdf   ddfdZd	ede
eef   ddfdZde
eef   ddfdZy)r  a  
    A class to watch and handle crawl job events via WebSocket connection.

    Attributes:
        id (str): The ID of the crawl job to watch
        app (V1FirecrawlApp): The V1FirecrawlApp instance
        data (List[Dict[str, Any]]): List of crawled documents/data
        status (str): Current status of the crawl job
        ws_url (str): WebSocket URL for the crawl job
        event_handlers (dict): Dictionary of event type to list of handler functions
    r   appc                     || _         || _        g | _        d| _        |j                  j                  dd       d| | _        g g g d| _        y )Nr   httpwsr  )doner   document)r   r  r   r   rW  replacews_urlevent_handlers)r`  r   r  s      r#   ra  zV1CrawlWatcher.__init__  sU    *,	 ,,VT:;:bTJ
r2   rX  Nc                   K   t        j                  | j                  ddd| j                  j                   fg      4 d{   }| j                  |       d{    ddd      d{    y7 .7 7 	# 1 d{  7  sw Y   yxY ww)zU
        Establishes WebSocket connection and starts listening for messages.
        Nr  r  )max_sizeadditional_headers
websocketsconnectr  r  rV  _listenr`  	websockets     r#   r  zV1CrawlWatcher.connect  s      %%KK!0GDHH<L<L;M2N OP
 	* 	* ,,y)))	* 	* 	*
 *	* 	* 	* 	*sZ   ABA2BA8A4A8!B,A6-B4A86B8B
>B?B
Bc                    K   |2 3 d{   }t        j                  |      }| j                  |       d{    87 37 6 yw)z
        Listens for incoming WebSocket messages and handles them.

        Args:
            websocket: The WebSocket connection object
        NrB   loads_handle_messager`  r  rL  msgs       r#   r  zV1CrawlWatcher._listen  H      ' 	, 	,'**W%C&&s+++	,+ '1   AA=A*A?AAAA
event_typehandlerc                 ^    || j                   v r| j                   |   j                  |       yy)z
        Adds an event handler function for a specific event type.

        Args:
            event_type (str): Type of event to listen for ('done', 'error', or 'document')
            handler (Callable): Function to handle the event
        N)r  append)r`  r  r  s      r#   add_event_listenerz!V1CrawlWatcher.add_event_listener  s0     ,,,
+227; -r2   detailc                 Z    || j                   v r| j                   |   D ]
  } ||        yy)z
        Dispatches an event to all registered handlers for that event type.

        Args:
            event_type (str): Type of event to dispatch
            detail (Dict[str, Any]): Event details/data to pass to handlers
        N)r  )r`  r  r  r  s       r#   dispatch_eventzV1CrawlWatcher.dispatch_event  s8     ,,,..z:    -r2   r  c                   K   |d   dk(  r<d| _         | j                  d| j                   | j                  | j                  d       y|d   dk(  r@d| _         | j                  d| j                   | j                  |d   | j                  d       y|d   dk(  rl|d	   d
   | _         | j                  j	                  |d	   j                  d	g              | j                  D ]!  }| j                  d|| j                  d       # y|d   dk(  rA| j                  j                  |d	          | j                  d|d	   | j                  d       yyw)z
        Handles incoming WebSocket messages based on their type.

        Args:
            msg (Dict[str, Any]): The message to handle
        r   r  r]   r   r   r   r   r^   r   r   r   r   catchupr   r   r  r   r   Nr   r  r   r   r  r  r  r`  r  docs      r#   r  zV1CrawlWatcher._handle_message  2     v;& %DK4;;		Y]Y`Y`(ab[G#"DKDKK]`ah]iquqxqx)yz[I%f+h/DKIIS[__VR89yy N##JDGG0LMN[J&IIS[)
S[,PQ '   EErX  N)r+   r,   r-   r.   r0   rU  ra  r  r  r   r   r   r  r  r  r1   r2   r#   r  r    s    


3 

^ 

	*	,	<S 	<8T#s(^DTVZDZ;[ 	<`d 	<
  
 d38n 
  
 Rc3h RD Rr2   r  c            0           e Zd ZdZddedef fdZ	 	 	 ddededeeef   d	eeeef      d
e	de
deeef   fdZ	 dded	eeef   deeef   d
e	de
deeef   fdZ	 ddedeeef   d
e	de
deeef   f
dZdej                  deddfdZde	dedededef
dZdefdZdefdZ	 	 ddedee   dee   ddfdZ	 	 ddee   dee   dee   ddfdZdddddddddddddddddd ded!eeed"         deeeef      d#eee      d$eee      d%ee   d&ee	   d'ee	   d(ee   d)ee   d*ee   d+ee   d,ee   d-eed.      d/ee   d0ee   d1ee   d2eeee e!e"e#e$e%e&e'e(f	         de)e   f&d3Z*dddddddddddddddddd4dd5dee   d!eeed6         deeeef      d#eee      d$eee      d%ee   d&ee	   d'ee	   d(ee   d)ee   d*ee   d+ee   d,ee   d-eed.      d0ee   d1ee   d2eeee e!e"e#e$e%e&e'e(f	         d7ee+   d8ee	   dee   de,f*d9Z-dddddddddddddddddddd:dee   d!eeed6         deeeef      d#eee      d$eee      d%ee   d&ee	   d'ee	   d(ee   d)ee   d*ee   d+ee   d,ee   d-eed.      d0ee   d1ee   d2eeee e!e"e#e$e%e&e'e(f	         d7ee+   d;ee   dee   de.f*d<Z/ddddddddddddddddd4dd=ded>eee      d?eee      d@ee	   dAee	   dBee	   dCee   dDee   dEee   dFee   dGee0   dHeeee1f      dIee   dJee   dKee   dLee	   dMee   d8ee	   dee   de2f(dNZ3ddddddddddddddddd4dd=ded>eee      d?eee      d@ee	   dAee	   dBee	   dCee   dDee   dEee   dFee   dGee0   dHeeee1f      dIee   dJee   dKee   dLee	   dMee   d8ee	   dee   de4f(dOZ5dPede2fdQZ6ddPedeeef   d8e	de2fdRZ7dddddddddSdedTee   dFee   dUee   dVee   dBee	   d'ee	   d(ee   dee8   de9fdWZ:	 dddddXdXdXddYdeee      dZee   d[ee   d\ee   dEee   d]ee   d^ee   d7eeeef      de;e   fd_Z<dPede,fd`Z=dPede>fdaZ?dPede>fdbZ@dPedeeef   fdcZAddede;e   fdeZB	 dddddXdXdXddYdeee      dZee   d[ee   d\ee   dEee   d]ee   d^ee   d7eeeef      de;e   fdfZCddddgdedhee	   diee   djee   deDf
dkZEdddddldedhee	   diee   dmee   djee   deFfdnZGdPedeDfdoZHdddddddddpdqed@ee	   dree	   dhee	   dsee   d\ee   dtee   dueeIeeef   gdf      dveeIeeef   gdf      deJfdwZKdddddddxdqed@ee	   dree	   dhee	   dsee   d\ee   dtee   deeef   fdyZLdPedeJfdzZMdddddddddd{	dqedBee	   d|ee   d}ee   d~ee   dee   d(ee   d'ee	   dGee0   deeeeef   eNf      deOfdZP xZQS )AsyncV1FirecrawlAppz
    Asynchronous version of V1FirecrawlApp that implements async methods using aiohttp.
    Provides non-blocking alternatives to all V1FirecrawlApp operations.
    rV  rW  c                 (    t         |   ||       y )N)rV  rW  superra  )r`  rV  rW  	__class__s      r#   ra  zAsyncV1FirecrawlApp.__init__  s    ':r2   NmethodrJ   r\   r   r/  r0  rX  c           	      &  K   t        j                         4 d{   }t        |      D ]  }	 |j                  ||||      4 d{   }	|	j                  dk(  r5t        j                  |d|z  z         d{    	 ddd      d{    f|	j                  dk\  r| j                  |	d| d       d{    |	j                          d{   cddd      d{    c cddd      d{    S  t        d	      7 7 7 7 7 T7 >7 07 # 1 d{  7  sw Y   nxY w# t         j                  $ r9}
||dz
  k(  r|
t        j                  |d|z  z         d{  7   Y d}
~
Id}
~
ww xY w# 1 d{  7  sw Y   yxY ww)
a  
        Generic async request method with exponential backoff retry logic.

        Args:
            method (str): The HTTP method to use (e.g., "GET" or "POST").
            url (str): The URL to send the request to.
            headers (Dict[str, str]): Headers to include in the request.
            data (Optional[Dict[str, Any]]): The JSON data to include in the request body (only for POST requests).
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        N)r  rJ   r\   rB   r2  r  i,  zmake z requestr   zMax retries exceeded)aiohttpClientSessionr3  requestr   asyncior  r  rB   ClientErrorr   )r`  r  rJ   r\   r   r/  r0  sessionr4  r  r  s              r#   _async_requestz"AsyncV1FirecrawlApp._async_request  s    6 ((* 	4 	4g > II&%3d  /   5 5!#??c1")--!w,0O"PPP$5 5 5 $??c1"&"4"4Xvhh?W"XXX%-]]_45 5 5	4 	4 	4I 233!	45 Q	5 Y45	45 5 5 5 ** I'A+-!--!w,(GHHHI	4 	4 	4s-  FDFE<D-D
	D-.D:D

;D D-DD-E<(D9D
:DD
DD-!D
"D-&E<(F4D5F:E<FD-
DD-DDD-FD(DD($D-+E<-E9 (E4(E+
)E4.E<4E99E<<FFF
Fc                 J   K   | j                  d|||||       d{   S 7 w)a  
        Make an async POST request with exponential backoff retry logic.

        Args:
            url (str): The URL to send the POST request to.
            data (Dict[str, Any]): The JSON data to include in the request body.
            headers (Dict[str, str]): Headers to include in the request.
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        POSTNr  )r`  rJ   r   r\   r/  r0  s         r#   _async_post_requestz'AsyncV1FirecrawlApp._async_post_request  s)     * ((gtWn]]]]   #!#c                 J   K   | j                  d||d||       d{   S 7 w)a  
        Make an async GET request with exponential backoff retry logic.

        Args:
            url (str): The URL to send the GET request to.
            headers (Dict[str, str]): Headers to include in the request.
            retries (int): Maximum number of retry attempts (default: 3).
            backoff_factor (float): Factor to calculate delay between retries (default: 0.5).
                Delay will be backoff_factor * (2 ** retry_count).

        Returns:
            Dict[str, Any]: The parsed JSON response from the server.

        Raises:
            aiohttp.ClientError: If the request fails after all retries.
            Exception: If max retries are exceeded or other errors occur.
        GETNr  )r`  rJ   r\   r/  r0  s        r#   _async_get_requestz&AsyncV1FirecrawlApp._async_get_request5  s)     ( ((WdG^\\\\r	  r  r  c                 P  K   	 |j                          d{   }|j                  dd      }|j                  dd      }| j                  |j                  |||       d{   }t        j                  |      7 c#  t        j                  d|j                         xY w7 Bw)aR  
        Handle errors from async API responses with detailed error messages.

        Args:
            response (aiohttp.ClientResponse): The response object from the failed request
            action (str): Description of the action that was being attempted

        Raises:
            aiohttp.ClientError: With a detailed error message based on the response status:
                - 402: Payment Required
                - 408: Request Timeout
                - 409: Conflict
                - 500: Internal Server Error
                - Other: Unexpected error with status code
        Nr   rC  rD  rE  ?Failed to parse Firecrawl error response as JSON. Status code: )rB   r  r  r  r   _get_async_error_messager`  r  r  
error_datarI  rJ  rL  s          r#   r  z!AsyncV1FirecrawlApp._handle_errorK  s      	{'}}.J&NN74PQM&NN96]^M 55hoov}^kll!!'** /	{%%(ghphwhwgx&yzzls8   B&A= A;(A=  !B&!B$"B&;A= =$B!!B&r  rI  rJ  c                 2   K   | j                  ||||      S wa  
        Generate a standardized error message based on HTTP status code for async operations.
        
        Args:
            status_code (int): The HTTP status code from the response
            action (str): Description of the action that was being performed
            error_message (str): The error message from the API response
            error_details (str): Additional error details from the API response
            
        Returns:
            str: A formatted error message
        rF  rN  s        r#   r  z,AsyncV1FirecrawlApp._get_async_error_messagef        &&{FM=YY   c                    K   | j                         }| j                  | j                   d|       d{   }t        di |S 7 w)z8Get current credit usage and billing period (v1, async).r  Nr1   )r~  r  rW  r  r`  r\   resps      r#   r  z$AsyncV1FirecrawlApp.get_credit_usageu  sP     ''),,||n12
 
 %,t,,	
   3AAAc                    K   | j                         }| j                  | j                   d|       d{   }t        di |S 7 w)z7Get current token usage and billing period (v1, async).r  Nr1   )r~  r  rW  r"  r  s      r#   r  z#AsyncV1FirecrawlApp.get_token_usage~  sP     ''),,||n01
 
 $+d++	
r  r#  r  AsyncV1CrawlWatcherc                    K   | j                  |||       d{   }|j                  d      rd|v rt        |d   |       S t        d      7 3w)a  
        Initiate an async crawl job and return an AsyncV1CrawlWatcher to monitor progress via WebSocket.

        Args:
          url (str): Target URL to start crawling from
          params (Optional[V1CrawlParams]): See V1CrawlParams model for configuration:
            URL Discovery:
            * includePaths - Patterns of URLs to include
            * excludePaths - Patterns of URLs to exclude
            * maxDepth - Maximum crawl depth
            * maxDiscoveryDepth - Maximum depth for finding new URLs
            * limit - Maximum pages to crawl

            Link Following:
            * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead
            * crawlEntireDomain - Follow parent directory links
            * allowExternalLinks - Follow external domain links  
            * ignoreSitemap - Skip sitemap.xml processing

            Advanced:
            * scrapeOptions - Page scraping configuration
            * webhook - Notification webhook settings
            * deduplicateSimilarURLs - Remove similar URLs
            * ignoreQueryParameters - Ignore URL parameters
            * regexOnFullURL - Apply regex to full URLs
          idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
          AsyncV1CrawlWatcher: An instance to monitor the crawl job via WebSocket

        Raises:
          Exception: If crawl job fails to start
        Nr   r   r  )r  r  r  r   )r`  rJ   r#  r  r  s        r#   r  z'AsyncV1FirecrawlApp.crawl_url_and_watch  sY     L  $33CQQi(T^-C&~d';TBB788	 R   AA4Ar  c                    K   | j                  |||       d{   }|j                  d      rd|v rt        |d   |       S t        d      7 3w)a  
        Initiate an async batch scrape job and return an AsyncV1CrawlWatcher to monitor progress.

        Args:
            urls (List[str]): List of URLs to scrape
            params (Optional[V1ScrapeParams]): See V1ScrapeParams model for configuration:

              Content Options:
              * formats - Content formats to retrieve
              * includeTags - HTML tags to include
              * excludeTags - HTML tags to exclude
              * onlyMainContent - Extract main content only
              
              Request Options:
              * headers - Custom HTTP headers
              * timeout - Request timeout (ms)
              * mobile - Use mobile user agent
              * proxy - Proxy type
              
              Extraction Options:
              * extract - Content extraction config
              * jsonOptions - JSON extraction config
              * actions - Actions to perform
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            AsyncV1CrawlWatcher: An instance to monitor the batch scrape job via WebSocket

        Raises:
            Exception: If batch scrape job fails to start
        Nr   r   r  )r  r  r  r   )r`  r  r#  r  batch_responses        r#   r  z/AsyncV1FirecrawlApp.batch_scrape_urls_and_watch  sY     H  $;;D&/ZZi(T^-C&~d';TBB>??	 [r  rs   )rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   rn   rk   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rz   ri  rO   rj  rR   c                .  K   | j                  |d       | j                         }|dt         d}|r||d<   |r||d<   |r||d<   |r||d<   |||d	<   |r||d
<   |r||d<   |	r|	j                  dd      |d<   |
|
|d<   |||d<   |||d<   |||d<   |r||d<   |||d<   |d| j	                  |      }t        |t              rd|v r| j	                  |d         |d<   t        |t              r|n|j                  dd      |d<   |d| j	                  |      }t        |t              rd|v r| j	                  |d         |d<   t        |t              r|n|j                  dd      |d<   |r6|D cg c]'  }t        |t              r|n|j                  dd      ) c}|d<   d|v r)|d   r$d|d   v r| j	                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j	                  |d   d         |d   d<   d}| j                  | j                   | ||       d{   }|j                  d      rd|v rt        di |d   S d|v rt        d|d          |j                  dt        |            }t        d|       c c}w 7 jw)aA  
        Scrape a single URL asynchronously.

        Args:
          url (str): Target URL to scrape
          formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)
          headers (Optional[Dict[str, str]]): Custom HTTP headers
          include_tags (Optional[List[str]]): HTML tags to include
          exclude_tags (Optional[List[str]]): HTML tags to exclude
          only_main_content (Optional[bool]): Extract main content only
          wait_for (Optional[int]): Wait for a specific element to appear
          timeout (Optional[int]): Request timeout (ms)
          location (Optional[V1LocationConfig]): Location configuration
          mobile (Optional[bool]): Use mobile user agent
          skip_tls_verification (Optional[bool]): Skip TLS verification
          remove_base64_images (Optional[bool]): Remove base64 images
          block_ads (Optional[bool]): Block ads
          proxy (Optional[Literal["basic", "stealth", "auto"]]): Proxy type (basic/stealth)
          extract (Optional[V1JsonConfig]): Content extraction settings
          json_options (Optional[V1JsonConfig]): JSON extraction settings
          actions (Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]]): Actions to perform
          **kwargs: Additional parameters to pass to the API

        Returns:
            V1ScrapeResponse with:
            * success - Whether scrape was successful
            * markdown - Markdown content if requested
            * html - HTML content if requested
            * rawHtml - Raw HTML content if requested
            * links - Extracted links if requested
            * screenshot - Screenshot if requested
            * extract - Extracted data if requested
            * json - JSON data if requested
            * error - Error message if scrape failed

        Raises:
            Exception: If scraping fails
        rp  rq  rr  rn   r\   ro   rp   Nrq   rr   rt   Trs  ru   rv   rw   rx   ry   r   r   re   rO   r   rR   rw  r   r   r   r{  r1   )r}  r~  r  r  r  r  r  rW  r  r   r   r0   )r`  rJ   rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   ri  rO   rj  rR   r  r  r  r  r  r  error_contents                             r#   rp  zAsyncV1FirecrawlApp.scrape_url  sp    z 	fl3((* #G9-
 '.M)$'.M)$+7M-(+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'" (1M*%..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$%-	*BxS`ajSkGk151I1I-XaJbckJl1mM)$X.M)mM.Jx[hiv[wOw595M5Mm\iNjksNt5uM-(2  11||nXJ'
 
 <<	"v'9#7hv&677 ;HW<M;NOPP %LL#h-@M;M?KLL+ (U
s    E4J6,J"BJ(J)A+Jr  )rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  r  r   r  c                  K   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |$|D cg c]  }|j                  d
d
       c}|d<   ||j                  d
d
      |d<   |j	                  |       t        di |}|j                  d
d
      }||d<   dt         |d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   | j                  |      }| j                  | j                   d||       d{   }|j                  d      r-	 |j                  d      }| j                  |||       d{   S | j                  |d       yc c}w 7 Z#  t        d      xY w7 .w) a  
        Asynchronously scrape multiple URLs and monitor until completion.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeStatusResponse with:
            * Scraping status and progress
            * Scraped content for each URL
            * Success/error information

        Raises:
            Exception: If batch scrape fails
        r  Nrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   rR   r   r  rq  r  r  r   r   r|  r  r1   )r}  r  r  r  r  r   r  r~  r  rW  r  r   _async_monitor_job_statusr  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   r  r  r  r  r  r  r  r  r   s                               r#   r  z%AsyncV1FirecrawlApp.batch_scrape_urlsc  s    z 	f&9: '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-(dk'lZ`TPT(U'lM)$%*ZZDZ%QM'" 	V$ &66"''D'I"F"-gY 7H#I(>8{[dOeCe/3/G/GT]H^_gHh/iK	"8,K'K,F8WbcpWqKq373K3KKXeLfgoLp3qK&x0 ''811||n,-
 
 <<	"P\\$' 77G]SSSx)ABC (m(
P"MOOSsC   EK
J1/C,K
J6K
2J8 K
KK
8KK
)rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   rn  r  rn  c                  K   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |	|	j                  d
d
      |d<   |
|
|d<   |||d<   |||d<   |||d<   |||d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |d| j                  |      }t        |t              rd|v r| j                  |d         |d<   t        |t              r|n|j                  d
d
      |d<   |r6|D cg c]'  }t        |t              r|n|j                  d
d
      ) c}|d<   ||j                  d
d
      |d<   |||d<   |j	                  |       t        d i |}|j                  d
d
      }||d<   dt         |d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   d|v r)|d   r$d|d   v r| j                  |d   d         |d   d<   | j                  |      }| j                  | j                   d||       d{   }|j                  d      dk(  r	 t        d i |j                         S | j                  |d       d{    yc c}w 7 R#  t        d      xY w7 w)!a,  
        Initiate a batch scrape job asynchronously.

        Args:
            urls (List[str]): URLs to scrape
            formats (Optional[List[Literal]]): Content formats to retrieve
            headers (Optional[Dict[str, str]]): Custom HTTP headers
            include_tags (Optional[List[str]]): HTML tags to include
            exclude_tags (Optional[List[str]]): HTML tags to exclude
            only_main_content (Optional[bool]): Extract main content only
            wait_for (Optional[int]): Wait time in milliseconds
            timeout (Optional[int]): Request timeout in milliseconds
            location (Optional[LocationConfig]): Location configuration
            mobile (Optional[bool]): Use mobile user agent
            skip_tls_verification (Optional[bool]): Skip TLS verification
            remove_base64_images (Optional[bool]): Remove base64 encoded images
            block_ads (Optional[bool]): Block advertisements
            proxy (Optional[Literal]): Proxy type to use
            extract (Optional[JsonConfig]): Content extraction config
            json_options (Optional[JsonConfig]): JSON extraction config
            actions (Optional[List[Union]]): Actions to perform
            agent (Optional[AgentOptions]): Agent configuration
            zero_data_retention (Optional[bool]): Whether to delete data after 24 hours
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1BatchScrapeResponse with:
            * success - Whether job started successfully
            * id - Unique identifier for the job
            * url - Status check URL
            * error - Error message if start failed

        Raises:
            Exception: If job initiation fails
        r  Nrn   r\   ro   rp   rq   rr   rt   Trs  ru   rv   rw   rx   ry   r   re   rO   r   rR   r   rv  r  rq  r  r  r  rz  r|  r  r1   )r}  r  r  r  r  r   r  r~  r  rW  r  r   rB   r   r  )r`  r  rn   r\   rb  rc  rd  re  rt   ru   rv   rf  rg  rh  r   rO   rj  rR   r   rn  r  r  r  r  r  r  r  s                              r#   r  z+AsyncV1FirecrawlApp.async_batch_scrape_urls  s    | 	f&?@ '.M)$'.M)$#+7M-(#+7M-((/@M+,'/M)$'.M)$(0tRV(WM*%&,M(# ,3HM/0+2FM./ (1M*%%*M'"..w7G'4(X-@$($<$<WX=N$O!2<Wd2KwQXQ]Q]gkz~Q]QM)$#33LAL,-(l2J)-)A)A,xBX)YX&;ElTX;Y<_k_p_pz~  NR_p  `SM-( MT  (U  CI*VT2JPVP[P[eix|P[P}(}  (UM)$%*ZZDZ%QM'"*1DM-. 	V$ &66"''D'I"F"-gY 7H#I(>8{[dOeCe/3/G/GT]H^_gHh/iK	"8,K'K,F8WbcpWqKq373K3KKXeLfgoLp3qK&x0 ''811||n,-
 
 <<&#-P,?x}}?? $$X/GHHHE (U,
P"MOOHsC   EK,KC3K4K5KK	 &K<K=K	KK)r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  c                  K   | j                  |d       i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d	<   |	|	|d
<   |
|
|d<   ||j                  dd      |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |j                  |       t        di |}|j                  dd      }||d<   dt         |d<   | j                  |      }| j                  | j                   d||       d{   }|j                  d      r-	 |j                  d      }| j                  |||       d{   S | j                  |d       d{    y7 ]#  t        d      xY w7 17 w)a  
        Crawl a website starting from a URL.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[V1ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, V1WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            delay (Optional[int]): Delay in seconds between scrapes
            allow_subdomains (Optional[bool]): Follow subdomains
            poll_interval (Optional[int]): Seconds between status checks (default: 2)
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlStatusResponse with:
            * Crawling status and progress
            * Crawled page contents
            * Success/error information

        Raises:
            Exception: If crawl fails
        r  Nr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   rJ   rq  r  r  r   r   r|  r  r1   )r}  r  r  r   r  r~  r  rW  r  r   r$  r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r\   r  r   s                              r#   r  zAsyncV1FirecrawlApp.crawl_urlz  s2    v 	fk2 $+8L($+8L( '0L$*0CL,-$)L!*0CL,-!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+ 	F# %4|4"''D'I E"-gY 7H''811\\N)
$k7< < <<	"P\\$' 77G]SSS$$X/@AAA<P"MOOSAsH   DFE+F$E- 5FE=F%E?&F-E::F?Fc                  K   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   n|||d<   |	|	|d	<   |
|
|d
<   ||j                  dd      |d<   |||d<   |||d<   |||d<   |||d<   |||d<   |||d<   |j                  |       t        di |}|j                  dd      }||d<   dt         |d<   | j	                  |      }| j                  | j                   d||       d{   }|j                  d      r	 t        di |S | j                  |d       d{    y7 <#  t        d      xY w7 w)a  
        Start an asynchronous crawl job.

        Args:
            url (str): Target URL to start crawling from
            include_paths (Optional[List[str]]): Patterns of URLs to include
            exclude_paths (Optional[List[str]]): Patterns of URLs to exclude
            max_depth (Optional[int]): Maximum crawl depth
            max_discovery_depth (Optional[int]): Maximum depth for finding new URLs
            limit (Optional[int]): Maximum pages to crawl
            allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead
            crawl_entire_domain (Optional[bool]): Follow parent directory links
            allow_external_links (Optional[bool]): Follow external domain links
            ignore_sitemap (Optional[bool]): Skip sitemap.xml processing
            scrape_options (Optional[ScrapeOptions]): Page scraping configuration
            webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings
            deduplicate_similar_urls (Optional[bool]): Remove similar URLs
            ignore_query_parameters (Optional[bool]): Ignore URL parameters
            regex_on_full_url (Optional[bool]): Apply regex to full URLs
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests
            **kwargs: Additional parameters to pass to the API

        Returns:
            V1CrawlResponse with:
            * success - Whether crawl started successfully
            * id - Unique identifier for the crawl job
            * url - Status check URL for the crawl
            * error - Error message if start failed

        Raises:
            Exception: If crawl initiation fails
        Nr   r   r   r   r   r   r   r   r   Trs  r   r   r   r   r   r   r   rJ   rq  r  r  r   r|  r  r1   )r  r  r   r  r~  r  rW  r  r   r   r  )r`  rJ   r  r  r  r  r   r  r  rQ  r  r  r   r  r  r  r   r  r  r  r  r  r  r  r\   r  s                             r#   r  z#AsyncV1FirecrawlApp.async_crawl_url  s
    p  $+8L($+8L( '0L$*0CL,-$)L!*0CL,-!-1EL-.+1EL-.%,:L)%,:,?,?\`,?,aL)&-L##/5ML12".4KL01(->L)*$)L!'.>L*+ 	F# %4|4"''D'I E"-gY 7H ''811\\N)
$


 
 <<	"P&222 $$X/@AAA
P"MOOAs6   C9E;D8<E
D: E2E
3E:EEr   c           
      $  K   | j                         }d| }| j                  | j                   | |       d{   }|j                  d      dk(  rd|v r|d   }d|v r}t	        |d         dk(  rnk|j                  d      }|st
        j                  d       nB| j                  ||       d{   }|j                  |j                  dg              |}d|v r}||d<   t        |j                  d      |j                  d	      |j                  d      |j                  d
      |j                  d      |j                  d      d|v rdnd      }d|v r|j                  d      |_	        d|v r|j                  d      |_
        |S 7 P7 ܭw)a7  
        Check the status and results of an asynchronous crawl job.

        Args:
            id (str): Unique identifier for the crawl job

        Returns:
            V1CrawlStatusResponse containing:
            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of pages crawled
            * total - Total pages to crawl
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of crawled documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        r  Nr   r]   r   r   r   r  r   r   r   r   FT)r   r   r]   r   r   r   r   )r~  r  rW  r  r  r   r   r  r   r   r   	r`  r   r\   r  r  r   r  r  r  s	            r#   r  z&AsyncV1FirecrawlApp.check_crawl_statusg  s    2 '')t$ 33||nXJ'
 

 ??8$3$"6*+;v./14*v6H#'HI&*&=&=h&P PIKK	fb 9:"+K + '+F#(??8,//'*!ook2#6!ook2($3E
 k!(__W5HN[ 'OOF3HMI
 !Qs)   9FFA5F1F2+FB.FFc                 z  K   	 | j                  | j                   d| |       d{   }|j                  d      dk(  rd|v r|d   }d|v r}t        |d         dk(  rnk|j                  d      }|st        j                  d       nB| j                  ||       d{   }|j                  |j                  dg              |}d|v r}||d<   t        di |S t        d	      |j                  d      d
v r(t        j                  t        |d             d{    nt        d|d          07 7 7 w)a  
        Monitor the status of an asynchronous job until completion.

        Args:
            id (str): The ID of the job to monitor
            headers (Dict[str, str]): Headers to include in status check requests
            poll_interval (int): Seconds between status checks (default: 2)

        Returns:
            V1CrawlStatusResponse: The job results if completed successfully

        Raises:
            Exception: If the job fails or an error occurs during status checks
        r  Nr   r]   r   r   r   r  z&Job completed but no data was returnedr;  r  z#Job failed or was stopped. Status: r1   )r  rW  r  r  r   r   r  r   r   r  r  rA  )r`  r   r\   r  r  r   r  r  s           r#   r$  z-AsyncV1FirecrawlApp._async_monitor_job_status  sU      $ 7 7<<.
2$/! K
 x(K7[(&v.D K/{623q8!#.??6#:'"NN+LM!*.*A*A(G*T$T	IMM&"$=>&/ !K/ +/K'0?;??#$LMM*.nnmmCq$9:::"EkRZF[E\ ]^^5  %U ;s;   &D;D4A5D;D7+D;AD;D9D;7D;9D;)r   r  r  r  r   rt   ru   r#  r   r  r  c                8  K   i }
|	r"|
j                  |	j                  dd             |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d	<   ||j                  dd      |
d
<   t        di |
}|j                  dd      }||d<   dt         |d<   d}| j	                  | j
                   | |dd| j                   i       d{   }|j                  d      rd|v rt        di |S d|v rt        d|d          t        d|       7 Gw)a  
        Asynchronously map and discover links from a URL.

        Args:
          url (str): Target URL to map
          params (Optional[V1MapParams]): See V1MapParams model:
            Discovery Options:
            * search - Filter pattern for URLs
            * ignoreSitemap - Skip sitemap.xml
            * includeSubdomains - Include subdomain links
            * sitemapOnly - Only use sitemap.xml
            
            Limits:
            * limit - Max URLs to return
            * timeout - Request timeout (ms)

        Returns:
          V1MapResponse with:
          * Discovered URLs
          * Success/error status

        Raises:
          Exception: If mapping fails
        Trs  Nr   r   r   r   r   rt   ru   rJ   rq  r  r  r  r  r6  r   rN   r   zFailed to map URL. Error: r1   )
r  r  r   r  r  rW  rV  r  r   r   )r`  rJ   r   r  r  r  r   rt   ru   r#  r  r  r  r  r  s                  r#   r  zAsyncV1FirecrawlApp.map_url  s    H 
fkk4dkKL #)Jx %*8J').@J*+#(4J}%"'Jw$+Jy!%-]]Dt]%TJz" #0Z0"''D'I E"-gY 7H 11||nXJ'$~&>? 2 
 
 <<	"w(': ,8,, 8'9J8KLMM8
CDD
s   CDDADFr  r*   re   rP  rR  rS  c                  K   ddl }	|	j                  dt        d       | j                         }
|s|st	        d      |s|st	        d      |r| j                  |      }|xs g ||||dt                d	}|r||d
<   |r||d<   |r||d<   | j                  | j                   d||
       d{   }|j                  d      r|j                  d      }|st        d      	 | j                  | j                   d| |
       d{   }|d   dk(  rt        di |S |d   dv rt        d|d    d|d          t        j                  d       d{    xt        d|j                  d             7 7 r7 &w)aQ  
        Asynchronously extract structured information from URLs.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration

        Returns:
          V1ExtractResponse with:
          * Structured data matching schema
          * Source information if requested
          * Success/error status

        Raises:
          ValueError: If prompt/schema missing or extraction fails
        r   Nr  r  r  r  r  rq  r  r*   r   r   r  r   r   r  r  r   r]   r  r  r	  r   r
  r1   )r  r  r  r~  r^  r  r$   r  rW  r  r   r  r  r  r  )r`  r  r*   re   rP  rQ  rR  rS  r   r  r\   r  r  r  r  s                  r#   rO   zAsyncV1FirecrawlApp.extract!  s    N 	%  	 	
 '')fBCCF@AA--f5F JB"60'#KM?3
 %+L"+8L($)L!11||nK(
 
 <<	"\\$'F KLL$($;$;||nL9% 
 x(K7,;{;; *.EE#l;x3H2IS^_fSgRh$ijjmmA&&&  8g9N8OPQQ1
 's8   B.F 0E:1AF 	E<
AF E>#F <F >F c           
        K   | j                         }d| }| j                  | j                   | |       d{   }|d   dk(  rd|v r|d   }d|v r}t        |d         dk(  rnk|j	                  d      }|st
        j                  d       nB| j                  ||       d{   }|j                  |j	                  dg              |}d|v r}||d<   t        |j	                  d      |j	                  d	      |j	                  d      |j	                  d
      |j	                  d      |j	                  d            }d|v r|d   |d<   d|v r|d   |d<   dd|v rdi|S di|S 7 .7 ƭw)a2  
        Check the status of an asynchronous batch scrape job.

        Args:
            id (str): The ID of the batch scrape job

        Returns:
            V1BatchScrapeStatusResponse containing:
            Status Information:
            * status - Current state (scraping/completed/failed/cancelled)
            * completed - Number of URLs scraped
            * total - Total URLs to scrape
            * creditsUsed - API credits consumed
            * expiresAt - Data expiration timestamp
            
            Results:
            * data - List of scraped documents
            * next - URL for next page of results (if paginated)
            * success - Whether status check succeeded
            * error - Error message if failed

        Raises:
            Exception: If status check fails
        r  Nr   r]   r   r   r   r  r   r   r   r  r   r   FT)	r~  r  rW  r  r  r   r   r  r   r)  s	            r#   r  z-AsyncV1FirecrawlApp.check_batch_scrape_status  s    2 '')&rd+ 33||nXJ'
 

 x K/$"6*+;v./14*v6H#'HI&*&=&=h&P PIKK	fb 9:"+K + '+F#.??8,//'*!ook2#6!ook2(
 k! +G 4HW[ *62HV ; 6u

 	
<@

 	
G
 !Qs)   9E.E)A)E.%E,&+E.BE.,E.c                    K   | j                         }| j                  | j                   d| d|       d{   S 7 w)a]  
        Get information about errors from an asynchronous batch scrape job.

        Args:
          id (str): The ID of the batch scrape job

        Returns:
          V1CrawlErrorsResponse containing:
            errors (List[Dict[str, str]]): List of errors with fields:
              * id (str): Error ID
              * timestamp (str): When the error occurred
              * url (str): URL that caused the error
              * error (str): Error message
          * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
          Exception: If error check fails
        r  r  Nr~  r  rW  r`  r   r\   s      r#   r  z-AsyncV1FirecrawlApp.check_batch_scrape_errors  sK     & ''),,||n-bT9
 
 	
 
   6?=?c                    K   | j                         }| j                  | j                   d| d|       d{   S 7 w)aa  
        Get information about errors from an asynchronous crawl job.

        Args:
            id (str): The ID of the crawl job

        Returns:
            V1CrawlErrorsResponse containing:
            * errors (List[Dict[str, str]]): List of errors with fields:
                - id (str): Error ID
                - timestamp (str): When the error occurred
                - url (str): URL that caused the error
                - error (str): Error message
            * robotsBlocked (List[str]): List of URLs blocked by robots.txt

        Raises:
            Exception: If error check fails
        r  r  Nr/  r0  s      r#   r  z&AsyncV1FirecrawlApp.check_crawl_errors  sJ     & ''),,||nJrd'2
 
 	
 
r1  c                   K   | j                         }t        j                         4 d{   }|j                  | j                   d| |      4 d{   }|j                          d{   cddd      d{    cddd      d{    S 7 i7 @7 *7 7 # 1 d{  7  sw Y   nxY wddd      d{  7   y# 1 d{  7  sw Y   yxY ww)r  Nr  r6  )r~  r  r  r9  rW  rB   )r`  r   r\   r  r  s        r#   r  z AsyncV1FirecrawlApp.cancel_crawl  s      '')((* 	- 	-g~~j&Ew~W - -[c%]]_,- - -	- 	- 	--,-	-- - -	- 	- 	- 	- 	-s   )CBC'CBCB.B/B2C>B?CCBCCBCCB1	%B(&B1	-C4C?C CCCCCr  c                    K   | j                         }	 | j                  | j                   d| |       d{   S 7 # t        $ r}t	        t        |            d}~ww xY ww)a>  
        Check the status of an asynchronous extraction job.

        Args:
            job_id (str): The ID of the extraction job

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any
            * warning (Optional[str]): Warning message if any
            * sources (Optional[List[str]]): Source URLs if requested

        Raises:
            ValueError: If status check fails
        r  Nr~  r  rW  r   r^  r0   )r`  r  r\   r  s       r#   r  z&AsyncV1FirecrawlApp.get_extract_status  sm     $ '')	%00<<.VH5     	%SV$$	%1   A$$? =? A$? 	A!AA!!A$c          	        K   | j                         }	|s|st        d      |s|st        d      |r| j                  |      }t        |xs g ||||dt               }
|r||
d<   |r||
d<   |r||
d<   	 | j                  | j                   d|
|	       d	{   S 7 # t        $ r}t        t        |            d	}~ww xY ww)
a  
        Initiate an asynchronous extraction job without waiting for completion.

        Args:
            urls (Optional[List[str]]): URLs to extract from
            prompt (Optional[str]): Custom extraction prompt
            schema (Optional[Any]): JSON schema/Pydantic model
            system_prompt (Optional[str]): System context
            allow_external_links (Optional[bool]): Follow external links
            enable_web_search (Optional[bool]): Enable web search
            show_sources (Optional[bool]): Include source URLs
            agent (Optional[Dict[str, Any]]): Agent configuration
            idempotency_key (Optional[str]): Unique key to prevent duplicate requests

        Returns:
            V1ExtractResponse[Any] with:
            * success (bool): Whether request succeeded
            * data (Optional[Any]): Extracted data matching schema
            * error (Optional[str]): Error message if any

        Raises:
            ValueError: If job initiation fails
        r  r  rq  r  r*   r   r   r  N)	r~  r^  r  r  r  r  rW  r   r0   )r`  r  r*   re   rP  rQ  rR  rS  r   r\   r  r  s               r#   r  z!AsyncV1FirecrawlApp.async_extract,  s     D '')fBCCF@AA--f5F(3-$ 	*
 %+L"+8L($)L!	%11<<.,   
  	%SV$$	%s<   A3C6#B  BB  CB   	C)B==CCr  r  r  r  r  r  c                  K   i }|||d<   |||d<   |||d<   | j                  ||||       d{   }|j                  d      rd|vr|S |d   }	 | j                  |       d{   }|d   d	k(  r|S |d   d
k(  rt        d|j                  d             |d   dk7  rnt	        j
                  d       d{    ot        ddd
d      S 7 7 k7 w)a  
        Generate LLMs.txt for a given URL and monitor until completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            V1GenerateLLMsTextStatusResponse containing:
            * success (bool): Whether generation completed successfully
            * status (str): Status of generation (processing/completed/failed)
            * data (Dict[str, str], optional): Generated text with fields:
                - llmstxt (str): Generated LLMs.txt content
                - llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            Exception: If generation fails
        Nr4  r5  r  r8  r   r   r   r]   r^   z#LLMs.txt generation failed. Error: r   r  r  Fr   r  r  )r!  r  r"  r   r  r  rM  )	r`  rJ   r  r  r  r#  r  r  r   s	            r#   r$  z&AsyncV1FirecrawlApp.generate_llms_textr  s7    :  (F9%%3F>"*.AF*+66) 3	 7 
 
 ||I&$h*>O$??GGFh;.!X-"EfjjQXFYEZ [\\!\1--"""  0Ev  @H  TV  W  	W-
 H #s4   0CC5C(C)AC=C>CCCr  r6  c                \  K   i }|||d<   |||d<   |||d<   t        ||||      }| j                         }d|i|j                  dd      }d	t         |d
<   	 | j	                  | j
                   d||       d{   S 7 # t        $ r}	t        t        |	            d}	~	ww xY ww)a@  
        Initiate an asynchronous LLMs.txt generation job without waiting for completion.

        Args:
            url (str): Target URL to generate LLMs.txt from
            max_urls (Optional[int]): Maximum URLs to process (default: 10)
            show_full_text (Optional[bool]): Include full text in output (default: False)
            cache (Optional[bool]): Whether to use cached content if available (default: True)
            experimental_stream (Optional[bool]): Enable experimental streaming

        Returns:
            V1GenerateLLMsTextResponse containing:
            * success (bool): Whether job started successfully
            * id (str): Unique identifier for the job
            * error (str, optional): Error message if start failed

        Raises:
            ValueError: If job initiation fails
        Nr4  r5  r  r  rJ   Trs  rq  r  r&  )	r2  r~  r  r  r  rW  r   r^  r0   )
r`  rJ   r  r  r6  r  r#  r\   r'  r  s
             r#   r!  z,AsyncV1FirecrawlApp.async_generate_llms_text  s     6  (F9%%3F>"*.AF*+)'"5	
 '')CQ6;;4;#PQ	 +G95	(	%11<<.,   
  	%SV$$	%s<   AB,#B  BB B,B 	B)B$$B))B,c                    K   | j                         }	 | j                  | j                   d| |       d{   S 7 # t        $ r}t	        t        |            d}~ww xY ww)a  
        Check the status of an asynchronous LLMs.txt generation job.

        Args:
            id (str): The ID of the generation job

        Returns:
            V1GenerateLLMsTextStatusResponse containing:
            * success (bool): Whether generation completed successfully
            * status (str): Status of generation (processing/completed/failed)
            * data (Dict[str, str], optional): Generated text with fields:
                - llmstxt (str): Generated LLMs.txt content
                - llmsfulltxt (str, optional): Full version if requested
            * error (str, optional): Error message if generation failed
            * expiresAt (str): When the generated data expires

        Raises:
            ValueError: If status check fails
        r+  Nr5  r`  r   r\   r  s       r#   r"  z3AsyncV1FirecrawlApp.check_generate_llms_text_status  sm     ( '')	%00<<.RD1     	%SV$$	%r6  )r  rO  r  rP  rP  /_AsyncV1FirecrawlApp__experimental_stream_stepsrR  rS  r  rO  rP  r=  rR  rS  c                  K   i }
|||
d<   |||
d<   |||
d<   |||
d<   |||
d<   |||
d<   t        di |
}
| j                  ||||||       d{   }|j                  d	      rd
|vr|S |d
   }d}d}	 | j                  |       d{   }|r)d|v r%|d   |d }|D ]
  } ||        t	        |d         }|	r)d|v r%|d   |d }|D ]
  } |	|        t	        |d         }|d   dk(  r|S |d   dk(  rt        d|j                  d             |d   dk7  rnt        j                  d       d{    t        dd      S 7 7 7 wrU  )	r9  rW  r  rX  r  r   r  r  rB  )r`  r  r  rO  r  rP  rP  r=  rR  rS  rY  r  r  rZ  r[  r   r\  r]  r^  r_  s                       r#   r`  z!AsyncV1FirecrawlApp.deep_research  s    P  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AA11!+' 2 
 
 ||I&$h*>O$::6BBF|v5!'!56I6J!K . *H)*&)&*>&?#Y&0$Y/0A0BC) &Ff%&$'y(9$:!h;.!X-"?

7@S?T UVV!\1--"""- 0 ,EAlmmO
  C* #s7   AEE9EEB*E9E:EEE)r  rO  r  rP  rP  r=  c                ~  K   i }|||d<   |||d<   |||d<   |||d<   |||d<   |||d<   t        di |}| j                         }	d|i|j                  d	d	
      }
dt         |
d<   	 | j	                  | j
                   d|
|	       d{   S 7 # t        $ r}t        t        |            d}~ww xY ww)rb  Nr   r;  r4  r=  r   rV  r  Trs  rq  r  rc  r1   )	r9  r~  r  r  r  rW  r   r^  r0   )r`  r  r  rO  r  rP  rP  r=  rY  r\   r'  r  s               r#   rW  z'AsyncV1FirecrawlApp.async_deep_researchc  s    >  *3OJ'!+5OK()1OI&&0?O,-$.;ON+&2<WO89.AA'')e^';';TX\';']^	 +G95	(	%11<<. 12   
  	%SV$$	%s<   A+B=.#B BB B=B 	B:!B55B::B=c                    K   | j                         }	 | j                  | j                   d| |       d{   S 7 # t        $ r}t	        t        |            d}~ww xY ww)rg  rh  Nr5  r<  s       r#   rX  z.AsyncV1FirecrawlApp.check_deep_research_status  sn     2 '')	%00<<. 22$7     	%SV$$	%r6  )	r   r  r  r  rX   ru   rt   r  r#  r  r  r  rX   c       	           K   i }|
rDt        |
t              r|j                  |
       n"|j                  |
j                  dd             |||d<   |||d<   |||d<   |||d<   |||d<   |||d	<   |||d
<   |	|	j                  dd      |d<   |j                  |       t        dd|i|}|j                  dd      }dt         |d<   | j                  | j                   d|dd| j                   i       d{   S 7 w)a  
        Asynchronously search for content using Firecrawl.

        Args:
            query (str): Search query string
            limit (Optional[int]): Max results (default: 5)
            tbs (Optional[str]): Time filter (e.g. "qdr:d")
            filter (Optional[str]): Custom result filter
            lang (Optional[str]): Language code (default: "en")
            country (Optional[str]): Country code (default: "us") 
            location (Optional[str]): Geo-targeting
            timeout (Optional[int]): Request timeout in milliseconds
            scrape_options (Optional[ScrapeOptions]): Result scraping configuration
            params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters
            **kwargs: Additional keyword arguments for future compatibility

        Returns:
            SearchResponse: Response containing:
                * success (bool): Whether request succeeded
                * data (List[FirecrawlDocument]): Search results
                * warning (Optional[str]): Warning message if any
                * error (Optional[str]): Error message if any

        Raises:
            Exception: If search fails or response cannot be parsed
        Trs  Nr   r  r  r  rX   ru   rt   r   r  rq  r  r  r  r  r1   )r  r  r  r
  r  r  rW  rV  )r`  r  r   r  r  r  rX   ru   rt   r  r#  r  r  r  r  s                  r#   r   zAsyncV1FirecrawlApp.search  sa    R &$'$$V,$$V[[$T[%RS %*M'"?#&M% &,M(#$(M&!'.M)$(0M*%'.M)$%-;-@-@$]a-@-bM/* 	V$ &CEC]C"''D'I"-gY 7H--||nJ'~67
 
 	
 
s   DD
DD
)r\  )Nr|  r  r  r  )r  r  )Rr+   r,   r-   r.   r0   ra  r   r   r   r   r   r  r  r  r  ClientResponser  r  r  r  r"  r  r   r  r   r   r  r	   r   rW   r   r   r   r   r   r   r   r   r   r   r   r   rp  r'   r   r  r   r  ri   r[   r   r  r   r  r  r$  r   r   r  r  rO   r  r   r  r  r  r  r  rM  r$  rG  r!  r"  r   rB  r`  rW  rX  r
  r  r   __classcell__r  s   @r#   r  r    sT   
; ;c ; .2$'+4+4 +4 #s(^	+4
 4S>*+4 +4 "+4 -1cN+4^ 7:^^"&sCx.^;?S>^^.3^>B38n^2 7:]]%)#s(^]].3]>B38n],+G,B,B +C +TX +6Z# Zs Z[^ Zor Zwz Z-(= -,'; , /3-1	*9*9 ]+*9 &c]	*9 7L	*9^ 04-1	(@s)(@ ^,(@ &c]	(@ 7L	(@\ mq04040404&*%*37%)4837(,OS(,.237 EI)DMDM d7  ,g  $h  i  j	DM
 d38n-DM #49-DM #49-DM  (~DM smDM c]DM /0DM TNDM $,D>DM #+4.DM  ~DM  G$JKL!DM"  ~#DM$ l+%DM& #<0'DM( d57I=Zgiv  yG  IW  Yr  t  *  $@  A  B)DM* *#.+DMT W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*.'()--HC3iHC $w  (Q   R  S  T	HC
 $sCx.)HC tCy)HC tCy)HC $D>HC 3-HC #HC +,HC HC  (~HC 'tnHC D>HC   FGH!HC" ,'#HC$ |,%HC& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'HC( ')HC*  }+HC, "#-HC0 
%1HC^ W[,0,0,0,0"&!&/3!%04/3$(KO*./3 AE*..2)--JI3iJI $w  (Q   R  S  T	JI
 $sCx.)JI tCy)JI tCy)JI $D>JI 3-JI #JI +,JI JI  (~JI 'tnJI D>JI   FGH!JI" ,'#JI$ |,%JI& $u\3E}Vcer  uC  ES  Un  p{  &{   |  }  ~'JI( ')JI* &d^+JI, "#-JI0 
1JI` .2-1#'-1#/3.2/3)-489=3726,0#+/'()-+uBuB  S	*	uB
  S	*uB C=uB &c]uB }uB 'tnuB &d^uB 'tnuB !uB !1uB %_ 456uB #+4.uB  "*$!uB" $D>#uB$ }%uB& #4.'uB(  })uB* "#+uB. 
/uBx .2-1#'-1#/3.2/3)-489=3726,0#+/'()-+sBsB  S	*	sB
  S	*sB C=sB &c]sB }sB 'tnsB &d^sB 'tnsB !sB !1sB %_ 456sB #+4.sB  "*$!sB" $D>#sB$ }%sB& #4.'sB(  })sB* "#+sB. 
/sBj@3 @3H @D)_# )_S#X )__b )_  lA )_^ !%)--1'+#!&/3(,KEKE 	KE
 !KE %TNKE tnKE }KE #KE +,KE %KE 2?KE^ )-dR %)$(+/3805+0.2dR49%dR SM	dR
 SMdR $C=dR #+4.dR  (~dR #4.dR DcN+dR 8I7MdRLB
# B
:U B
H
# 
:O 
2
3 
3H 
2-S -T#s(^ -(%s %7H7M %: )-D% %)$(+/3805+0.2D%49%D% SM	D%
 SMD% $C=D% #+4.D%  (~D% #4.D% DcN+D% 8I7MD%T '+-126;W;W sm	;W
 %TN;W "*$;W <\;WB '+-1$(265%5% sm	5%
 %TN5% D>5% "*$5% <V5%n% %@` %B (,(,&*-1+/:>FJDH^n^n  }	^n
 !^n sm^n &c]^n $C=^n *2$^n "(DcN+;T+A"BC^n  $sCx.)94)? @A^n Nj^nH (,(,&*-1+/:>:%:%  }	:%
 !:% sm:% &c]:% $C=:% *2$:% DHS>:%x %3  %;W  %L $(!%$("&%)&*%*8<FJN
N
 C=	N

 #N
 SMN
 3-N
 c]N
 smN
 c]N
 %_5N
 U4S>>#ABCN
 *N
r2   r  c            
            e Zd ZdZdedef fdZddZddZd	e	ee
f   ddfd
Zdej                  deddfdZdededededef
dZ xZS )r  zQ
    Async version of V1CrawlWatcher that properly handles async operations.
    r   r  c                 &    t         |   ||       y r  r  )r`  r   r  r  s      r#   ra  zAsyncV1CrawlWatcher.__init__  s    S!r2   rX  Nc                   K   t        j                  | j                  dd| j                  j                   fg      4 d{   }| j                  |       d{    ddd      d{    y7 .7 7 	# 1 d{  7  sw Y   yxY ww)z[
        Establishes async WebSocket connection and starts listening for messages.
        r  r  )r  Nr  r  s     r#   r  zAsyncV1CrawlWatcher.connect  s      %%KK!0GDHH<L<L;M2N OP
 	* 	* ,,y)))		* 	* 	* *		* 	* 	* 	*sZ   A BA1BA7A3A7 B+A5,B3A75B7B	=B >B	Bc                    K   |2 3 d{   }t        j                  |      }| j                  |       d{    87 37 6 yw)z
        Listens for incoming WebSocket messages and handles them asynchronously.

        Args:
            websocket: The WebSocket connection object
        Nr  r  s       r#   r  zAsyncV1CrawlWatcher._listen"  r  r  r  c                   K   |d   dk(  r<d| _         | j                  d| j                   | j                  | j                  d       y|d   dk(  r@d| _         | j                  d| j                   | j                  |d   | j                  d       y|d   dk(  rl|d	   d
   | _         | j                  j	                  |d	   j                  d	g              | j                  D ]!  }| j                  d|| j                  d       # y|d   dk(  rA| j                  j                  |d	          | j                  d|d	   | j                  d       yyw)z
        Handles incoming WebSocket messages based on their type asynchronously.

        Args:
            msg (Dict[str, Any]): The message to handle
        r   r  r]   r  r   r^   r  r  r   r   r  r  Nr  r  s      r#   r  z#AsyncV1CrawlWatcher._handle_message-  r  r  r  r  c                 d  K   	 |j                          d{   }|j                  dd      }|j                  dd      }| j
                  j                  |j                  |||       d{   }t        j                  |      7 m#  t        j                  d|j                         xY w7 Bw)z9
        Handle errors from async API responses.
        Nr   rC  rD  rE  r  )rB   r  r  r  r   r  r  r  s          r#   r  z!AsyncV1CrawlWatcher._handle_errorC  s     	{'}}.J&NN74PQM&NN96]^M
 99(//6S`bopp!!'** /	{%%(ghphwhwgx&yzz qs8   B0B B(B  +B0+B.,B0B $B++B0r  rI  rJ  c                 2   K   | j                  ||||      S wr  r  rN  s        r#   r  z,AsyncV1CrawlWatcher._get_async_error_messageS  r  r  r  )r+   r,   r-   r.   r0   r  ra  r  r  r   r   r  r  rB  r  r   r  rC  rD  s   @r#   r  r    s    "3 "%8 "*	,Rc3h RD R,+G,B,B +C +TX + Z# Zs Z[^ Zor Zwz Zr2   r  )Tr.   loggingr   r  typingr   r   r   r   r   r   r	   r
   r   rB   r   r   r  rF   r  r  r  	getLoggerr   Loggerr/   r$   r  r%   	BaseModelr'   r4   r9   r=   rI   rW   r[   rc   ri   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r
  r  r  r  r  r"  r$  r*  r-  r0  r2  r9  r@  rB  rG  rI  rM  rU  r  r  r  r1   r2   r#   <module>rQ     s]    	  X X X   	     +'++K8 8
 -CLJ!X'' !
(H.. (h(( 
C8-- C:(,,gaj : *x)) *
Uh(( Uh00 $h(( $(#8%% #"++ "H&& 
H&& 
H&& 
#X'' #X''  2 2 
"($$ "(X'' (+8%% +._ . *1-wqz  ,H.. ,	$("4"4 	$+H&& +( h((  	$H.. 	$8%% H.. 
	0($$ 	0 H&&  
4h(( 
4
&**GAJ 
&
4X'' 
4 x))  -** -H.. -x)) -8-- H$6$6 1h&8&8 18#5#5 0X%7%7 01x11 1	68-- 	6 X//  8#5#5  !3!3  &8+=+= &x'9'9  x))  +h(( +b*  b* JUYR YRvh
. h
T9OZ. OZr2   