
    Sjש                     l   d Z ddlZddlmZmZmZmZmZmZm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z1 dd	l/m2Z3 dd
l/m4Z5 ddl/m6Z7 ddl/m8Z9 dd
l/m4Z: ddl/m;Z< ddl/m=Z> ddl/m?Z@ ddl/mAZB ddlCmDZD  G d d      ZEy)zs
Main Firecrawl v2 API client.

This module provides the main client class that orchestrates all v2 functionality.
    N)OptionalListDictAnyCallableUnionLiteral   ) ClientConfigScrapeOptionsDocumentSearchRequest
SearchDataSourceOptionCategoryOptionCrawlRequestCrawlResponseCrawlJobCrawlParamsRequest	PDFParserCrawlParamsDataWebhookConfigAgentWebhookConfigCrawlErrorsResponseActiveCrawlsResponse
MapOptionsMapDataFormatOption
WaitActionScreenshotActionClickActionWriteActionPressActionScrollActionScrapeActionExecuteJavascriptAction	PDFActionLocationPaginationConfigAgentOptions)
HttpClient)FirecrawlError)scrape)crawl)batch)search)map)usage)extract)agent)browserWatcherc            <       H   e Zd ZdZededefd       Z	 	 	 	 	 ddee   dedee	   d	e
d
e	f
dZddddddddddddddddddddddedeed      deeeef      deee      deee      dee   dee
   dee
   dee   deeee   eeeef      f      deeed         ded   dee   dee   dee   dee   dee   dee   d ee
   d!ee   d"ee   def,d#Zdddddddddd$	d%ed&eee      d'eee      d(ee
   d)ee   dee   d*ee   dee
   d+ee   d"ee   defd,Zddddddd-dd-d-d-ddddd-d-d.dddd/ded0ee   d1eee      d2eee      d3ee
   d4eed5      d6ee   d7ed(ee
   d8ed9ed:ed;ee
   d<ee
   d=eeeef      d+ee   d>ed?ed@e
dee
   dAee	   d"ee   def.dBZddddddd-dd-d-d-ddddd-d-ddCded0ee   d1eee      d2eee      d3ee
   d4eed5      d6ee   d7ed(ee
   d8ed9ed:ed;ee
   d<ee
   d=eeeef      d+ee   d>ed?ed"ee   def(dDZ	 dddEdFedGee   dAee	   defdHZddEdIedAee	   defdJZdKede fdLZ!de"fdMZ#de"fdNZ$dddddddddOdedPee   dQee   d7ee   d(ee
   d4eed5      dee
   d"ee   dee%   de&fdRZ'dKedefdSZ(ded0ede)fdTZ*	 ddddddddddddU
dVeee      d0ee   dWeeee+f      dXee   d9ee   dYee   dZee   d+ed[   d*ee   d"ee   d\ee,   fd]Z-	 dddddddddd.dddd^dVeee      d0ee   dWeeee+f      dXee   d9ee   dYee   dZee   d+ed[   d*ee   d@e
dee
   d"ee   d\ee,   fd_Z.ddddddddddddddddddddddddddd`dVee   deed      deeeef      deee      deee      dee   dee
   dee
   dee   deeee   eeeef      f      deeed         ded   dee   dee   dee   dee   dee   dee   d ee
   d!ee   d=eeeef      daee   d*ee   d<ee
   d?ee   d"ee   dbee   f6dcZ/	 ddFedGee   fddZ0ddEdIedAee	   fdeZ1dFedefdfZ2dFefdgZ3dFefdhZ4	 ddddddddidVeee      d0edWee+   d"ee   djee
   dkee   dleedm      d=eeee5f      fdnZ6	 dddd.ddddddodVeee      d0edWee+   d"ee   d@e
dee
   djee
   dkee   dleedm      d=eeee5f      fdpZ7dFefdqZ8dFedefdrZ9ds Z:dt Z;du Z<ddvefdwZ=ddvefdxZ>dy Z?ddddzd{ee
   d|ee
   d}ee   fd~Z@ddddededed   dee
   fdZAdefdZBdddeed      fdZCdd.dddFeded   d@e
dee
   deDf
dZEddddddddddddddddddddddddddd.dddVee   deed      deeeef      deee      deee      dee   dee
   dee
   dee   deeee   eeeef      f      deeed         ded   dee   dee   dee   dee   dee   dee   d ee
   d!ee   d=eeeef      daee   d*ee   d<ee
   d?ee   d"ee   dbee   d@e
dee
   f:dZFy)FirecrawlClientz|
    Main Firecrawl v2 API client.

    This client provides a clean, modular interface to all Firecrawl functionality.
    urlreturnc                 &    d| j                         v S )Nzapi.firecrawl.dev)lower)r:   s    F/root/.hermes/venv/lib/python3.12/site-packages/firecrawl/v2/client.py_is_cloud_servicez!FirecrawlClient._is_cloud_service@   s    "ciik11    Napi_keyapi_urltimeoutmax_retriesbackoff_factorc                     |t        j                  d      }| j                  |      r|st        d      t	        |||||      | _        t        |||||      | _        y)a  
        Initialize the Firecrawl client.

        Args:
            api_key: Firecrawl API key (or set FIRECRAWL_API_KEY env var)
            api_url: Base URL for the Firecrawl API
            timeout: Request timeout in seconds
            max_retries: Maximum number of retries for failed requests
            backoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)
        NFIRECRAWL_API_KEYzlAPI key is required for the cloud API. Set FIRECRAWL_API_KEY environment variable or pass api_key parameter.)rA   rB   rC   rD   rE   )rC   rD   rE   )osgetenvr?   
ValueErrorr   configr+   http_client)selfrA   rB   rC   rD   rE   s         r>   __init__zFirecrawlClient.__init__D   su    $ ?ii 34G!!'*7- 
 ##)
 &#)
r@   )formatsheadersinclude_tagsexclude_tagsonly_main_contentrC   wait_formobileparsersactionslocationskip_tls_verificationremove_base64_images	fast_modeuse_mock	block_adsproxymax_agestore_in_cacheintegrationrO   r   rP   rQ   rR   rS   rT   rU   rV   rW   )	r   r    r!   r"   r#   r$   r%   r&   r'   rX   r(   rY   rZ   r[   r\   r]   r^   r_   r`   ra   c                   t        d ||||||||	|
|||||||||||fD              rrt        di t        di d|d|d|d|d|d|d|d	|	d
|
d|d|d|d|d|d|d|d|d|d|d|j                         D ci c]
  \  }}||| c}}nd}t	        j
                  | j                  ||      S c c}}w )ab  
        Scrape a single URL and return the document.
        Args:
            url: URL to scrape
            formats: List of formats to scrape
            headers: Dictionary of headers to use
            include_tags: List of tags to include
            exclude_tags: List of tags to exclude
            only_main_content: Whether to only scrape the main content
            timeout: Timeout in milliseconds
            wait_for: Wait for a specific element to be present
            mobile: Whether to use mobile mode
            parsers: List of parsers to use
            actions: List of actions to perform
            location: Location to scrape
            skip_tls_verification: Whether to skip TLS verification
            remove_base64_images: Whether to remove base64 images
            fast_mode: Whether to use fast mode
            use_mock: Whether to use mock mode
            block_ads: Whether to block ads
            proxy: Proxy to use
            max_age: Maximum age of the cache
            store_in_cache: Whether to store the result in the cache
        Returns:
            Document
        c              3   $   K   | ]  }|d u 
 y wN .0vs     r>   	<genexpr>z)FirecrawlClient.scrape.<locals>.<genexpr>   s       \1$  \   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   Nre   )anyr   dictitemsscrape_moduler-   rL   )rM   r:   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   krh   optionss                            r>   r-   zFirecrawlClient.scrapeo   s   T   \'<Whjqs{  ~D  FM  OV  X`  bw  yM  OX  Zb  dm  ot  v}  M  OZ  )[  \  \/   
 $ !!! *! *	!
 #4!  ! "! !  !  ! "! '<! &:! $! "!  $!!" #!$  %!&  .'!( ()!* eg+(1* + !t (
. bf/ 	0 ##D$4$4c7CC/(s   B<)	sources
categorieslimittbsrX   ignore_invalid_urlsrC   scrape_optionsra   queryrq   rr   rs   rt   ru   rv   c       	         l    t        |||||||||	|

      }t        j                  | j                  |      S )a  
        Search for documents.
        
        Args:
            query: Search query string
            limit: Maximum number of results to return (default: 5)
            tbs: Time-based search filter
            location: Location string for search
            timeout: Request timeout in milliseconds (default: 300000)
            page_options: Options for scraping individual pages
            
        Returns:
            SearchData containing the search results
        )
rw   rq   rr   rs   rt   rX   ru   rC   rv   ra   )r   search_moduler0   rL   )rM   rw   rq   rr   rs   rt   rX   ru   rC   rv   ra   requests               r>   r0   zFirecrawlClient.search   sG    8  ! 3)#
 ##D$4$4g>>r@   F   )promptexclude_pathsinclude_pathsmax_discovery_depthsitemapignore_sitemapignore_query_parametersrs   crawl_entire_domainallow_external_linksallow_subdomainsdelaymax_concurrencywebhookrv   regex_on_full_urlzero_data_retentionpoll_intervalrC   request_timeoutra   r|   r}   r~   r   r   )onlyincludeskipr   r   r   r   r   r   r   r   r   r   r   r   c                    |}|||rdnd}i d|d|d|d|d|d|d	|	d
|
d|d|d|d|d|d|d|d|d|}|||d<   t        di |}t        j                  | j                  ||||      S )aR  
        Start a crawl job and wait for it to complete.

        Args:
            url: Target URL to start crawling from
            prompt: Optional prompt to guide the crawl
            exclude_paths: Patterns of URLs to exclude
            include_paths: Patterns of URLs to include
            max_discovery_depth: Maximum depth for finding new URLs
            sitemap: Sitemap usage mode ("only" | "include" | "skip")
            ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)
            ignore_query_parameters: Ignore URL parameters
            limit: Maximum pages to crawl
            crawl_entire_domain: Follow parent directory links
            allow_external_links: Follow external domain links
            allow_subdomains: Follow subdomains
            delay: Delay in seconds between scrapes
            max_concurrency: Maximum number of concurrent scrapes
            webhook: Webhook configuration for notifications
            scrape_options: Page scraping configuration
            regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
            zero_data_retention: Whether to delete data after 24 hours
            poll_interval: Seconds between status checks
            timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
            request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout
            
        Returns:
            CrawlJob when job completes
            
        Raises:
            ValueError: If request is invalid
            Exception: If the crawl fails to start or complete
            TimeoutError: If timeout is reached
        r   r   r:   r|   r}   r~   r   r   rs   r   r   r   r   r   r   rv   r   r   ra   r   )r   rC   r   re   )r   crawl_moduler.   rL   )rM   r:   r|   r}   r~   r   r   r   r   rs   r   r   r   r   r   r   rv   r   r   r   rC   r   ra   resolved_sitemaprequest_kwargsrz   s                             r>   r.   zFirecrawlClient.crawl   s=   x ##(B)7vY
3
f
 ]
 ]	

 "#6
 &'>
 U
 "#6
 #$8
  0
 U
 
 w
 n
  !2
  "#6!
" ;#
& '(8N9%00!!'+
 	
r@   )r|   r}   r~   r   r   r   r   rs   r   r   r   r   r   r   rv   r   r   ra   c                    |}|||rdnd}i d|d|d|d|d|d|d	|	d
|
d|d|d|d|d|d|d|d|d|}|||d<   t        di |}t        j                  | j                  |      S )a  
        Start an asynchronous crawl job.

        Args:
            url: Target URL to start crawling from
            prompt: Optional prompt to guide the crawl
            exclude_paths: Patterns of URLs to exclude
            include_paths: Patterns of URLs to include
            max_discovery_depth: Maximum depth for finding new URLs
            sitemap: Sitemap usage mode ("only" | "include" | "skip")
            ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)
            ignore_query_parameters: Ignore URL parameters
            limit: Maximum pages to crawl
            crawl_entire_domain: Follow parent directory links
            allow_external_links: Follow external domain links
            allow_subdomains: Follow subdomains
            delay: Delay in seconds between scrapes
            max_concurrency: Maximum number of concurrent scrapes
            webhook: Webhook configuration for notifications
            scrape_options: Page scraping configuration
            regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
            zero_data_retention: Whether to delete data after 24 hours
            
        Returns:
            CrawlResponse with job information
            
        Raises:
            ValueError: If request is invalid
            Exception: If the crawl operation fails to start
        r   r   r:   r|   r}   r~   r   r   rs   r   r   r   r   r   r   rv   r   r   ra   r   re   )r   r   start_crawlrL   )rM   r:   r|   r}   r~   r   r   r   r   rs   r   r   r   r   r   r   rv   r   r   ra   r   r   rz   s                          r>   r   zFirecrawlClient.start_crawlG  s.   j ##(B)7vY
3
f
 ]
 ]	

 "#6
 &'>
 U
 "#6
 #$8
  0
 U
 
 w
 n
  !2
  "#6!
" ;#
& '(8N9%00''(8(8'BBr@   r   job_idpagination_configc                H    t        j                  | j                  |||      S )av  
        Get the status of a crawl job.
        
        Args:
            job_id: ID of the crawl job
            pagination_config: Optional configuration for pagination behavior
            request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination 
                is enabled (default) and there are multiple pages of results, this timeout applies to 
                each page request separately, not to the entire operation
            
        Returns:
            CrawlJob with current status and data
            
        Raises:
            Exception: If the status check fails
        )r   r   )r   get_crawl_statusrL   )rM   r   r   r   s       r>   r   z FirecrawlClient.get_crawl_status  s)    . ,,/+	
 	
r@   next_urlc                F    t        j                  | j                  ||      S )a7  
        Fetch a single page of crawl results using a next URL.

        Args:
            next_url: Opaque next URL from a prior crawl status response
            request_timeout: Timeout (in seconds) for the HTTP request

        Returns:
            CrawlJob with the page data and next URL (if any)
        r   )r   get_crawl_status_pagerL   rM   r   r   s      r>   r   z%FirecrawlClient.get_crawl_status_page  s&      11+
 	
r@   crawl_idc                 B    t        j                  | j                  |      S )a  
        Retrieve error details and robots.txt blocks for a given crawl job.
        
        Args:
            crawl_id: The ID of the crawl job
        
        Returns:
            CrawlErrorsResponse containing per-URL errors and robots-blocked URLs
        )r   get_crawl_errorsrL   rM   r   s     r>   r   z FirecrawlClient.get_crawl_errors  s     ,,T-=-=xHHr@   c                 @    t        j                  | j                        S )z
        Get a list of currently active crawl jobs.
        
        Returns:
            ActiveCrawlsResponse containing a list of active crawl jobs.
        )r   get_active_crawlsrL   rM   s    r>   r   z!FirecrawlClient.get_active_crawls  s     --d.>.>??r@   c                 "    | j                         S )z
        List currently active crawl jobs for the authenticated team.
        
        Returns:
            ActiveCrawlsResponse containing the list of active crawl jobs
        )r   r   s    r>   active_crawlszFirecrawlClient.active_crawls  s     %%''r@   r0   include_subdomainsr   rs   r   rC   ra   rX   r0   r   c                    t        d ||||||||	fD              rt        ||||||nd|||	      nd}
t        j                  | j                  ||
      S )a/  Map a URL and return discovered links.

        Args:
            url: Root URL to explore
            search: Optional substring filter for discovered links
            include_subdomains: Whether to include subdomains
            ignore_query_parameters: Whether to ignore query parameters when mapping
            limit: Maximum number of links to return
            sitemap: Sitemap usage mode ("only" | "include" | "skip")
            timeout: Request timeout in milliseconds

        Returns:
            MapData containing the discovered links
        c              3   $   K   | ]  }|d u 
 y wrd   re   rf   s     r>   ri   z&FirecrawlClient.map.<locals>.<genexpr>  s       O1$  Orj   Nr   r   )rk   r   
map_moduler1   rL   )rM   r:   r0   r   r   rs   r   rC   ra   rX   rp   s              r>   r1   zFirecrawlClient.map  s    H   O1CE\^celnu  xC  EM  )N  O  O 1$;&2G	#	
 UY 	 ~~d..W==r@   c                 B    t        j                  | j                  |      S )z
        Cancel a crawl job.
        
        Args:
            crawl_id: The ID of the crawl job to cancel
            
        Returns:
            bool: True if the crawl was cancelled, False otherwise
        )r   cancel_crawlrL   r   s     r>   r   zFirecrawlClient.cancel_crawl  s     (()9)98DDr@   c                 \    t        ||      }t        j                  | j                  |      S )zDerive crawl parameters from natural-language prompt.

        Args:
            url: Root URL
            prompt: Instruction describing how to crawl

        Returns:
            CrawlParamsData with normalized crawl configuration
        )r:   r|   )r   r   crawl_params_previewrL   )rM   r:   r|   rz   s       r>   r   z$FirecrawlClient.crawl_params_preview   s)     %V<001A1A7KKr@   
r|   schemasystem_promptr   enable_web_searchshow_sourcesrv   ru   ra   r4   urlsr   r   r   r   r   r4   c       
         X    t        j                  | j                  |||||||||	|
|      S )a  Start an extract job (non-blocking).

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls: URLs to extract from (optional)
            prompt: Natural-language instruction for extraction
            schema: Target JSON schema for the output
            system_prompt: Optional system instruction
            allow_external_links: Allow hyperlinks in output
            enable_web_search: Whether to augment with web search
            show_sources: Include per-field/source mapping when available
            scrape_options: Scrape options applied prior to extraction
            ignore_invalid_urls: Skip invalid URLs instead of failing
            integration: Integration tag/name
            agent: Agent configuration
        Returns:
            Response payload with job id/status (poll with get_extract_status)
        r   )extract_modulestart_extractrL   )rM   r   r|   r   r   r   r   r   rv   ru   ra   r4   s               r>   r   zFirecrawlClient.start_extract-  sB    J ++'!5/%) 3#
 	
r@   r|   r   r   r   r   r   rv   ru   r   rC   ra   r4   c                \    t        j                  | j                  |||||||||	|
|||      S )a  Extract structured data and wait until completion.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            urls: URLs to extract from (optional)
            prompt: Natural-language instruction for extraction
            schema: Target JSON schema for the output
            system_prompt: Optional system instruction
            allow_external_links: Allow hyperlinks in output
            enable_web_search: Whether to augment with web search
            show_sources: Include per-field/source mapping when available
            scrape_options: Scrape options applied prior to extraction
            ignore_invalid_urls: Skip invalid URLs instead of failing
            poll_interval: Seconds between status checks
            timeout: Maximum seconds to wait (None for no timeout)
            integration: Integration tag/name
            agent: Agent configuration
        Returns:
            Final extract response when completed
        r   )r   r3   rL   )rM   r   r|   r   r   r   r   r   rv   ru   r   rC   ra   r4   s                 r>   r3   zFirecrawlClient.extracta  sH    R %%'!5/%) 3'#
 	
r@   )rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   append_to_idru   r   r   ra   idempotency_keyr   r   c                   t        d ||||||||	|
||||||||||fD              rot        di t        di d|d|d|d|d|d|d|d	|	d
|
d|d|d|d|d|d|d|d|d|d|j                         D ci c]
  \  }}||| c}}nd}t	        j
                  | j                  |||||||||
      S c c}}w )a  Start a batch scrape job over multiple URLs (non-blocking).

        Args:
            urls: List of URLs to scrape
            formats: Output formats to collect per URL
            headers: HTTP headers
            include_tags: HTML tags to include
            exclude_tags: HTML tags to exclude
            only_main_content: Restrict scraping to main content
            timeout: Per-request timeout in milliseconds
            wait_for: Wait condition in milliseconds
            mobile: Emulate mobile viewport
            parsers: Parser list (e.g., ["pdf"]) 
            actions: Browser actions to perform
            location: Location settings
            skip_tls_verification: Skip TLS verification
            remove_base64_images: Remove base64 images from output
            fast_mode: Prefer faster scraping modes
            use_mock: Use a mock data source (internal/testing)
            block_ads: Block ads during scraping
            proxy: Proxy setting
            max_age: Cache max age
            store_in_cache: Whether to store results in cache
            webhook: Webhook configuration
            append_to_id: Append to an existing batch job
            ignore_invalid_urls: Skip invalid URLs without failing
            max_concurrency: Max concurrent scrapes
            zero_data_retention: Delete data after 24 hours
            integration: Integration tag/name
            idempotency_key: Header used to deduplicate starts

        Returns:
            Response payload with job id (poll with get_batch_scrape_status)
        c              3   $   K   | ]  }|d u 
 y wrd   re   rf   s     r>   ri   z5FirecrawlClient.start_batch_scrape.<locals>.<genexpr>         O1$  Orj   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   N)rp   r   r   ru   r   r   ra   r   re   )rk   r   rl   rm   batch_modulestart_batch_scraperL   )rM   r   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   r   ru   r   r   ra   r   ro   rh   rp   s                                  r>   r   z"FirecrawlClient.start_batch_scrape  s   n   O'<Whjqs{  ~D  FM  OV  X`  bw  yM  OX  Zb  dm  ot  v}  M  )N  O  O-   
 $ !!! *! *	!
 #4!  ! "! !  !  ! "! '<! &:! $! "!  $!!" #!$  %!&  .'!( eg)(1( ) !t (
, UY- 	0 ..% 3+ 3#+
 	
/(s    C c                 F    t        j                  | j                  ||      S )a  Get current status and any scraped data for a batch job.

        Args:
            job_id: Batch job ID
            pagination_config: Optional configuration for pagination behavior

        Returns:
            Status payload including counts and partial data
        )r   )r   get_batch_scrape_statusrL   )rM   r   r   s      r>   r   z'FirecrawlClient.get_batch_scrape_status  s&     33/
 	
r@   c                F    t        j                  | j                  ||      S )aB  Fetch a single page of batch scrape results using a next URL.

        Args:
            next_url: Opaque next URL from a prior batch scrape status response
            request_timeout: Timeout (in seconds) for the HTTP request

        Returns:
            BatchScrapeJob with the page data and next URL (if any)
        r   )r   get_batch_scrape_status_pagerL   r   s      r>   r   z,FirecrawlClient.get_batch_scrape_status_page  s&     88+
 	
r@   c                 B    t        j                  | j                  |      S )zCancel a running batch scrape job.

        Args:
            job_id: Batch job ID

        Returns:
            True if the job was cancelled
        )r   cancel_batch_scraperL   rM   r   s     r>   r   z#FirecrawlClient.cancel_batch_scrape*  s     //0@0@&IIr@   c                 B    t        j                  | j                  |      S )zRetrieve error details for a batch scrape job.

        Args:
            job_id: Batch job ID

        Returns:
            Errors and robots-blocked URLs for the job
        )batch_methodsget_batch_scrape_errorsrL   r   s     r>   r   z'FirecrawlClient.get_batch_scrape_errors5  s     44T5E5EvNNr@   c                 B    t        j                  | j                  |      S )a  Get the current status (and data if completed) of an extract job.

        .. deprecated::
            The extract endpoint is in maintenance mode and its use is discouraged.
            Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
            to find a replacement.

        Args:
            job_id: Extract job ID

        Returns:
            Extract response payload with status and optional data
        )r   get_extract_statusrL   r   s     r>   r   z"FirecrawlClient.get_extract_status@  s     001A1A6JJr@   )r   ra   max_creditsstrict_constrain_to_urlsmodelr   r   r   r   )zspark-1-prozspark-1-minic                R    t        j                  | j                  ||||||||	      S )aT  Start an agent job (non-blocking).

        Args:
            urls: URLs to process (optional)
            prompt: Natural-language instruction for the agent
            schema: Target JSON schema for the output (dict or Pydantic BaseModel)
            integration: Integration tag/name
            max_credits: Maximum credits to use (optional)
            model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")
            webhook: Webhook URL or configuration for notifications
        Returns:
            Response payload with job id/status (poll with get_agent_status)
        )r|   r   ra   r   r   r   r   )agent_modulestart_agentrL   )	rM   r   r|   r   ra   r   r   r   r   s	            r>   r   zFirecrawlClient.start_agentP  s8    2 ''##%=

 
	
r@   )r   ra   r   rC   r   r   r   r   c       	         V    t        j                  | j                  |||||||||	|
      S )a  Run an agent and wait until completion.

        Args:
            urls: URLs to process (optional)
            prompt: Natural-language instruction for the agent
            schema: Target JSON schema for the output (dict or Pydantic BaseModel)
            integration: Integration tag/name
            poll_interval: Seconds between status checks
            timeout: Maximum seconds to wait (None for no timeout)
            max_credits: Maximum credits to use (optional)
            model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")
            webhook: Webhook URL or configuration for notifications
        Returns:
            Final agent response when completed
        )	r|   r   ra   r   rC   r   r   r   r   )r   r4   rL   )rM   r   r|   r   ra   r   rC   r   r   r   r   s              r>   r4   zFirecrawlClient.agentu  s>    : !!#'#%=
 	
r@   c                 B    t        j                  | j                  |      S )zGet the current status (and data if completed) of an agent job.

        Args:
            job_id: Agent job ID

        Returns:
            Agent response payload with status and optional data
        )r   get_agent_statusrL   r   s     r>   r   z FirecrawlClient.get_agent_status  s     ,,T-=-=vFFr@   c                 B    t        j                  | j                  |      S )zCancel a running agent job.

        Args:
            job_id: Agent job ID

        Returns:
            True if the agent was cancelled
        )r   cancel_agentrL   r   s     r>   r   zFirecrawlClient.cancel_agent  s     (()9)96BBr@   c                 @    t        j                  | j                        S )zCGet current concurrency and maximum allowed for this team/key (v2).)usage_methodsget_concurrencyrL   r   s    r>   r   zFirecrawlClient.get_concurrency      ,,T-=-=>>r@   c                 @    t        j                  | j                        S )z-Get remaining credits for this team/key (v2).)r   get_credit_usagerL   r   s    r>   r   z FirecrawlClient.get_credit_usage      --d.>.>??r@   c                 @    t        j                  | j                        S )z$Get recent token usage metrics (v2).)r   get_token_usagerL   r   s    r>   r   zFirecrawlClient.get_token_usage  r   r@   
by_api_keyc                 B    t        j                  | j                  |      S )z!Get historical credit usage (v2).)r   get_credit_usage_historicalrL   rM   r   s     r>   r   z+FirecrawlClient.get_credit_usage_historical  s    889I9I:VVr@   c                 B    t        j                  | j                  |      S )z Get historical token usage (v2).)r   get_token_usage_historicalrL   r   s     r>   r   z*FirecrawlClient.get_token_usage_historical  s    778H8H*UUr@   c                 @    t        j                  | j                        S )z*Get metrics about the team's scrape queue.)r   get_queue_statusrL   r   s    r>   r   z FirecrawlClient.get_queue_status  r   r@   ttlactivity_ttlstream_web_viewr   r   r   c                H    t        j                  | j                  |||      S )aJ  Create a new browser session.

        Args:
            ttl: Total time-to-live in seconds (30-3600, default 300)
            activity_ttl: Inactivity TTL in seconds (10-3600)
            stream_web_view: Whether to enable webview streaming

        Returns:
            BrowserCreateResponse with session id and CDP URL
        r   )browser_moduler5   rL   )rM   r   r   r   s       r>   r5   zFirecrawlClient.browser  s)    " %%%+	
 	
r@   bashlanguagerC   
session_idcoder   )pythonnoder   c                J    t        j                  | j                  ||||      S )aa  Execute code in a browser session.

        Args:
            session_id: Browser session ID
            code: Code to execute
            language: Programming language ("python", "node", or "bash")
            timeout: Execution timeout in seconds (1-300, default 30)

        Returns:
            BrowserExecuteResponse with execution result
        r   )r   browser_executerL   )rM   r   r   r   rC   s        r>   r   zFirecrawlClient.browser_execute  s,    & --
 	
r@   c                 B    t        j                  | j                  |      S )zDelete a browser session.

        Args:
            session_id: Browser session ID

        Returns:
            BrowserDeleteResponse
        )r   delete_browserrL   )rM   r   s     r>   r   zFirecrawlClient.delete_browser  s     ,,T-=-=zJJr@   statusr   )active	destroyedc                D    t        j                  | j                  |      S )zList browser sessions.

        Args:
            status: Filter by session status ("active" or "destroyed")

        Returns:
            BrowserListResponse with list of sessions
        r   )r   list_browsersrL   )rM   r   s     r>   r  zFirecrawlClient.list_browsers  s#     ++
 	
r@   r.   kindr   rC   r  )r.   r/   c                "    t        | ||||      S )aA  Create a watcher for crawl or batch jobs.

        Args:
            job_id: Job ID to watch
            kind: Job kind ("crawl" or "batch")
            poll_interval: Seconds between status checks
            timeout: Maximum seconds to watch (None for no timeout)

        Returns:
            Watcher instance
        r  r6   )rM   r   r  r   rC   s        r>   watcherzFirecrawlClient.watcher  s    & tV$mU\]]r@   )rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   r   ru   r   r   ra   r   r   wait_timeoutr  c                   t        d ||||||||	|
||||||||||fD              rot        di t        di d|d|d|d|d|d|d|d	|	d
|
d|d|d|d|d|d|d|d|d|d|j                         D ci c]
  \  }}||| c}}nd} t	        j
                  | j                  || |||||||||      S c c}}w )zE
        Start a batch scrape job and wait until completion.
        c              3   $   K   | ]  }|d u 
 y wrd   re   rf   s     r>   ri   z/FirecrawlClient.batch_scrape.<locals>.<genexpr>n  r   rj   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   N)
rp   r   r   ru   r   r   ra   r   r   rC   re   )rk   r   rl   rm   r   batch_scraperL   )!rM   r   rO   rP   rQ   rR   rS   rC   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   r   r   ru   r   r   ra   r   r   r  ro   rh   rp   s!                                    r>   r	  zFirecrawlClient.batch_scrape4  s   t   O'<Whjqs{  ~D  FM  OV  X`  bw  yM  OX  Zb  dm  ot  v}  M  )N  O  O-   
 $ !!! *! *	!
 #4!  ! "! !  !  ! "! '<! &:! $! "!  $!!" #!$  %!&  .'!( eg)(1( ) !t (
, UY- 	0 ((% 3+ 3#+' 
 	
/(s    C)Nzhttps://api.firecrawl.devN   g      ?rd   )F)G__name__
__module____qualname____doc__staticmethodstrboolr?   r   floatintrN   r   r   r   r   r   r-   r   r   r   r   r0   r	   r   r   r.   r   r   r)   r   r   r   r   r   r   r   r(   r   r1   r   r   r   r   r*   r   r3   r   r   r   r   r   r   r   r   r4   r   r   r   r   r   r   r   r   r5   r   r   r  r7   r  r	  re   r@   r>   r9   r9   9   s`    2s 2t 2 2
 "&2#' #)
#)
 )
 %	)

 )
 )
^ 37,0,0,0,0!%"&!%KO AE)-04/3$("&$(#!%)-%)/KDKD $~./	KD
 $sCx.)KD tCy)KD tCy)KD $D>KD #KD 3-KD KD %S	4c9n0E+F FGHKD $u  &{   |  }  ~KD :&KD  (~KD  'tn!KD" D>#KD$ 3-%KD& D>'KD( })KD* #+KD, !-KD. c]/KD0 
1KDb 1559#!"&.2!%26%))?)? $|,-	)?
 T.12)? })? c])? 3-)? &d^)? #)? !/)? c])? 
)?^ !%-1-1-1@D)-(-#$)%*!&#)-7;26"'$)!%+/%)1^
^
 	^

  S	*^
  S	*^
 &c]^
 '";<=^
 !^
 "&^
 }^
 "^
 #^
 ^
 }^
  "#!^
" %] 234#^
$ !/%^
&  '^
( ")^
* +^
, #-^
. "%/^
0 c]1^
2 
3^
H !%-1-1-1@D)-(-#$)%*!&#)-7;26"'$)%)+QCQC 	QC
  S	*QC  S	*QC &c]QC '";<=QC !QC "&QC }QC "QC #QC QC }QC  "#!QC" %] 234#QC$ !/%QC&  'QC( ")QC* c]+QC, 
-QCl 9=

 ,0

 $$45

 "%
 

D ,0	

 "%	

 

,
I 
I1D 
I@#7 @(3 ( !%-126#@D!%%)'+&>&> 	&>
 %TN&> "*$&> }&> '";<=&> #&> c]&> 8$&> 
&>P
ES 
ET 
EL LS L_ L %)2
 !%+/'+/3,0'+48.2%)(,2
tCy!2
 	2

 c3h(2
  }2
 'tn2
 $D>2
 tn2
 !12
 &d^2
 c]2
 %2
l %)8
 !%+/'+/3,0'+48.2!%%)(,8
tCy!8
 	8

 c3h(8
  }8
 'tn8
 $D>8
 tn8
 !18
 &d^8
 8
 #8
 c]8
 %8
| 37,0,0,0,0!%"&!%KO AE)-04/3$("&$(#!%)-7;&*.2)-.2%))-;d
3id
 $~./	d

 $sCx.)d
 tCy)d
 tCy)d
 $D>d
 #d
 3-d
 d
 %S	4c9n0E+F FGHd
 $u  &{   |  }  ~d
 :&d
  (~d
  'tn!d
" D>#d
$ 3-%d
& D>'d
( })d
* #+d
, !-d
. %] 234/d
0 sm1d
2 &d^3d
4 "#5d
6 &d^7d
8 c]9d
: "#;d
R 9=

 $$45
0 ,0	

 "%	
*	J# 	J$ 	J	Oc 	OK K$ %)#

 !%%)%)37BF<@#
tCy!#
 	#

 #
 c]#
 c]#
 #+4.#
  =>?#
 %%7 789#
N %))

 !%%)!%%)37BF<@)
tCy!)
 	)

 )
 c])
 )
 #)
 c])
 #+4.)
  =>?)
 %%7 789)
V	Gs 	G	C3 	C4 	C?@?Wd WVT V@ "&**.
 c]
 sm	

 "$
: 7=!%

 

 23
 #
6	K 	K <@
 !678
, +2!%^^ &'	^
 ^ #^ 
^2 37,0,0,0,0!%"&!%KO AE)-04/3$("&$(#!%)-7;&*.2)-.2%))-&*?I
3iI
 $~./	I

 $sCx.)I
 tCy)I
 tCy)I
 $D>I
 #I
 3-I
 I
 %S	4c9n0E+F FGHI
 $u  &{   |  }  ~I
 :&I
  (~I
  'tn!I
" D>#I
$ 3-%I
& D>'I
( })I
* #+I
, !-I
. %] 234/I
0 sm1I
2 &d^3I
4 "#5I
6 &d^7I
8 c]9I
: "#;I
< =I
> sm?I
r@   r9   )Fr  rH   typingr   r   r   r   r   r   r	   typesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   utils.http_clientr+   utils.error_handlerr,   methodsr-   rn   r.   r   r/   r   r0   ry   r1   r   r   r2   r   r3   r   r4   r   r5   r   r  r7   r9   re   r@   r>   <module>r     s{    
 F F F! ! ! ! ! ! ! ! !D * / , * * , & + + . * . D
 D
r@   