
    4vj0|                     |   d Z ddlZddlZddlZdZd Zd Zd Zd Zd Z	d	 Z
d
 ZdddZd Zd ZdddZd Zd Zd Zd Zd Z	 dCddddddZd Zd Zd Zd d!d"d#d$d%Zd& Zdd'd(Zddd)d*Zd d!d"d#dd+d,Zd- Zd. Zd/ Z d0 Z!d1 Z"d2 Z#d3 Z$d4d5d6d7Z%d4d8d5d9d:Z&dd;d<Z'd5d4d8d=d>Z(dd5d?d@Z)dA Z*dB Z+y)Da  
Copyright (C) 2023 Artifex Software, Inc.

This file is part of PyMuPDF.

PyMuPDF is free software: you can redistribute it and/or modify it under the
terms of the GNU Affero General Public License as published by the Free
Software Foundation, either version 3 of the License, or (at your option)
any later version.

PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.

You should have received a copy of the GNU Affero General Public License
along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>

Alternative licensing terms are available from the licensor.
For commercial licensing, see <https://www.artifex.com/> or contact
Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
CA 94129, USA, for further information.

---------------------------------------------------------------------

PyMuPDF table grid refinement (opt-in extension).

Provides the refine_grid / refine_grid_structure / refine_grid_rows API that
refines a detected table's cell grid using page text and vector-graphics
geometry. Re-exported by pymupdf.table; never runs on the default
find_tables() path.
    Ng      @c                    g }| j                  d      }|j                  dg       D ]  }|j                  d      dk7  r|j                  dg       D ]  }|j                  d      }|j                  d      }|j                  dg       D ]x  }d	j                  d
 |j                  dg       D              }|j                         s<|j	                  |j                  dg       D 	cg c]  }	t        |	       c}	|||d       z   |S c c}	w )a
  Flattened rawdict text spans carrying line-direction metadata.

    Only used to locate vertical/rotated text so those words can be replaced by
    span-level bboxes in _refine_page_words. Minimal fields (bbox/text/dir/wmode)
    -- the only ones the consumers read.rawdictblockstyper   linesdirwmodespans c              3   R   K   | ]  }t        |j                  d d             ! yw)cr   N)strget.0chars     H/root/.hermes/venv/lib/python3.12/site-packages/pymupdf/_table_refine.py	<genexpr>z(_refine_rawdict_spans.<locals>.<genexpr>F   s      X$s488C#45Xs   %'charsbbox)r   textr   r	   )get_textr   joinstripappendfloat)
pager
   rawblockline	directionr	   spanr   vs
             r   _refine_rawdict_spansr$   7   s    E
--	
"C2& 99V!IIgr* 	DIHHW%E"- wwX$((7TVBWXXzz|3788FB3G Haq H $(!&			$ L !Is   "Dc                 (   | j                  d      dvry| j                  d      }t        |t        t        f      rt	        |      dk  ry	 t        t        |d               }t        t        |d               }||kD  S # t        t        f$ r Y yw xY w)	zTrue when a rawdict span's direction indicates non-horizontal text flow.

    wmode != 0 is vertical; otherwise compare |dir_x| vs |dir_y| (a missing or
    unparsable direction counts as horizontal).r	   Nr   Tr      Fr      )	r   
isinstancelisttuplelenabsr   	TypeError
ValueError)r"   r!   dxdys       r   _refine_is_vertical_or_rotatedr2   T   s    
 xx	)Ii$/3y>A3Ey|$%y|$% 7N z" s   .A? ?BBc                 t   t        | dd      }||S | j                  d      D cg c]a  }t        |d         j                         rCt	        |d         t	        |d         t	        |d         t	        |d         t        |d         fc }}t        |       D cg c];  }t        |      r.t        |j                  d	      xs d
      j                         r|= }}|r+|D cg c]*  }t        j                  |j                  d      xs g       , }}g }|D ]R  }|\  }	}
}}}t        j                  |	|z   dz  |
|z   dz        t        fd|D              rB|j                  |       T t        ||      D ]  \  }}|j                  r|j                  t	        |j                        t	        |j                         t	        |j"                        t	        |j$                        t        |d	         f        t'        |d       }	 t)        | d|       |S c c}w c c}w c c}w # t*        $ r Y |S w xY w)a1  Center-point word list for grid refinement, cached on the page object.

    Extract page words once (page.get_text("words")), then substitute vertical/
    rotated text: any horizontal word whose center falls inside a rotated span's
    bbox is dropped, and each rotated span is appended as a single word._refine_words_cacheNwords   r   r(   r'      r   r   r         ?c              3   Z   K   | ]"  }|j                   r|j                         $ y wN)is_emptycontains)r   rectcenters     r   r   z%_refine_page_words.<locals>.<genexpr>   s      XT$--4==(Xs   ++c                     | d   | d   fS Nr(   r    items    r   <lambda>z$_refine_page_words.<locals>.<lambda>   s    tAwQ.@     key)getattrr   r   r   r   r$   r2   r   pymupdfRectPointanyr   zipr;   x0y0x1y1sortedsetattr	Exception)r   cachewr5   r"   rotated_spansrotated_rectskeptwordwx0wy0wx1wy1_r=   r>   s                  @r   _refine_page_wordsr`   f   s&    D/6E w'qt9?? 
qteAaDk51;adS1YGE  *$/)$/C8H8NB4O4U4U4W 	M 
 JWX$dhhv&6&<"=XX 	D$(!Cc3]]C#I#4sSyC6GHFX]XXKK	 m]; 	JD$}}KKtwwtwwtwwtwwQTUYZ`UaQbc	 t!@A+U3 L=
 Y"  Ls%   A&HA H $/H%H* *	H76H7c                 f    | |z   dz  }||z   dz  }	||cxk  xr |k  nc xr ||	cxk  xr |k  S c S )zWord-to-cell membership by CENTER point (inclusive), not clip-overlap.

    A word straddling a column line is claimed by exactly one cell.r8   rA   )
r[   r\   r]   r^   rN   rO   rP   rQ   cxcys
             r   _refine_word_in_rectrd      s@     )s	B
)s	B>r>,bBn"n,n,rE   c                 >   t        |j                        t        |j                        t        |j                        t        |j                        f\  }}}}t        |       D 	
cg c]!  \  }}}}	}
t        ||||	||||      r||||	|
f# c}
}	}}}S c c}
}	}}}w )zGPage words whose center lies in rect, as (x0, y0, x1, y1, text) tuples.)r   rN   rO   rP   rQ   r`   rd   )r   r=   rN   rO   rP   rQ   r[   r\   r]   r^   r   s              r   _refine_words_in_rectrf      s    477^U477^U477^U477^SNBB );4(@ $Cc3S#sBBC 
c3T"  s   )&Bc                 \    t        t        j                  d| j                                     S )z8Loose value-like predicate: the text contains any digit.\d)boolresearchr   r   s    r   _refine_has_digitrm      s    		%.//rE   c                     | j                         }t        t        j                  d|            xr t	        d |D               S )z=Strict value-like predicate: contains a digit and no letters.rh   c              3   <   K   | ]  }|j                           y wr:   )isalphar   s     r   r   z-_refine_has_digit_no_alpha.<locals>.<genexpr>   s     7\47\s   )r   ri   rj   rk   rL   )r   strippeds     r   _refine_has_digit_no_alpharr      s7    zz|H		%*+\C7\S[7\4\0\\rE   ffffff?)	thresholdc                    | y	 | d d \  }}}t        |      |kD  xr t        |      |kD  xr t        |      |kD  S # t         $ r Y yw xY w)NTr7   )rT   r   )fillrt   rgbs        r   _refine_is_whiterz      sa    |r(1a 8iQE!Hy$8QU1X	=QQ  s   	< 	AAc                 `   |#t        j                  |      }|j                  s|S | D cg c]?  }|D ]8  }|4t        j                  |      j                  st        j                  |      : A }}}|sy t        j                  |d         }|dd  D ]  }|j                  |        |S c c}}w Nr   r(   )rI   rJ   r;   include_rect)cells
table_bboxr=   rowcellrectsrC   s          r   _refine_table_rectr      s    ||J'}}K  GLL$6$?$? 	TE  <<a!Dab	  $ Ks   AB*c                   g }t        | j                  j                        }| j                         D ]  }t	        |j                  d            r|j                  dg       D ]  }|d   dk7  rt        j                  |d         }|j                          |j                  d|z  k\  rG|j                  |k  s|j                  |k  rf||z  j                  rv|j                  |         |S )Nrv   itemsr   rj   r(   rs   )r   r=   widthget_drawingsrz   r   rI   rJ   	normalizeheightr;   r   )r   
table_rectmin_dimout
page_widthdrawingrC   r=   s           r   _refine_raw_shaded_rectsr      s    
Ctyy'J$$& GKK/0KK, 	DAw$<<Q(DNNzzTJ..zzG#t{{W'<z!++JJt	 JrE   g?)contain_fracc                    g }t        |       D ]O  \  j                         dk  rt        fdt        |       D              }|r?|j                         Q |S )Nr   c              3      K   | ]?  \  }}|k7  xr1 |z  j                         z  k\  xr |j                         d z  kD   A yw)g?N)get_area)r   other_indexotherarear   indexr=   s      r   r   z0_refine_cell_background_rects.<locals>.<genexpr>   s`      
 #U 5  /'')\D-@@/ 4$;./
s   AA)	enumerater   rL   r   )	raw_rectsr   keepnestedr   r   r=   s    `  @@@r   _refine_cell_background_rectsr      sk    D + t}}19 
 '0	&:	
 
 KK KrE   c                
   g }g }t        |       D ]H  }|r3||d   z
  |kD  r(|j                  t        |      t        |      z         g }|j                  |       J |r&|j                  t        |      t        |      z         |S )N)rR   r   sumr,   )values	toleranceclusterscurrentvalues        r   _refine_clusterr      s    HG uwr{*Y6OOCL3w<78Gu	
 Gs7|34OrE   c                    t               }t               }| j                         D ]K  }|j                  d      dv }|j                  dg       D ]  }|d   }|dk(  r|d   |d   }	}t        |j                  |	j                  z
        dk  rt        |j
                  |	j
                  z
        d	kD  rx|j                  d
z
  |j                  |	j                  z   dz  cxk  r|j                  d
z   k  r9n n6|j                  t        |j                  |	j                  z   dz  d             nt        |j
                  |	j
                  z
        dk  rt        |j                  |	j                  z
        d	kD  rw|j                  d
z
  |j
                  |	j
                  z   dz  cxk  r|j                  d
z   k  r8n n5|j                  t        |j
                  |	j
                  z   dz  d             |dk7  rt        j                  |d         }
|
j                          |
|z  j                  r|
j                   d
k  rF|
j"                  d	kD  r7|j                  t        |
j                  |
j                  z   dz  d             3|
j"                  d
k  rF|
j                   d	kD  r7|j                  t        |
j                  |
j                  z   dz  d             |s|j                  t        |
j                  d             |j                  t        |
j                  d             |j                  t        |
j                  d             |j                  t        |
j                  d             " N t%        |      t%        |      fS )Nr   )sfsr   r   lr(   r'         ?g      $@       @r8   rj   )setr   r   r-   yxrO   rQ   addroundrN   rP   rI   rJ   r   r;   r   r   rR   )r   r   xsysr   strokedrC   kindp1p2r=   s              r   _refine_border_linesr      s   	B	B$$& !*++f%4KK, 	*D7Ds{a$q'Brtt$s*BDD244K(4/"+rttbdd{c.AWJMMTWDWWFF5"$$+!4a89rtt$s*BDD244K(4/"+rttbdd{c.AWJMMTWDWWFF5"$$+!4a89t|<<Q(DNNz!++{{S TZZ$%6udgg/36:;c!dkkD&8udgg/36:;uTWWa()uTWWa()uTWWa()uTWWa()?	*!*D ":vbz!!rE   c                ^    | D cg c]  t        fd|D              r c}S c c}w )Nc              3   @   K   | ]  }t        |z
        k    y wr:   )r-   )r   borderedger   s     r   r   z$_refine_drop_near.<locals>.<genexpr>(  s      -dRXc$-.@I.M-ds   rL   )edgesbordersr   r   s     ``r   _refine_drop_nearr   '  s%    "eT#-d\c-d*dDeees   **c                    | D ]\  \  }}}}	}
t        |
      j                         s#||z   dz  }||	z   dz  }||z   |cxk  r	||z
  k  sEn H||z   |cxk  r
||z
  k  sZ y ^ y)Nr8   TF)r   r   )r5   rN   rO   rP   rQ   marginr[   r\   r]   r^   r   rb   rc   s                r   _refine_has_textr   +  s    $)  S#sD4y Ci3Ci3;)b6k)b6kB.Lf.L /M rE   c          
      R   t        fd|D              }|rg|}d }	t        t        |      dz
        D ]   }
t        | |||
   |||
dz            r|
}	 n |		 |S |	t        |      k  r|j	                  |	       n |	dz
  dk\  r|j	                  |	dz
         n	 |S |r|S )Nc              3   L   K   | ]  }z   |cxk  r	z
  k  sn n|  y wr:   rA   )r   r   r   rO   rQ   s     r   r   z3_refine_content_filter_row_edges.<locals>.<genexpr>7  s%     SD"v+2RrF{2R4Ss   $
$r(   r   r   )rR   ranger,   r   pop)r5   rO   rQ   
candidatesrN   rP   r   r   boundsempty_indexr   s    ``   `    r    _refine_content_filter_row_edgesr   6  s    SJSSE
!u!b!3v;?+ 	E#E2ve}b&QRBS\bc#	  L U#IIk"1_!IIkAo&L  LrE   r   g      @g      @)cluster_tolerancer   r   border_tolerancec                V   t        ||      }||s|S t        | ||      }t        |      }	|	s|S t        |	D 
cg c]3  }
t	        |
j
                        t	        |
j                        fD ]  }| 5 c}}
|      }t        | |      \  }}t        |||      }|s|S t        |       }g }d}|D ]e  }|D cg c]  }|t        j                  |       }}|s|j                  |       =t        d |D              }t        d |D              }t        d |D              }t        d |D              }t        |||||||	      }|s|j                  |       |g||}t!        t#        |      d
z
        D ]  }g }|D ]o  }||j                  d       t        j                  |      }
|j                  t	        |
j$                        ||   t	        |
j&                        ||d
z      g       q |j                  |        |t#        |      dz
  z  }h |dk  r|S |S c c}}
w c c}w )a;  Split rows the line grid merged but a cell background rectangle separates.

    Cell background-shading rectangles give row boundaries the border lines do
    not; candidate y-edges from those rectangles (minus edges that coincide with
    real borders, minus edges that would cut an empty band) subdivide each row.N)r   )r   r   c              3   F   K   | ]  }t        |j                          y wr:   )r   rO   r   r   s     r   r   z%_split_shaded_rows.<locals>.<genexpr>s       2T%.2   !c              3   F   K   | ]  }t        |j                          y wr:   )r   rQ   r   s     r   r   z%_split_shaded_rows.<locals>.<genexpr>t  r   r   c              3   F   K   | ]  }t        |j                          y wr:   )r   rN   r   s     r   r   z%_split_shaded_rows.<locals>.<genexpr>u  r   r   c              3   F   K   | ]  }t        |j                          y wr:   )r   rP   r   s     r   r   z%_split_shaded_rows.<locals>.<genexpr>v  r   r   r   r(   r'   )r   r   r   r   r   rO   rQ   r   r   r`   rI   rJ   r   minmaxr   r   r,   rN   rP   )r   r~   r   r   r   r   r   r   r   background_rectsr=   r   y_edges
_border_xs	border_ysr5   	new_cells
added_rowsr   r   livery0ry1rx0rx1cutsr   
band_indexbands                                r   _split_shaded_rowsr   J  sF    $E:6J(z7KI4Y?*X$dggdgg7WXtXX#G 1zBJ	>NOGt$EIJ &/2Gtd6FT"GGS!2T222T222T222T22/sC#s[abS!""c"Fa0 
	#JD <KK%"<<-DKKtww
);U477^VT^abTbMcd T"
	# 	c&kAo%
3&6 QQ 	Y Hs   8H 
6H&>H&c                     t        |       S r:   )rr   rl   s    r   _refine_value_liker     s    %d++rE   c                     t        |       S r:   )rm   rl   s    r   _refine_span_value_liker     s    T""rE   c          	         t        j                  |      }g }t        | |      D ]  \  }}}}	}
t        |
      j	                         }|s%t        |      t        |      z   dz  }t        |      t        |	      z   dz  }|j                  |cxk  r|j                  k  syn ||j                  |cxk  r|j                  k  sn |j                  t        |      t        |      ||f        t        |      dk  ry|j                  d        g }|d   g}|dd D ]9  }|d   |d	   d   z
  |kD  r|j                  |       |g})|j                  |       ; |j                  |       |D ]  }|j                  d
        t        t        |      dz
        D ]p  }||dz      d   ||   d   z
  |k  rdj                  d |d|dz    D              }dj                  d ||dz   d D              }t        |      sct        |      so  y  y)zTrue if a cell holds two value-like groups separated by a wide gap on one
    text line -- the signal that a single grid column packs several columns.r   r'   Fc                     | d   S )Nr'   rA   rB   s    r   rD   z*_refine_cell_span_signal.<locals>.<lambda>  s
    Q rE   rF   r   r(   Nr   c                     | d   S r&   rA   rB   s    r   rD   z*_refine_cell_span_signal.<locals>.<lambda>  s
    47 rE    c              3   &   K   | ]	  }|d      ywr7   NrA   r   rZ   s     r   r   z+_refine_cell_span_signal.<locals>.<genexpr>  s     BDGB   c              3   &   K   | ]	  }|d      ywr   rA   r   s     r   r   z+_refine_cell_span_signal.<locals>.<genexpr>  s     CT!WCr   T)rI   rJ   rf   r   r   r   rN   rP   rO   rQ   r   r,   sortr   r   r   )r   r   gapline_tolerancer=   r5   rN   rO   rP   rQ   r   rq   rb   rc   r   r   rZ   r    r   leftrights                        r   _refine_cell_span_signalr     s    <<DE 5dD A ?BBt9??$Bi%)#s*Bi%)#s*77b#DGG#2(@(@LL%)U2YH=>? 5zA~	JJ'J(EQxjGab	 !7WR[^#n4LL!fGNN4 ! 
LL 		*	+3t9q=) 	EEAIq!DKN2c988B[uqy0ABBDHHCeaik1BCCE&t,1H1O	 rE   g      (@g      @g333333?r'   r   r   support_ratiomin_supportc                   t        |      }t        |t        ||z              i }|D ]@  }t        |      D ]0  \  }	}
|
	t	        | |
||      s|j                  |	d      dz   ||	<   2 B t        fd|j                         D              S )zReturn the column indices that fire the under-segmentation gate: columns
    where enough rows carry the two-value-group signal (a support threshold).)r   r   r   r(   c              3   4   K   | ]  \  }}|k\  s|  y wr:   rA   )r   colcountsupport_thresholds      r   r   z._refine_detect_span_columns.<locals>.<genexpr>  s     W*#uEEV<V#Ws   )r,   r   intr   r   r   rR   r   )r   r~   r   r   r   r   rowscol_hitsr   	col_indexr   r  s              @r   _refine_detect_span_columnsr    s    
 u:DK]T-A)BCH E(~ 	EOIt|'dN[&.ll9a&@1&D#		EE W(8WWWrE   c           	      ^   g }|D ]$  }g }|t        |      k  r||   t        j                  ||         }t        | |      D ]  \  }}}	}
}t	        |      j                         s#t        |      t        |	      z   dz  }t        |      t        |
      z   dz  }|j                  |cxk  r|j                  k  swn z|j                  |cxk  r|j                  k  sn |j                  t        |      t        |	      t	        |      f        |j                  t        |             ' |S )Nr   )r,   rI   rJ   rf   r   r   r   rN   rP   rO   rQ   r   rR   )r   r~   r   r  r   r5   r=   rN   rO   rP   rQ   r   rb   rc   s                 r   _refine_column_wordsr    s   D #S>c#h2<<C)D(=dD(I D$BB4y(Bi%)+s2Bi%)+s277b+DGG+20H0HLL%)U2YD	!BCD 	F5M"# KrE   bridge_tolerancec                   t        |       D cg c]  \  }}|D ]  \  }}}|||f  }}}}}}t        |      dk  rg S t        d |D              }t        d |D              }	|	|z
  dk  rg S g }
d }|}||	k  rj|D ch c]  \  }}}||cxk  r|k  sn n| }}}}t        |      |kD  r|)|
j	                  |d   |d   f       d }n|||gn|d   |g}|dz  }||	k  rj||
j	                  |d   |d   f       g }|
D ]  \  }}||dz   k  s||	dz
  k\  r||z   dz  t        fd| D              }| D cg c]  }t        fd	|D              r| }}dx}}dx}}dx}}g }g }|D ]  }|D cg c]  \  }}}|k  s| }}}}|D cg c]  \  }}}|k\  s| }}}}|r8|dz  }|dz  }|t        d
j                  |            z  }|j                  |       |sz|dz  }|dz  }|t        d
j                  |            z  }|j                  |        |dk  s|dk  r*|dkD  r6t        d
j                  |            rt        d
j                  |            se|r||z  dk  s
|r
||z  dk  r{|j	                          t        |      S c c}}}}}w c c}}}w c c}w c c}}}w c c}}}w )Nr'   c              3   (   K   | ]
  \  }}}|  y wr:   rA   )r   rN   r_   s      r   r   z!_refine_cut_xs.<locals>.<genexpr>  s     *HB1R*   c              3   (   K   | ]
  \  }}}|  y wr:   rA   )r   r_   rP   s      r   r   z!_refine_cut_xs.<locals>.<genexpr>  s     *HAr1R*r  r(   r   r   r   c              3   L   K   | ]  }t        fd |D              sd  yw)c              3   F   K   | ]  \  }}}|cxk  xr |k  nc   y wr:   rA   r   rN   rP   r_   cut_xs       r   r   z+_refine_cut_xs.<locals>.<genexpr>.<genexpr>  %     =eR[RTVXZ[b5o2oo=e   !r(   Nr   )r   r5   r  s     r   r   z!_refine_cut_xs.<locals>.<genexpr>  s     f5#=e_d=e:eQfs   $$c              3   F   K   | ]  \  }}}|cxk  xr |k  nc   y wr:   rA   r  s       r   r   z!_refine_cut_xs.<locals>.<genexpr>  r  r  r   r8   )r   r,   r   r   r   r   rL   r   r   extendrR   )
rows_wordsr
  	row_indexr5   rN   rP   r_   	intervalslohichannelsr   r   crossing_rowsr   	left_edge
right_edgecrossing_count	body_rows	left_rows
right_rowsleft_okright_okleft_nright_nleft_valuesright_valuesr   r   r   r  s                                 @r   _refine_cut_xsr*    s=   =Fz=Rll)9EfklWbXZ\^`a"b)$l$lIl
9~		*	*	*B	*	*	*B	Bw{	HG
A
r'<EUU'8r2yaRTUU} 00"WQZ 89 'q!fgaj!_G	S r' WQZ01D!) "	:Q*Q"6Z'3.fJff )3fu#=e_d=e:eUf	f!""	J( 	+E-2BB\RTbEkDBDB.3CClb"drU{TCECQ	!-chhtn==""4(a
1.sxx??##E*	+ q=JNA%chh{&;<DVWZW_W_`lWmDn6)C/WGASVYAYEE"H $<{ m V& g CCs5   JJ(J(=J/J/8J4J4J;$J;r
  allowed_colsc          
         t        d |D        d      }|d n
t        |      }i }t        |      D ])  }|||vr
t        t	        | ||      |      }|s%|||<   + |s|S g }	|D ]  }
g }t        |
      D ]  \  }}|j                  |      }|s|j                  |       +|"|j                  d gt        |      dz   z         Ot        j                  |      \  }}}}|g|D cg c]  }||cxk  r|k  sn n| c}z   |gz   }t        t        |      dz
        D ]   }|j                  ||   |||dz      |g       "  |	j                  |        |	S c c}w )Nc              3   2   K   | ]  }t        |        y wr:   )r,   )r   r   s     r   r   z(_refine_split_columns.<locals>.<genexpr>"  s     +cS+s   r   )defaultr	  r(   )r   r   r   r*  r  r   r   r   r  r,   rI   rJ   )r   r~   r
  r,  ncolsactive_colscol_cutsr   r   r   r   new_rowr   rN   rO   rP   rQ   r  r   r   s                       r   _refine_split_columnsr4  !  s   +U+Q7E&.$C4EKHU| !"s+'=24DWgh HSM! I ""3 
	GIC<<$Dt$vTQ78!(d!3BBTHuURUHHB4O"3r7Q;/ GENNBuIr2eai="#EFG
	G 	!"  Is   $E
5E
)r   r   r   r   r
  c                \    t        | |||||      }|s|S t        | ||t        |            S )zSplit under-segmented columns, but only those that fire the value-group
    gate -- so ordinary single-value columns are never disturbed.r   r+  )r  r4  r   )r   r~   r   r   r   r   r
  
fired_colss           r   _split_undersegmented_columnsr7  B  sE     -%#J  e&6S_ rE   c                 l    | D cg c]!  }|D ]  }|t        j                  |       # c}}S c c}}w r:   )rI   rJ   )r~   r   r   s      r   _refine_rects_from_cellsr9  ^  s1    */T3CTD4CSGLLTTTTs   00c                 r    | sy t        j                  | d         }| dd  D ]  }|j                  |        |S r|   )rI   rJ   r}   )r   r=   rC   s      r   _refine_union_rectr;  b  sB    <<a!Dab	  $ KrE   c                 H    t        | d g       }t        d |D              }|S )Nc                 <    t        d | D              t        |       fS )Nc              3   &   K   | ]	  }|d  y wNr(   rA   r   s     r   r   z?_refine_best_column_bounds.<locals>.<lambda>.<locals>.<genexpr>n  s     C4$2BQCs   )r   r,   )r   s    r   rD   z,_refine_best_column_bounds.<locals>.<lambda>n  s    C3CCSXN rE   )rG   r/  c              3      K   | ]^  }|Zt        j                  |      }|j                  |j                  kD  r,t	        |j                        t	        |j                        f ` y wr:   )rI   rJ   rP   rN   r   )r   r   r=   s      r   r   z-_refine_best_column_bounds.<locals>.<genexpr>q  sQ      \\$'D77TWW	 
twwtww(s   A$A&)r   rR   )r~   best_rowr   s      r   _refine_best_column_boundsrB  k  s5    NH
   F MrE   c                    | sg S t        | d       } | d   gg}| d   d   | d   d   z   dz  }| dd D ]E  }|d   |d   z   dz  }||z
  t        kD  r|j                  |g       n|d   j                  |       |}G g }|D ]{  }|j                  t        d	 |D              t        d
 |D              t	        d |D              t	        d |D              dj                  d t        |d       D              d       } |S )zyGroup center-point words into body "lines" by a fixed center-y gap,
    returning each line's union bbox and joined text.c                 (    | d   | d   z   dz  | d   fS )Nr(   r7   r   r   rA   rB   s    r   rD   z,_refine_cluster_words_by_y.<locals>.<lambda>  s!    T!WtAw->#,EtAw+O rE   rF   r   r(   r7   r   Nr   c              3   &   K   | ]	  }|d      yw)r   NrA   r   s     r   r   z-_refine_cluster_words_by_y.<locals>.<genexpr>       6d$q'6r   c              3   &   K   | ]	  }|d      yw)r(   NrA   r   s     r   r   z-_refine_cluster_words_by_y.<locals>.<genexpr>  rF  r   c              3   &   K   | ]	  }|d      yw)r'   NrA   r   s     r   r   z-_refine_cluster_words_by_y.<locals>.<genexpr>  rF  r   c              3   &   K   | ]	  }|d      ywr   rA   r   s     r   r   z-_refine_cluster_words_by_y.<locals>.<genexpr>  rF  r   r   c              3   &   K   | ]	  }|d      yw)r6   NrA   r   s     r   r   z-_refine_cluster_words_by_y.<locals>.<genexpr>  s      _Ta _r   c                     | d   S r&   rA   rB   s    r   rD   z,_refine_cluster_words_by_y.<locals>.<lambda>  s    VZ[\V] rE   )rN   rO   rP   rQ   r   )rR   _REFINE_LINE_GAPr   r   r   r   )r5   r   last_centerrZ   r>   r   clusters          r   _refine_cluster_words_by_yrO  {  s$    	5OPEq
|H8A;q!,3Kab	 q'DG#s*K"22OOTF#RL% E 	
6g666g666g666g66 _VGI]5^ __	
	
 LrE   c                 
   t               }t        | d         }t        | d         }||z   dz  }t        |      D ]F  \  }\  }}t        ||      t	        ||      z
  }	|	dkD  s||cxk  r|k  s3n 6|j                  |       H |S )NrN   rP   r   r   )r   r   r   r   r   r   )
r    
col_boundscolsline_x0line_x1r>   r   rN   rP   overlaps
             r   _refine_line_columnsrV    s    5DDJGDJG3&F$Z0 xBgr"S"%55Q;",",HHUO KrE   c           	         g }t        |j                        t        |j                        t        |j                        t        |j                        f\  }}}}t        |       D ]  \  }}}	}
}||	z   dz  }||
z   dz  }||cxk  r|k  s%n (||cxk  r|k  s4n 7t        |      j                         }|sS|j                  t        |      t        |      t        |	      t        |
      |f        |S )Nr8   )	r   rN   rO   rP   rQ   r`   r   r   r   )r   r=   r5   r   r   r   r   rN   rO   rP   rQ   r   rb   rc   s                 r   _refine_page_words_in_rectrX    s    EtwwtwwtwwtwwWCc3 24 8 QBB2g_2g_"r 0S 0t9??$DeBirE"IuRy$OPQ LrE   c                   | sg S t        |       }t        |d   d         t        |d   d         gg}|dd  D ]  \  }}|d   }|d   t        |      z
  }t        |d   |d   z
  t        |      t        |      z
        }|dkD  r?||z  |k\  r7t        |d   t        |            |d<   t        |d   t        |            |d<   |j	                  t        |      t        |      g        |D cg c]	  \  }}||f c}}S c c}}w )Nr   r(   r   )rR   r   r   r   r   )	r   fracr   mergedrO   rQ   r   rU  
min_heights	            r   _refine_merge_overlapping_linesr]    s   	5MEU1Xa[!5q!#567F) 2B*!*uRy(gaj0%)eBi2GH
>g
2d:WQZr3GAJWQZr3GAJMM59eBi012 $**RRH***s   .D g333333?r(   clean_thresholdheader_row_countc          
         t        |      }|r(t        dt        t        |      t	        |                  nd}t        dt	        |      |z
        }t	        |      |k  ryt	        |      dk  ryt        t        ||d             }||j                  ryt        | |      }t        |      }|syt        dt	        |      dz   dz        }	g }
|D ]7  }t        t        ||            }t	        |      |	k\  s'|
j                  |       9 t	        |
      t	        |      z  }||k  ryt	        |
      |k  ry|j                  |j                  |j                  |j                   g|D cg c]  \  }}t#        |d      t#        |d      g c}}|
D cg c]  }t%        |d         t%        |d         g  c}||dS c c}}w c c}w )ay  Decide whether the body rows are over-merged. Returns the geometry needed
    to re-cut them (body bbox, column bounds, per-record y-bounds) or None.

    Fires only when the body text clusters into clean multi-column "record" lines
    (clean_ratio >= clean_threshold) and there are more records than existing body
    rows -- i.e. several records collapsed into one grid row.r(   r   Nr'   rO   rQ   )	body_bboxrQ  record_line_boundsexisting_body_rowsr`  )rB  r   r   r  r,   r;  r9  r;   rX  rO  rR   rV  r   rN   rO   rP   rQ   r   r   )r   r~   r_  r`  rQ  rd  	body_rectr5   r   min_colsrecord_linesr    rR  clean_ratiorN   rP   s                   r   _refine_detect_row_overmergeri    s    ,E2JINs1c#&6"7UDETUQE
-= =>
5z%%
:"#;EBRBS<T#UVII..&tY7E&u-E1s:*q01HL &*4<=t9 %&
 l#c%j0K_$
<..  llILL),,	MBLMBb!eBl3MR^_$d4j 15d3DE_0, M_s   %"F8#F>gffffff?r_  merge_overlap_fracr`  c          
      (   t        | |||      }||S t        j                  |d         }|d   D cg c]  \  }}t        |      t        |      f }	}}|d   D 
cg c]  \  }
}t        |
      t        |      f }}
}t	        ||      }t        |      t        |d         k  r|S |D 
cg c]  \  }
}|
|z   dz   }}
}t        |j                        g}t        ||d	d       D ]  \  }}|j                  ||z   dz          |j                  t        |j                               |dt        |d
          }g }t        t        |            D ]  }||   }
||d	z      }g }|	D ]~  \  }}t        j                  ||
||      }|j                  t        |j                        t        |j                        t        |j                        t        |j                        g        |j                  |        g ||S c c}}w c c}}
w c c}}
w )a8  Re-cut over-merged body rows into one grid row per detected record.

    Keeps the header rows intact, then rebuilds the body as evenly-bounded rows
    (cut at the midpoints between consecutive record centers) across the detected
    column bounds. header_row_count is the number of leading header rows to keep.r^  Nrb  rQ  rc  )rZ  rd  r   r(   r`  )ri  rI   rJ   r   r]  r,   r  rO   rM   r   rQ   r   rN   rP   )r   r~   r_  rk  r`  metarb  rN   rP   rQ  rO   rQ   raw_line_boundsline_boundscentersr   prevnxtheader_rowsnew_bodyr  r3  r=   s                          r   _split_overmerged_rowsru    s    (e_GWD |T+./I7;L7IJVR59eBi(JJJ<@AU<VW&"bb	59-WOW1/HZ[K
;3t$89::-8962rR39G99<< !E'!"+. )	cdSjC'()	LLy||$%7#d#5678KH3{+, !	99q=!  	]FB<<BB/DNNE$''NE$''NE$''NERVRYRYN[\	] 	 ! %[$8$$/ KW
 :s    H H0Hr   c                8    t        | ||      }t        | |      }|S )aV  Structural half of refine_grid: split shaded rows, then under-segmented
    columns. Exposed separately so a caller that must know the header/body
    boundary of the post-structure grid can insert that computation between the
    two phases. table_bbox, when given, bounds the shaded-rectangle search;
    otherwise the cells' union is used.)r   r7  )r   r~   r   s      r   refine_grid_structurerx    s#     tUJ7E)$6ELrE   )r`  r_  rk  c                "    t        | ||||      S )a  Row half of refine_grid: split body rows that collapsed several records
    into one grid row. header_row_count is the number of leading header rows to
    keep intact; callers that resolve a header region pass it in, and the default
    of 1 is a conservative single-header assumption.rj  )ru  )r   r~   r`  r_  rk  s        r   refine_grid_rowsrz  &  s     
 "'-) rE   )r   r`  c                >    t        | ||      }t        | ||      }|S )a  Refine a detected table's cell grid and return the refined grid.

    ``cells`` is a row-major grid: a list of rows, each a list of ``[x0, y0, x1,
    y1]`` cell rectangles (``None`` for a gap). The grid is refined in three
    stages -- split rows that cell background shading separates, split columns
    that jam several values into one cell, and split body rows that merged
    several records -- using the page's words and vector graphics. The result is
    a new grid in the same format (rows may be added and rows may widen).

    ``table_bbox`` optionally bounds the table (else the cells' union is used).
    ``header_row_count`` is the number of leading header rows to preserve when
    re-cutting body rows (default 1). This is the all-in-one convenience wrapper;
    a caller needing the header boundary of the intermediate grid can instead
    call :func:`refine_grid_structure` then :func:`refine_grid_rows`.
    rv  )r`  )rx  rz  )r   r~   r   r`  s       r   refine_gridr|  4  s&      "$*EET5;KLELrE   c           	         | sg S t        | D ch c]  }|d   	 c}      }t        |      D ci c]  \  }}||
 }}}g }t        | d       }t        j                  |d       D ]k  \  }}	dgt	        |      z  }
|	D ]A  }t        |d         t        |d         t        |d         t        |d         g|
||d      <   C |j                  |
       m |S c c}w c c}}w )	zConvert a Table's flat cell list into the row-major grid refine_grid wants.

    Mirrors Table.rows: columns are the sorted distinct left edges, rows are the
    cells grouped by top edge, each padded with None where a column has no cell.r   c                     | d   | d   fS r@   rA   r   s    r   rD   z'_refine_cells_to_grid.<locals>.<lambda>S  s    1Q41, rE   rF   c                     | d   S r?  rA   r  s    r   rD   z'_refine_cells_to_grid.<locals>.<lambda>T  s
    !A$ rE   Nr(   r'   r7   )rR   r   	itertoolsgroupbyr,   r   r   )r~   r   r   ir   r  gridordered_y	row_cellsr   r   s               r   _refine_cells_to_gridr  I  s    
 		u%!1%	&B"+B-0$!QA0I0DU 67G"**7G Ifs2w 	gD',T!W~uT!W~uTRSW~W\]abc]dWe&fC	$q'"#	gC	
 K &0s
   CCc                     | D cg c]A  }|D ]:  }|6t        |d         t        |d         t        |d         t        |d         f< C c}}S c c}}w )zIFlatten a refined grid back to a Table's flat (x0, y0, x1, y1) cell list.r   r(   r'   r7   )r   )r  r   r   s      r   _refine_grid_to_cellsr  \  sh       
tAwtAwtAwtAwHH  s   AAr:   ),__doc__r  rj   rI   rL  r$   r2   r`   rd   rf   rm   rr   rz   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r*  r4  r7  r9  r;  rB  rO  rV  rX  r]  ri  ru  rx  rz  r|  r  r  rA   rE   r   <module>r     sf  B  	   :$'T-0
] )- R&* >A "
%"Pf. A
 AJ,#$P SQRX"" 45 >B <=4 J 	8U <	
+" BFXY +^ %)TTU&%T 6:  78bf  ,0! *&rE   