
    `gj_                        d Z ddlmZ ddlZddlmZmZ ddlmZ erddlm	Z	 	 ddl
mZ  G d	 d
      Zy# e$ r ddlZY w xY w)u   Hybrid keyword/BM25 retrieval for the memory store.

Ported from KIK memory_agent.py — combines FTS5 full-text search with
Jaccard similarity reranking and trust-weighted scoring.
    )annotationsN)datetimetimezone)TYPE_CHECKING   )MemoryStore)holographicc                  X   e Zd ZdZ	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 d	 	 	 	 	 	 	 	 	 ddZ	 	 d	 	 	 	 	 	 	 ddZ	 	 d	 	 	 	 	 	 	 ddZ	 	 d	 	 	 	 	 	 	 ddZ	 	 	 d	 	 	 	 	 	 	 ddZ		 	 d	 	 	 	 	 	 	 dd	Z
	 	 	 	 	 	 	 	 	 	 dd
Zedd       Z eh d      Zedd       Zedd       ZddZy)FactRetrieverz:Multi-strategy fact retrieval with trust-weighted scoring.c                    || _         || _        || _        |dkD  rt        j                  sd}d}d}|| _        || _        || _        y )Nr   g333333?皙?        )store	half_lifehrr_dimhrr
_HAS_NUMPY
fts_weightjaccard_weight
hrr_weight)selfr   temporal_decay_half_lifer   r   r   r   s          W/root/.hermes/venv/lib/python3.12/site-packages/plugins/memory/holographic/retrieval.py__init__zFactRetriever.__init__   sM     
1 >#..J NJ$,$    Nc                   | j                  ||||dz        }|sg S | j                  |      }g }|D ]j  }| j                  |d         }	| j                  |j                  dd            }
|	|
z  }| j                  ||      }|j                  dd      }| j                  dkD  rf|j                  d      rUt        j                  |d         }t        j                  || j                        }t        j                  ||      d	z   d
z  }nd}| j                  |z  | j                  |z  z   | j                  |z  z   }||d   z  }| j                  dkD  r6|| j                  |j                  d      xs |j                  d            z  }||d<   |j                  |       m |j                  d d       |d| }|D ]  }|j!                  dd        |S )u  Hybrid search: FTS5 candidates → Jaccard rerank → trust weighting.

        Pipeline:
        1. FTS5 search: Get limit*3 candidates from SQLite full-text search
        2. Jaccard boost: Token overlap between query and fact content
        3. Trust weighting: final_score = relevance * trust_score
        4. Temporal decay (optional): decay = 0.5^(age_days / half_life)

        Returns list of dicts with fact data + 'score' field, sorted by score desc.
           contenttags fts_rankr   r   
hrr_vector      ?       @      ?trust_score
updated_at
created_atscorec                    | d   S Nr)    xs    r   <lambda>z&FactRetriever.search.<locals>.<lambda>k   
    !G* r   TkeyreverseN)_fts_candidates	_tokenizeget_jaccard_similarityr   r   bytes_to_phasesencode_textr   
similarityr   r   r   _temporal_decayappendsortpop)r   querycategory	min_trustlimit
candidatesquery_tokensscoredfactcontent_tokens
tag_tokens
all_tokensjaccard	fts_scorefact_vec	query_vechrr_sim	relevancer)   resultss                       r   searchzFactRetriever.search0   s   $ ))%9eaiP
I ~~e, 	 D!^^DO<N(<=J'*4J..|ZHGS1I "txx'=..tL/ABOOE4<<@	>>)X>DK 94--78//G34I
 ] 33E ~~!--dhh|.D.^Q]H^__!DMMM$;	 @ 	,d;%. 	)DHH\4(	)r   c                   t         j                  s| j                  |||      S | j                  j                  }t        j
                  d| j                        }t        j
                  |j                         | j                        }t        j                  ||      }|rjd| }|j                  d|f      j                         }	|	rBt        j                  |	d         }
t        j                  |
|      }| j                  |||      S d}g }|r|dz  }|j                  |       |j                  d| d	|      j                         }|s| j                  |||      S g }|D ]  }t!        |      }t        j                  |j#                  d
            }t        j                  ||      }t        j
                  d| j                        }t        j                  t        j$                  |d   | j                        |      }t        j&                  ||      }|dz   dz  |d   z  |d<   |j                  |        |j)                  d d       |d| S )u:  Compositional entity query using HRR algebra.

        Unbinds entity from memory bank to extract associated content.
        This is NOT keyword search — it uses algebraic structure to find facts
        where the entity plays a structural role.

        Falls back to FTS5 search if numpy unavailable.
        r@   rB   __hrr_role_entity__zcat:z3SELECT vector FROM memory_banks WHERE bank_name = ?vectorWHERE hrr_vector IS NOT NULL AND category = ?
            SELECT fact_id, content, category, tags, trust_score,
                   retrieval_count, helpful_count, created_at, updated_at,
                   hrr_vector
            FROM facts
            
            r"   __hrr_role_content__r   r#   r$   r&   r)   c                    | d   S r+   r,   r-   s    r   r/   z%FactRetriever.probe.<locals>.<lambda>   r0   r   Tr1   N)r   r   rQ   r   _connencode_atomr   lowerbindexecutefetchoner8   unbind_score_facts_by_vectorr<   fetchalldictr>   r9   r:   r=   )r   entityr@   rB   connrole_entity
entity_vec	probe_key	bank_namebank_rowbank_vec	extractedwhereparamsrowsrE   rowrF   rL   residualrole_contentcontent_vecsims                          r   probezFactRetriever.prober   s7    ~~;;v;FFzz oo&;T\\J__V\\^T\\B
HHZ5	 xj)I||E hj  ..x/ABJJx;	22 3  
 /((EMM(#||
 G  	
 (* 	 ;;v;FF 
	 C9D**488L+ABHzz(I6H??+A4<<PL((3??4	?DLL#QS_`K..;7C 3Y#-]0CCDMMM$
	  	,d;fu~r   c                   t         j                  s| j                  |||      S | j                  j                  }t        j
                  |j                         | j                        }d}g }|r|dz  }|j                  |       |j                  d| d|      j                         }|s| j                  |||      S g }	|D ]  }
t        |
      }t        j                  |j                  d            }t        j                  ||      }t        j
                  d| j                        }t        j
                  d| j                        }t        j                  ||      }t        j                  ||      }t!        ||      }|d	z   d
z  |d   z  |d<   |	j                  |        |	j#                  d d       |	d| S )uk  Discover facts that share structural connections with an entity.

        Unlike probe (which finds facts *about* an entity), related finds
        facts that are connected through shared context — e.g., other entities
        mentioned alongside this one, or content that overlaps structurally.

        Falls back to FTS5 search if numpy unavailable.
        rS   rV   rW   rX   rY   r"   rT   rZ   r#   r$   r&   r)   c                    | d   S r+   r,   r-   s    r   r/   z'FactRetriever.related.<locals>.<lambda>  r0   r   Tr1   N)r   r   rQ   r   r\   r]   r^   r   r<   r`   rd   re   r8   r>   rb   r:   maxr=   )r   rf   r@   rB   rg   ri   ro   rp   rq   rE   rr   rF   rL   rs   rh   rt   entity_role_simcontent_role_simbest_sims                      r   relatedzFactRetriever.related   s    ~~;;v;FFzz __V\\^T\\B
 /((EMM(#||
 G  	
 (* 	 ;;v;FF  	 C9D**488L+ABH zz(J7H //*?NK??+A4<<PL!nnX{CO"~~hE?,<=H%^s2T-5HHDMMM$#	 & 	,d;fu~r   c                .   t         j                  r|s%dj                  |      }| j                  |||      S | j                  j
                  }t        j                  d| j                        }g }|D ]W  }t        j                  |j                         | j                        }	t        j                  |	|      }
|j                  |
       Y d}g }|r|dz  }|j                  |       |j                  d| d|      j                         }|s%dj                  |      }| j                  |||      S t        j                  d| j                        }g }|D ]  }t        |      }t        j                  |j                  d	            }g }|D ]?  }
t        j                   ||
      }t        j"                  ||      }|j                  |       A t%        |      }|d
z   dz  |d   z  |d<   |j                  |        |j'                  d d       |d| S )u  Multi-entity compositional query — vector-space JOIN.

        Given multiple entities, algebraically intersects their structural
        connections to find facts related to ALL of them simultaneously.
        This is compositional reasoning that no embedding DB can do.

        Example: reason(["peppi", "backend"]) finds facts where peppi AND
        backend both play structural roles — without keyword matching.

        Falls back to FTS5 search if numpy unavailable.
         rS   rT   rV   rW   rX   rY   rZ   r"   r#   r$   r&   r)   c                    | d   S r+   r,   r-   s    r   r/   z&FactRetriever.reason.<locals>.<lambda>O  r0   r   Tr1   N)r   r   joinrQ   r   r\   r]   r   r^   r_   r<   r`   rd   re   r8   r>   rb   r:   minr=   )r   entitiesr@   rB   r?   rg   rh   entity_residualsrf   ri   rj   ro   rp   rq   rt   rE   rr   rF   rL   entity_scoresrs   rv   min_sims                          r   reasonzFactRetriever.reason  s   " ~~XHHX&E;;uxu;EEzzoo&;T\\J  	/FFJ[9I##I.	/ /((EMM(#||
 G  	
 (* 	 HHX&E;;uxu;EE
 '=t||L 	 C9D**488L+ABHM- *	::h	:nnX|<$$S)*
 -(G$s]c1D4GGDMMM$	  	,d;fu~r   c                   t         j                  sg S | j                  j                  }d}g }|r|dz  }|j	                  |       |j                  d| d|      j                         }t        |      dk  rg S d}t        |      |kD  rt        |d d	      }|d
| }i }	|D ]L  }
|
d   }|j                  d|f      j                         }|D ch c]  }|d   j                          c}|	|<   N |D cg c]  }t        |       }}g }t        t        |            D ]  }t        |dz   t        |            D ]d  }||   ||   }}|	j                  |d   t                     }|	j                  |d   t                     }|r|sM||z  rt        ||z        t        ||z        z  nd}|dk  rwt        j                  |d         }t        j                  |d         }t        j                  ||      }|d|dz   dz  z
  z  }||k\  s|j!                         D ci c]  \  }}|dk7  s|| }}}|j!                         D ci c]  \  }}|dk7  s|| }}}|j	                  ||t#        |d      t#        |d      t#        |d      t        ||z        d       g  |j%                  d d	       |d
| S c c}w c c}w c c}}w c c}}w )u  Find potentially contradictory facts via entity overlap + content divergence.

        Two facts contradict when they share entities (same subject) but have
        low content-vector similarity (different claims). This is automated
        memory hygiene — no other memory system does this.

        Returns pairs of facts with a contradiction score.
        Falls back to empty list if numpy unavailable.
        zWHERE f.hrr_vector IS NOT NULLz AND f.category = ?z
            SELECT f.fact_id, f.content, f.category, f.tags, f.trust_score,
                   f.created_at, f.updated_at, f.hrr_vector
            FROM facts f
            rY      i  c                    | d   xs | d   S )Nr'   r(   r,   )rs    r   r/   z*FactRetriever.contradict.<locals>.<lambda>  s    ao.P< r   Tr1   Nfact_idz
                SELECT e.name FROM entities e
                JOIN fact_entities fe ON fe.entity_id = e.entity_id
                WHERE fe.fact_id = ?
                namer   r   333333?r"   r#   r$   r   )fact_afact_bentity_overlapcontent_similaritycontradiction_scoreshared_entitiesc                    | d   S )Nr   r,   r-   s    r   r/   z*FactRetriever.contradict.<locals>.<lambda>  s    !,A*B r   )r   r   r   r\   r<   r`   rd   lensortedr^   re   ranger6   setr8   r:   itemsroundr=   )r   r@   	thresholdrB   rg   ro   rp   rq   _MAX_CONTRADICT_FACTSfact_entitiesrr   fidentity_rowsr   factscontradictionsijf1f2ents1ents2r   v1v2content_simr   kvf1_cleanf2_cleans                                  r   
contradictzFactRetriever.contradictR  s)    ~~Izz 1**EMM(#|| G 	 
 (* 	 t9q=I
 !$t9,,$$PZ^_D../D .0 
	JCi.C,,
  hj  >I!I!F)//"3!IM#
	J #''Qa''s5z" #	A1q5#e*- "q58B%))"Y-?%))"Y-?E NSUZ]UU]!3c%%-6H!Had!C' ((L)9:((L)9:!nnR4 '5{S?PTW>W8W&X#&)313QAqL?P1QHQ13QAqL?P1QHQ"))"*"**/*B.3K.C/45H!/L+1%%-+@+ 7"#	J 	 BDQfu%%Y "J (:  RQs$   J>4KK
,K
K
K
c                   | j                   j                  }d}g }|r|dz  }|j                  |       |j                  d| d|      j	                         }g }|D ]i  }	t        |	      }
t        j                  |
j                  d            }t        j                  ||      }|dz   dz  |
d   z  |
d	<   |j                  |
       k |j                  d
 d       |d| S )z-Score facts by similarity to a target vector.rV   rW   rX   rY   r"   r#   r$   r&   r)   c                    | d   S r+   r,   r-   s    r   r/   z6FactRetriever._score_facts_by_vector.<locals>.<lambda>  r0   r   Tr1   N)r   r\   r<   r`   rd   re   r   r8   r>   r:   r=   )r   
target_vecr@   rB   rg   ro   rp   rq   rE   rr   rF   rL   rv   s                r   rc   z$FactRetriever._score_facts_by_vector  s     zz.((EMM(#||
 G  	
 (* 	  	 C9D**488L+ABH..X6C 3Y#-]0CCDMMM$	  	,d;fu~r   c                   | j                   j                  }g }dg}|j                  | j                  |             |r"|j                  d       |j                  |       |j                  d       |j                  |       dj	                  |      }d| d}	|j                  |       	 |j                  |	|      j                         }
|
sg S |
D cg c]  }t        |d          }}|rt        |      nd}t        |d	      }g }t        |
|      D ];  \  }}t        |      }|j                  dd
       ||z  |d<   |j                  |       = |S # t        $ r g cY S w xY wc c}w )zGet raw FTS5 candidates from the store.

        Uses the store's database connection directly for FTS5 MATCH
        with rank scoring. Normalizes FTS5 rank to [0, 1] range.
        zfacts_fts MATCH ?zf.category = ?zf.trust_score >= ?z AND z
            SELECT f.*, facts_fts.rank as fts_rank_raw
            FROM facts_fts
            JOIN facts f ON f.fact_id = facts_fts.rowid
            WHERE zA
            ORDER BY facts_fts.rank
            LIMIT ?
        fts_rank_rawr#   gư>Nr!   )r   r\   r<   _sanitize_fts_queryr   r`   rd   	Exceptionabsrz   zipre   r>   )r   r?   r@   rA   rB   rg   rp   where_clauses	where_sqlsqlrq   rr   	raw_ranksmax_rankrP   raw_rankrF   s                    r   r4   zFactRetriever._fts_candidates  sz    zz ,-
 	d..u56  !12MM(#12i LL/	 + 		 	e	<<V,557D
 I :>>#S^,->	>%.3y>Cx& y1 	!MC9DHH^T*'(2DNN4 		! )  	I	 ?s   + E E$E! E!c                    | s
t               S t               }| j                         j                         D ]'  }|j                  d      }|s|j	                  |       ) |S )zSimple whitespace tokenization with lowercasing.

        Strips common punctuation. No stemming/lemmatization (Phase 1).
        .,;:!?"'()[]{}#@<>)r   r^   splitstripadd)texttokenswordcleaneds       r   r5   zFactRetriever._tokenize$  sX     5LJJL&&( 	$Djj!67G

7#	$ r   >}   ar   amanasatbebydoheifinisitmemynoofonorsotoupweallandanyarebutcandiddonfewforhadhasherhimhishowitsnornotnowoffouroutownshethetoowaswhowhyyoubeenbothdoesdowneachfromhaveherehersintojustmoremostonceonlyoursoversamesomesuchthanthatthemthentheythisverywerewhatwhenwhomwillwithyouraboutaboveafteragainbeingcoulddoingothertheirtherethesethoseunderuntilro   whichwhilewouldyoursbeforeduringhavingitselfmyselfshouldtheirsbecausebetweenfurtherherselfhimselfthroughyourself	ourselves
themselves
yourselvesc                P   |syd}g }|j                         j                         D ]j  }|j                  d      j                  t        j                  dd|            }t        |      dk  rG|| j                  v rV|j                  d| d       l |s|S dj                  |      S )a  Convert a natural-language query to an FTS5-safe OR expression.

        FTS5 treats a multi-word MATCH argument as AND-joined by default,
        which tanks recall on prose queries. This helper:
          - tokenizes the query
          - drops stopwords and short (<2 char) tokens
          - strips FTS5 special characters from each token
          - OR-joins the survivors

        If nothing remains (pathological query), falls back to the raw
        query so the caller sees zero results instead of a SQL error.
        r    z"()*^:-+r   r   "z OR )
r^   r   r   	translatestr	maketransr   _FTS_STOPWORDSr<   r   )clsr?   _FTS_SPECIALr   rawr   s         r   r   z!FactRetriever._sanitize_fts_queryI  s      ";;=&&( 
	*Cii 56AAb"l3G 7|a#,,, MMAgYa.)
	* L{{6""r   c                \    | r|syt        | |z        }t        | |z        }|dkD  r||z  S dS )u6   Jaccard similarity coefficient: |A ∩ B| / |A ∪ B|.r   r   )r   )set_aset_bintersectionunions       r   r7   z!FactRetriever._jaccard_similaritym  s>     E55=)EEM"',qy|e#9c9r   c                   | j                   r|sy	 t        |t              r&t        j                  |j                  dd            }n|}|j                   |j                  t        j                        }t        j                  t        j                        |z
  j                         dz  }|dk  ryt        j                  d|| j                   z        S # t        t        f$ r Y yw xY w)zExponential decay: 0.5^(age_days / half_life_days).

        Returns 1.0 if decay is disabled or timestamp is missing.
        r#   Zz+00:00)tzinfoiQ r   r%   )r   
isinstancerH  r   fromisoformatreplacerU  r   utcr   total_secondsmathpow
ValueError	TypeError)r   timestamp_strtsage_dayss       r   r;   zFactRetriever._temporal_decayv  s    
 ~~]	--++M,A,A#x,PQ"yy ZZx||Z4 X\\2R7FFH5PH!|88CDNN!:;;I& 		s   B C 2"C C'&C')r   r   r   r   i   )r   r   r   intr   floatr   rc  r   rc  r   rb  )Nr   
   )
r?   rH  r@   
str | NonerA   rc  rB   rb  return
list[dict])Nrd  )rf   rH  r@   re  rB   rb  rf  rg  )r   z	list[str]r@   re  rB   rb  rf  rg  )r@   re  r   rc  rB   rb  rf  rg  )r   z'np.ndarray'r@   re  rB   rb  rf  rg  )r   rH  rf  zset[str])r?   rH  rf  rH  )rO  r   rP  r   rf  rc  )r_  re  rf  rc  )__name__
__module____qualname____doc__r   rQ   rw   r~   r   r   rc   r4   staticmethodr5   	frozensetrJ  classmethodr   r7   r;   r,   r   r   r   r      sF   D
 )* #%% #&% 	%
 % % %4  $@@ @ 	@
 @ 
@J  $	LL L 	L
 
Lb  $	BB B 	B
 
BN  $	LL L 	L
 
L`  $	h&h& h& 	h&
 
h&Z  $	# # # 	#
 
#JAA A 	A
 A 
AF  $    N$ !# !#F : :r   r   )rk  
__future__r   r[  r   r   typingr   r   r   r    r	   r   ImportErrorr   r,   r   r   <module>rr     sF    #  '  "$
x	 x		  s   5 	A A