
    KiP                        d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZ  ej                  e      Z	 	 d	 	 	 	 	 	 	 	 	 	 	 dd	Zdd
Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 d	 	 	 	 	 	 	 	 	 ddZy)z0Tool 1: build_or_update_graph + run_postprocess.    )annotationsN)Any   )
full_buildincremental_update   )
_get_storec                   g }||d<   |dk(  r|S 	 | j                         }|D ]\  }|d   |d   |d   |d   |d   f\  }}	}
}}|
dv rd	|	 d
|xs d d}|r|d| z  }n|
dk(  rd|	 }n|	}| j                  ||dd        ^ | j                          d|d<   	 ddlm}  ||       }||d<   d|d<   |dk(  r|S | xr t        |      }	 |rddlm}  || |      }nddlm} ddlm}  ||       } || |      }||d <   	 |rdd#lm}  || |      }ndd$lm} dd%lm}  ||       } || |      }||d&<   	 t1        |        d|d)<   | j5                  d,t7        j8                  d-             | j5                  d|       |S # t        j                  t
        t        f$ rK}t        j                  d|       |j                  dt        |      j                   d|        Y d}~Hd}~ww xY w# t        j                  t        f$ rK}t        j                  d|       |j                  dt        |      j                   d|        Y d}~d}~ww xY w# t        j                  t        f$ rK}t        j                  d!|       |j                  d"t        |      j                   d|        Y d}~d}~ww xY w# t        j                  t        f$ rK}t        j                  d'|       |j                  d(t        |      j                   d|        Y d}~d}~ww xY w# t        j                  t2        f$ rK}t        j                  d*|       |j                  d+t        |      j                   d|        Y d}~9d}~ww xY w).zRun post-build steps based on *postprocess* level.

    When *full_rebuild* is False and *changed_files* are available,
    uses incremental flow/community detection for faster updates.

    Returns a list of warning strings (empty on success).
    postprocess_levelnoner   r   r         FunctionTestdef ( ) -> Classclass N   Tsignatures_updated Signature computation failed: %sSignature computation failed: : rebuild_fts_indexfts_indexedfts_rebuiltFTS index rebuild failed: %sFTS index rebuild failed: minimal)incremental_trace_flowsstore_flowstrace_flowsflows_detectedFlow detection failed: %sFlow detection failed: )incremental_detect_communitiesdetect_communitiesstore_communitiescommunities_detectedCommunity detection failed: %sCommunity detection failed: summaries_computedzSummary computation failed: %szSummary computation failed: last_postprocessed_at%Y-%m-%dT%H:%M:%S)get_nodes_without_signatureupdate_node_signaturecommitsqlite3OperationalError	TypeErrorKeyErrorloggerwarningappendtype__name__code_review_graph.searchr   ImportErrorboolcode_review_graph.flowsr%   r'   r)   code_review_graph.communitiesr-   r/   r1   _compute_summaries	Exceptionset_metadatatimestrftime)storebuild_resultpostprocessfull_rebuildchanged_fileswarningsrowsrownode_idnamekindparamsretsiger   	fts_countuse_incrementalr%   count_store_flows_trace_flowsflowsr-   _detect_communities_store_communitiescommss                              i/home/jay/workspace/scripts/.codegraph-venv/lib/python3.12/site-packages/code_review_graph/tools/build.py_run_postprocessrg      s    H(3L$%fR002 	<CAAAAA0,GT4 ++TF!FLb>3T#<'Ctfo''Tc;	< 	-1)*
N>%e,	&/]#&*]#
 i '&>4+>OKG+E=AEKK 'E .E).%&
P 35-HE (.E&ue4E/4+,P5!-1)*
 
/B!C 
*K8OO $$i: R91=8a9I9I8J"QCPQQR $$k2 N5q94T!W5E5E4FbLMMN, $$k2 K2A61$q'2B2B1C2aSIJJK, $$k2 P7;6tAw7G7G6H1#NOOP $$i0 P7;6tAw7G7G6H1#NOOPs}   BE: G& 4I <4J4 1L :G#A GG#&I
?A II
J1&A J,,J14LA LLM?4A M::M?c                  9 ddl }ddlm} ddlm} | j
                  }	 |j                  d       |j                  d        |t              }|j                  d      D ]  }||d   xx   |d   z  cc<    |j                  d	      D ]  }||d   xx   |d   z  cc<     |t              }|j                  d
      D ]7  }|d   |d   |d   }
}	}||   j                  |	|j                  |
d      f       9  |t              } |t              }|j                  d      D ]<  }|d   |d   }}|||   vs||   j                  |       ||   j                  |       > |j                  d      j                         }|D ]  }|d   |d   |d   |d   f\  }}}}t        |j                  |g       d d      }|j                  |dd D cg c]  }|d   	 c}      }|j                  |g       dd }d}|r9 ||      }d|v r-d|v r'|j!                  dd      d   j#                  d      d   nd}|j                  d||||||xs df        |j%                          	 |j                  d       |j                  d       |j                  d      j                         }t               }g }|D ]t  }|j                  |d          |d   r|j-                  |d         ng }|j                  |       |dd D ]  }|j                  |        |sa|j                  |d          v i }|r|t        |      }t/        dt1        |      d      D ]X  } || | dz    }!dj3                  d |!D              }"|j                  d|" d |!      j                         }#|#D ]  }$|$d   ||$d   <    Z t5        ||      D ]  \  }}|d   |d   |d   }'}&}%|d   |d   |d   }*})}(|j                  |'t7        |'            }+g },|r|,j                  |+       t1        |      dkD  r/|dd D ]'  }|j                  |      }-|-s|,j                  |-       ) t1        |      dkD  r+|j                  |d         }.|.r|.|,vr|,j                  |.       |j                  d!|%|&|+|j                  |,      |(|)|*f        |j%                          	 |j                  d       |j                  d"       i }/|j                  d#      D ]  }|d   |/|d   <    i }0|j                  d$      D ]  }|d   |0|d   <    |j                  d%      j                         }1h d&}2|1D ]  }3|3d   |3d   |3d   }	}
}|/j                  |
d      }4|0j                  |
d      }5|5dkD  rd'nd(}6|	j9                         9t;        9fd)|2D              rdnd}7d*}8|4d+kD  r|8d,z  }8n
|4dkD  r|8d-z  }8|6d(k(  r|8d,z  }8|7r|8d.z  }8t=        |8d/      }8|j                  d0||
|8|4|6|7f        |j%                          yc c}w # t&        j(                  $ r |j+                          Y w xY w# t&        j(                  $ r |j+                          Y w xY w# t&        j(                  $ r |j+                          Y yw xY w)1a  Populate community_summaries, flow_snapshots, and risk_index tables.

    Uses batched aggregate queries and in-memory grouping instead of
    per-community/per-node loops. On graphs with ~100k edges this
    reduces the work from ``O(nodes + communities)`` SQLite round trips
    each doing their own B-tree scan to a handful of ``GROUP BY``
    queries, turning what used to be an effective hang into a few
    seconds.

    Each summary block (community_summaries, flow_snapshots, risk_index)
    is wrapped in an explicit transaction so the DELETE + INSERT sequence
    is atomic.  If a table doesn't exist yet the block is silently skipped.
    r   N)defaultdict)commonprefixzBEGIN IMMEDIATEzDELETE FROM community_summarieszFSELECT source_qualified, COUNT(*) FROM edges GROUP BY source_qualifiedr   zFSELECT target_qualified, COUNT(*) FROM edges GROUP BY target_qualifiedzfSELECT community_id, name, qualified_name FROM nodes WHERE community_id IS NOT NULL AND kind != 'File'r   zHSELECT community_id, file_path FROM nodes WHERE community_id IS NOT NULLz9SELECT id, name, size, dominant_language FROM communitiesr   c                    | d   S )Nr    )ncs    rf   <lambda>z$_compute_summaries.<locals>.<lambda>   s
    r!u     T)keyreverse      r   /zINSERT OR REPLACE INTO community_summaries (community_id, name, purpose, key_symbols, size, dominant_language) VALUES (?, ?, ?, ?, ?, ?)zDELETE FROM flow_snapshotszZSELECT id, name, entry_point_id, criticality, node_count, file_count, path_json FROM flows   r   i  ,c              3      K   | ]  }d   yw)?Nrl   ).0_s     rf   	<genexpr>z%_compute_summaries.<locals>.<genexpr>  s     ';';s   z2SELECT id, qualified_name FROM nodes WHERE id IN (r   zINSERT OR REPLACE INTO flow_snapshots (flow_id, name, entry_point, critical_path, criticality, node_count, file_count) VALUES (?, ?, ?, ?, ?, ?, ?)zDELETE FROM risk_indexz[SELECT target_qualified, COUNT(*) FROM edges WHERE kind = 'CALLS' GROUP BY target_qualifiedz_SELECT source_qualified, COUNT(*) FROM edges WHERE kind = 'TESTED_BY' GROUP BY source_qualifiedzVSELECT id, qualified_name, name FROM nodes WHERE kind IN ('Function', 'Class', 'Test')>   sqlauthcryptlogintokensecretexecutesessionpassword
credential
permissiontesteduntestedc              3  &   K   | ]  }|v  
 y w)Nrl   )rz   kw
name_lowers     rf   r|   z%_compute_summaries.<locals>.<genexpr>N  s     @bz)@s   g        
   g333333?g333333?g?g      ?zINSERT OR REPLACE INTO risk_index (node_id, qualified_name, risk_score, caller_count, test_coverage, security_relevant, last_computed) VALUES (?, ?, ?, ?, ?, ?, datetime('now')))jsoncollectionsri   os.pathrj   _connr   intlistrA   getsetaddfetchallsorteddumpsrsplitsplitr:   r;   r<   rollbackloadsrangelenjoinzipstrloweranymin):rN   _jsonri   rj   connedge_countsrU   nodes_by_commcidrW   qnfiles_by_comm
seen_filesfpcommunity_rowsrcnamecsizeclangmembersmkey_symspathspurposeprefix	flow_rows
needed_idsparsed_pathspath_idsnid
id_to_nameid_listibatchplaceholders	node_rowsnrfidfnameep_idcritncountfcountep_namecritical_pathnmlastcaller_countstested_counts
risk_nodessecurity_kwncaller_countr   coveragesec_relevantriskr   s:                                                            @rf   rI   rI      s    '$;;DL&'67 '2#&6<<(
 	*C A3q6)		*
 <<(
 	*C A3q6)		* ;Fd:K<<@
 	FC  FCFCFrC#%%t[__R-C&DE	F /:$.?*5c*:
<<-
 	.C !fc!fCC(3##B'c"))"-	. G

(* 	   	A'(tQqT1Q41'=$Cu !!#r*$G
 {{'"1+#>QAaD#>?H "%%c2.s3EG%e,&= &= c1-a066s;B?.0 
 LL, eWhu{C	/	: 	
A&'12LL/
 (* 	  #u
(* 	-ANN1Q4 ,-aDu{{1Q4(bH)!} $s#$x|,	- &(
 :&G1c'lC0 	.!c'*"xx';U';; LL$$0>4 (*	 
 $ .B(*1Jr!u%.	. y,7 	KAx !!adAaDC#$Q41qt&&D nnUCJ7G')M$$W-x=1$'!} 5'^^C0)0045 x=1$%>>(2,7DM 9%,,T2LLG eWekk-&@vv'	!	. 	
;&'-. )+<<=
 	+C %(FM#a&!		+ )+<<A
 	+C %(FM#a&!		+ \\:
 (* 	
  	AaD!A$!TC(,,R3L"&&r1-F#)A:x:HJ@K@@a  Db !:%tS>DLL= b$hE'	4 	k $?( ## H ## | ## s`   DX 4BX 
X
BX B/X7 DX7 .BX7 5EY! X #X43X47#YY!#ZZc                   t        |      \  }}	 | r)t        |||      }ddd|d    d|d    d|d    d	d
|}nVt        |||      }|d   dk(  rddd|d||j                          S ddd|d    d|d    d|d    d|d    d|d    dd
|}| s|j	                  d      nd}	t        |||| |	      }
|
r|
|d<   ||j                          S # |j                          w xY w)u  Build or incrementally update the code knowledge graph.

    Args:
        full_rebuild: If True, re-parse every file. If False (default),
                      only re-parse files changed since ``base``.
        repo_root: Path to the repository root. Auto-detected if omitted.
        base: Git ref for incremental diff (default: HEAD~1).
        postprocess: Post-processing level after build:
            ``"full"`` (default) — signatures, FTS, flows, communities.
            ``"minimal"`` — signatures + FTS only (fast, keeps search working).
            ``"none"`` — skip all post-processing (raw parse only).
        recurse_submodules: If True, include files from git submodules
            via ``git ls-files --recurse-submodules``. When None
            (default), falls back to the CRG_RECURSE_SUBMODULES
            environment variable. Default: disabled.

    Returns:
        Summary with files_parsed/updated, node/edge counts, and errors.
    okfullzFull build complete: parsed files_parsedz files, created total_nodesz nodes and total_edgesz edges.)status
build_typesummary)basefiles_updatedr   incrementalz)No changes detected. Graph is up to date.)r   r   r   r   zIncremental update: z files re-parsed, z edges updated. Changed: rR   z. Dependents also updated: dependent_files.N)rQ   rR   rS   )r	   r   r   closer   rg   )rQ   	repo_rootr   rP   recurse_submodulesrN   rootresultrO   changedrS   s              rf   build_or_update_graphr   f  so   4 Y'KE4.e-?@F$26.3I2J K%m45[m,-W6		 	L (e$?Fo&!+""/J)4	
 < 	- +*6/+B*CCUm,-[m,- .  & 78 90067H0I/J!	M	 L 6B&**_-t#<%W
 '/L$s   A	C *AC C!c                   t        |      \  }}ddi}g }	 	 |j                         }|D ]\  }	|	d   |	d   |	d   |	d   |	d   f\  }
}}}}|dv rd	| d
|xs d d}|r|d| z  }n|dk(  rd| }n|}|j                  |
|dd        ^ |j                          d|d<   |r	 ddlm}  ||      }||d<   | r#	 ddlm} ddlm}  ||      } |||      }||d<   |r#	 dd lm} dd!lm}  ||      } |||      }||d"<   |j-                  d%t/        j0                  d&             d'|d(<   |r||d)<   ||j3                          S # t        j
                  t        t        f$ rK}t        j                  d|       |j                  dt        |      j                   d|        Y d}~d}~ww xY w# t        j
                  t        f$ rK}t        j                  d|       |j                  dt        |      j                   d|        Y d}~Xd}~ww xY w# t        j
                  t        f$ rK}t        j                  d|       |j                  dt        |      j                   d|        Y d}~d}~ww xY w# t        j
                  t        f$ rK}t        j                  d#|       |j                  d$t        |      j                   d|        Y d}~d}~ww xY w# |j3                          w xY w)*a  Run post-processing steps on an existing graph.

    Useful for running expensive steps (flows, communities) separately
    from the build, or for re-running after the graph has been updated
    with ``postprocess="none"``.

    Args:
        flows: Run flow detection. Default: True.
        communities: Run community detection. Default: True.
        fts: Rebuild FTS index. Default: True.
        repo_root: Repository root path. Auto-detected if omitted.

    Returns:
        Summary of what was computed.
    r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Nr   Tr   r   r   r   r   r    r"   r#   r&   r(   r*   r+   r,   r.   r0   r2   r3   r4   r6   r7   zPost-processing complete.r   rS   )r	   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   r   rE   rG   r'   r)   rH   r/   r1   rK   rL   rM   r   )rb   communitiesftsr   rN   _rootr   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r   r]   r`   ra   tracedr_   rc   rd   re   s                             rf   run_postprocessr     s1   * i(LE5&-FHE	V446D @FCFCFCFCF40tVS // a"~Q7CcU|+W_"4&/CC++GS#Y?@ LLN+/F'(
 VF-e4	(1}%
 	SOO%e,$UF3+0'(
 X ,E2*5%816-.
 	#T]]3F%G	
 8y!)F:e (()X> 	VNN=qAOO<T!W=M=M<NbQRPSTUU	V ,,k: V=qA"<T!W=M=M<NbQRPS TUUV ,,k: S:A>"9$q':J:J9K2aS QRRS  ,,k: X?C">tAw?O?O>PPRSTRU VWWX 	s   BE  K!  F, 3K! 6"H K! "I: =2K!  F)A F$K! $F))K! ,HA HK! HK! I7,A I2,K! 2I77K! :KA KK! KK! !K3)FN)rN   r   rO   dict[str, Any]rP   r   rQ   rF   rR   zlist[str] | Nonereturnz	list[str])rN   r   r   None)FNzHEAD~1r   N)rQ   rF   r   
str | Noner   r   rP   r   r   zbool | Noner   r   )TTTN)
rb   rF   r   rF   r   rF   r   r   r   r   )__doc__
__future__r   loggingr;   rL   typingr   r   r   r   _commonr	   	getLoggerrC   r?   rg   rI   r   r   rl   ro   rf   <module>r      s    6 "     8 			8	$ &*mm m m 	m
 $m m`cN  &*III I 	I
 $I IZ  	^^^ 
^ 	^
 ^ro   