diff --git a/graphify/build.py b/graphify/build.py index 7f195fe2f4..0acdde7651 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -29,6 +29,7 @@ import unicodedata from pathlib import Path import networkx as nx +from graphify.edges import effective_weight from .ids import make_id, normalize_id as _normalize_id from .paths import default_graph_json as _default_graph_json from .paths import is_absolute_any_platform as _is_abs @@ -1167,6 +1168,10 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if not math.isfinite(_num_val) or _num_val < 0: _num_val = 1.0 attrs[_num_key] = _num_val + # Persist the score consumers should use for ranking, clustering, and + # rendering. Raw relation weight alone makes inferred edges look as + # strong as explicitly extracted edges. + attrs["effective_weight"] = effective_weight(attrs) # Backfill source_file from the endpoint nodes (every node carries one). # Semantic/LLM edges occasionally omit it, which downstream validation # flags and leaves query results with no file reference (#1279). diff --git a/graphify/cluster.py b/graphify/cluster.py index 6822107001..b3e478becd 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -6,6 +6,7 @@ import json import sys import networkx as nx +from graphify.edges import effective_weight def _suppress_output(): @@ -32,17 +33,35 @@ def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: from corrupting terminal scroll buffers on Windows PowerShell 5.1. """ stable = nx.Graph() - stable.add_nodes_from(sorted(G.nodes(), key=str)) - edge_rows = sorted( - G.edges(data=True), + # NetworkX's Louvain implementation uses sets internally. Integer node + # IDs have stable hashing across Python processes; mapping back after the + # partition removes process-randomized string-hash ordering. + ordered_nodes = sorted(G.nodes(), key=str) + node_to_int = {node: index for index, node in enumerate(ordered_nodes)} + int_to_node = {index: node for node, index in node_to_int.items()} + stable.add_nodes_from(range(len(ordered_nodes))) + edge_rows = [] + for src, tgt, attrs in G.edges(data=True): + src_i, tgt_i = node_to_int[src], node_to_int[tgt] + if src_i > tgt_i: + src_i, tgt_i = tgt_i, src_i + edge_rows.append((src_i, tgt_i, attrs)) + edge_rows.sort( key=lambda row: ( - str(row[0]), - str(row[1]), + row[0], + row[1], json.dumps(row[2], sort_keys=True, ensure_ascii=False, default=str), - ), + ) ) - for src, tgt, attrs in edge_rows: - stable.add_edge(src, tgt, **attrs) + for src_i, tgt_i, attrs in edge_rows: + weighted_attrs = dict(attrs) + # NetworkX's Louvain implementation reads the canonical ``weight`` + # attribute internally, even when a custom weight name is requested by + # the public wrapper. Feed it the confidence-adjusted score explicitly. + weighted_attrs["weight"] = effective_weight(attrs) + # Endpoints were canonicalized before sorting so each adjacency mapping + # has the same order in every process. + stable.add_edge(src_i, tgt_i, **weighted_attrs) try: from graspologic.partition import leiden @@ -63,18 +82,27 @@ def _partition(G: nx.Graph, resolution: float = 1.0) -> dict[str, int]: result = leiden(stable, **kwargs) finally: sys.stderr = old_stderr - return result + return {int_to_node[node]: cid for node, cid in result.items()} except ImportError: pass # Fallback: networkx louvain (available since networkx 2.7). # Inspect kwargs to stay compatible across NetworkX versions — max_level # was added in a later release and prevents hangs on large sparse graphs. - kwargs: dict = {"seed": 42, "threshold": 1e-4, "resolution": resolution} + kwargs: dict = { + "seed": 42, + "threshold": 1e-4, + "resolution": resolution, + "weight": "effective_weight", + } if "max_level" in inspect.signature(nx.community.louvain_communities).parameters: kwargs["max_level"] = 10 communities = nx.community.louvain_communities(stable, **kwargs) - return {node: cid for cid, nodes in enumerate(communities) for node in nodes} + return { + int_to_node[node]: cid + for cid, nodes in enumerate(communities) + for node in nodes + } _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split @@ -162,11 +190,11 @@ def cluster( # Compute hub exclusion set before removing anything so degree is based on full graph hub_nodes: set[str] = set() if exclude_hubs_percentile is not None: - degrees = sorted(d for _, d in G.degree()) + degrees = sorted(_weighted_degree(G, node) for node in G.nodes()) if degrees: idx = max(0, int(len(degrees) * exclude_hubs_percentile / 100) - 1) threshold = degrees[idx] - hub_nodes = {n for n, d in G.degree() if d > threshold} + hub_nodes = {n for n in G if _weighted_degree(G, n) > threshold} # Leiden warns and drops isolates - handle them separately # Also exclude hub nodes from partitioning so they don't pull unrelated @@ -265,6 +293,10 @@ def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: return actual / possible if possible > 0 else 0.0 +def _weighted_degree(G: nx.Graph, node: str) -> float: + return sum(effective_weight(data) for _, _, data in G.edges(node, data=True)) + + def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()} diff --git a/graphify/edges.py b/graphify/edges.py new file mode 100644 index 0000000000..b514a2de9d --- /dev/null +++ b/graphify/edges.py @@ -0,0 +1,48 @@ +"""Shared edge typing, confidence, and filtering helpers.""" +from __future__ import annotations + +from math import isfinite +from typing import Mapping + + +_CONFIDENCE_DEFAULTS = { + "EXTRACTED": 1.0, + "INFERRED": 0.5, + "AMBIGUOUS": 0.2, +} + + +def confidence_score(edge: Mapping[str, object]) -> float: + """Return a bounded confidence score, with schema-safe defaults.""" + raw = edge.get("confidence_score") + if raw is None: + raw = _CONFIDENCE_DEFAULTS.get(str(edge.get("confidence", "EXTRACTED")), 1.0) + try: + value = float(raw) + except (TypeError, ValueError): + value = 1.0 + return min(1.0, max(0.0, value)) if isfinite(value) else 0.0 + + +def effective_weight(edge: Mapping[str, object]) -> float: + """Return relation strength adjusted by provenance confidence.""" + try: + relation_weight = float(edge.get("weight", 1.0)) + except (TypeError, ValueError): + relation_weight = 1.0 + if not isfinite(relation_weight) or relation_weight < 0: + relation_weight = 1.0 + return relation_weight * confidence_score(edge) + + +def passes_edge_filter( + edge: Mapping[str, object], + *, + min_effective_weight: float = 0.0, + relations: set[str] | None = None, +) -> bool: + """Whether an edge is eligible for a consumer's filtered view.""" + relation = edge.get("relation") + if relations is not None and relation not in relations: + return False + return effective_weight(edge) >= min_effective_weight diff --git a/graphify/export.py b/graphify/export.py index 50ed388a41..5660cc3f71 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -17,6 +17,7 @@ from graphify.analyze import _node_community_map from graphify.build import edge_data from graphify.paths import stem_filename_budget +from graphify.edges import effective_weight, passes_edge_filter from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 @@ -317,6 +318,7 @@ def _json_sort_key(item: dict) -> str: if "confidence_score" not in link: conf = link.get("confidence", "EXTRACTED") link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) + link["effective_weight"] = effective_weight(link) # Restore original edge direction. Undirected NetworkX storage may # canonicalize endpoint order, flipping `calls` and other directional # edges in graph.json. The build path stashes the true endpoints in @@ -549,6 +551,7 @@ def to_obsidian( output_dir: str, community_labels: dict[int, str] | None = None, cohesion: dict[int, float] | None = None, + min_effective_weight: float = 0.0, ) -> int: """Export graph as an Obsidian vault - one .md file per node with [[wikilinks]], plus one _COMMUNITY_name.md overview note per community (sorted to top by underscore prefix). @@ -556,6 +559,9 @@ def to_obsidian( Open the output directory as a vault in Obsidian to get an interactive graph view with community colors and full-text search over node metadata. + ``min_effective_weight`` filters weak/inferred connections consistently with + the HTML exporter. The default of ``0.0`` preserves the historical export. + Returns the number of node notes + community notes written. """ out = Path(output_dir) @@ -592,6 +598,9 @@ def _owned_write(rel_name: str, content: str) -> bool: # every note write raises FileNotFoundError (#2655). No-op on POSIX. _stem_limit = stem_filename_budget(out, reserve=_DEDUP_SUFFIX_RESERVE) + def _edge_allowed(edata: dict) -> bool: + return passes_edge_filter(edata, min_effective_weight=min_effective_weight) + # Map node_id → safe filename so wikilinks stay consistent. # Deduplicate: if two nodes produce the same filename, append a numeric suffix. node_filename = _dedup_node_filenames( @@ -602,6 +611,8 @@ def _owned_write(rel_name: str, content: str) -> bool: def _dominant_confidence(node_id: str) -> str: confs = [] for u, v, edata in G.edges(node_id, data=True): + if not _edge_allowed(edata): + continue confs.append(edata.get("confidence", "EXTRACTED")) if not confs: return "EXTRACTED" @@ -654,7 +665,10 @@ def _dominant_confidence(node_id: str) -> str: lines += ["---", "", f"# {label}", ""] # Outgoing edges as wikilinks - neighbors = list(G.neighbors(node_id)) + neighbors = [ + neighbor for neighbor in G.neighbors(node_id) + if _edge_allowed(edge_data(G, node_id, neighbor)) + ] if neighbors: lines.append("## Connections") for neighbor in sorted(neighbors, key=lambda n: G.nodes[n].get("label", n)): @@ -679,6 +693,8 @@ def _dominant_confidence(node_id: str) -> str: for cid in communities: inter_community_edges[cid] = {} for u, v in G.edges(): + if not _edge_allowed(edge_data(G, u, v)): + continue cu = node_community.get(u) cv = node_community.get(v) if cu is not None and cv is not None and cu != cv: @@ -692,7 +708,9 @@ def _community_reach(node_id: str) -> int: neighbor_cids = { node_community[nb] for nb in G.neighbors(node_id) - if nb in node_community and node_community[nb] != node_community.get(node_id) + if nb in node_community + and node_community[nb] != node_community.get(node_id) + and _edge_allowed(edge_data(G, node_id, nb)) } return len(neighbor_cids) @@ -800,7 +818,14 @@ def _community_name(cid) -> str: # Top bridge nodes - highest degree nodes that connect to other communities bridge_nodes = [ - (node_id, G.degree(node_id), _community_reach(node_id)) + ( + node_id, + sum( + 1 for neighbor in G.neighbors(node_id) + if _edge_allowed(edge_data(G, node_id, neighbor)) + ), + _community_reach(node_id), + ) for node_id in members if _community_reach(node_id) > 0 ] diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py index 0f0560b50d..71bb2816c5 100644 --- a/graphify/exporters/html.py +++ b/graphify/exporters/html.py @@ -8,6 +8,7 @@ import json import networkx as nx from graphify.security import sanitize_label +from graphify.edges import effective_weight, passes_edge_filter MAX_NODES_FOR_VIZ = 5_000 @@ -399,6 +400,7 @@ def to_html( member_counts: dict[int, int] | None = None, node_limit: int | None = None, learning_overlay: dict | None = None, + min_effective_weight: float = 0.0, ) -> None: """Generate an interactive vis.js HTML visualization of the graph. @@ -425,6 +427,8 @@ def to_html( meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) edge_counts = _Counter() for u, v in G.edges(): + if not passes_edge_filter(G.edges[u, v], min_effective_weight=min_effective_weight): + continue cu, cv = node_to_community.get(u), node_to_community.get(v) if cu is not None and cv is not None and cu != cv: edge_counts[(min(cu, cv), max(cu, cv))] += 1 @@ -461,7 +465,8 @@ def to_html( }) meta.graph["hyperedges"] = remapped to_html(meta, meta_communities, output_path, - community_labels=community_labels, member_counts=mc) + community_labels=community_labels, member_counts=mc, + min_effective_weight=min_effective_weight) print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") print("Tip: run with --obsidian for full node-level detail.") return @@ -557,6 +562,8 @@ def to_html( # for `calls` and `rationale_for` in the rendered graph (#563). vis_edges = [] for u, v, data in G.edges(data=True): + if not passes_edge_filter(data, min_effective_weight=min_effective_weight): + continue confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") true_src = data.get("_src", u) @@ -570,6 +577,7 @@ def to_html( "width": 2 if confidence == "EXTRACTED" else 1, "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, "confidence": confidence, + "effective_weight": effective_weight(data), }) # Build community legend data diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 21fd2ca3a3..f130589726 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,5 +1,7 @@ import json import sys +import os +import subprocess import networkx as nx from pathlib import Path from graphify.build import build_from_json @@ -98,3 +100,40 @@ def test_remap_communities_to_previous_assigns_deterministic_new_ids(): assert list(remapped.keys()) == [0, 1] assert remapped[0] == ["x", "y", "z"] assert remapped[1] == ["m"] + + +def test_cluster_uses_effective_edge_weight_and_excludes_weighted_hubs(): + G = nx.Graph() + G.add_edge("hub", "a", weight=1.0, confidence="INFERRED", confidence_score=0.2) + G.add_edge("hub", "b", weight=1.0, confidence="EXTRACTED", confidence_score=1.0) + G.add_edge("a", "b", weight=1.0, confidence="EXTRACTED", confidence_score=1.0) + communities = cluster(G, exclude_hubs_percentile=50) + assert {n for nodes in communities.values() for n in nodes} == set(G) + + +def test_partition_is_stable_for_string_nodes_across_runs(): + G = nx.Graph() + for left, right in (("alpha", "beta"), ("beta", "gamma"), ("delta", "epsilon")): + G.add_edge(left, right, weight=1.0, confidence="EXTRACTED", confidence_score=1.0) + first = cluster(G) + second = cluster(G) + assert first == second + + +def test_partition_is_stable_across_fresh_processes(): + script = """ +import json +import networkx as nx +from graphify.cluster import cluster +G = nx.Graph() +for left, right in ((\"alpha\", \"beta\"), (\"beta\", \"gamma\"), (\"delta\", \"epsilon\")): + G.add_edge(left, right, weight=1.0, confidence=\"EXTRACTED\", confidence_score=1.0) +print(json.dumps(cluster(G), sort_keys=True)) +""" + env = dict(os.environ) + env["PYTHONPATH"] = str(Path(__file__).parents[1]) + runs = [ + subprocess.check_output([sys.executable, "-c", script], env=env, text=True) + for _ in range(2) + ] + assert runs[0] == runs[1] diff --git a/tests/test_confidence.py b/tests/test_confidence.py index 299548aca3..78ee357b82 100644 --- a/tests/test_confidence.py +++ b/tests/test_confidence.py @@ -62,6 +62,12 @@ def test_inferred_edges_score_in_range(): assert found, "No INFERRED edges found in test fixture" +def test_effective_weight_multiplies_relation_weight_by_confidence(): + G = build_from_json(_make_extraction()) + inferred = next(d for _, _, d in G.edges(data=True) if d.get("confidence") == "INFERRED") + assert inferred["effective_weight"] == 0.75 * 0.8 + + def test_ambiguous_edges_score_at_most_04(): """AMBIGUOUS edges must have confidence_score <= 0.4.""" G = build_from_json(_make_extraction()) @@ -96,6 +102,7 @@ def test_confidence_score_round_trip(): score = link["confidence_score"] assert isinstance(score, float), f"confidence_score should be float, got {type(score)}" assert 0.0 <= score <= 1.0, f"confidence_score={score} out of range" + assert "effective_weight" in link def test_to_json_defaults_missing_confidence_score(): diff --git a/tests/test_export.py b/tests/test_export.py index d86e413ec1..c9ee625601 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -3,6 +3,7 @@ import re import tempfile from pathlib import Path +import networkx as nx from graphify.build import build_from_json from graphify.cluster import cluster from graphify.export import to_json, to_cypher, to_graphml, to_html, to_canvas, to_obsidian @@ -336,6 +337,33 @@ def test_to_html_contains_nodes_and_edges(): assert "RAW_EDGES" in content +def test_to_html_filters_edges_by_effective_weight(): + G = nx.Graph() + G.add_nodes_from(["a", "b", "c"]) + G.add_edge("a", "b", relation="strong", confidence="EXTRACTED", confidence_score=1.0, weight=1.0) + G.add_edge("a", "c", relation="weak", confidence="INFERRED", confidence_score=0.2, weight=1.0) + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "graph.html" + to_html(G, {0: ["a", "b", "c"]}, str(out), min_effective_weight=0.5) + content = out.read_text() + assert '"label": "strong"' in content + assert '"label": "weak"' not in content + + +def test_to_obsidian_filters_weak_connections_by_effective_weight(): + G = nx.Graph() + G.add_node("a", label="Alpha") + G.add_node("b", label="Beta") + G.add_node("c", label="Gamma") + G.add_edge("a", "b", relation="strong", confidence="EXTRACTED", confidence_score=1.0, weight=1.0) + G.add_edge("a", "c", relation="weak", confidence="INFERRED", confidence_score=0.2, weight=1.0) + with tempfile.TemporaryDirectory() as tmp: + to_obsidian(G, {0: ["a", "b", "c"]}, tmp, min_effective_weight=0.5) + alpha = (Path(tmp) / "Alpha.md").read_text() + assert "[[Beta]]" in alpha + assert "[[Gamma]]" not in alpha + + def test_to_html_member_counts_accepted(): """to_html accepts member_counts without raising.""" G = make_graph()