0.4.0: Add match clustering

4 files changed, 313 insertions, 25 deletions

diff --git a/README.md b/README.md
index 4d1ffbc..f3d6cc9 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,8 @@ previous run (tracked per-file, per-inode) and then exits.
 - INI-style configuration (`pylogsentinel.conf`).
 - Multiple named log sets (e.g. [logs.default], [logs.errors]) with per-set discovery via static paths or shell command
 - Per-rule regular expressions with optional flags (`/pattern/i` style).
-- Per-rule or actions executed as shell commands with rich contextual environment variables.
+- Per-rule actions executed as shell commands with rich contextual environment variables.
+- Intelligent match clustering: nearby matches for the same rule are grouped into a single event to avoid redundant alerts.
 - Reliable state tracking via inode-based files (`<inode>` in `state_dir`).
 - Safe handling of truncation & rotation (offset reset if file shrinks).
 - Strict lock file (`LOCK`) prevents concurrent overlapping runs.
@@ -128,14 +129,16 @@ Each action defines a shell `cmd` executed when a matching rule triggers. The sp
 
 Environment variables available to the command:
 
-| Variable           | Description                                                            |
-| ------------------ | ---------------------------------------------------------------------- |
-| `RULE_ID`          | The rule identifier (e.g. `error`).                                    |
-| `RULE_PATTERN`     | The normalized pattern string (e.g. `/error/i`).                       |
-| `RULE_DESCRIPTION` | Description text or placeholder string if not provided.                |
-| `FILE`             | Absolute path of the log file where the match occurred.                |
-| `LINE`             | 1-based absolute line number within the file at time of scan.          |
-| `CONTEXT`          | Concatenated lines around the match (default radius 2 before & after). |
+| Variable           | Description                                                                                          |
+| ------------------ | ---------------------------------------------------------------------------------------------------- |
+| `RULE_ID`          | The rule identifier (e.g. `error`).                                                                  |
+| `RULE_PATTERN`     | The normalized pattern string (e.g. `/error/i`).                                                     |
+| `RULE_DESCRIPTION` | Description text or placeholder string if not provided.                                              |
+| `FILE`             | Absolute path of the log file where the match occurred.                                              |
+| `LINE`             | 1-based absolute line number of the first match in the cluster.                                      |
+| `CONTEXT`          | Contiguous block of lines spanning the cluster with surrounding context (default radius 5).           |
+| `MATCH_COUNT`      | Number of matching lines in the cluster (e.g. `1` for a single match, `3` for three grouped matches). |
+| `MATCHED_LINES`    | Comma-separated 1-based line numbers of all matches in the cluster (e.g. `42` or `42,44,47`).        |
 
 Your `cmd` can reference these with typical shell expansion, for example:
 
@@ -144,6 +147,14 @@ Your `cmd` can reference these with typical shell expansion, for example:
 cmd = printf "%s\n%s\n" "$RULE_DESCRIPTION" "$CONTEXT" | mail -s "Alert $RULE_ID: $FILE:$LINE" ops@example.com
 ```
 
+### Match Clustering
+
+When multiple lines match the same rule within a short span, they are grouped into a single cluster rather than triggering separate actions for each match. This avoids redundant alerts for cascading errors.
+
+Two matches are merged into the same cluster if the number of non-matching lines between them is at most the context radius (default 5). The cluster's context block spans from `context_radius` lines before the first match to `context_radius` lines after the last match — one contiguous block including any non-matching lines in between.
+
+Example with `context_radius = 5`: if errors appear on lines 10, 13, and 16 of a file, they form a single cluster (gaps of 2 and 2, both ≤ 5). The action fires once with `CONTEXT` covering lines 5–21, `MATCH_COUNT=3`, and `MATCHED_LINES=10,13,16`.
+
 ## State Tracking
 
 For each processed file:
diff --git a/pylogsentinel/core.py b/pylogsentinel/core.py
index 1b6ce13..5ee033e 100644
--- a/pylogsentinel/core.py
+++ b/pylogsentinel/core.py
@@ -67,6 +67,8 @@ class MatchEvent:
     line_number: int
     context: str
     matched_text: str
+    match_count: int = 1
+    matched_lines: str = ""
 
 
 def parse_size(value: str) -> int:
@@ -516,6 +518,15 @@ def read_new_lines(
     return lines, new_offset, new_last_line_number
 
 
+def _rule_applies(rule: Rule, file_path: str) -> bool:
+    """Check if a rule applies to the given file path."""
+    for base in rule.log_base_paths:
+        b = base.rstrip(os.sep)
+        if file_path == b or file_path.startswith(b + os.sep):
+            return True
+    return False
+
+
 def generate_matches(
     file_path: str,
     lines: list[str],
@@ -524,33 +535,71 @@ def generate_matches(
     context_radius: int = CONTEXT_RADIUS,
 ) -> Iterator[MatchEvent]:
     """
-    Iterate over lines and yield MatchEvent objects for each rule match.
-    A rule is only applied if the file belongs to its configured log set
-    (file path equal to or under one of the set's base paths).
+    Iterate over lines and yield one MatchEvent per cluster of nearby matches.
+
+    Matches for the same rule are grouped into clusters: consecutive matches
+    whose gap (number of non-matching lines between them) is at most
+    context_radius are merged into one cluster.  Each cluster produces a
+    single MatchEvent whose context spans from context_radius lines before
+    the first match to context_radius lines after the last match.
     """
     total_lines = len(lines)
+
+    # Collect per-rule match indices in one pass.
+    # Key: rule_id, Value: list of (line_index, matched_text)
+    rule_matches: dict[str, list[tuple[int, str]]] = {}
+
     for idx, line in enumerate(lines):
-        absolute_line_number = initial_line_number + idx + 1
         for rule in rules.values():
-            applies = False
-            for base in rule.log_base_paths:
-                b = base.rstrip(os.sep)
-                if file_path == b or file_path.startswith(b + os.sep):
-                    applies = True
-                    break
-            if not applies:
+            if not _rule_applies(rule, file_path):
                 continue
             m = rule.compiled.search(line)
             if not m:
                 continue
-            start_ctx = max(0, idx - context_radius)
-            context = "".join(lines[start_ctx : idx + 1])
+            rule_matches.setdefault(rule.rule_id, []).append((idx, m.group(0)))
+
+    # Build clusters per rule and yield one MatchEvent per cluster.
+    for rule in rules.values():
+        matches = rule_matches.get(rule.rule_id)
+        if not matches:
+            continue
+
+        # matches are already in line order; group into clusters.
+        clusters: list[list[tuple[int, str]]] = []
+        current_cluster: list[tuple[int, str]] = [matches[0]]
+
+        for i in range(1, len(matches)):
+            prev_idx = current_cluster[-1][0]
+            curr_idx = matches[i][0]
+            gap = curr_idx - prev_idx - 1  # non-matching lines between
+            if gap <= context_radius:
+                current_cluster.append(matches[i])
+            else:
+                clusters.append(current_cluster)
+                current_cluster = [matches[i]]
+        clusters.append(current_cluster)
+
+        for cluster in clusters:
+            first_idx = cluster[0][0]
+            last_idx = cluster[-1][0]
+
+            ctx_start = max(0, first_idx - context_radius)
+            ctx_end = min(total_lines - 1, last_idx + context_radius)
+            context = "".join(lines[ctx_start : ctx_end + 1])
+
+            first_line_number = initial_line_number + first_idx + 1
+            matched_lines_str = ",".join(
+                str(initial_line_number + idx + 1) for idx, _ in cluster
+            )
+
             yield MatchEvent(
                 rule=rule,
                 file_path=file_path,
-                line_number=absolute_line_number,
+                line_number=first_line_number,
                 context=context,
-                matched_text=m.group(0),
+                matched_text=cluster[0][1],
+                match_count=len(cluster),
+                matched_lines=matched_lines_str,
             )
 
 
@@ -574,6 +623,8 @@ def execute_action(
             "FILE": event.file_path,
             "LINE": str(event.line_number),
             "CONTEXT": event.context,
+            "MATCH_COUNT": str(event.match_count),
+            "MATCHED_LINES": event.matched_lines,
         }
     )
     if env_overrides:
diff --git a/pyproject.toml b/pyproject.toml
index a79322b..92a152a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "pylogsentinel"
-version = "0.3.2"
+version = "0.4.0"
 description = "Lightweight cron-friendly log monitoring utility with regex rules and per-rule shell actions"
 readme = "README.md"
 requires-python = ">=3.11"
diff --git a/tests/test_generate_matches.py b/tests/test_generate_matches.py
new file mode 100644
index 0000000..f9dd828
--- /dev/null
+++ b/tests/test_generate_matches.py
@@ -0,0 +1,226 @@
+"""Tests for generate_matches clustering logic."""
+
+from __future__ import annotations
+
+import re
+import unittest
+
+from pylogsentinel.core import (
+    MatchEvent,
+    Rule,
+    generate_matches,
+)
+
+
+def _make_rule(
+    rule_id: str = "test",
+    pattern: str = "error",
+    flags: int = re.IGNORECASE,
+    base_paths: list[str] | None = None,
+) -> Rule:
+    """Helper to build a Rule for testing."""
+    if base_paths is None:
+        base_paths = ["/var/log"]
+    return Rule(
+        rule_id=rule_id,
+        pattern=f"/{pattern}/i",
+        description="test rule",
+        action_id="default",
+        compiled=re.compile(pattern, flags),
+        log_set_ids=["default"],
+        log_base_paths=base_paths,
+    )
+
+
+FILE_PATH = "/var/log/app.log"
+
+
+class TestSingleMatch(unittest.TestCase):
+    """A single match produces one event with surrounding context."""
+
+    def test_single_match_mid_file(self):
+        lines = [f"line {i}\n" for i in range(20)]
+        lines[10] = "ERROR happened here\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        ev = events[0]
+        self.assertEqual(ev.line_number, 11)  # 1-based
+        self.assertEqual(ev.match_count, 1)
+        self.assertEqual(ev.matched_lines, "11")
+        # Context: lines 7-13 (idx 7 to 13 inclusive)
+        expected_ctx = "".join(lines[7:14])
+        self.assertEqual(ev.context, expected_ctx)
+
+    def test_single_match_near_start(self):
+        lines = [f"line {i}\n" for i in range(10)]
+        lines[1] = "ERROR at line 1\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        ev = events[0]
+        # Context start clamped to 0
+        expected_ctx = "".join(lines[0:5])
+        self.assertEqual(ev.context, expected_ctx)
+
+    def test_single_match_near_end(self):
+        lines = [f"line {i}\n" for i in range(10)]
+        lines[9] = "ERROR at the end\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        ev = events[0]
+        # Context end clamped to last line
+        expected_ctx = "".join(lines[6:10])
+        self.assertEqual(ev.context, expected_ctx)
+
+
+class TestClusteringMerge(unittest.TestCase):
+    """Matches within context_radius lines of each other merge."""
+
+    def test_two_matches_within_radius(self):
+        """Matches 2 lines apart (gap=1) with radius=3 → one cluster."""
+        lines = [f"line {i}\n" for i in range(20)]
+        lines[8] = "ERROR first\n"
+        lines[10] = "ERROR second\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        ev = events[0]
+        self.assertEqual(ev.match_count, 2)
+        self.assertEqual(ev.matched_lines, "9,11")
+        self.assertEqual(ev.line_number, 9)  # first match
+        # Context: idx 5 to 13
+        expected_ctx = "".join(lines[5:14])
+        self.assertEqual(ev.context, expected_ctx)
+
+    def test_three_matches_chained(self):
+        """Three matches each within radius of the previous → one cluster."""
+        lines = [f"line {i}\n" for i in range(30)]
+        lines[5] = "ERROR one\n"
+        lines[8] = "ERROR two\n"
+        lines[11] = "ERROR three\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        ev = events[0]
+        self.assertEqual(ev.match_count, 3)
+        self.assertEqual(ev.matched_lines, "6,9,12")
+        # Context: idx 2 to 14
+        expected_ctx = "".join(lines[2:15])
+        self.assertEqual(ev.context, expected_ctx)
+
+
+class TestClusteringSplit(unittest.TestCase):
+    """Matches far apart produce separate events."""
+
+    def test_two_matches_beyond_radius(self):
+        """Gap of 5 with radius=3 → two clusters."""
+        lines = [f"line {i}\n" for i in range(30)]
+        lines[5] = "ERROR first\n"
+        lines[11] = "ERROR second\n"  # gap = 5 lines (idx 6-10)
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 2)
+        self.assertEqual(events[0].match_count, 1)
+        self.assertEqual(events[0].line_number, 6)
+        self.assertEqual(events[1].match_count, 1)
+        self.assertEqual(events[1].line_number, 12)
+
+    def test_gap_exactly_at_radius(self):
+        """Gap == radius → merged (gap <= context_radius)."""
+        lines = [f"line {i}\n" for i in range(20)]
+        lines[5] = "ERROR first\n"
+        lines[9] = "ERROR second\n"  # gap = 3 (idx 6,7,8)
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0].match_count, 2)
+
+    def test_gap_one_beyond_radius(self):
+        """Gap == radius + 1 → split."""
+        lines = [f"line {i}\n" for i in range(20)]
+        lines[5] = "ERROR first\n"
+        lines[10] = "ERROR second\n"  # gap = 4 (idx 6,7,8,9)
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 2)
+
+
+class TestMultipleRules(unittest.TestCase):
+    """Different rules produce independent clusters even on same lines."""
+
+    def test_two_rules_same_line(self):
+        lines = [f"line {i}\n" for i in range(10)]
+        lines[5] = "ERROR WARN something\n"
+        rule_error = _make_rule(rule_id="error", pattern="ERROR")
+        rule_warn = _make_rule(rule_id="warn", pattern="WARN")
+        rules = {"error": rule_error, "warn": rule_warn}
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, rules, context_radius=2)
+        )
+        self.assertEqual(len(events), 2)
+        rule_ids = {ev.rule.rule_id for ev in events}
+        self.assertEqual(rule_ids, {"error", "warn"})
+
+
+class TestRuleNotApplicable(unittest.TestCase):
+    """Rule with non-matching base paths produces no events."""
+
+    def test_rule_does_not_apply(self):
+        lines = ["ERROR something\n"]
+        rule = _make_rule(base_paths=["/other/path"])
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 0)
+
+
+class TestInitialLineNumber(unittest.TestCase):
+    """Verify absolute line numbers use initial_line_number offset."""
+
+    def test_offset_line_numbers(self):
+        lines = ["ERROR line\n"]
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 100, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        # initial_line_number=100 means 100 lines already processed
+        # idx=0 → absolute = 100 + 0 + 1 = 101
+        self.assertEqual(events[0].line_number, 101)
+
+
+class TestMatchedText(unittest.TestCase):
+    """matched_text field contains the first match's text."""
+
+    def test_matched_text_first_in_cluster(self):
+        lines = [f"line {i}\n" for i in range(10)]
+        lines[3] = "first ERROR here\n"
+        lines[5] = "second ERROR there\n"
+        rule = _make_rule()
+        events = list(
+            generate_matches(FILE_PATH, lines, 0, {"test": rule}, context_radius=3)
+        )
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0].matched_text, "ERROR")
+
+
+if __name__ == "__main__":
+    unittest.main()