Validate result views on real evidence and expose file-hash matches
This commit is contained in:
1 parent
7ee77c3fc2
commit
ff8dc9ee7a
4 files changed
+39
No files matched your search
@@ -80,6 +80,8 @@ index. Prepared input copies, if needed, remain in `inputs/`.
|
||||
|
||||
`details/findings.jsonl` includes context and processing errors as well as matches;
|
||||
use `FINDINGS.csv` to browse matched events. A match is not proof of execution.
|
||||
Original-file SHA-256 matches are counted separately in `SUMMARY.txt`; their
|
||||
evidence is in `details/file_hash_hits.json`, even when no log event matches.
|
||||
The CSV keeps authorized-test and reference statuses visible. Spreadsheet-sensitive
|
||||
cells are prefixed with an apostrophe; exact original values remain in JSON.
|
||||
|
||||
|
||||
@@ -140,3 +140,25 @@ prepared inputs remain under inputs/. Tests cover separation of context from
|
||||
matches, preservation of JSON evidence, authorized-test labels, spreadsheet
|
||||
formula protection and no-matches wording. Direct engine.scan callers retain
|
||||
the flat layout. A synthetic CLI smoke check confirmed the new directory layout.
|
||||
|
||||
REAL-DATA OUTPUT PRESENTATION CHECK — 2026-10-05
|
||||
|
||||
The CLI was run against 19,625 referenced original lines from two independent
|
||||
private datasets, with each dataset's own IOC pack. Source files were verified
|
||||
against their recorded hashes before and after processing. This was a bounded
|
||||
presentation regression, not a new full-dataset scan.
|
||||
|
||||
The runs produced 14,633 and 2,461 matched-event CSV rows respectively. Every
|
||||
CSV event ID, source reference, indicator status and defanged indicator was
|
||||
checked against the full JSON event. Summary counts and partial status agreed
|
||||
with the JSON summary. The JSON retained 123 and 1,448 additional context/error
|
||||
events excluded from the matched-event CSV. Decoding output was present in
|
||||
13,019 and 344 matched events; authorized-test labels were preserved.
|
||||
|
||||
Both runs exited 2 with partial coverage: 12,510 and 1,584 failed decoding
|
||||
candidates remain reported. No matching errors occurred in these runs. Private
|
||||
source mapping and validation results remain outside the public release.
|
||||
|
||||
A subsequent presentation correction exposes original-file SHA-256 matches in
|
||||
SUMMARY.txt even without an event match. Three focused presentation tests passed,
|
||||
including this separate hash-only case; Ruff, formatting and strict mypy passed.
|
||||
@@ -76,6 +76,7 @@ def present_results(output: Path, summary: dict[str, Any]) -> None:
|
||||
f"Processing status: {summary.get('status', 'unknown')}",
|
||||
f"Records read: {stats.get('records', 0)}",
|
||||
f"Events matching a rule or indicator: {counts['matches']}",
|
||||
f"Original-file SHA-256 indicator matches: {stats.get('file_hash_hits', 0)}",
|
||||
f"Matching events including an authorized-test indicator: {counts['authorized_test']}",
|
||||
f"Update/version records: {stats.get('update_records', 0)}",
|
||||
"",
|
||||
@@ -86,6 +87,7 @@ def present_results(output: Path, summary: dict[str, Any]) -> None:
|
||||
"2. IOCS.csv — search candidates for SIEM; not an automatic blocklist.",
|
||||
"3. details/ — full JSON events, decoding, update records, correlations and checksums.",
|
||||
" details/findings.jsonl also retains context and processing-error events.",
|
||||
" File-hash matches are in details/file_hash_hits.json, separate from event matches.",
|
||||
"",
|
||||
"HOW TO INTERPRET THIS",
|
||||
"",
|
||||
|
||||
@@ -47,3 +47,16 @@ class PresentationTests(unittest.TestCase):
|
||||
(output / "findings.jsonl").write_text("")
|
||||
present_results(output, {"status": "complete"})
|
||||
self.assertIn("No matches does not prove", (output / "SUMMARY.txt").read_text())
|
||||
|
||||
def test_file_hash_hit_is_visible_without_an_event_match(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
output = Path(temporary)
|
||||
(output / "findings.jsonl").write_text("")
|
||||
hits = [{"source": "sample.bin", "basis": "original file SHA-256"}]
|
||||
(output / "file_hash_hits.json").write_text(json.dumps(hits))
|
||||
present_results(output, {"status": "complete", "counts": {"file_hash_hits": 1}})
|
||||
text = (output / "SUMMARY.txt").read_text()
|
||||
self.assertIn("Events matching a rule or indicator: 0", text)
|
||||
self.assertIn("Original-file SHA-256 indicator matches: 1", text)
|
||||
self.assertIn("details/file_hash_hits.json", text)
|
||||
self.assertEqual(json.loads((output / "details/file_hash_hits.json").read_text()), hits)
|
||||
Reference in new issue
Block a user