Initial release: offline NetScaler triage with Sigma rules and MIT license

src/netscaler_triage/indicators.py

@@ -0,0 +1,94 @@
+"""Extract observables, not verdicts. Original values remain suitable for exact matching."""
+
+import ipaddress
+import re
+from typing import Any
+from urllib.parse import urlsplit
+
+
+def defang(value: str, indicator_type: str) -> str:
+    if indicator_type in {"url", "domain", "ipv4", "ipv6", "ip"}:
+        value = re.sub(r"^https:", "hxxps:", value, flags=re.I)
+        value = re.sub(r"^http:", "hxxp:", value, flags=re.I)
+        value = value.replace(".", "[.]").replace(":", "[:]")
+    return value
+
+
+def extract_indicators(text: str) -> list[dict[str, Any]]:
+    if len(text) > 16777216:
+        raise ValueError("indicator input exceeds 16 MiB; process log lines individually")
+    found: dict[tuple[str, str], dict[str, Any]] = {}
+
+    def add(kind: str, value: str, role: str = "observable", start: int | None = None) -> None:
+        key = (kind, value)
+        if key not in found:
+            found[key] = dict(
+                type=kind,
+                value=value,
+                defanged=defang(value, kind),
+                role=role,
+                observation="Text occurrence; not evidence of a connection or execution.",
+                start=start,
+            )
+
+    urls = list(re.finditer(r"\bhttps?://[^\s<>\"'`]+", text, re.I))
+    for match in urls:
+        value = match.group().rstrip("),;}")
+        add("url", value, "url_in_text", match.start())
+        try:
+            hostname = urlsplit(value).hostname
+            if hostname:
+                try:
+                    parsed = ipaddress.ip_address(hostname)
+                    add(
+                        "ipv4" if parsed.version == 4 else "ipv6",
+                        hostname,
+                        "url_host",
+                        match.start(),
+                    )
+                except ValueError:
+                    if "." in hostname and re.fullmatch(r"[A-Za-z0-9.-]+", hostname):
+                        add("domain", hostname, "url_host", match.start())
+        except ValueError:
+            pass
+    for match in re.finditer(
+        r"(?<![\w.])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?![\w.])|(?<![\w:])(?:[A-Fa-f0-9]{0,4}:){2,}[A-Fa-f0-9:.]*(?![\w:])",
+        text,
+    ):
+        value = match.group()
+        try:
+            parsed = ipaddress.ip_address(value)
+            add("ipv4" if parsed.version == 4 else "ipv6", value, "address_in_text", match.start())
+        except ValueError:
+            continue
+    for match in re.finditer(
+        r"(?<![A-Za-z0-9])(?:[a-fA-F0-9]{64}|[a-fA-F0-9]{40}|[a-fA-F0-9]{32})(?![A-Za-z0-9])", text
+    ):
+        value = match.group()
+        add(
+            {32: "md5", 40: "sha1", 64: "sha256"}[len(value)],
+            value,
+            "hash_shaped_text",
+            match.start(),
+        )
+    # Domain-shaped filenames are ambiguous: require URL, DNS/Host context or an email address.
+    domain_pattern = r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}"
+    for match in re.finditer(
+        r"(?:\b(?:host|hostname|dns|domain|server_name)\s*[:=]\s*|@)("
+        + domain_pattern
+        + r")(?![\w.-])",
+        text,
+        re.I,
+    ):
+        add("domain", match.group(1), "named_host_in_text", match.start(1))
+    url_index = 0
+    for match in re.finditer(r"(?<![\w/:])/(?!/)[^\s\"'`<>|;{}()]+", text):
+        while url_index < len(urls) and urls[url_index].end() <= match.start():
+            url_index += 1
+        if (
+            url_index < len(urls)
+            and urls[url_index].start() <= match.start() < urls[url_index].end()
+        ):
+            continue
+        add("path", match.group(), "path_in_text", match.start())
+    return list(found.values())