95 lines
3.6 KiB
Python
95 lines
3.6 KiB
Python
"""Extract observables, not verdicts. Original values remain suitable for exact matching."""
|
|
|
|
import ipaddress
|
|
import re
|
|
from typing import Any
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
def defang(value: str, indicator_type: str) -> str:
|
|
if indicator_type in {"url", "domain", "ipv4", "ipv6", "ip"}:
|
|
value = re.sub(r"^https:", "hxxps:", value, flags=re.I)
|
|
value = re.sub(r"^http:", "hxxp:", value, flags=re.I)
|
|
value = value.replace(".", "[.]").replace(":", "[:]")
|
|
return value
|
|
|
|
|
|
def extract_indicators(text: str) -> list[dict[str, Any]]:
|
|
if len(text) > 16777216:
|
|
raise ValueError("indicator input exceeds 16 MiB; process log lines individually")
|
|
found: dict[tuple[str, str], dict[str, Any]] = {}
|
|
|
|
def add(kind: str, value: str, role: str = "observable", start: int | None = None) -> None:
|
|
key = (kind, value)
|
|
if key not in found:
|
|
found[key] = dict(
|
|
type=kind,
|
|
value=value,
|
|
defanged=defang(value, kind),
|
|
role=role,
|
|
observation="Text occurrence; not evidence of a connection or execution.",
|
|
start=start,
|
|
)
|
|
|
|
urls = list(re.finditer(r"\bhttps?://[^\s<>\"'`]+", text, re.I))
|
|
for match in urls:
|
|
value = match.group().rstrip("),;}")
|
|
add("url", value, "url_in_text", match.start())
|
|
try:
|
|
hostname = urlsplit(value).hostname
|
|
if hostname:
|
|
try:
|
|
parsed = ipaddress.ip_address(hostname)
|
|
add(
|
|
"ipv4" if parsed.version == 4 else "ipv6",
|
|
hostname,
|
|
"url_host",
|
|
match.start(),
|
|
)
|
|
except ValueError:
|
|
if "." in hostname and re.fullmatch(r"[A-Za-z0-9.-]+", hostname):
|
|
add("domain", hostname, "url_host", match.start())
|
|
except ValueError:
|
|
pass
|
|
for match in re.finditer(
|
|
r"(?<![\w.])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?![\w.])|(?<![\w:])(?:[A-Fa-f0-9]{0,4}:){2,}[A-Fa-f0-9:.]*(?![\w:])",
|
|
text,
|
|
):
|
|
value = match.group()
|
|
try:
|
|
parsed = ipaddress.ip_address(value)
|
|
add("ipv4" if parsed.version == 4 else "ipv6", value, "address_in_text", match.start())
|
|
except ValueError:
|
|
continue
|
|
for match in re.finditer(
|
|
r"(?<![A-Za-z0-9])(?:[a-fA-F0-9]{64}|[a-fA-F0-9]{40}|[a-fA-F0-9]{32})(?![A-Za-z0-9])", text
|
|
):
|
|
value = match.group()
|
|
add(
|
|
{32: "md5", 40: "sha1", 64: "sha256"}[len(value)],
|
|
value,
|
|
"hash_shaped_text",
|
|
match.start(),
|
|
)
|
|
# Domain-shaped filenames are ambiguous: require URL, DNS/Host context or an email address.
|
|
domain_pattern = r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}"
|
|
for match in re.finditer(
|
|
r"(?:\b(?:host|hostname|dns|domain|server_name)\s*[:=]\s*|@)("
|
|
+ domain_pattern
|
|
+ r")(?![\w.-])",
|
|
text,
|
|
re.I,
|
|
):
|
|
add("domain", match.group(1), "named_host_in_text", match.start(1))
|
|
url_index = 0
|
|
for match in re.finditer(r"(?<![\w/:])/(?!/)[^\s\"'`<>|;{}()]+", text):
|
|
while url_index < len(urls) and urls[url_index].end() <= match.start():
|
|
url_index += 1
|
|
if (
|
|
url_index < len(urls)
|
|
and urls[url_index].start() <= match.start() < urls[url_index].end()
|
|
):
|
|
continue
|
|
add("path", match.group(), "path_in_text", match.start())
|
|
return list(found.values())
|