112 lines
4.1 KiB
Python
112 lines
4.1 KiB
Python
"""Exact and wildcard queries over a bucketed dataset."""
|
|
|
|
import re
|
|
import subprocess
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
|
|
|
|
def postprocess(results: list[str]) -> list[str]:
|
|
"""Normalise ``email:password`` lines into a sorted, de-duplicated list."""
|
|
emails_passwords: set[str] = set()
|
|
for txt_line in results:
|
|
if '@' not in txt_line: # does it contain an email address?
|
|
continue
|
|
if ':' in txt_line: # '[email protected]:iskandar89
|
|
separator = ':'
|
|
elif ';' in txt_line: # '[email protected];ol1391ga
|
|
separator = ';'
|
|
else:
|
|
continue
|
|
try:
|
|
email, password = txt_line.strip().split(separator)
|
|
except ValueError:
|
|
continue
|
|
emails_passwords.add(f'{email}:{password}')
|
|
return sorted(emails_passwords)
|
|
|
|
|
|
def glob_to_regex(pattern: str) -> str:
|
|
"""Convert a glob to a regex anchored to the email field.
|
|
|
|
``*`` and ``?`` never cross the ``:`` separator.
|
|
ponytail: ``*`` matches within the email only; a wildcard that must cross
|
|
the separator needs a different index.
|
|
"""
|
|
parts = []
|
|
for char in pattern:
|
|
if char == '*':
|
|
parts.append('[^:]*')
|
|
elif char == '?':
|
|
parts.append('[^:]')
|
|
else:
|
|
parts.append(re.escape(char))
|
|
return '^' + ''.join(parts) + ':'
|
|
|
|
|
|
class QueryAPI:
|
|
"""Read-only lookups: O(1) exact match or a ripgrep wildcard scan."""
|
|
|
|
@staticmethod
|
|
@lru_cache(maxsize=4096) # ponytail: stale if the index changes mid-run
|
|
def scanning(directory: Path) -> tuple[dict[str, Path], dict[str, Path]]:
|
|
"""Split a directory's entries into (files, subdirectories)."""
|
|
file_list: dict[str, Path] = {}
|
|
dir_list: dict[str, Path] = {}
|
|
for entry in directory.iterdir():
|
|
if entry.is_file():
|
|
file_list[entry.name] = entry
|
|
else:
|
|
dir_list[entry.name] = entry
|
|
return file_list, dir_list
|
|
|
|
@staticmethod
|
|
def search_in_file(path: Path, email: str) -> list[str]:
|
|
"""Return matching lines for ``email`` from a single bucket file."""
|
|
assert path.is_file(), path
|
|
try:
|
|
output = subprocess.check_output(
|
|
['rg', '--no-line-number', '-ie', '^' + re.escape(email), str(path)], text=True)
|
|
except subprocess.CalledProcessError:
|
|
return []
|
|
return [line for line in output.splitlines() if line]
|
|
|
|
@staticmethod
|
|
def recursive_search(path: Path, characters: list[str], index: int) -> list[str]:
|
|
"""Descend the bucket tree one character at a time, then search."""
|
|
if index >= len(characters):
|
|
return []
|
|
file_list, dir_list = QueryAPI.scanning(path)
|
|
current = characters[index].lower()
|
|
if current in file_list:
|
|
return QueryAPI.search_in_file(file_list[current], ''.join(characters))
|
|
if current in dir_list:
|
|
return QueryAPI.recursive_search(dir_list[current], characters, index + 1)
|
|
if 'symbols' in file_list:
|
|
return QueryAPI.search_in_file(file_list['symbols'], ''.join(characters))
|
|
return []
|
|
|
|
@staticmethod
|
|
def query(data_dir: str, request: str, query_type: str) -> list[str]:
|
|
"""Exact email lookup using the bucket path (O(1) file read)."""
|
|
if len(request) <= 3:
|
|
return []
|
|
if query_type == 'email' and '@' not in request:
|
|
return []
|
|
return postprocess(QueryAPI.recursive_search(Path(data_dir), list(request), 0))
|
|
|
|
@staticmethod
|
|
def wildcard(data_dir: str, pattern: str) -> list[str]:
|
|
"""Glob search across the whole dataset with ripgrep.
|
|
|
|
ponytail: full scan of the tree (ripgrep is fast enough); add a
|
|
domain-sharded index only if wildcard latency becomes a problem.
|
|
"""
|
|
try:
|
|
output = subprocess.check_output(
|
|
['rg', '--no-filename', '--no-line-number', '-i', '-e',
|
|
glob_to_regex(pattern), str(Path(data_dir))], text=True)
|
|
except subprocess.CalledProcessError:
|
|
return []
|
|
return postprocess(output.splitlines())
|