From 3b7011f4334860d208fbefe22b9dfe370e7f08ba Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Wed, 23 Sep 2026 20:01:02 +0200 Subject: [PATCH] Bound crack memory: cap wordlist sort, chunk target loading - prepare_candidates: use sort -S 1G --parallel 4 (was unbounded, OOM-prone on large --passwords-from datasets) - crack: add --max-hashes to stream large hash files in bounded chunks, so the in-RAM target index no longer scales with the whole input - docs + regression test for chunked cracking --- README.md | 5 ++ collectionparser/cli.py | 15 ++++- collectionparser/crack.py | 112 ++++++++++++++++++++++++++------------ tests/crack_test.py | 23 ++++++++ 4 files changed, 119 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 3d5180f..6f3f0df 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,11 @@ sources are merged through `sort -u`. Only unsalted MD5/SHA1/SHA256 are attackable — bcrypt is ignored. Feed `cracked.txt` back into the index with `split`/`merge`. +The **target index is held in RAM** (roughly 250 bytes per hash), so a +multi-hundred-million-hash file will exhaust memory. Pass `--max-hashes N` +(e.g. `--max-hashes 8000000`) to stream the input in bounded chunks at the cost +of rescanning the wordlist per chunk. + ## Measured performance Single large-file test on Collections 2-5 (3 files, 19.94 GB, 3 workers): diff --git a/collectionparser/cli.py b/collectionparser/cli.py index 9baa6cb..c38174b 100644 --- a/collectionparser/cli.py +++ b/collectionparser/cli.py @@ -99,8 +99,20 @@ def export(dataset: str, out: str): ) @click.option("--out", required=True, type=output_file()) @click.option("--workers", default=0, type=int, show_default=True, help="0 = all CPUs.") +@click.option( + "--max-hashes", + default=0, + type=int, + show_default=True, + help="Cap hashes held in RAM at once (0 = all); larger inputs are chunked.", +) def crack( - hashes_path: str, wordlist: tuple, passwords_from: tuple, out: str, workers: int + hashes_path: str, + wordlist: tuple, + passwords_from: tuple, + out: str, + workers: int, + max_hashes: int, ): from collectionparser.crack import crack as run_crack @@ -110,6 +122,7 @@ def crack( wordlists=wordlist, passwords_from=passwords_from, workers=workers or None, + max_hashes=max_hashes or None, ) diff --git a/collectionparser/crack.py b/collectionparser/crack.py index 4adecd3..3d2d723 100644 --- a/collectionparser/crack.py +++ b/collectionparser/crack.py @@ -8,9 +8,11 @@ Only unsalted MD5/SHA1/SHA256 are attackable; bcrypt is intentionally not supported (a wordlist attack on it is infeasible). Memory: the target index is built once and shared with the workers by forking -(copy-on-write), so it costs one copy, not one per worker. ponytail: platforms -without ``fork`` fall back to rebuilding it in each worker (RAM x workers); a -shared mmap index is the upgrade path for enormous hash files. +(copy-on-write), so it costs one copy, not one per worker. It still scales with +the number of hashes, so pass ``max_hashes`` to stream the input in bounded +chunks on memory-limited hosts. ponytail: platforms without ``fork`` fall back +to rebuilding it in each worker (RAM x workers); a shared mmap index is the +upgrade path for enormous hash files. """ import gzip @@ -21,6 +23,7 @@ import shutil import subprocess import urllib.request from concurrent.futures import ProcessPoolExecutor +from itertools import islice from pathlib import Path from collectionparser.api import find_files @@ -71,8 +74,13 @@ def prepare_candidates(wordlists, passwords_from, work_dir: Path) -> Path: env = {**os.environ, "LC_ALL": "C", "TMPDIR": str(work_dir)} with ( open(unique, "wb") as sink_file, + # Match api.sort_unique: cap the sort buffer so a huge --passwords-from + # dataset spills to disk instead of OOMing the host. subprocess.Popen( - ["sort", "-u"], stdin=subprocess.PIPE, stdout=sink_file, env=env + ["sort", "-S", "1G", "--parallel", "4", "-u"], + stdin=subprocess.PIPE, + stdout=sink_file, + env=env, ) as process, ): with process.stdin as sink: @@ -129,25 +137,34 @@ def _scan_stripe(args: tuple) -> list[bytes]: return matches -def crack( - hashes_path: str, - out_path: str, - wordlists=(), - passwords_from=(), - workers: int | None = None, - work_dir: str | None = None, -) -> int: - """Attack the hashes with the candidate wordlists; write ``email:password``.""" +def _iter_hash_chunks(hashes_path, max_hashes, work_dir): + """Yield hash-file paths of at most ``max_hashes`` lines, or the whole file.""" + if not max_hashes or max_hashes <= 0: + yield str(hashes_path) + return + index = 0 + with open(hashes_path, "rb") as reader: + while True: + lines = list(islice(reader, max_hashes)) + if not lines: + break + chunk = Path(work_dir) / f"hashes.chunk.{index:05d}.txt" + chunk.write_bytes(b"".join(lines)) + try: + yield str(chunk) + finally: + chunk.unlink(missing_ok=True) + index += 1 + + +def _attack(hashes_path, writer, wordlist, workers) -> int: + """Attack one hashes file, writing ``email:password`` matches to ``writer``.""" global _TARGETS _TARGETS = load_targets(hashes_path) targets = _TARGETS if not targets: - print("No crackable MD5/SHA1/SHA256 hashes found.") + print(f"No crackable MD5/SHA1/SHA256 hashes found in {hashes_path}.") return 0 - work_dir = Path(work_dir) if work_dir else Path(out_path).parent - wordlist = prepare_candidates(list(wordlists), list(passwords_from), work_dir) - - workers = max(1, workers or os.cpu_count() or 1) n_hashes = sum( len(emails) for by_digest in targets.values() for emails in by_digest.values() ) @@ -157,24 +174,49 @@ def crack( ) jobs = [(str(hashes_path), str(wordlist), i, workers) for i in range(workers)] + total = 0 + if workers == 1: + results = map(_scan_stripe, jobs) + else: + # fork shares the parent's _TARGETS by copy-on-write. + context = multiprocessing.get_context("fork") if hasattr(os, "fork") else None + executor = ProcessPoolExecutor(max_workers=workers, mp_context=context) + results = executor.map(_scan_stripe, jobs) + try: + for matches in results: + for line in matches: + writer.write(line) + total += 1 + finally: + if workers != 1: + executor.shutdown() + return total + + +def crack( + hashes_path: str, + out_path: str, + wordlists=(), + passwords_from=(), + workers: int | None = None, + work_dir: str | None = None, + max_hashes: int | None = None, +) -> int: + """Attack the hashes with the candidate wordlists; write ``email:password``. + + ``max_hashes`` bounds how many hash lines are held in the in-RAM target + index at once; larger inputs are streamed in chunks that spill to ``work_dir``. + A single global index of hundreds of millions of hashes needs tens of GB, so + set this on memory-limited hosts. ponytail: the wordlist is rescanned per + chunk, so a smaller ``max_hashes`` trades RAM for repeated passes. + """ + work_dir = Path(work_dir) if work_dir else Path(out_path).parent + wordlist = prepare_candidates(list(wordlists), list(passwords_from), work_dir) + workers = max(1, workers or os.cpu_count() or 1) + total = 0 with open(out_path, "wb") as writer: - if workers == 1: - results = map(_scan_stripe, jobs) - else: - # fork shares the parent's _TARGETS by copy-on-write. - context = ( - multiprocessing.get_context("fork") if hasattr(os, "fork") else None - ) - executor = ProcessPoolExecutor(max_workers=workers, mp_context=context) - results = executor.map(_scan_stripe, jobs) - try: - for matches in results: - for line in matches: - writer.write(line) - total += 1 - finally: - if workers != 1: - executor.shutdown() + for chunk_path in _iter_hash_chunks(hashes_path, max_hashes, work_dir): + total += _attack(chunk_path, writer, wordlist, workers) print(f"Cracked {total:,} hashes -> {out_path}.") return total diff --git a/tests/crack_test.py b/tests/crack_test.py index 5840819..9f8b279 100644 --- a/tests/crack_test.py +++ b/tests/crack_test.py @@ -72,6 +72,29 @@ class CrackTest(unittest.TestCase): self.assertEqual(1, total) self.assertEqual([b"a@x.com:reused"], out.read_bytes().splitlines()) + def test_crack_chunks_hash_file(self): + with tempfile.TemporaryDirectory() as tmp: + hashes = Path(tmp) / "hashes.txt" + hashes.write_bytes( + b"a@x.com:" + hashlib.md5(b"hunter2").hexdigest().encode() + b"\n" + b"b@y.com:" + hashlib.sha1(b"secret").hexdigest().encode() + b"\n" + ) + wordlist = Path(tmp) / "words.txt" + wordlist.write_bytes(b"hunter2\nsecret\n") + out = Path(tmp) / "cracked.txt" + total = crack( + str(hashes), + str(out), + wordlists=[str(wordlist)], + workers=1, + max_hashes=1, + ) + self.assertEqual(2, total) + self.assertEqual( + {b"a@x.com:hunter2", b"b@y.com:secret"}, + set(out.read_bytes().splitlines()), + ) + if __name__ == "__main__": unittest.main()