Bound crack memory: cap wordlist sort, chunk target loading

- prepare_candidates: use sort -S 1G --parallel 4 (was unbounded, OOM-prone
  on large --passwords-from datasets)
- crack: add --max-hashes to stream large hash files in bounded chunks, so the
  in-RAM target index no longer scales with the whole input
- docs + regression test for chunked cracking

README.md

@@ -139,6 +139,11 @@ sources are merged through `sort -u`. Only unsalted MD5/SHA1/SHA256 are
 attackable — bcrypt is ignored. Feed `cracked.txt` back into the index with
 `split`/`merge`.
 
+The **target index is held in RAM** (roughly 250 bytes per hash), so a
+multi-hundred-million-hash file will exhaust memory. Pass `--max-hashes N`
+(e.g. `--max-hashes 8000000`) to stream the input in bounded chunks at the cost
+of rescanning the wordlist per chunk.
+
 ## Measured performance
 
 Single large-file test on Collections 2-5 (3 files, 19.94 GB, 3 workers):

collectionparser/cli.py

@@ -99,8 +99,20 @@ def export(dataset: str, out: str):
 )
 @click.option("--out", required=True, type=output_file())
 @click.option("--workers", default=0, type=int, show_default=True, help="0 = all CPUs.")
+@click.option(
+    "--max-hashes",
+    default=0,
+    type=int,
+    show_default=True,
+    help="Cap hashes held in RAM at once (0 = all); larger inputs are chunked.",
+)
 def crack(
-    hashes_path: str, wordlist: tuple, passwords_from: tuple, out: str, workers: int
+    hashes_path: str,
+    wordlist: tuple,
+    passwords_from: tuple,
+    out: str,
+    workers: int,
+    max_hashes: int,
 ):
     from collectionparser.crack import crack as run_crack
 
@@ -110,6 +122,7 @@ def crack(
         wordlists=wordlist,
         passwords_from=passwords_from,
         workers=workers or None,
+        max_hashes=max_hashes or None,
     )
 
 

collectionparser/crack.py

@@ -8,9 +8,11 @@ Only unsalted MD5/SHA1/SHA256 are attackable; bcrypt is intentionally not
 supported (a wordlist attack on it is infeasible).
 
 Memory: the target index is built once and shared with the workers by forking
-(copy-on-write), so it costs one copy, not one per worker. ponytail: platforms
-without ``fork`` fall back to rebuilding it in each worker (RAM x workers); a
-shared mmap index is the upgrade path for enormous hash files.
+(copy-on-write), so it costs one copy, not one per worker. It still scales with
+the number of hashes, so pass ``max_hashes`` to stream the input in bounded
+chunks on memory-limited hosts. ponytail: platforms without ``fork`` fall back
+to rebuilding it in each worker (RAM x workers); a shared mmap index is the
+upgrade path for enormous hash files.
 """
 
 import gzip
@@ -21,6 +23,7 @@ import shutil
 import subprocess
 import urllib.request
 from concurrent.futures import ProcessPoolExecutor
+from itertools import islice
 from pathlib import Path
 
 from collectionparser.api import find_files
@@ -71,8 +74,13 @@ def prepare_candidates(wordlists, passwords_from, work_dir: Path) -> Path:
     env = {**os.environ, "LC_ALL": "C", "TMPDIR": str(work_dir)}
     with (
         open(unique, "wb") as sink_file,
+        # Match api.sort_unique: cap the sort buffer so a huge --passwords-from
+        # dataset spills to disk instead of OOMing the host.
         subprocess.Popen(
-            ["sort", "-u"], stdin=subprocess.PIPE, stdout=sink_file, env=env
+            ["sort", "-S", "1G", "--parallel", "4", "-u"],
+            stdin=subprocess.PIPE,
+            stdout=sink_file,
+            env=env,
         ) as process,
     ):
         with process.stdin as sink:
@@ -129,25 +137,34 @@ def _scan_stripe(args: tuple) -> list[bytes]:
     return matches
 
 
-def crack(
-    hashes_path: str,
-    out_path: str,
-    wordlists=(),
-    passwords_from=(),
-    workers: int | None = None,
-    work_dir: str | None = None,
-) -> int:
-    """Attack the hashes with the candidate wordlists; write ``email:password``."""
+def _iter_hash_chunks(hashes_path, max_hashes, work_dir):
+    """Yield hash-file paths of at most ``max_hashes`` lines, or the whole file."""
+    if not max_hashes or max_hashes <= 0:
+        yield str(hashes_path)
+        return
+    index = 0
+    with open(hashes_path, "rb") as reader:
+        while True:
+            lines = list(islice(reader, max_hashes))
+            if not lines:
+                break
+            chunk = Path(work_dir) / f"hashes.chunk.{index:05d}.txt"
+            chunk.write_bytes(b"".join(lines))
+            try:
+                yield str(chunk)
+            finally:
+                chunk.unlink(missing_ok=True)
+            index += 1
+
+
+def _attack(hashes_path, writer, wordlist, workers) -> int:
+    """Attack one hashes file, writing ``email:password`` matches to ``writer``."""
     global _TARGETS
     _TARGETS = load_targets(hashes_path)
     targets = _TARGETS
     if not targets:
-        print("No crackable MD5/SHA1/SHA256 hashes found.")
+        print(f"No crackable MD5/SHA1/SHA256 hashes found in {hashes_path}.")
         return 0
-    work_dir = Path(work_dir) if work_dir else Path(out_path).parent
-    wordlist = prepare_candidates(list(wordlists), list(passwords_from), work_dir)
-
-    workers = max(1, workers or os.cpu_count() or 1)
     n_hashes = sum(
         len(emails) for by_digest in targets.values() for emails in by_digest.values()
     )
@@ -157,24 +174,49 @@ def crack(
     )
 
     jobs = [(str(hashes_path), str(wordlist), i, workers) for i in range(workers)]
+    total = 0
+    if workers == 1:
+        results = map(_scan_stripe, jobs)
+    else:
+        # fork shares the parent's _TARGETS by copy-on-write.
+        context = multiprocessing.get_context("fork") if hasattr(os, "fork") else None
+        executor = ProcessPoolExecutor(max_workers=workers, mp_context=context)
+        results = executor.map(_scan_stripe, jobs)
+    try:
+        for matches in results:
+            for line in matches:
+                writer.write(line)
+                total += 1
+    finally:
+        if workers != 1:
+            executor.shutdown()
+    return total
+
+
+def crack(
+    hashes_path: str,
+    out_path: str,
+    wordlists=(),
+    passwords_from=(),
+    workers: int | None = None,
+    work_dir: str | None = None,
+    max_hashes: int | None = None,
+) -> int:
+    """Attack the hashes with the candidate wordlists; write ``email:password``.
+
+    ``max_hashes`` bounds how many hash lines are held in the in-RAM target
+    index at once; larger inputs are streamed in chunks that spill to ``work_dir``.
+    A single global index of hundreds of millions of hashes needs tens of GB, so
+    set this on memory-limited hosts. ponytail: the wordlist is rescanned per
+    chunk, so a smaller ``max_hashes`` trades RAM for repeated passes.
+    """
+    work_dir = Path(work_dir) if work_dir else Path(out_path).parent
+    wordlist = prepare_candidates(list(wordlists), list(passwords_from), work_dir)
+    workers = max(1, workers or os.cpu_count() or 1)
+
     total = 0
     with open(out_path, "wb") as writer:
-        if workers == 1:
-            results = map(_scan_stripe, jobs)
-        else:
-            # fork shares the parent's _TARGETS by copy-on-write.
-            context = (
-                multiprocessing.get_context("fork") if hasattr(os, "fork") else None
-            )
-            executor = ProcessPoolExecutor(max_workers=workers, mp_context=context)
-            results = executor.map(_scan_stripe, jobs)
-        try:
-            for matches in results:
-                for line in matches:
-                    writer.write(line)
-                    total += 1
-        finally:
-            if workers != 1:
-                executor.shutdown()
+        for chunk_path in _iter_hash_chunks(hashes_path, max_hashes, work_dir):
+            total += _attack(chunk_path, writer, wordlist, workers)
     print(f"Cracked {total:,} hashes -> {out_path}.")
     return total

tests/crack_test.py

@@ -72,6 +72,29 @@ class CrackTest(unittest.TestCase):
             self.assertEqual(1, total)
             self.assertEqual([b"[email protected]:reused"], out.read_bytes().splitlines())
 
+    def test_crack_chunks_hash_file(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            hashes = Path(tmp) / "hashes.txt"
+            hashes.write_bytes(
+                b"[email protected]:" + hashlib.md5(b"hunter2").hexdigest().encode() + b"\n"
+                b"[email protected]:" + hashlib.sha1(b"secret").hexdigest().encode() + b"\n"
+            )
+            wordlist = Path(tmp) / "words.txt"
+            wordlist.write_bytes(b"hunter2\nsecret\n")
+            out = Path(tmp) / "cracked.txt"
+            total = crack(
+                str(hashes),
+                str(out),
+                wordlists=[str(wordlist)],
+                workers=1,
+                max_hashes=1,
+            )
+            self.assertEqual(2, total)
+            self.assertEqual(
+                {b"[email protected]:hunter2", b"[email protected]:secret"},
+                set(out.read_bytes().splitlines()),
+            )
+
 
 if __name__ == "__main__":
     unittest.main()