boring stuff

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 *.py[cod]
 .pytest_cache/
+.ruff_cache/
 .coverage
 htmlcov/
 .tox/
@@ -13,9 +14,21 @@ dist/
 
 # Local datasets and pipeline scratch/state files
 data/
+dataset/
 scratch/
 *.jsonl
 build_state.json
-progress_*.txt
+stats.md
+progress*.txt
+success*.txt
+failure*.txt
+hashes*.txt
+failures.txt
+candidates*.txt
+cracked*.txt
+rockyou*.txt*
+*.part.*
+*.sorted
+*.tmp
 
 .DS_Store

LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2026 collectionparser contributors
+
+Derived from 3.7-billion-passwords-tools
+(https://github.com/philipperemy/3.7-billion-passwords-tools)
+Copyright (c) 2019 Philippe Remy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -3,6 +3,10 @@
 Command line tools to parse, deduplicate, and query multi-billion password
 collections (Collections 1-5, AntiPublic, Breach Compilation).
 
+> **Responsible use.** This project ships **no data**. Use it only on datasets
+> you own or are authorized to process, for security research and defense.
+> Do not commit datasets, hashes, or cracked credentials.
+
 Processing produces a deduped `a/b/c` bucket tree that answers exact lookups in
 O(1) and wildcard queries with a parallel ripgrep scan:
 
@@ -164,10 +168,20 @@ Run `collectionparser --help` or `collectionparser <command> --help`.
 | `sort` | Sort and deduplicate every file in a query-friendly folder. |
 | `merge` | Merge one query-friendly folder into another, then sort/dedup. |
 
+## Credits
+
+Derived from and inspired by
+[philipperemy/3.7-billion-passwords-tools](https://github.com/philipperemy/3.7-billion-passwords-tools)
+by [Philippe Remy](https://github.com/philipperemy) — the original parser and
+`a/b/c` index. This project rewrites it for memory safety, resumable batched
+builds, wildcard queries, hash collection/cracking and JSONL export.
+
+Thanks, Philippe.
+
 ## Development
 
 ```bash
 pip install -e '.[dev]'
-ruff check .      # lint
+ruff format . && ruff check .   # style + lint
-pytest            # tests
+pytest                          # tests
 ```

pyproject.toml

@@ -7,11 +7,14 @@ name = "collectionparser"
 version = "3.0.0"
 description = "Parse, deduplicate, and query multi-billion password collections."
 readme = "README.md"
+license = { text = "MIT" }
 requires-python = ">=3.11"
+authors = [{ name = "collectionparser contributors" }]
 keywords = ["passwords", "breach", "email", "dataset", "search"]
 classifiers = [
     "Environment :: Console",
     "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Topic :: Security",
@@ -28,6 +31,9 @@ dev = ["pytest>=8", "ruff>=0.6"]
 [project.scripts]
 collectionparser = "collectionparser.cli:cli"
 
+[project.urls]
+"Original project" = "https://github.com/philipperemy/3.7-billion-passwords-tools"
+
 [tool.setuptools.packages.find]
 include = ["collectionparser*"]