From cbfa73c4b3f78ccdfcc7886284565c2960675d1d Mon Sep 17 00:00:00 2001 From: StasonJatham Date: Sat, 19 Sep 2026 14:56:03 +0200 Subject: [PATCH] boring stuff --- .gitignore | 17 +++++++++++++++-- LICENSE | 25 +++++++++++++++++++++++++ README.md | 18 ++++++++++++++++-- pyproject.toml | 6 ++++++ 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore index 0236f61..07a30a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ __pycache__/ *.py[cod] .pytest_cache/ +.ruff_cache/ .coverage htmlcov/ .tox/ @@ -13,9 +14,21 @@ dist/ # Local datasets and pipeline scratch/state files data/ +dataset/ scratch/ *.jsonl build_state.json -progress_*.txt +stats.md +progress*.txt +success*.txt +failure*.txt +hashes*.txt +failures.txt +candidates*.txt +cracked*.txt +rockyou*.txt* +*.part.* +*.sorted +*.tmp -.DS_Store \ No newline at end of file +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9d6eb1c --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +MIT License + +Copyright (c) 2026 collectionparser contributors + +Derived from 3.7-billion-passwords-tools +(https://github.com/philipperemy/3.7-billion-passwords-tools) +Copyright (c) 2019 Philippe Remy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 76abf3c..b9a32e6 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Command line tools to parse, deduplicate, and query multi-billion password collections (Collections 1-5, AntiPublic, Breach Compilation). +> **Responsible use.** This project ships **no data**. Use it only on datasets +> you own or are authorized to process, for security research and defense. +> Do not commit datasets, hashes, or cracked credentials. + Processing produces a deduped `a/b/c` bucket tree that answers exact lookups in O(1) and wildcard queries with a parallel ripgrep scan: @@ -164,10 +168,20 @@ Run `collectionparser --help` or `collectionparser --help`. | `sort` | Sort and deduplicate every file in a query-friendly folder. | | `merge` | Merge one query-friendly folder into another, then sort/dedup. | +## Credits + +Derived from and inspired by +[philipperemy/3.7-billion-passwords-tools](https://github.com/philipperemy/3.7-billion-passwords-tools) +by [Philippe Remy](https://github.com/philipperemy) — the original parser and +`a/b/c` index. This project rewrites it for memory safety, resumable batched +builds, wildcard queries, hash collection/cracking and JSONL export. + +Thanks, Philippe. + ## Development ```bash pip install -e '.[dev]' -ruff check . # lint -pytest # tests +ruff format . && ruff check . # style + lint +pytest # tests ``` diff --git a/pyproject.toml b/pyproject.toml index e70bda4..dc632ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,14 @@ name = "collectionparser" version = "3.0.0" description = "Parse, deduplicate, and query multi-billion password collections." readme = "README.md" +license = { text = "MIT" } requires-python = ">=3.11" +authors = [{ name = "collectionparser contributors" }] keywords = ["passwords", "breach", "email", "dataset", "search"] classifiers = [ "Environment :: Console", "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Topic :: Security", @@ -28,6 +31,9 @@ dev = ["pytest>=8", "ruff>=0.6"] [project.scripts] collectionparser = "collectionparser.cli:cli" +[project.urls] +"Original project" = "https://github.com/philipperemy/3.7-billion-passwords-tools" + [tool.setuptools.packages.find] include = ["collectionparser*"]