From 36e65df95e042169de9f0d687fb044b543d9ea37 Mon Sep 17 00:00:00 2001 From: Philippe Remy Date: Mon, 23 Mar 2020 21:01:46 +0900 Subject: [PATCH] origin commit --- Makefile | 16 +++ README.md | 175 ++++++++++++++++++++++++ breach/__init__.py | 0 breach/api.py | 143 ++++++++++++++++++++ breach/cli.py | 103 ++++++++++++++ breach/cparser.pyx | 179 ++++++++++++++++++++++++ breach/parser.py | 314 +++++++++++++++++++++++++++++++++++++++++++ breach/query.py | 194 ++++++++++++++++++++++++++ breach/splitter.py | 80 +++++++++++ breach/utils.py | 142 +++++++++++++++++++ requirements.txt | 4 + setup.py | 26 ++++ tests/parser_test.py | 141 +++++++++++++++++++ tests/query_test.py | 33 +++++ tests/utils_test.py | 36 +++++ 15 files changed, 1586 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 breach/__init__.py create mode 100644 breach/api.py create mode 100644 breach/cli.py create mode 100644 breach/cparser.pyx create mode 100644 breach/parser.py create mode 100644 breach/query.py create mode 100644 breach/splitter.py create mode 100644 breach/utils.py create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/parser_test.py create mode 100644 tests/query_test.py create mode 100644 tests/utils_test.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..75e2f6a --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +all: + make install + +install: + pip install cython pytest + python setup.py build_ext --inplace + pip install . --upgrade + +uninstall: + pip uninstall -y breach + +test: + pytest tests + +clean: + rm -rf *.out breach/*.c *.bin *.exe *.o *.a breach/*.so build *.html __pycache__ breach/__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 0000000..e6d341a --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +# Collections 1, Collections 2-5 and AntiPublic Parser + +Command line tools to manipulate the data from those multi-billion passwords collections. + +The full processing will take a couple of days and will generate a file structure that can be queried almost in o(1). + +``` +$ john.applessed@apple.com +john.applessed@apple.com:toto123 +``` + + +### Set Up + +Create a virtual environment and install the package. + +``` +virtualenv -p python3 venv +source venv/bin/activate +make install +``` + +## Extraction + +https://superuser.com/questions/1308374/how-to-extract-all-tar-gz-files-present-in-multiple-folders-at-a-time-to-another + +```bash +find "$1" -name '*.tar.gz' -execdir tar -xzvf '{}' -C extracted \; +find . -name "*.rar" -exec unrar x -o+ {} \; +``` + +## Processing + +Processing the Collection 1 is much faster than the Collections 2-5. The estimates for Collections 2-5 are reported below. + +The parsing took around 20 hours on my server (CPU i7-8700K, 32GB of memory). I didn't have a large enough SSD to store all the temporary computations so everything +was done on a standard HDD. A faster disk will surely make the processing faster. + +The sorting/removing duplicates step took 15 hours in total. + +The splitting into the smaller files (this file struct makes every query almost instantaneous) took a couple of hours at most. + +In total, expect around 2 days to process the Collections 2-5. + +```bash +breach parse --path /path/to/extracted --success_file success.txt --failure_file failure.txt --cython_acceleration +rm -rf tmp && mkdir tmp # you need like 750GB in tmp/. By default /tmp/ is not enough for this! +cat success.txt | pv -cN cut | sort -T tmp -u -S 90% --parallel=12 | pv -cN cut > success_sorted.txt +breach split --file success_sorted.txt --out data +``` + +## Converting the format of BreachCompilation to the new format + +The dataset is available here: https://github.com/philipperemy/tensorflow-1.4-billion-password-analysis + +It's easy to convert the large `BreachCompilation` dataset to this format by running those commands. + +Expect those commands to take some time (less than a day). + +```bash +find /path/to/BreachCompilation/ -type f -exec cat {} + > breach_compilation.txt +rm -rf tmp && mkdir tmp # By default /tmp/ is not enough for this! +cat breach_compilation.txt | pv -cN cut | sort -T tmp -u -S 90% --parallel=12 | pv -cN cut > breach_compilation_sorted.txt +breach split --file breach_compilation_sorted.txt --out data_breach_compilation_sorted +``` + +From there a simple `breach merge` will be enough to merge it to the Collections 1 & 2 to 5. + +## Merging all the datasets together + +Run the Collection 1 and Collections 2-5 through the processing step described above. + +You will have two directories: `/path/to/collections1_data` and `/path/to/collections2_5_data`. + +Additionally, if you have the other dataset BreachCompilation, you will have another directory `/path/to/data_breach_compilation_sorted`, generated by the step above. + +The merge is destructive so it's better to create a copy of the output first and then merge each one into the output. + +``` +cp -rf /path/to/collections1_data /path/to/big_dataset +breach merge --src /path/to/collections2_5_data --dest /path/to/big_dataset +breach merge --src /path/to/data_breach_compilation_sorted --dest /path/to/big_dataset +``` + +## Usage + +The manual of the command line tool can be fetched by running `breach dumphelp`. + +``` +Usage: cli [OPTIONS] COMMAND [ARGS]... + +Options: + --debug / --no-debug + --help Show this message and exit. + +Commands: + chunk chunk large TXT files into smaller files. + clean Cleans a query friendly folder PATH. Move incorrect records and + sort the files. + dumphelp + evaluate Evaluates some metrics such as precision/recall (e.g. is OLD into + NEW). + merge Merges dataset SRC into dataset DEST. + parse Parses an unstructured folder PATH of many files and generates two + files: SUCCESS_FILE and FAILURE_FILE. All valid email:password + will go to SUCCESS_FILE. + sort Sorts a query friendly folder PATH. Target is itself. + split Converts a large FILE to a query friendly folder OUT (e.g. a/b/c). + Use RESTART_FROM to resume from the i-th line. + test Infers passwords of a list of emails defined in FILE with a query + friendly folder DATASET. + +Usage: cli dumphelp [OPTIONS] + +Options: + --help Show this message and exit. + +Usage: cli split [OPTIONS] + +Options: + --file FILE [required] + --out DIRECTORY [required] + --restart_from INTEGER [default: 0] + --help Show this message and exit. + +Usage: cli chunk [OPTIONS] + +Options: + --path DIRECTORY [required] + --size INTEGER [default: 50] + --help Show this message and exit. + +Usage: cli sort [OPTIONS] + +Options: + --path DIRECTORY [required] + --help Show this message and exit. + +Usage: cli clean [OPTIONS] + +Options: + --path DIRECTORY [required] + --help Show this message and exit. + +Usage: cli test [OPTIONS] + +Options: + --file FILE [required] + --dataset [breach_compilation|collections_1|collections_2_5|all] + [required] + --help Show this message and exit. + +Usage: cli parse [OPTIONS] + +Options: + --path DIRECTORY [required] + --success_file FILE [required] + --failure_file FILE [required] + --cython_acceleration / --no-cython_acceleration + --help Show this message and exit. + +Usage: cli merge [OPTIONS] + +Options: + --src DIRECTORY [required] + --dest DIRECTORY [required] + --help Show this message and exit. + +Usage: cli evaluate [OPTIONS] + +Options: + --old DIRECTORY [required] + --new DIRECTORY [required] + --help Show this message and exit. + ``` diff --git a/breach/__init__.py b/breach/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/breach/api.py b/breach/api.py new file mode 100644 index 0000000..304c43a --- /dev/null +++ b/breach/api.py @@ -0,0 +1,143 @@ +import os +import random +import shutil +import subprocess +from glob import glob +from pathlib import Path + +from tqdm import tqdm + +from breach.parser import parse_to_files +from breach.query import QueryAPI, get_query_manager +from breach.splitter import get_output_file, write_line_to_file +from breach.utils import alpha_num_lookup, shuffle, read_lines + + +class API: + + @staticmethod + def evaluate(old, new): + hit_count = 0 + total_count = 0 + files = [p for p in Path(old).expanduser().glob('**/*') if p.is_file()] + for i in range(1000): + lines = read_lines(random.choice(files)) + shuffle(lines) + random_lines = lines[0:10] + for random_line in random_lines: + password_list = QueryAPI.query(new, random_line.split(':')[0], 'email') + total_count += 1 + if len(password_list) == 0: + print(f'OLD: {random_line}, NEW: miss, HIT_RATE = {hit_count / total_count:.3f}.') + else: + hit_count += 1 + print(f'OLD: {random_line}, NEW: hit, HIT_RATE = {hit_count / total_count:.3f}.') + + @staticmethod + def merge(src, dest): + src_files = find_files(src) + dest_files = find_files(dest) + for src_index, src_file in src_files.items(): + if src_index in dest_files: + merge(src_file, dest_files[src_index]) + else: + move(src_file, dest + src_index) + + @staticmethod + def parse(path, success_file, failure_file, cython_acceleration=False): + parse_to_files(path, success_file, failure_file, cython_acceleration) + + @staticmethod + def sort(path): + path = str(Path(path).expanduser()) + paths = [Path(a) for a in glob(path + '/**/*', recursive=True) if Path(a).is_file() + and Path(a).suffix != '.sorted'] + for path in tqdm(paths, desc='Unique > Sorting > Replacing existing files'): + output_path = path.with_suffix('.sorted') + lines = read_lines(path) + unique_sorted_lines = sorted(set(lines)) + with open(output_path, 'w', encoding='utf8') as w: + w.write('\n'.join(unique_sorted_lines) + '\n') + shutil.move(output_path, path) + + @staticmethod + def clean(path): + files = list(find_files(Path(path).expanduser()).values()) + shuffle(files) + alpha = alpha_num_lookup() + num_files_moved = 0 + files_handlers = {} # u limit here! + with tqdm(files, desc='clean') as bar: + for file in bar: + lines_to_delete = [] + lines = read_lines(file) + for line in lines: + line_split = line.split(':') + ground_truth_file = get_output_file(alpha, line_split[0]) + key = str(file).replace(path, '') + if key.startswith('/'): + key = key[1:] + if ground_truth_file != key: + output_file = Path(path) / ground_truth_file + # print(f'MOVE: {line_split[0]} -> {output_file}.') + write_line_to_file(files_handlers, line_split, output_file, buffer_scale=10) + lines_to_delete.append(line) + num_files_moved += 1 + if num_files_moved % 100_000 == 0: + truncated_file = str(file).replace(path, "") + truncated_output_file = str(output_file).replace(path, "") + bar.set_description(f'clean. {num_files_moved:,} moved. ' + f'move: {line_split[0]}: {truncated_file}' + f' -> {truncated_output_file}') + lines2 = sorted(set(lines) - set(lines_to_delete)) + if len(lines) != len(lines2): + with open(str(file), 'w', encoding='utf8') as w: + w.write('\n'.join(lines2) + '\n') + + @staticmethod + def test(file: str, dataset: str): + qm = get_query_manager(dataset) + cracked_rate = 0 + lines = read_lines(file) + for line in lines: + results = qm.perform_query(line.strip(), 'email') + if len(results) > 0: + passwords = [p.split(':', 1)[-1] for p in results] + print('FOUND:', line, ':'.join(passwords)) + cracked_rate += 1 + print(f'Cracked rate: {cracked_rate / len(lines)}') + + @staticmethod + def chunk_files(path: str, max_size: int): + # split -b 50000000 file.txt file.txt. + # -d to have .00 and .01 is not compatible with MacOS... + files = list(find_files(Path(path).expanduser()).values()) + for f in tqdm(files, desc='chunk large files'): + if f.stat().st_size / 1e6 > max_size: + cmd = ['split', '-b', str(int(max_size * 1e6)), str(f), str(f) + '_'] + print(f'File size: {f.stat().st_size / 1e6:.2f} Mb. ' + ' '.join(cmd)) + subprocess.check_output(cmd) + f.unlink() + + +def find_files(path: str): + path2 = Path(path).expanduser() + files = [p for p in path2.glob('**/*') if p.is_file()] + index = {str(f).replace(str(path2), ''): f for f in files} + assert len(index) == len(files) + return index + + +def merge(src, dst): + print(f'merge {src} {dst}') + with open(src, 'r', encoding='utf8', errors='ignore') as r: + content1 = r.read() + with open(dst, 'a', encoding='utf8', errors='ignore') as w: + w.write(content1) + print(f'del {src}') + os.remove(src) + + +def move(src, dst): + print(f'mv {src} {dst}') + shutil.move(src, dst) diff --git a/breach/cli.py b/breach/cli.py new file mode 100644 index 0000000..6b6a8d8 --- /dev/null +++ b/breach/cli.py @@ -0,0 +1,103 @@ +import logging +from pathlib import Path + +import click + +from breach.api import API +from breach.query import DATASET_CHOICES +from breach.splitter import split_file +from breach.utils import Ct, num_open_files_limit + + +# Extraction: nohup find . -name '*.tar.gz' -execdir tar --keep-newer-files -xzvf '{}' -C ../extracted \; & + +def recursive_help(cmd, parent=None): + ctx = click.core.Context(cmd, info_name=cmd.name, parent=parent) + print(cmd.get_help(ctx)) + print() + commands = getattr(cmd, 'commands', {}) + for sub in commands.values(): + recursive_help(sub, ctx) + + +@click.group() +@click.option('--debug/--no-debug') +@click.pass_context +def cli(ctx, debug): + if debug: + click.echo('Debugging mode enabled.') + logging.basicConfig(format='%(asctime)12s - %(levelname)s - %(message)s', level=logging.INFO) + if not debug and num_open_files_limit() < 60_000 and ctx.invoked_subcommand in ['split', 'clean']: + click.echo('ERROR: Increase the limit of the number of open files to at least 60,000.') + click.echo('ERROR: ulimit -n 60000') + exit(1) + + +@cli.command() +def dumphelp(): + recursive_help(cli) + + +@cli.command(short_help='Converts a large FILE to a query friendly folder OUT (e.g. a/b/c). ' + 'Use RESTART_FROM to resume from the i-th line.') +@click.option('--file', required=True, type=Ct.input_file()) +@click.option('--out', required=True, type=Ct.output_dir()) +@click.option('--restart_from', default=0, type=int, show_default=True) +def split(file, out, restart_from): + split_file(file, out, restart_from) + + +@cli.command(short_help='chunk large TXT files into smaller files.') +@click.option('--path', required=True, type=Ct.input_dir()) +@click.option('--size', default=50, type=int, show_default=True) # MB. +def chunk(path, size): + API.chunk_files(path, size) + + +@cli.command(short_help='Sorts a query friendly folder PATH. Target is itself.') +@click.option('--path', required=True, type=Ct.input_dir()) +def sort(path): + API.sort(path) + + +@cli.command(short_help='Cleans a query friendly folder PATH. Move incorrect records and sort the files.') +@click.option('--path', required=True, type=Ct.input_dir()) +def clean(path): + API.clean(path) + + +@cli.command(short_help='Infers passwords of a list of emails defined in FILE with a query friendly folder DATASET.') +@click.option('--file', required=True, type=Ct.input_file()) +# @click.option('--dataset', required=True, type=click.Choice(DATASET_CHOICES, case_sensitive=False)) +@click.option('--dataset', required=True) +def test(file: str, dataset: str): + API.test(file, dataset) + + +@cli.command(short_help='Parses an unstructured folder PATH of many files and generates two files: ' + 'SUCCESS_FILE and FAILURE_FILE. All valid email:password will go to SUCCESS_FILE.') +@click.option('--path', required=True, type=Ct.output_dir()) +@click.option('--success_file', required=True, type=Ct.output_file()) +@click.option('--failure_file', required=True, type=Ct.output_file()) +@click.option('--cython_acceleration/--no-cython_acceleration', default=False) +def parse(path, success_file, failure_file, cython_acceleration): + API.parse(Path(path).expanduser(), success_file, failure_file, cython_acceleration) + + +@cli.command(short_help='Merges dataset SRC into dataset DEST. Duplicates are removed in the second pass.') +@click.option('--src', required=True, type=Ct.input_dir()) +@click.option('--dest', required=True, type=Ct.input_dir()) +def merge(src: str, dest: str): + API.merge(src, dest) + API.clean(dest) + + +@cli.command(short_help='Evaluates some metrics such as precision/recall (e.g. is OLD into NEW).') +@click.option('--old', required=True, type=Ct.input_dir()) +@click.option('--new', required=True, type=Ct.input_dir()) +def evaluate(old: str, new: str): + API.evaluate(old, new) + + +if __name__ == '__main__': + cli() diff --git a/breach/cparser.pyx b/breach/cparser.pyx new file mode 100644 index 0000000..991c03f --- /dev/null +++ b/breach/cparser.pyx @@ -0,0 +1,179 @@ +import re +import string + +import cython +from validate_email import validate_email + +from breach.utils import PasswordType + +cdef inline c_is_md5(str password): + if len(password) == 32: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + +cdef inline c_is_sha1(str password): + if len(password) == 40: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + +cdef inline c_is_sha_256(str password): + if len(password) == 64: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + +cdef inline c_is_b_crypt(str password): + if 50 < len(password) < 70: + b_crypt_pattern_cond = password[0:4] == "$2a$" or password[0:4] == "$2b$" or password[0:4] == "$2y$" + if b_crypt_pattern_cond: + return True + return False + +cdef inline c_get_password_type(str password): + if c_is_md5(password): + return PasswordType.MD5 + elif c_is_sha1(password): + return PasswordType.SHA1 + elif c_is_sha_256(password): + return PasswordType.SHA256 + elif c_is_b_crypt(password): + return PasswordType.B_CRYPT + else: + return PasswordType.DECRYPTED + +@cython.boundscheck(False) +def c_parseline(str s): + # case mail@mail.com:password + # case mail@mail.com;password + # case mail@mail.com password + valid_separators = [':', ';', ' ', '\t', ' ', ' ', ', ', ',', ';;', '::', ','] + # https://github.com/swiftmailer/swiftmailer/issues/239 + b = s.strip() + b = ''.join(list(filter(lambda x: x in string.printable, b))) + while '..' in b: # TODO: could be move just for the email. + b = b.replace('..', '') + if b.startswith('mailto:'): + b = b[len('mailto:'):] + if b.startswith('E-mail:'): + b = b[len('E-mail:'):] + if b.startswith(':'): + b = b[1:] + if b.endswith(':'): + b = b[:-1] + + # TODO: could be move just for the email. but we dont need to be 100% accurate here. + b = b.replace('.@', '@') # just typo. + b = b.replace('@.', '@') # just typo. + b = b.replace(' @ ', '@') # just typo. + b = b.replace('@@', '@') # just typo. + + if ':' in b and ';' in b: + b = b.replace(';', ':') + + # FLIP=FALSE + for valid_separator in valid_separators: + result = c_parse_line(b, valid_separator, flip=False) + if result is not None: + return result + # FLIP=TRUE + for valid_separator in valid_separators: + result = c_parse_line(b, valid_separator, flip=True) + if result is not None: + return result + # print(f'DISCARD: {s.strip()}') + return None + +@cython.boundscheck(False) +cdef inline c_our_validate_email(str candidate): + parts = candidate.split('@') + if len(parts) != 2: + return False + if '.' not in parts[-1]: # .com or .fr + return False + return validate_email(candidate) + +@cython.boundscheck(False) +cdef c_parse_line(str row, str separator, bint flip=False): + cdef bint is_sep_in_row = separator in row + cdef bint sep_count_cond = False + cdef str mail = '' + cdef str password = '' + if not is_sep_in_row: # return fast. + return None + if separator != ' ': + parts = row.split(separator) + sep_count_cond = len(parts) == 2 + if len(parts) > 2: + row = separator.join(parts[-2:]) + # sh_karimian@yahoo.com:kiyan1365:16 + # wgwendlee@aol.com:godis2good: + sep_count_cond = True + elif is_sep_in_row: + sep_count_cond = True + else: + sep_count_cond = False + if sep_count_cond: + mail, password = row.split(separator, 1) + if flip: + password, mail = mail, password + password = password.replace(' ', '') + if len(password) > 30: # Password of 30 chars look like this: adfsfdsgdfghkfdghdsdfgdfgdfgdf. + return None + if len(password) == 0: + return None + if '\n\n' in password: + return None + if set(password) == {'?'}: # - gauntletskd@gmail.com:?????? ??? ????? + return None + if password.count('.') >= 3 and \ + re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', password) is not None: + return None + mail = mail.replace(' ', '') + # https://grox.net/utils/encoding.html + if '@' not in mail: + mail = mail.replace('%40', '@') + # - thakhuy.step129@hotmail.com.kkk111:125.26.106.158,nopparat + # count first because it's faster than compiling a regexp. + if len(mail) == 0: + return None + last_char = mail[-1] + first_char = mail[0] + # misseme21@yahoo.jp\ ilovebois3 + if last_char in ['<', '.', '\\', '>', '_']: + mail = mail[:-1] + # .y.heine@gmail.com atohiyamamot [exists] + if first_char in ['.', '<']: + mail = mail[1:] + if mail.endswith('@com'): + # iemongozalu@gmail@com mizo1210 + mail = mail.replace('@com', '.com') + if not c_our_validate_email(mail): + # some people replace neanta,buru@ezweb.ne,jp + mail = mail.replace(',', '.') + parts = mail.split('@') + if len(parts) > 2: + mail = '@'.join(parts[0:2]) + # TODO: do it for most common domain names. + if mail.count('@hotmail.com') == 2: # taa06@hotmail.com@hotmail.com:8765432f + mail = mail.replace('@hotmail.com', '', 1) + if c_our_validate_email(mail) and not c_our_validate_email(password): + password_type = c_get_password_type(password) + return mail, password, password_type + return None diff --git a/breach/parser.py b/breach/parser.py new file mode 100644 index 0000000..6677b30 --- /dev/null +++ b/breach/parser.py @@ -0,0 +1,314 @@ +import os +import re +import string +from glob import glob +from pathlib import Path +from time import time + +from breach.cparser import c_parseline +from tqdm import tqdm +from validate_email import validate_email + +from breach.utils import chunks, read_lines, PasswordType +from breach.utils import parallel_function, shuffle + + +# slow functions. + +def our_validate_email(candidate: str): + if candidate.count('@') != 1: + return False + if '.' not in candidate.split('@')[-1]: # .com or .fr + return False + return validate_email(candidate) + + +class LineParser: + + @staticmethod + def _parse_line(row, separator, flip=False): + if separator not in row: # return fast. + return None + if separator != ' ': + parts = row.split(separator) + sep_count_cond = len(parts) == 2 + if len(parts) > 2: + row = separator.join(parts[-2:]) + # sh_karimian@yahoo.com:kiyan1365:16 + # wgwendlee@aol.com:godis2good: + sep_count_cond = True + elif separator in row: + sep_count_cond = True + else: + sep_count_cond = False + if sep_count_cond: + mail, password = row.split(separator, 1) + if flip: + password, mail = mail, password + password = password.replace(' ', '') + if len(password) > 30: # Password of 30 chars look like this: adfsfdsgdfghkfdghdsdfgdfgdfgdf. + return None + if len(password) == 0: + return None + if '\n\n' in password: + return None + if set(password) == {'?'}: # - gauntletskd@gmail.com:?????? ??? ????? + return None + if password.count('.') >= 3 and \ + re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', password) is not None: + return None + mail = mail.replace(' ', '') + # https://grox.net/utils/encoding.html + if '@' not in mail: + mail = mail.replace('%40', '@') + # - thakhuy.step129@hotmail.com.kkk111:125.26.106.158,nopparat + # count first because it's faster than compiling a regexp. + + if len(mail) == 0: + return None + + last_char = mail[-1] + first_char = mail[0] + # misseme21@yahoo.jp\ ilovebois3 + if last_char in ['<', '.', '\\', '>', '_']: + mail = mail[:-1] + # .y.heine@gmail.com atohiyamamot [exists] + if first_char in ['.', '<']: + mail = mail[1:] + if mail.endswith('@com'): + # iemongozalu@gmail@com mizo1210 + mail = mail.replace('@com', '.com') + if not our_validate_email(mail): + # some people replace neanta,buru@ezweb.ne,jp + mail = mail.replace(',', '.') + parts = mail.split('@') + if len(parts) > 2: + mail = '@'.join(parts[0:2]) + # TODO: do it for most common domain names. + if mail.count('@hotmail.com') == 2: # taa06@hotmail.com@hotmail.com:8765432f + mail = mail.replace('@hotmail.com', '', 1) + if our_validate_email(mail) and not our_validate_email(password): + password_type = LineParser.get_password_type(password) + return mail, password, password_type + return None + + @staticmethod + def is_md5(password): + if len(password) == 32: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + + @staticmethod + def is_sha1(password): + if len(password) == 40: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + + @staticmethod + def is_sha_256(password): + if len(password) == 64: + for e in password: + if not ((ord('0') <= ord(e) <= ord('9')) or + (ord('A') <= ord(e) <= ord('F')) or + (ord('a') <= ord(e) <= ord('f'))): + return False + return True + else: + return False + + @staticmethod + def is_b_crypt(password): + b_crypt_pattern_cond = password[0:4] == "$2a$" or password[0:4] == "$2b$" or password[0:4] == "$2y$" + if 50 < len(password) < 70 and b_crypt_pattern_cond: + return True + return False + + @staticmethod + def get_password_type(password): + if LineParser.is_md5(password): + return PasswordType.MD5 + elif LineParser.is_sha1(password): + return PasswordType.SHA1 + elif LineParser.is_sha_256(password): + return PasswordType.SHA256 + elif LineParser.is_b_crypt(password): + return PasswordType.B_CRYPT + else: + return PasswordType.DECRYPTED + + @staticmethod + def parseline(s: str): + # case mail@mail.com:password + # case mail@mail.com;password + # case mail@mail.com password + valid_separators = [':', ';', ' ', '\t', ' ', ' ', ', ', ',', ';;', '::', ','] + # https://github.com/swiftmailer/swiftmailer/issues/239 + b = s.strip() + b = ''.join(list(filter(lambda x: x in string.printable, b))) + while '..' in b: # TODO: could be move just for the email. + b = b.replace('..', '') + if b.startswith('mailto:'): + b = b[len('mailto:'):] + if b.startswith('E-mail:'): + b = b[len('E-mail:'):] + if b.startswith(':'): + b = b[1:] + if b.endswith(':'): + b = b[:-1] + + # TODO: could be move just for the email. but we dont need to be 100% accurate here. + b = b.replace('.@', '@') # just typo. + b = b.replace('@.', '@') # just typo. + b = b.replace(' @ ', '@') # just typo. + b = b.replace('@@', '@') # just typo. + + if ':' in b and ';' in b: + b = b.replace(';', ':') + + # FLIP=FALSE + for valid_separator in valid_separators: + result = LineParser._parse_line(b, valid_separator, flip=False) + if result is not None: + return result + # FLIP=TRUE + for valid_separator in valid_separators: + result = LineParser._parse_line(b, valid_separator, flip=True) + if result is not None: + return result + # print(f'DISCARD: {s.strip()}') + return None + + +class Processor: + + def __init__(self, w=None, f=None, cython_acceleration=False): # for big files -> writes to disk directly. + if cython_acceleration: + print('Cython acceleration enabled.') + self.bind = c_parseline + else: + print('Cython acceleration disabled.') + self.bind = LineParser.parseline + self.w = w # cannot be serialized for multi processing. + self.f = f # cannot be serialized for multi processing. + + def process(self, filename: Path, is_big_file: bool = False): + success_list = [] + failure_list = [] + with open(str(filename), 'r', encoding='utf8', errors='ignore') as r: + buf_size = 10 ** 6 # for very large files. + while True: + lines = r.readlines(buf_size) + if len(lines) == 0: + break + for line in lines: + result = self.bind(line) + if result is not None: + mail, password, password_type = result + if password_type == PasswordType.DECRYPTED: + # mail is case insensitive. + success_str = ':'.join([str(mail).lower().strip(), password.strip()]) + '\n' + if is_big_file: + self.w.write(success_str) + else: + success_list.append(success_str) + else: + # stop writing binary data to file... + fail_line = ''.join(list(filter(lambda x: x in string.printable, line))) + if is_big_file: + self.f.write(fail_line) + else: + failure_list.append(fail_line) + return success_list, failure_list + + +def parse_to_files(input_dir, success_filename, failure_filename, cython_acceleration): + files = [p for p in Path(input_dir).glob('**/*') if p.is_file()] + shuffle(files) + # restart from where we stopped. + progress_filename = 'progress.txt' + if os.path.exists(progress_filename): + lines = read_lines(progress_filename) + else: + lines = [] + lines = {n: 1 for n in lines} + files = [f for f in files if str(f) not in lines] + shuffle(files) + # files.sort(key=lambda f: os.stat(f).st_size) # smallest to largest + + success_total = 0 + failure_total = 0 + num_threads = os.cpu_count() + start_time = time() + with open(progress_filename, 'a+') as p: + with open(failure_filename, 'a+', encoding='utf8') as f: + with open(success_filename, 'a+', encoding='utf8') as w: + large_files_processor = Processor(w=w, f=f, cython_acceleration=cython_acceleration) + small_files_processor = Processor(cython_acceleration=cython_acceleration) + with tqdm(files) as bar: + for chunk in chunks(files, num_threads): + # more than 300MB? + any_large_files = any([c.stat().st_size / 1e6 > 300 for c in chunk]) + if any_large_files: + # single thread and write directly to the files. + for chunky in chunk: + large_files_processor.process(chunky) + else: + # as many threads as we want. + results = parallel_function(small_files_processor.process, chunk, len(chunk)) + + for single_result in results: + success_list, failure_list = single_result + + for single_success in success_list: + w.write(single_success) + for single_success in failure_list: + f.write(single_success) + + success_total += len(success_list) + failure_total += len(failure_list) + + bar.update(len(chunk)) + bar.set_description(f'success = {success_total:,} failure = {failure_total:,}') + for single_chunk in chunk: + p.write(str(single_chunk) + '\n') + print(f'Time elapsed {time() - start_time:.3f} seconds.') + + +def statistics(input_dir): + files = glob(os.path.expanduser(input_dir) + '/**/*.*', recursive=True) + files = [f for f in files if not f.endswith('.tar.gz')] + shuffle(files) + parsed_lines = 0 + with open('stats.txt', 'w') as w: + w.write(','.join(['success', 'failure', 'failure_rate', 'filename']) + '\n') + for filename in files: + success = 0 + failure = 0 + lines = read_lines(filename) + parsed_lines += len(lines) + for line in lines: + result = LineParser.parseline(line) + if result is not None: + success += 1 + else: + failure += 1 + try: + failure_rate = failure / (success + failure) * 100 + except ZeroDivisionError: + failure_rate = 0.0 + w.write(','.join([str(success), str(failure), f'{failure_rate:.3f}', filename]) + '\n') + print(f'Parsed lines = {parsed_lines:,}.') + print(f'filename = {filename}, success = {success:,}, ' + f'failure = {failure:,}, failure rate = {failure_rate:.3f}%') diff --git a/breach/query.py b/breach/query.py new file mode 100644 index 0000000..a7bdb80 --- /dev/null +++ b/breach/query.py @@ -0,0 +1,194 @@ +import logging +import os +import subprocess +from pathlib import Path + +logger = logging.getLogger(__name__) + + +# https://pypi.org/project/ripgrepy/ + +class BreachCompilationConstants: + EMAIL_INDEX = '/media/philippe/DATA/datasets/BreachCompilation/data' + PASSWORD_INDEX = '/media/philippe/DATA/datasets/BreachCompilation/passwords' + + +class Collections1: + EMAIL_INDEX = '/media/philippe/8TB/datasets/passwords2/data' + PASSWORD_INDEX = None + + +class Collections_2_5: + EMAIL_INDEX = '/media/philippe/8TB/datasets/passwords2/data' + PASSWORD_INDEX = None + + +DATASET_CHOICES = ['breach_compilation', 'collections_1', 'collections_2_5', 'all'] + + +def get_query_manager(dataset_choice: str): + if dataset_choice.lower() == 'breach_compilation': + return QueryManager(BreachCompilationConstants) + elif dataset_choice.lower() == 'collections_1': + return QueryManager(Collections1) + elif dataset_choice.lower() == 'collections_2_5': + return QueryManager(Collections_2_5) + elif os.path.exists(dataset_choice): + class Custom: + EMAIL_INDEX = dataset_choice + PASSWORD_INDEX = None + + return QueryManager(Custom) + else: + return QueryMultiManager() # all + + +def mask_pass_justin_rule(password: str, num_asterisks=-1) -> str: + if len(password) >= 8: + return password[0:2] + (len(password) - 3) * '*' + password[-1] + return password[0] + '*' * (len(password) - 2) + password[-1] + + +def mask_pass(password: str, num_asterisks=2) -> str: + if len(password) <= num_asterisks: + return '*' * len(password) + if len(password) >= 4: + i = len(password) // 2 + even = num_asterisks % 2 == 0 + half = num_asterisks // 2 + if even: + left = half + right = half + else: + left = half + right = half + 1 + return password[0:i - left] + '*' * num_asterisks + password[i + right:] + else: + return password + + +def extract_emails_and_passwords(txt_lines): + emails_passwords = [] + for txt_line in txt_lines: + try: + if '@' in txt_line: # does it contain an email address? + if all([char in txt_line for char in [':', ';']]): # which separator is it? : or ;? + separator = ':' + elif ':' in txt_line: # '_---madc0w---_@live.com:iskandar89 + separator = ':' + elif ';' in txt_line: # '_---lelya---_@mail.ru;ol1391ga + separator = ';' + else: + continue + + strip_txt_line = txt_line.strip() + email, password = strip_txt_line.split(separator) + emails_passwords.append((email, password)) + except Exception: + pass + return emails_passwords + + +def postprocess(results, num_asterisks_to_mask_password=0): + response = [b for b in extract_emails_and_passwords(results)] + if num_asterisks_to_mask_password > 0: + response = [(b[0], mask_pass_justin_rule(b[1])) for b in response] + response = [':'.join(b) for b in response] + response = sorted(set(response)) + return response + + +class QueryAPI: + + @staticmethod + def scanning(directory: Path): + file_list, dir_list = {}, {} + for x in directory.iterdir(): + if x.is_file(): + file_list[x.name] = x + else: + dir_list[x.name] = x + return file_list, dir_list + + @staticmethod + def search_in_file(path: Path, email): + assert path.is_file(), path + try: + results = subprocess.check_output(['rg', '--no-line-number', '-ie', '^' + + email, str(path)]).decode('utf8').split('\n') + results = [a for a in results if a != ''] + except subprocess.CalledProcessError: + results = [] + return results + + @staticmethod + def recursive_search(path: Path, characters: list, index: int): + fl, dl = QueryAPI.scanning(path) + cur = characters[index].lower() + if cur in fl: + return QueryAPI.search_in_file(fl[cur], ''.join(characters)) + elif cur in dl: + return QueryAPI.recursive_search(dl[cur], characters, index + 1) + elif 'symbols' in fl: + return QueryAPI.search_in_file(fl['symbols'], ''.join(characters)) + else: + raise Exception('Hello.') + + @staticmethod + def query(data_dir: str, request: str, query_type: str, num_asterisks_to_mask_password: int = 0): + if len(request) <= 3: + return [] + if query_type == 'email' and '@' not in request: + return [] + response = QueryAPI.recursive_search(Path(data_dir), list(request), 0) + return postprocess(response, num_asterisks_to_mask_password) + + +class QueryManagerStub: + def __init__(self, constants): + self.default_email_index = constants.EMAIL_INDEX + self.default_password_index = constants.PASSWORD_INDEX + self.counter = 0 + + def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0): + self.counter += 1 + if self.counter % 2 == 0: + return [f'{request}:hello123', f'{request}:hello123456'] + else: + return [] + + +class QueryManager: + + def __init__(self, constants): + self.default_email_index = constants.EMAIL_INDEX + self.default_password_index = constants.PASSWORD_INDEX + + def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0): + if query_type == 'email': + index = self.default_email_index + elif query_type == 'password': + index = self.default_password_index + else: + raise Exception({'error': 'Index is either email or password.'}) + results = [] + if ' ' in request: # batch. + for single_req in request.split(' '): + results.extend(QueryAPI.query(index, single_req, query_type, num_asterisks_to_mask_password)) + else: + results.extend(QueryAPI.query(index, request, query_type, num_asterisks_to_mask_password)) + return results + + +class QueryMultiManager: + + def __init__(self): + self.qm1 = QueryManager(BreachCompilationConstants) + self.qm2 = QueryManager(Collections1) + self.qm3 = QueryManager(Collections_2_5) + + def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0): + results = self.qm1.perform_query(request, query_type, num_asterisks_to_mask_password) + results.extend(self.qm2.perform_query(request, query_type, num_asterisks_to_mask_password)) + results.extend(self.qm3.perform_query(request, query_type, num_asterisks_to_mask_password)) + return sorted(set(results)) diff --git a/breach/splitter.py b/breach/splitter.py new file mode 100644 index 0000000..5bca487 --- /dev/null +++ b/breach/splitter.py @@ -0,0 +1,80 @@ +import csv +import io +import sys +from time import time + +from breach.utils import alpha_num_lookup, create_dir, create_dir_for_file, RingDictionary + +csv.field_size_limit(sys.maxsize) + + +def split_file(breach_compilation_file, + output_dir='out', + restart_from=0): + # It seems like this function is not 100% correct as we need to clean the dataset after this. + # It's not a big deal but I don't understand why we have to do that. + # Could probably be to a limitation to the large number of files we open at the same time x increase of buffer. + create_dir(output_dir) + alpha = alpha_num_lookup() + files_handlers = RingDictionary() + t1 = time() + with open(breach_compilation_file, 'r', encoding='utf8', errors='ignore') as r: + for i, line in enumerate(r): + if i % 10_000_000 == 0: + elapsed_time = time() - t1 + t1 = time() + print(f'|- Processed {i:,} rows in {elapsed_time:.2f} seconds.') + if i < restart_from: + continue + try: # faster than if: https://stackoverflow.com/questions/1835756/using-try-vs-if-in-python + email, password = line.strip().split(':', 1) + except ValueError: + continue + + output_file = get_output_file(alpha, email) + output_file = output_dir + '/' + output_file + write_line_to_file(files_handlers, line, output_file) + + +def write_line_to_file(files_handlers, line, output_file, buffer_scale=100): + try: # faster than if: https://stackoverflow.com/questions/1835756/using-try-vs-if-in-python + files_handlers[output_file].write(line) + except KeyError: + create_dir_for_file(output_file) + files_handlers[output_file] = open(file=output_file, + mode='a+', + encoding='utf8', + buffering=buffer_scale * io.DEFAULT_BUFFER_SIZE) + files_handlers[output_file].write(line) + + +def get_output_file(alpha_lookup, email): + try: # most likely case. + letter_0 = email[0].lower() + letter_1 = email[1].lower() + letter_2 = email[2].lower() + except IndexError: + email_len = len(email) + if email_len == 0: + letter_0 = '' + letter_1 = '' + letter_2 = '' + elif email_len == 1: + letter_0 = email[0].lower() + letter_1 = '' + letter_2 = '' + elif email_len == 2: + letter_0 = email[0].lower() + letter_1 = email[1].lower() + letter_2 = '' + else: + raise Exception('Error in program.') + if letter_0 not in alpha_lookup: + output_file = 'symbols' + elif letter_1 not in alpha_lookup: + output_file = letter_0 + '/symbols' + elif letter_2 not in alpha_lookup: + output_file = letter_0 + '/' + letter_1 + '/symbols' + else: + output_file = letter_0 + '/' + letter_1 + '/' + letter_2 + return output_file diff --git a/breach/utils.py b/breach/utils.py new file mode 100644 index 0000000..4fc6859 --- /dev/null +++ b/breach/utils.py @@ -0,0 +1,142 @@ +import logging +import os +import random +import subprocess +from datetime import datetime +from multiprocessing import Pool + +import click + + +def shuffle(lst): + random.seed(123) + random.shuffle(lst) + + +def num_open_files_limit(): + return int(subprocess.check_output('ulimit -n', shell=True).decode().strip()) + + +def read_lines(filename: str): + with open(filename, 'r', encoding='utf8', errors='ignore') as r: + return r.read().strip().split('\n') + + +def create_dir(output_dir: str): + if len(output_dir) > 0 and not os.path.exists(output_dir): + os.makedirs(output_dir) + + +def create_dir_for_file(filename: str): + create_dir(os.path.dirname(filename)) + + +def alpha_num_lookup(): + alpha = list('abcdefghijklmnopqrstuvwxyz0123456789') + alpha_lookup = {a: 1 for a in alpha} + assert len(alpha) == 36 + return alpha_lookup + + +def init_logger(name): + format_str = '%(asctime)12s - %(levelname)s - %(message)s' + formatter = logging.Formatter(format_str) + log_dir = 'logs' + create_dir(log_dir) + log_filename = os.path.join(log_dir, name + '_' + datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + '.log') + logging.basicConfig(format=format_str, filename=log_filename, level=logging.INFO) + console = logging.StreamHandler() + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) + + +def parallel_function(f, sequence, num_threads=None): + if num_threads == 1: + return [f(x) for x in sequence] + pool = Pool(processes=num_threads) + result = pool.map(f, sequence) + cleaned = [x for x in result if x is not None] + pool.close() + pool.join() + return cleaned + + +def chunks(lst, n): + """Yield successive n-sized chunks from l.""" + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +class Ct: + + @staticmethod + def input_file(writable=False): + return click.Path(exists=True, file_okay=True, dir_okay=False, + writable=writable, readable=True, resolve_path=True) + + @staticmethod + def input_dir(writable=False): + return click.Path(exists=True, file_okay=False, dir_okay=True, + writable=writable, readable=True, resolve_path=True) + + @staticmethod + def output_file(): + return click.Path(exists=False, file_okay=True, dir_okay=False, + writable=True, readable=True, resolve_path=True) + + @staticmethod + def output_dir(): + return click.Path(exists=False, file_okay=False, dir_okay=True, + writable=True, readable=True, resolve_path=True) + + +class PasswordType: + MD5 = 'MD5' + SHA1 = 'SHA1' + SHA256 = 'SHA256' + B_CRYPT = 'B_CRYPT' + DECRYPTED = 'DECRYPTED' + + +class RingDictionary: + + def __init__(self, max_num_keys=100, debug=False): + self.d = {} # output_file -> file_handler. + self.k = [] + self.max_num_keys = max_num_keys + self.debug = debug + + def __getitem__(self, key): + return self.d[key] + + def __del__(self): + for k in self.k: + self.d[k].flush() + if self.debug: + print(f'[RingDictionary] Flush stream: {k}.') + + def __setitem__(self, key, item): + try: + self.d[key] + except KeyError: + if len(self.k) >= self.max_num_keys: + first_item = self.k.pop(0) + try: + self.d[first_item].flush() # for streams. + if self.debug: + print(f'[RingDictionary] Flush stream: {first_item}.') + except Exception: + pass + del self.d[first_item] + if self.debug: + print(f'[RingDictionary] Drop: {first_item}.') + if self.debug: + print(f'[RingDictionary] Add: {key}.') + self.k.append(key) + self.d[key] = item + + def keys(self): + return self.d.keys() + + def values(self): + return self.d.values() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6855e47 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +tqdm +validate-email +click +cython \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..884d040 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +import sys + +from Cython.Build import cythonize +from setuptools import setup, find_packages + +if sys.version_info < (3, 6): + raise Exception('Must be using Python 3.6 or above.') + +# Install cython first. +setup(name='breach', + version='2.2', + author='Philippe Remy', + ext_modules=cythonize("breach/cparser.pyx", language_level="3"), + packages=find_packages(), + install_requires=[ + 'tqdm', + 'validate-email', + 'click' + ], + include_package_data=True, + entry_points={ + 'console_scripts': [ + 'breach=breach.cli:cli', + ], + } + ) diff --git a/tests/parser_test.py b/tests/parser_test.py new file mode 100644 index 0000000..4b27dbc --- /dev/null +++ b/tests/parser_test.py @@ -0,0 +1,141 @@ +import unittest +from time import time + +from breach.parser import LineParser +from breach.splitter import get_output_file +from breach.utils import alpha_num_lookup + +from breach.cparser import c_parseline + +p_fast = c_parseline +p_slow = LineParser.parseline + +p = p_fast + + +class ParserTest(unittest.TestCase): + + def test_parse_1(self): + self.assertIsNotNone(p('hello@gmail.com:123')) + self.assertIsNotNone(p('hello@@gmail.com:123')) + self.assertIsNotNone(p('hello @ gmail.com:123')) + self.assertIsNotNone(p('hello @@ gmail.com:123')) + self.assertIsNotNone(p('hello @@ gmail.com:123:')) + self.assertIsNone(p('keyless.immo@gmail.com:')) + self.assertIsNotNone(p('premy.enseirb@gmail.com 2296')) + self.assertIsNotNone(p('hello @@ gmail.com,123')) + self.assertIsNotNone(p('123,hello @@ gmail.com')) + self.assertIsNone(p('hello@gmail.com123')) + self.assertIsNone(p('gauntletskd@gmail.com:?????? ??? ?????')) + self.assertIsNone(p('geoffroy.fogeoff4q@yahoo.co.uk.depth66charge:85.242.201.81,felix')) + self.assertIsNone(p('thakhuy.step129@hotmail.com.kkk111:125.26.106.158,nopparat')) + self.assertIsNone(p('bitebackbulldog@gmail.com:INSERT INTO `users' + '`(`id`, `verify`, `username`, `email`, `password')) + self.assertIsNotNone(p('aferrer@e-vende.com:Login')) + self.assertIsNotNone(p('misseme21@yahoo.jp ilovebois3')) + self.assertIsNotNone(p('.y.heine@gmail.com atohiyamamot')) + self.assertIsNotNone(p('taa06@hotmail.com@hotmail.com:8765432f')) + self.assertIsNotNone(p('neanta,buru@ezweb.ne,jp:123')) + self.assertIsNotNone(p('iemongozalu@gmail@com mizo1210')) + self.assertIsNotNone(p('violet28@hotmail.co.jp@hotmail.com:sayonara')) + self.assertIsNone(p('iemongozalu@gmail@com:')) + self.assertIsNone(p('iemongozalu@gmail@com,')) + self.assertIsNotNone(p('josephinejekyl@hotmail.co.uk chi chi 123')) + self.assertIsNotNone(p('h.umehara@seedinc.co.jp@34589@@@@@@:19820429')) + self.assertIsNotNone(p('heavenz_klein@zpost.plala.or.jp@33584@@@@@@ 03260702')) + + self.assertIsNotNone(p(':')) + self.assertIsNotNone(p('patricksullivan88@hotmail.jp< gorillague')) + self.assertIsNone(p('@bk.ru:peng7788525')) + self.assertIsNone(p('@mail.ru:andrew123')) + self.assertIsNone(p('test@bk.ru:lippens')) + self.assertIsNone(p('ESSENTIALWAY')) + self.assertIsNone(p('ESTARRON-ESPINOSA')) + self.assertIsNone(p('ERLONMIRANDA:sha98ne')) + self.assertIsNotNone(p('hookturn54@hotmail.com:3318flindo2003@yahoo.ca:victory')) + self.assertIsNotNone(p('mbssundaram@gmail.com;;shanmuga6')) + self.assertIsNotNone(p('mbssundaram@gmail.com::shanmuga6')) + self.assertIsNotNone(p('mbssundaram@gmail.com::hell@')) + self.assertIsNotNone(p('mbssundaram@gmail.com::hell@:')) + self.assertIsNotNone(p('Aliulfat 05@gmail.com:4849040')) + self.assertIsNotNone(p('harry gill@ yaho.com:7087096771')) + self.assertIsNotNone(p('shaheen .jafri @gmail.com:destinyuntold')) + self.assertIsNotNone(p('mail@ottino snc.it:04921540011')) + self.assertIsNone(p('sssssssandhya@gmail.:puleyar1')) + + self.assertIsNotNone(p('cjywzy%40eyou.com:Aebdd797')) + self.assertIsNotNone(p('finanza.rapido2013%40yahoo.it:123456')) + self.assertIsNotNone(p('habitacasa3%40yahoo.it:cesira89')) + self.assertIsNotNone(p('harry gill@ yaho.com:7087096771')) + self.assertIsNone(p('uwe kopka :biochemie')) + + # multi-line. + self.assertIsNotNone(p('INLOVE12:::1920-05-02:MENDHER4LIFE@yahoo.com:INLOVE12')) + self.assertIsNotNone(p('Nicesweetboobs:1983-10-05:happy.house@outlook.com:ilovekiss')) + self.assertIsNotNone(p('Katiya_ivanova1997@mail.ru:Katiya_ivanova1997@mail.ru;97tata1')) + + multi_lines = [ + 'piabersas@libero.it;stefano.ripamonti@rpmtlc.it:CAROLINAPIANTA', + 'Nicesweetboobs:1983-10-05:happy.house@outlook.com:ilovekiss', + '190.40.252.191,XabelciberX,abelciber@latinmail.com,gatita9685567', + '201.245.164.50,andres vergara,anfeveres@hotmail.com,barcelona10', + '65.94.11.229,stephane,fraisinoo@hotmail.com,ste264', + 'INLOVE12:::1920-05-02:MENDHER4LIFE@yahoo.com:INLOVE12', + 'agaliha84:::1969-07-02:jumbalaya684@privacyharbor.com:agaliha84', + 'happy8360:::1963-09-12:shabby407@yahoo.com:jdj407', + 'tinkertailor99:1947-10-12:terry@anymouse.com:77880577', + 'Cash5178:1970-11-06:littlejohn2dajr@yahoo.com:hulk1234', + 'Creecher0275:1972-04-22:creech.peter@yahoo.com:complete', + 'Katiya_ivanova1997@mail.ru:Katiya_ivanova1997@mail.ru;97tata1', + 'krasotka.1.zwezda@mail.ru:krasotka.1.zwezda@mail.ru;zwezda', + 'SavvySummer:1973-03-17:savvysummer1@gmail.com:sunshine0699', + 'RaWRrr:1990-12-02:ogaboga_12@hotmail.com:nitsua', + 'piabersas@libero.it;stefano.ripamonti@rpmtlc.it:CAROLINAPIANTA', + 'raffaele@puntolingue.it;info@hotelbrumansalerno.it:RRAHOTEL', + 'silent4463:1980-10-14:yancey.robert10@yahoo.com:salinas' + ] + + for multi_line in multi_lines: + self.assertIsNotNone(p(multi_line)) + + self.assertIsNone(p('300396 matelymaly rune_scape_player@hotmail.com 222.155.173.78 ' + '3d2c5414a041f518b03b750e62e4ebc3:Nhc')) + self.assertIsNone(p('05-02-2012;5721869;melanie2032003@yahoo.fr:f02368945726d5fc2a14eb576f7276c0')) + self.assertIsNotNone(p('diet3rix@gmail.com::g1u3ck')) + self.assertIsNotNone(p('asuss_85@mail.ru:asuss_85@mail.ru:2904198')) + self.assertIsNotNone(p('sanda ostojic.todor@gmail.com:vasafultv')) + self.assertIsNone(p('jsjjd.@Com.:love melon')) + + def test_parse_2(self): + t = time() + for i in range(20000): # 10,000 * 13 = 130,000 in 4.0 seconds. + # => 32B in in 10 days. That's a lot. + p('hello@gmail.com:123') + p('hello@@gmail.com:123') + p('hello @ gmail.com:123') + p('hello @@ gmail.com:123') + p('hello @@ gmail.com:123:') + p('hello @@ gmail.com,123') + p('123,hello @@ gmail.com') + p('hello@gmail.com123') + p('gauntletskd@gmail.com:?????? ??? ?????') + p('geoffroy.fogeoff4q@yahoo.co.uk.depth66charge:85.242.201.81,felix') + p('thakhuy.step129@hotmail.com.kkk111:125.26.106.158,nopparat') + p('bitebackbulldog@gmail.com:INSERT INTO `users' + '`(`id`, `verify`, `username`, `email`, `password') + p('aferrer@e-vende.com:Login') + print(time() - t) + + def test_parse_3(self): + alpha = alpha_num_lookup() + self.assertEqual('j/o/s', get_output_file(alpha, 'josemariacanyero@hotmail.com')) + self.assertEqual('c/e/o', get_output_file(alpha, 'ceo21w@gmail.com')) + self.assertEqual('x/symbols', get_output_file(alpha, 'x_fear_238@hotmail.fr')) + self.assertEqual('x/a/symbols', get_output_file(alpha, 'xa_fear_238@hotmail.fr')) + self.assertEqual('x/symbols', get_output_file(alpha, 'x@hotmail.com')) + self.assertEqual('k/e/v', get_output_file(alpha, 'kevmedic13@yahoo.com')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/query_test.py b/tests/query_test.py new file mode 100644 index 0000000..dedca33 --- /dev/null +++ b/tests/query_test.py @@ -0,0 +1,33 @@ +import unittest + +from breach.query import mask_pass, mask_pass_justin_rule + + +class ParserTest(unittest.TestCase): + + def test_mask_password(self): + self.assertEqual('123456**654321', mask_pass('12345688654321', num_asterisks=2)) + self.assertEqual('123456***54321', mask_pass('12345688654321', num_asterisks=3)) + self.assertEqual('12345****54321', mask_pass('12345688654321', num_asterisks=4)) + self.assertEqual('12**********21', mask_pass('12345688654321', num_asterisks=10)) + self.assertEqual('1************1', mask_pass('12345688654321', num_asterisks=12)) + self.assertEqual('**************', mask_pass('12345688654321', num_asterisks=14)) + self.assertEqual('**************', mask_pass('12345688654321', num_asterisks=18)) + self.assertEqual('**************', mask_pass('12345688654321', num_asterisks=99)) + self.assertEqual('1234568*654321', mask_pass('12345688654321', num_asterisks=1)) + self.assertEqual('12345688654321', mask_pass('12345688654321', num_asterisks=0)) + + def test_mask_password_justin(self): + expected = 'il**********m' + initial = 'iloveicecream' + assert len(expected) == len(initial) + self.assertEqual(expected, mask_pass_justin_rule(initial, num_asterisks=-1)) + + expected = 'm****n' + initial = 'martin' + assert len(expected) == len(initial) + self.assertEqual(expected, mask_pass_justin_rule(initial, num_asterisks=-1)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/utils_test.py b/tests/utils_test.py new file mode 100644 index 0000000..d8dff02 --- /dev/null +++ b/tests/utils_test.py @@ -0,0 +1,36 @@ +import unittest + +from breach.utils import RingDictionary + + +class UtilsTest(unittest.TestCase): + + def test_ring_dictionary_1(self): + rd = RingDictionary(max_num_keys=3) + rd['a'] = 2 + self.assertListEqual(sorted(rd.keys()), ['a']) + self.assertListEqual(sorted(rd.k), ['a']) + rd['b'] = 2 + self.assertListEqual(sorted(rd.keys()), ['a', 'b']) + self.assertListEqual(sorted(rd.k), ['a', 'b']) + rd['c'] = 2 + self.assertListEqual(sorted(rd.keys()), ['a', 'b', 'c']) + self.assertListEqual(sorted(rd.k), ['a', 'b', 'c']) + rd['d'] = 2 + self.assertListEqual(sorted(rd.keys()), ['b', 'c', 'd']) + self.assertListEqual(sorted(rd.k), ['b', 'c', 'd']) + + rd = RingDictionary(max_num_keys=1) + rd['c'] = 1 + rd['c'] = 2 + rd['c'] = 3 + self.assertListEqual(sorted(rd.keys()), ['c']) + self.assertListEqual(sorted(rd.k), ['c']) + + rd['d'] = 3 + self.assertListEqual(sorted(rd.keys()), ['d']) + self.assertListEqual(sorted(rd.k), ['d']) + + +if __name__ == '__main__': + unittest.main()