origin commit

breach/api.py

@@ -0,0 +1,143 @@
+import os
+import random
+import shutil
+import subprocess
+from glob import glob
+from pathlib import Path
+
+from tqdm import tqdm
+
+from breach.parser import parse_to_files
+from breach.query import QueryAPI, get_query_manager
+from breach.splitter import get_output_file, write_line_to_file
+from breach.utils import alpha_num_lookup, shuffle, read_lines
+
+
+class API:
+
+    @staticmethod
+    def evaluate(old, new):
+        hit_count = 0
+        total_count = 0
+        files = [p for p in Path(old).expanduser().glob('**/*') if p.is_file()]
+        for i in range(1000):
+            lines = read_lines(random.choice(files))
+            shuffle(lines)
+            random_lines = lines[0:10]
+            for random_line in random_lines:
+                password_list = QueryAPI.query(new, random_line.split(':')[0], 'email')
+                total_count += 1
+                if len(password_list) == 0:
+                    print(f'OLD: {random_line}, NEW: miss, HIT_RATE = {hit_count / total_count:.3f}.')
+                else:
+                    hit_count += 1
+                    print(f'OLD: {random_line}, NEW: hit, HIT_RATE = {hit_count / total_count:.3f}.')
+
+    @staticmethod
+    def merge(src, dest):
+        src_files = find_files(src)
+        dest_files = find_files(dest)
+        for src_index, src_file in src_files.items():
+            if src_index in dest_files:
+                merge(src_file, dest_files[src_index])
+            else:
+                move(src_file, dest + src_index)
+
+    @staticmethod
+    def parse(path, success_file, failure_file, cython_acceleration=False):
+        parse_to_files(path, success_file, failure_file, cython_acceleration)
+
+    @staticmethod
+    def sort(path):
+        path = str(Path(path).expanduser())
+        paths = [Path(a) for a in glob(path + '/**/*', recursive=True) if Path(a).is_file()
+                 and Path(a).suffix != '.sorted']
+        for path in tqdm(paths, desc='Unique > Sorting > Replacing existing files'):
+            output_path = path.with_suffix('.sorted')
+            lines = read_lines(path)
+            unique_sorted_lines = sorted(set(lines))
+            with open(output_path, 'w', encoding='utf8') as w:
+                w.write('\n'.join(unique_sorted_lines) + '\n')
+            shutil.move(output_path, path)
+
+    @staticmethod
+    def clean(path):
+        files = list(find_files(Path(path).expanduser()).values())
+        shuffle(files)
+        alpha = alpha_num_lookup()
+        num_files_moved = 0
+        files_handlers = {}  # u limit here!
+        with tqdm(files, desc='clean') as bar:
+            for file in bar:
+                lines_to_delete = []
+                lines = read_lines(file)
+                for line in lines:
+                    line_split = line.split(':')
+                    ground_truth_file = get_output_file(alpha, line_split[0])
+                    key = str(file).replace(path, '')
+                    if key.startswith('/'):
+                        key = key[1:]
+                    if ground_truth_file != key:
+                        output_file = Path(path) / ground_truth_file
+                        # print(f'MOVE: {line_split[0]} -> {output_file}.')
+                        write_line_to_file(files_handlers, line_split, output_file, buffer_scale=10)
+                        lines_to_delete.append(line)
+                        num_files_moved += 1
+                        if num_files_moved % 100_000 == 0:
+                            truncated_file = str(file).replace(path, "")
+                            truncated_output_file = str(output_file).replace(path, "")
+                            bar.set_description(f'clean. {num_files_moved:,} moved. '
+                                                f'move: {line_split[0]}: {truncated_file}'
+                                                f' -> {truncated_output_file}')
+                lines2 = sorted(set(lines) - set(lines_to_delete))
+                if len(lines) != len(lines2):
+                    with open(str(file), 'w', encoding='utf8') as w:
+                        w.write('\n'.join(lines2) + '\n')
+
+    @staticmethod
+    def test(file: str, dataset: str):
+        qm = get_query_manager(dataset)
+        cracked_rate = 0
+        lines = read_lines(file)
+        for line in lines:
+            results = qm.perform_query(line.strip(), 'email')
+            if len(results) > 0:
+                passwords = [p.split(':', 1)[-1] for p in results]
+                print('FOUND:', line, ':'.join(passwords))
+                cracked_rate += 1
+        print(f'Cracked rate: {cracked_rate / len(lines)}')
+
+    @staticmethod
+    def chunk_files(path: str, max_size: int):
+        # split -b 50000000 file.txt file.txt.
+        # -d to have .00 and .01 is not compatible with MacOS...
+        files = list(find_files(Path(path).expanduser()).values())
+        for f in tqdm(files, desc='chunk large files'):
+            if f.stat().st_size / 1e6 > max_size:
+                cmd = ['split', '-b', str(int(max_size * 1e6)), str(f), str(f) + '_']
+                print(f'File size: {f.stat().st_size / 1e6:.2f} Mb. ' + ' '.join(cmd))
+                subprocess.check_output(cmd)
+                f.unlink()
+
+
+def find_files(path: str):
+    path2 = Path(path).expanduser()
+    files = [p for p in path2.glob('**/*') if p.is_file()]
+    index = {str(f).replace(str(path2), ''): f for f in files}
+    assert len(index) == len(files)
+    return index
+
+
+def merge(src, dst):
+    print(f'merge {src} {dst}')
+    with open(src, 'r', encoding='utf8', errors='ignore') as r:
+        content1 = r.read()
+    with open(dst, 'a', encoding='utf8', errors='ignore') as w:
+        w.write(content1)
+    print(f'del {src}')
+    os.remove(src)
+
+
+def move(src, dst):
+    print(f'mv {src} {dst}')
+    shutil.move(src, dst)

breach/cli.py

@@ -0,0 +1,103 @@
+import logging
+from pathlib import Path
+
+import click
+
+from breach.api import API
+from breach.query import DATASET_CHOICES
+from breach.splitter import split_file
+from breach.utils import Ct, num_open_files_limit
+
+
+# Extraction: nohup find . -name '*.tar.gz' -execdir tar --keep-newer-files -xzvf '{}' -C ../extracted \; &
+
+def recursive_help(cmd, parent=None):
+    ctx = click.core.Context(cmd, info_name=cmd.name, parent=parent)
+    print(cmd.get_help(ctx))
+    print()
+    commands = getattr(cmd, 'commands', {})
+    for sub in commands.values():
+        recursive_help(sub, ctx)
+
+
+@click.group()
+@click.option('--debug/--no-debug')
+@click.pass_context
+def cli(ctx, debug):
+    if debug:
+        click.echo('Debugging mode enabled.')
+    logging.basicConfig(format='%(asctime)12s - %(levelname)s - %(message)s', level=logging.INFO)
+    if not debug and num_open_files_limit() < 60_000 and ctx.invoked_subcommand in ['split', 'clean']:
+        click.echo('ERROR: Increase the limit of the number of open files to at least 60,000.')
+        click.echo('ERROR: ulimit -n 60000')
+        exit(1)
+
+
+@cli.command()
+def dumphelp():
+    recursive_help(cli)
+
+
+@cli.command(short_help='Converts a large FILE to a query friendly folder OUT (e.g. a/b/c). '
+                        'Use RESTART_FROM to resume from the i-th line.')
+@click.option('--file', required=True, type=Ct.input_file())
+@click.option('--out', required=True, type=Ct.output_dir())
+@click.option('--restart_from', default=0, type=int, show_default=True)
+def split(file, out, restart_from):
+    split_file(file, out, restart_from)
+
+
+@cli.command(short_help='chunk large TXT files into smaller files.')
+@click.option('--path', required=True, type=Ct.input_dir())
+@click.option('--size', default=50, type=int, show_default=True)  # MB.
+def chunk(path, size):
+    API.chunk_files(path, size)
+
+
+@cli.command(short_help='Sorts a query friendly folder PATH. Target is itself.')
+@click.option('--path', required=True, type=Ct.input_dir())
+def sort(path):
+    API.sort(path)
+
+
+@cli.command(short_help='Cleans a query friendly folder PATH. Move incorrect records and sort the files.')
+@click.option('--path', required=True, type=Ct.input_dir())
+def clean(path):
+    API.clean(path)
+
+
+@cli.command(short_help='Infers passwords of a list of emails defined in FILE with a query friendly folder DATASET.')
+@click.option('--file', required=True, type=Ct.input_file())
+# @click.option('--dataset', required=True, type=click.Choice(DATASET_CHOICES, case_sensitive=False))
+@click.option('--dataset', required=True)
+def test(file: str, dataset: str):
+    API.test(file, dataset)
+
+
+@cli.command(short_help='Parses an unstructured folder PATH of many files and generates two files: '
+                        'SUCCESS_FILE and FAILURE_FILE. All valid email:password will go to SUCCESS_FILE.')
+@click.option('--path', required=True, type=Ct.output_dir())
+@click.option('--success_file', required=True, type=Ct.output_file())
+@click.option('--failure_file', required=True, type=Ct.output_file())
+@click.option('--cython_acceleration/--no-cython_acceleration', default=False)
+def parse(path, success_file, failure_file, cython_acceleration):
+    API.parse(Path(path).expanduser(), success_file, failure_file, cython_acceleration)
+
+
+@cli.command(short_help='Merges dataset SRC into dataset DEST. Duplicates are removed in the second pass.')
+@click.option('--src', required=True, type=Ct.input_dir())
+@click.option('--dest', required=True, type=Ct.input_dir())
+def merge(src: str, dest: str):
+    API.merge(src, dest)
+    API.clean(dest)
+
+
+@cli.command(short_help='Evaluates some metrics such as precision/recall (e.g. is OLD into NEW).')
+@click.option('--old', required=True, type=Ct.input_dir())
+@click.option('--new', required=True, type=Ct.input_dir())
+def evaluate(old: str, new: str):
+    API.evaluate(old, new)
+
+
+if __name__ == '__main__':
+    cli()

breach/cparser.pyx

@@ -0,0 +1,179 @@
+import re
+import string
+
+import cython
+from validate_email import validate_email
+
+from breach.utils import PasswordType
+
+cdef inline c_is_md5(str password):
+    if len(password) == 32:
+        for e in password:
+            if not ((ord('0') <= ord(e) <= ord('9')) or
+                    (ord('A') <= ord(e) <= ord('F')) or
+                    (ord('a') <= ord(e) <= ord('f'))):
+                return False
+        return True
+    else:
+        return False
+
+cdef inline c_is_sha1(str password):
+    if len(password) == 40:
+        for e in password:
+            if not ((ord('0') <= ord(e) <= ord('9')) or
+                    (ord('A') <= ord(e) <= ord('F')) or
+                    (ord('a') <= ord(e) <= ord('f'))):
+                return False
+        return True
+    else:
+        return False
+
+cdef inline c_is_sha_256(str password):
+    if len(password) == 64:
+        for e in password:
+            if not ((ord('0') <= ord(e) <= ord('9')) or
+                    (ord('A') <= ord(e) <= ord('F')) or
+                    (ord('a') <= ord(e) <= ord('f'))):
+                return False
+        return True
+    else:
+        return False
+
+cdef inline c_is_b_crypt(str password):
+    if 50 < len(password) < 70:
+        b_crypt_pattern_cond = password[0:4] == "$2a$" or password[0:4] == "$2b$" or password[0:4] == "$2y$"
+        if b_crypt_pattern_cond:
+            return True
+    return False
+
+cdef inline c_get_password_type(str password):
+    if c_is_md5(password):
+        return PasswordType.MD5
+    elif c_is_sha1(password):
+        return PasswordType.SHA1
+    elif c_is_sha_256(password):
+        return PasswordType.SHA256
+    elif c_is_b_crypt(password):
+        return PasswordType.B_CRYPT
+    else:
+        return PasswordType.DECRYPTED
+
+@cython.boundscheck(False)
+def c_parseline(str s):
+    # case   [email protected]:password
+    # case   [email protected];password
+    # case   [email protected] password
+    valid_separators = [':', ';', ' ', '\t', '  ', '   ', ', ', ',', ';;', '::', ',']
+    # https://github.com/swiftmailer/swiftmailer/issues/239
+    b = s.strip()
+    b = ''.join(list(filter(lambda x: x in string.printable, b)))
+    while '..' in b:  # TODO: could be move just for the email.
+        b = b.replace('..', '')
+    if b.startswith('mailto:'):
+        b = b[len('mailto:'):]
+    if b.startswith('E-mail:'):
+        b = b[len('E-mail:'):]
+    if b.startswith(':'):
+        b = b[1:]
+    if b.endswith(':'):
+        b = b[:-1]
+
+    # TODO: could be move just for the email. but we dont need to be 100% accurate here.
+    b = b.replace('.@', '@')  # just typo.
+    b = b.replace('@.', '@')  # just typo.
+    b = b.replace(' @ ', '@')  # just typo.
+    b = b.replace('@@', '@')  # just typo.
+
+    if ':' in b and ';' in b:
+        b = b.replace(';', ':')
+
+    # FLIP=FALSE
+    for valid_separator in valid_separators:
+        result = c_parse_line(b, valid_separator, flip=False)
+        if result is not None:
+            return result
+    # FLIP=TRUE
+    for valid_separator in valid_separators:
+        result = c_parse_line(b, valid_separator, flip=True)
+        if result is not None:
+            return result
+    # print(f'DISCARD: {s.strip()}')
+    return None
+
+@cython.boundscheck(False)
+cdef inline c_our_validate_email(str candidate):
+    parts = candidate.split('@')
+    if len(parts) != 2:
+        return False
+    if '.' not in parts[-1]:  # .com or .fr
+        return False
+    return validate_email(candidate)
+
+@cython.boundscheck(False)
+cdef c_parse_line(str row, str separator, bint flip=False):
+    cdef bint is_sep_in_row = separator in row
+    cdef bint sep_count_cond = False
+    cdef str mail = ''
+    cdef str password = ''
+    if not is_sep_in_row:  # return fast.
+        return None
+    if separator != ' ':
+        parts = row.split(separator)
+        sep_count_cond = len(parts) == 2
+        if len(parts) > 2:
+            row = separator.join(parts[-2:])
+            # [email protected]:kiyan1365:16
+            # [email protected]:godis2good:
+            sep_count_cond = True
+    elif is_sep_in_row:
+        sep_count_cond = True
+    else:
+        sep_count_cond = False
+    if sep_count_cond:
+        mail, password = row.split(separator, 1)
+        if flip:
+            password, mail = mail, password
+        password = password.replace(' ', '')
+        if len(password) > 30:  # Password of 30 chars look like this: adfsfdsgdfghkfdghdsdfgdfgdfgdf.
+            return None
+        if len(password) == 0:
+            return None
+        if '\n\n' in password:
+            return None
+        if set(password) == {'?'}:  # - [email protected]:?????? ??? ?????
+            return None
+        if password.count('.') >= 3 and \
+                re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', password) is not None:
+            return None
+        mail = mail.replace(' ', '')
+        # https://grox.net/utils/encoding.html
+        if '@' not in mail:
+            mail = mail.replace('%40', '@')
+        # - [email protected]:125.26.106.158,nopparat
+        # count first because it's faster than compiling a regexp.
+        if len(mail) == 0:
+            return None
+        last_char = mail[-1]
+        first_char = mail[0]
+        # [email protected]\ ilovebois3
+        if last_char in ['<', '.', '\\', '>', '_']:
+            mail = mail[:-1]
+        # [email protected] atohiyamamot [exists]
+        if first_char in ['.', '<']:
+            mail = mail[1:]
+        if mail.endswith('@com'):
+            # iemongozalu@gmail@com mizo1210
+            mail = mail.replace('@com', '.com')
+        if not c_our_validate_email(mail):
+            # some people replace neanta,[email protected],jp
+            mail = mail.replace(',', '.')
+            parts = mail.split('@')
+            if len(parts) > 2:
+                mail = '@'.join(parts[0:2])
+        # TODO: do it for most common domain names.
+        if mail.count('@hotmail.com') == 2:  # [email protected]@hotmail.com:8765432f
+            mail = mail.replace('@hotmail.com', '', 1)
+        if c_our_validate_email(mail) and not c_our_validate_email(password):
+            password_type = c_get_password_type(password)
+            return mail, password, password_type
+    return None

breach/parser.py

@@ -0,0 +1,314 @@
+import os
+import re
+import string
+from glob import glob
+from pathlib import Path
+from time import time
+
+from breach.cparser import c_parseline
+from tqdm import tqdm
+from validate_email import validate_email
+
+from breach.utils import chunks, read_lines, PasswordType
+from breach.utils import parallel_function, shuffle
+
+
+# slow functions.
+
+def our_validate_email(candidate: str):
+    if candidate.count('@') != 1:
+        return False
+    if '.' not in candidate.split('@')[-1]:  # .com or .fr
+        return False
+    return validate_email(candidate)
+
+
+class LineParser:
+
+    @staticmethod
+    def _parse_line(row, separator, flip=False):
+        if separator not in row:  # return fast.
+            return None
+        if separator != ' ':
+            parts = row.split(separator)
+            sep_count_cond = len(parts) == 2
+            if len(parts) > 2:
+                row = separator.join(parts[-2:])
+                # [email protected]:kiyan1365:16
+                # [email protected]:godis2good:
+                sep_count_cond = True
+        elif separator in row:
+            sep_count_cond = True
+        else:
+            sep_count_cond = False
+        if sep_count_cond:
+            mail, password = row.split(separator, 1)
+            if flip:
+                password, mail = mail, password
+            password = password.replace(' ', '')
+            if len(password) > 30:  # Password of 30 chars look like this: adfsfdsgdfghkfdghdsdfgdfgdfgdf.
+                return None
+            if len(password) == 0:
+                return None
+            if '\n\n' in password:
+                return None
+            if set(password) == {'?'}:  # - [email protected]:?????? ??? ?????
+                return None
+            if password.count('.') >= 3 and \
+                    re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', password) is not None:
+                return None
+            mail = mail.replace(' ', '')
+            # https://grox.net/utils/encoding.html
+            if '@' not in mail:
+                mail = mail.replace('%40', '@')
+            # - [email protected]:125.26.106.158,nopparat
+            # count first because it's faster than compiling a regexp.
+
+            if len(mail) == 0:
+                return None
+
+            last_char = mail[-1]
+            first_char = mail[0]
+            # [email protected]\ ilovebois3
+            if last_char in ['<', '.', '\\', '>', '_']:
+                mail = mail[:-1]
+            # [email protected] atohiyamamot [exists]
+            if first_char in ['.', '<']:
+                mail = mail[1:]
+            if mail.endswith('@com'):
+                # iemongozalu@gmail@com mizo1210
+                mail = mail.replace('@com', '.com')
+            if not our_validate_email(mail):
+                # some people replace neanta,[email protected],jp
+                mail = mail.replace(',', '.')
+                parts = mail.split('@')
+                if len(parts) > 2:
+                    mail = '@'.join(parts[0:2])
+            # TODO: do it for most common domain names.
+            if mail.count('@hotmail.com') == 2:  # [email protected]@hotmail.com:8765432f
+                mail = mail.replace('@hotmail.com', '', 1)
+            if our_validate_email(mail) and not our_validate_email(password):
+                password_type = LineParser.get_password_type(password)
+                return mail, password, password_type
+        return None
+
+    @staticmethod
+    def is_md5(password):
+        if len(password) == 32:
+            for e in password:
+                if not ((ord('0') <= ord(e) <= ord('9')) or
+                        (ord('A') <= ord(e) <= ord('F')) or
+                        (ord('a') <= ord(e) <= ord('f'))):
+                    return False
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def is_sha1(password):
+        if len(password) == 40:
+            for e in password:
+                if not ((ord('0') <= ord(e) <= ord('9')) or
+                        (ord('A') <= ord(e) <= ord('F')) or
+                        (ord('a') <= ord(e) <= ord('f'))):
+                    return False
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def is_sha_256(password):
+        if len(password) == 64:
+            for e in password:
+                if not ((ord('0') <= ord(e) <= ord('9')) or
+                        (ord('A') <= ord(e) <= ord('F')) or
+                        (ord('a') <= ord(e) <= ord('f'))):
+                    return False
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def is_b_crypt(password):
+        b_crypt_pattern_cond = password[0:4] == "$2a$" or password[0:4] == "$2b$" or password[0:4] == "$2y$"
+        if 50 < len(password) < 70 and b_crypt_pattern_cond:
+            return True
+        return False
+
+    @staticmethod
+    def get_password_type(password):
+        if LineParser.is_md5(password):
+            return PasswordType.MD5
+        elif LineParser.is_sha1(password):
+            return PasswordType.SHA1
+        elif LineParser.is_sha_256(password):
+            return PasswordType.SHA256
+        elif LineParser.is_b_crypt(password):
+            return PasswordType.B_CRYPT
+        else:
+            return PasswordType.DECRYPTED
+
+    @staticmethod
+    def parseline(s: str):
+        # case   [email protected]:password
+        # case   [email protected];password
+        # case   [email protected] password
+        valid_separators = [':', ';', ' ', '\t', '  ', '   ', ', ', ',', ';;', '::', ',']
+        # https://github.com/swiftmailer/swiftmailer/issues/239
+        b = s.strip()
+        b = ''.join(list(filter(lambda x: x in string.printable, b)))
+        while '..' in b:  # TODO: could be move just for the email.
+            b = b.replace('..', '')
+        if b.startswith('mailto:'):
+            b = b[len('mailto:'):]
+        if b.startswith('E-mail:'):
+            b = b[len('E-mail:'):]
+        if b.startswith(':'):
+            b = b[1:]
+        if b.endswith(':'):
+            b = b[:-1]
+
+        # TODO: could be move just for the email. but we dont need to be 100% accurate here.
+        b = b.replace('.@', '@')  # just typo.
+        b = b.replace('@.', '@')  # just typo.
+        b = b.replace(' @ ', '@')  # just typo.
+        b = b.replace('@@', '@')  # just typo.
+
+        if ':' in b and ';' in b:
+            b = b.replace(';', ':')
+
+        # FLIP=FALSE
+        for valid_separator in valid_separators:
+            result = LineParser._parse_line(b, valid_separator, flip=False)
+            if result is not None:
+                return result
+        # FLIP=TRUE
+        for valid_separator in valid_separators:
+            result = LineParser._parse_line(b, valid_separator, flip=True)
+            if result is not None:
+                return result
+        # print(f'DISCARD: {s.strip()}')
+        return None
+
+
+class Processor:
+
+    def __init__(self, w=None, f=None, cython_acceleration=False):  # for big files -> writes to disk directly.
+        if cython_acceleration:
+            print('Cython acceleration enabled.')
+            self.bind = c_parseline
+        else:
+            print('Cython acceleration disabled.')
+            self.bind = LineParser.parseline
+        self.w = w  # cannot be serialized for multi processing.
+        self.f = f  # cannot be serialized for multi processing.
+
+    def process(self, filename: Path, is_big_file: bool = False):
+        success_list = []
+        failure_list = []
+        with open(str(filename), 'r', encoding='utf8', errors='ignore') as r:
+            buf_size = 10 ** 6  # for very large files.
+            while True:
+                lines = r.readlines(buf_size)
+                if len(lines) == 0:
+                    break
+                for line in lines:
+                    result = self.bind(line)
+                    if result is not None:
+                        mail, password, password_type = result
+                        if password_type == PasswordType.DECRYPTED:
+                            # mail is case insensitive.
+                            success_str = ':'.join([str(mail).lower().strip(), password.strip()]) + '\n'
+                            if is_big_file:
+                                self.w.write(success_str)
+                            else:
+                                success_list.append(success_str)
+                    else:
+                        # stop writing binary data to file...
+                        fail_line = ''.join(list(filter(lambda x: x in string.printable, line)))
+                        if is_big_file:
+                            self.f.write(fail_line)
+                        else:
+                            failure_list.append(fail_line)
+        return success_list, failure_list
+
+
+def parse_to_files(input_dir, success_filename, failure_filename, cython_acceleration):
+    files = [p for p in Path(input_dir).glob('**/*') if p.is_file()]
+    shuffle(files)
+    # restart from where we stopped.
+    progress_filename = 'progress.txt'
+    if os.path.exists(progress_filename):
+        lines = read_lines(progress_filename)
+    else:
+        lines = []
+    lines = {n: 1 for n in lines}
+    files = [f for f in files if str(f) not in lines]
+    shuffle(files)
+    # files.sort(key=lambda f: os.stat(f).st_size)  # smallest to largest
+
+    success_total = 0
+    failure_total = 0
+    num_threads = os.cpu_count()
+    start_time = time()
+    with open(progress_filename, 'a+') as p:
+        with open(failure_filename, 'a+', encoding='utf8') as f:
+            with open(success_filename, 'a+', encoding='utf8') as w:
+                large_files_processor = Processor(w=w, f=f, cython_acceleration=cython_acceleration)
+                small_files_processor = Processor(cython_acceleration=cython_acceleration)
+                with tqdm(files) as bar:
+                    for chunk in chunks(files, num_threads):
+                        # more than 300MB?
+                        any_large_files = any([c.stat().st_size / 1e6 > 300 for c in chunk])
+                        if any_large_files:
+                            # single thread and write directly to the files.
+                            for chunky in chunk:
+                                large_files_processor.process(chunky)
+                        else:
+                            # as many threads as we want.
+                            results = parallel_function(small_files_processor.process, chunk, len(chunk))
+
+                            for single_result in results:
+                                success_list, failure_list = single_result
+
+                                for single_success in success_list:
+                                    w.write(single_success)
+                                for single_success in failure_list:
+                                    f.write(single_success)
+
+                                success_total += len(success_list)
+                                failure_total += len(failure_list)
+
+                        bar.update(len(chunk))
+                        bar.set_description(f'success = {success_total:,} failure = {failure_total:,}')
+                        for single_chunk in chunk:
+                            p.write(str(single_chunk) + '\n')
+    print(f'Time elapsed {time() - start_time:.3f} seconds.')
+
+
+def statistics(input_dir):
+    files = glob(os.path.expanduser(input_dir) + '/**/*.*', recursive=True)
+    files = [f for f in files if not f.endswith('.tar.gz')]
+    shuffle(files)
+    parsed_lines = 0
+    with open('stats.txt', 'w') as w:
+        w.write(','.join(['success', 'failure', 'failure_rate', 'filename']) + '\n')
+        for filename in files:
+            success = 0
+            failure = 0
+            lines = read_lines(filename)
+            parsed_lines += len(lines)
+            for line in lines:
+                result = LineParser.parseline(line)
+                if result is not None:
+                    success += 1
+                else:
+                    failure += 1
+            try:
+                failure_rate = failure / (success + failure) * 100
+            except ZeroDivisionError:
+                failure_rate = 0.0
+            w.write(','.join([str(success), str(failure), f'{failure_rate:.3f}', filename]) + '\n')
+            print(f'Parsed lines = {parsed_lines:,}.')
+            print(f'filename = {filename}, success = {success:,}, '
+                  f'failure = {failure:,}, failure rate = {failure_rate:.3f}%')

breach/query.py

@@ -0,0 +1,194 @@
+import logging
+import os
+import subprocess
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+# https://pypi.org/project/ripgrepy/
+
+class BreachCompilationConstants:
+    EMAIL_INDEX = '/media/philippe/DATA/datasets/BreachCompilation/data'
+    PASSWORD_INDEX = '/media/philippe/DATA/datasets/BreachCompilation/passwords'
+
+
+class Collections1:
+    EMAIL_INDEX = '/media/philippe/8TB/datasets/passwords2/data'
+    PASSWORD_INDEX = None
+
+
+class Collections_2_5:
+    EMAIL_INDEX = '/media/philippe/8TB/datasets/passwords2/data'
+    PASSWORD_INDEX = None
+
+
+DATASET_CHOICES = ['breach_compilation', 'collections_1', 'collections_2_5', 'all']
+
+
+def get_query_manager(dataset_choice: str):
+    if dataset_choice.lower() == 'breach_compilation':
+        return QueryManager(BreachCompilationConstants)
+    elif dataset_choice.lower() == 'collections_1':
+        return QueryManager(Collections1)
+    elif dataset_choice.lower() == 'collections_2_5':
+        return QueryManager(Collections_2_5)
+    elif os.path.exists(dataset_choice):
+        class Custom:
+            EMAIL_INDEX = dataset_choice
+            PASSWORD_INDEX = None
+
+        return QueryManager(Custom)
+    else:
+        return QueryMultiManager()  # all
+
+
+def mask_pass_justin_rule(password: str, num_asterisks=-1) -> str:
+    if len(password) >= 8:
+        return password[0:2] + (len(password) - 3) * '*' + password[-1]
+    return password[0] + '*' * (len(password) - 2) + password[-1]
+
+
+def mask_pass(password: str, num_asterisks=2) -> str:
+    if len(password) <= num_asterisks:
+        return '*' * len(password)
+    if len(password) >= 4:
+        i = len(password) // 2
+        even = num_asterisks % 2 == 0
+        half = num_asterisks // 2
+        if even:
+            left = half
+            right = half
+        else:
+            left = half
+            right = half + 1
+        return password[0:i - left] + '*' * num_asterisks + password[i + right:]
+    else:
+        return password
+
+
+def extract_emails_and_passwords(txt_lines):
+    emails_passwords = []
+    for txt_line in txt_lines:
+        try:
+            if '@' in txt_line:  # does it contain an email address?
+                if all([char in txt_line for char in [':', ';']]):  # which separator is it? : or ;?
+                    separator = ':'
+                elif ':' in txt_line:  # '[email protected]:iskandar89
+                    separator = ':'
+                elif ';' in txt_line:  # '[email protected];ol1391ga
+                    separator = ';'
+                else:
+                    continue
+
+                strip_txt_line = txt_line.strip()
+                email, password = strip_txt_line.split(separator)
+                emails_passwords.append((email, password))
+        except Exception:
+            pass
+    return emails_passwords
+
+
+def postprocess(results, num_asterisks_to_mask_password=0):
+    response = [b for b in extract_emails_and_passwords(results)]
+    if num_asterisks_to_mask_password > 0:
+        response = [(b[0], mask_pass_justin_rule(b[1])) for b in response]
+    response = [':'.join(b) for b in response]
+    response = sorted(set(response))
+    return response
+
+
+class QueryAPI:
+
+    @staticmethod
+    def scanning(directory: Path):
+        file_list, dir_list = {}, {}
+        for x in directory.iterdir():
+            if x.is_file():
+                file_list[x.name] = x
+            else:
+                dir_list[x.name] = x
+        return file_list, dir_list
+
+    @staticmethod
+    def search_in_file(path: Path, email):
+        assert path.is_file(), path
+        try:
+            results = subprocess.check_output(['rg', '--no-line-number', '-ie', '^' +
+                                               email, str(path)]).decode('utf8').split('\n')
+            results = [a for a in results if a != '']
+        except subprocess.CalledProcessError:
+            results = []
+        return results
+
+    @staticmethod
+    def recursive_search(path: Path, characters: list, index: int):
+        fl, dl = QueryAPI.scanning(path)
+        cur = characters[index].lower()
+        if cur in fl:
+            return QueryAPI.search_in_file(fl[cur], ''.join(characters))
+        elif cur in dl:
+            return QueryAPI.recursive_search(dl[cur], characters, index + 1)
+        elif 'symbols' in fl:
+            return QueryAPI.search_in_file(fl['symbols'], ''.join(characters))
+        else:
+            raise Exception('Hello.')
+
+    @staticmethod
+    def query(data_dir: str, request: str, query_type: str, num_asterisks_to_mask_password: int = 0):
+        if len(request) <= 3:
+            return []
+        if query_type == 'email' and '@' not in request:
+            return []
+        response = QueryAPI.recursive_search(Path(data_dir), list(request), 0)
+        return postprocess(response, num_asterisks_to_mask_password)
+
+
+class QueryManagerStub:
+    def __init__(self, constants):
+        self.default_email_index = constants.EMAIL_INDEX
+        self.default_password_index = constants.PASSWORD_INDEX
+        self.counter = 0
+
+    def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0):
+        self.counter += 1
+        if self.counter % 2 == 0:
+            return [f'{request}:hello123', f'{request}:hello123456']
+        else:
+            return []
+
+
+class QueryManager:
+
+    def __init__(self, constants):
+        self.default_email_index = constants.EMAIL_INDEX
+        self.default_password_index = constants.PASSWORD_INDEX
+
+    def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0):
+        if query_type == 'email':
+            index = self.default_email_index
+        elif query_type == 'password':
+            index = self.default_password_index
+        else:
+            raise Exception({'error': 'Index is either email or password.'})
+        results = []
+        if ' ' in request:  # batch.
+            for single_req in request.split(' '):
+                results.extend(QueryAPI.query(index, single_req, query_type, num_asterisks_to_mask_password))
+        else:
+            results.extend(QueryAPI.query(index, request, query_type, num_asterisks_to_mask_password))
+        return results
+
+
+class QueryMultiManager:
+
+    def __init__(self):
+        self.qm1 = QueryManager(BreachCompilationConstants)
+        self.qm2 = QueryManager(Collections1)
+        self.qm3 = QueryManager(Collections_2_5)
+
+    def perform_query(self, request: str, query_type: str, num_asterisks_to_mask_password=0):
+        results = self.qm1.perform_query(request, query_type, num_asterisks_to_mask_password)
+        results.extend(self.qm2.perform_query(request, query_type, num_asterisks_to_mask_password))
+        results.extend(self.qm3.perform_query(request, query_type, num_asterisks_to_mask_password))
+        return sorted(set(results))

breach/splitter.py

@@ -0,0 +1,80 @@
+import csv
+import io
+import sys
+from time import time
+
+from breach.utils import alpha_num_lookup, create_dir, create_dir_for_file, RingDictionary
+
+csv.field_size_limit(sys.maxsize)
+
+
+def split_file(breach_compilation_file,
+               output_dir='out',
+               restart_from=0):
+    # It seems like this function is not 100% correct as we need to clean the dataset after this.
+    # It's not a big deal but I don't understand why we have to do that.
+    # Could probably be to a limitation to the large number of files we open at the same time x increase of buffer.
+    create_dir(output_dir)
+    alpha = alpha_num_lookup()
+    files_handlers = RingDictionary()
+    t1 = time()
+    with open(breach_compilation_file, 'r', encoding='utf8', errors='ignore') as r:
+        for i, line in enumerate(r):
+            if i % 10_000_000 == 0:
+                elapsed_time = time() - t1
+                t1 = time()
+                print(f'|- Processed {i:,} rows in {elapsed_time:.2f} seconds.')
+            if i < restart_from:
+                continue
+            try:  # faster than if: https://stackoverflow.com/questions/1835756/using-try-vs-if-in-python
+                email, password = line.strip().split(':', 1)
+            except ValueError:
+                continue
+
+            output_file = get_output_file(alpha, email)
+            output_file = output_dir + '/' + output_file
+            write_line_to_file(files_handlers, line, output_file)
+
+
+def write_line_to_file(files_handlers, line, output_file, buffer_scale=100):
+    try:  # faster than if: https://stackoverflow.com/questions/1835756/using-try-vs-if-in-python
+        files_handlers[output_file].write(line)
+    except KeyError:
+        create_dir_for_file(output_file)
+        files_handlers[output_file] = open(file=output_file,
+                                           mode='a+',
+                                           encoding='utf8',
+                                           buffering=buffer_scale * io.DEFAULT_BUFFER_SIZE)
+        files_handlers[output_file].write(line)
+
+
+def get_output_file(alpha_lookup, email):
+    try:  # most likely case.
+        letter_0 = email[0].lower()
+        letter_1 = email[1].lower()
+        letter_2 = email[2].lower()
+    except IndexError:
+        email_len = len(email)
+        if email_len == 0:
+            letter_0 = ''
+            letter_1 = ''
+            letter_2 = ''
+        elif email_len == 1:
+            letter_0 = email[0].lower()
+            letter_1 = ''
+            letter_2 = ''
+        elif email_len == 2:
+            letter_0 = email[0].lower()
+            letter_1 = email[1].lower()
+            letter_2 = ''
+        else:
+            raise Exception('Error in program.')
+    if letter_0 not in alpha_lookup:
+        output_file = 'symbols'
+    elif letter_1 not in alpha_lookup:
+        output_file = letter_0 + '/symbols'
+    elif letter_2 not in alpha_lookup:
+        output_file = letter_0 + '/' + letter_1 + '/symbols'
+    else:
+        output_file = letter_0 + '/' + letter_1 + '/' + letter_2
+    return output_file

breach/utils.py

@@ -0,0 +1,142 @@
+import logging
+import os
+import random
+import subprocess
+from datetime import datetime
+from multiprocessing import Pool
+
+import click
+
+
+def shuffle(lst):
+    random.seed(123)
+    random.shuffle(lst)
+
+
+def num_open_files_limit():
+    return int(subprocess.check_output('ulimit -n', shell=True).decode().strip())
+
+
+def read_lines(filename: str):
+    with open(filename, 'r', encoding='utf8', errors='ignore') as r:
+        return r.read().strip().split('\n')
+
+
+def create_dir(output_dir: str):
+    if len(output_dir) > 0 and not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+
+def create_dir_for_file(filename: str):
+    create_dir(os.path.dirname(filename))
+
+
+def alpha_num_lookup():
+    alpha = list('abcdefghijklmnopqrstuvwxyz0123456789')
+    alpha_lookup = {a: 1 for a in alpha}
+    assert len(alpha) == 36
+    return alpha_lookup
+
+
+def init_logger(name):
+    format_str = '%(asctime)12s - %(levelname)s - %(message)s'
+    formatter = logging.Formatter(format_str)
+    log_dir = 'logs'
+    create_dir(log_dir)
+    log_filename = os.path.join(log_dir, name + '_' + datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + '.log')
+    logging.basicConfig(format=format_str, filename=log_filename, level=logging.INFO)
+    console = logging.StreamHandler()
+    console.setFormatter(formatter)
+    logging.getLogger('').addHandler(console)
+
+
+def parallel_function(f, sequence, num_threads=None):
+    if num_threads == 1:
+        return [f(x) for x in sequence]
+    pool = Pool(processes=num_threads)
+    result = pool.map(f, sequence)
+    cleaned = [x for x in result if x is not None]
+    pool.close()
+    pool.join()
+    return cleaned
+
+
+def chunks(lst, n):
+    """Yield successive n-sized chunks from l."""
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
+
+class Ct:
+
+    @staticmethod
+    def input_file(writable=False):
+        return click.Path(exists=True, file_okay=True, dir_okay=False,
+                          writable=writable, readable=True, resolve_path=True)
+
+    @staticmethod
+    def input_dir(writable=False):
+        return click.Path(exists=True, file_okay=False, dir_okay=True,
+                          writable=writable, readable=True, resolve_path=True)
+
+    @staticmethod
+    def output_file():
+        return click.Path(exists=False, file_okay=True, dir_okay=False,
+                          writable=True, readable=True, resolve_path=True)
+
+    @staticmethod
+    def output_dir():
+        return click.Path(exists=False, file_okay=False, dir_okay=True,
+                          writable=True, readable=True, resolve_path=True)
+
+
+class PasswordType:
+    MD5 = 'MD5'
+    SHA1 = 'SHA1'
+    SHA256 = 'SHA256'
+    B_CRYPT = 'B_CRYPT'
+    DECRYPTED = 'DECRYPTED'
+
+
+class RingDictionary:
+
+    def __init__(self, max_num_keys=100, debug=False):
+        self.d = {}  # output_file -> file_handler.
+        self.k = []
+        self.max_num_keys = max_num_keys
+        self.debug = debug
+
+    def __getitem__(self, key):
+        return self.d[key]
+
+    def __del__(self):
+        for k in self.k:
+            self.d[k].flush()
+            if self.debug:
+                print(f'[RingDictionary] Flush stream: {k}.')
+
+    def __setitem__(self, key, item):
+        try:
+            self.d[key]
+        except KeyError:
+            if len(self.k) >= self.max_num_keys:
+                first_item = self.k.pop(0)
+                try:
+                    self.d[first_item].flush()  # for streams.
+                    if self.debug:
+                        print(f'[RingDictionary] Flush stream: {first_item}.')
+                except Exception:
+                    pass
+                del self.d[first_item]
+                if self.debug:
+                    print(f'[RingDictionary] Drop: {first_item}.')
+            if self.debug:
+                print(f'[RingDictionary] Add: {key}.')
+            self.k.append(key)
+            self.d[key] = item
+
+    def keys(self):
+        return self.d.keys()
+
+    def values(self):
+        return self.d.values()