315 lines
12 KiB
Python
315 lines
12 KiB
Python
import os
|
|
import re
|
|
import string
|
|
from glob import glob
|
|
from pathlib import Path
|
|
from time import time
|
|
|
|
from breach.cparser import c_parseline
|
|
from tqdm import tqdm
|
|
from validate_email import validate_email
|
|
|
|
from breach.utils import chunks, read_lines, PasswordType
|
|
from breach.utils import parallel_function, shuffle
|
|
|
|
|
|
# slow functions.
|
|
|
|
def our_validate_email(candidate: str):
|
|
if candidate.count('@') != 1:
|
|
return False
|
|
if '.' not in candidate.split('@')[-1]: # .com or .fr
|
|
return False
|
|
return validate_email(candidate)
|
|
|
|
|
|
class LineParser:
|
|
|
|
@staticmethod
|
|
def _parse_line(row, separator, flip=False):
|
|
if separator not in row: # return fast.
|
|
return None
|
|
if separator != ' ':
|
|
parts = row.split(separator)
|
|
sep_count_cond = len(parts) == 2
|
|
if len(parts) > 2:
|
|
row = separator.join(parts[-2:])
|
|
# [email protected]:kiyan1365:16
|
|
# [email protected]:godis2good:
|
|
sep_count_cond = True
|
|
elif separator in row:
|
|
sep_count_cond = True
|
|
else:
|
|
sep_count_cond = False
|
|
if sep_count_cond:
|
|
mail, password = row.split(separator, 1)
|
|
if flip:
|
|
password, mail = mail, password
|
|
password = password.replace(' ', '')
|
|
if len(password) > 30: # Password of 30 chars look like this: adfsfdsgdfghkfdghdsdfgdfgdfgdf.
|
|
return None
|
|
if len(password) == 0:
|
|
return None
|
|
if '\n\n' in password:
|
|
return None
|
|
if set(password) == {'?'}: # - [email protected]:?????? ??? ?????
|
|
return None
|
|
if password.count('.') >= 3 and \
|
|
re.search(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', password) is not None:
|
|
return None
|
|
mail = mail.replace(' ', '')
|
|
# https://grox.net/utils/encoding.html
|
|
if '@' not in mail:
|
|
mail = mail.replace('%40', '@')
|
|
# - [email protected]:125.26.106.158,nopparat
|
|
# count first because it's faster than compiling a regexp.
|
|
|
|
if len(mail) == 0:
|
|
return None
|
|
|
|
last_char = mail[-1]
|
|
first_char = mail[0]
|
|
# [email protected]\ ilovebois3
|
|
if last_char in ['<', '.', '\\', '>', '_']:
|
|
mail = mail[:-1]
|
|
# [email protected] atohiyamamot [exists]
|
|
if first_char in ['.', '<']:
|
|
mail = mail[1:]
|
|
if mail.endswith('@com'):
|
|
# iemongozalu@gmail@com mizo1210
|
|
mail = mail.replace('@com', '.com')
|
|
if not our_validate_email(mail):
|
|
# some people replace neanta,[email protected],jp
|
|
mail = mail.replace(',', '.')
|
|
parts = mail.split('@')
|
|
if len(parts) > 2:
|
|
mail = '@'.join(parts[0:2])
|
|
# TODO: do it for most common domain names.
|
|
if mail.count('@hotmail.com') == 2: # [email protected]@hotmail.com:8765432f
|
|
mail = mail.replace('@hotmail.com', '', 1)
|
|
if our_validate_email(mail) and not our_validate_email(password):
|
|
password_type = LineParser.get_password_type(password)
|
|
return mail, password, password_type
|
|
return None
|
|
|
|
@staticmethod
|
|
def is_md5(password):
|
|
if len(password) == 32:
|
|
for e in password:
|
|
if not ((ord('0') <= ord(e) <= ord('9')) or
|
|
(ord('A') <= ord(e) <= ord('F')) or
|
|
(ord('a') <= ord(e) <= ord('f'))):
|
|
return False
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
@staticmethod
|
|
def is_sha1(password):
|
|
if len(password) == 40:
|
|
for e in password:
|
|
if not ((ord('0') <= ord(e) <= ord('9')) or
|
|
(ord('A') <= ord(e) <= ord('F')) or
|
|
(ord('a') <= ord(e) <= ord('f'))):
|
|
return False
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
@staticmethod
|
|
def is_sha_256(password):
|
|
if len(password) == 64:
|
|
for e in password:
|
|
if not ((ord('0') <= ord(e) <= ord('9')) or
|
|
(ord('A') <= ord(e) <= ord('F')) or
|
|
(ord('a') <= ord(e) <= ord('f'))):
|
|
return False
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
@staticmethod
|
|
def is_b_crypt(password):
|
|
b_crypt_pattern_cond = password[0:4] == "$2a$" or password[0:4] == "$2b$" or password[0:4] == "$2y$"
|
|
if 50 < len(password) < 70 and b_crypt_pattern_cond:
|
|
return True
|
|
return False
|
|
|
|
@staticmethod
|
|
def get_password_type(password):
|
|
if LineParser.is_md5(password):
|
|
return PasswordType.MD5
|
|
elif LineParser.is_sha1(password):
|
|
return PasswordType.SHA1
|
|
elif LineParser.is_sha_256(password):
|
|
return PasswordType.SHA256
|
|
elif LineParser.is_b_crypt(password):
|
|
return PasswordType.B_CRYPT
|
|
else:
|
|
return PasswordType.DECRYPTED
|
|
|
|
@staticmethod
|
|
def parseline(s: str):
|
|
# case [email protected]:password
|
|
# case [email protected];password
|
|
# case [email protected] password
|
|
valid_separators = [':', ';', ' ', '\t', ' ', ' ', ', ', ',', ';;', '::', ',']
|
|
# https://github.com/swiftmailer/swiftmailer/issues/239
|
|
b = s.strip()
|
|
b = ''.join(list(filter(lambda x: x in string.printable, b)))
|
|
while '..' in b: # TODO: could be move just for the email.
|
|
b = b.replace('..', '')
|
|
if b.startswith('mailto:'):
|
|
b = b[len('mailto:'):]
|
|
if b.startswith('E-mail:'):
|
|
b = b[len('E-mail:'):]
|
|
if b.startswith(':'):
|
|
b = b[1:]
|
|
if b.endswith(':'):
|
|
b = b[:-1]
|
|
|
|
# TODO: could be move just for the email. but we dont need to be 100% accurate here.
|
|
b = b.replace('.@', '@') # just typo.
|
|
b = b.replace('@.', '@') # just typo.
|
|
b = b.replace(' @ ', '@') # just typo.
|
|
b = b.replace('@@', '@') # just typo.
|
|
|
|
if ':' in b and ';' in b:
|
|
b = b.replace(';', ':')
|
|
|
|
# FLIP=FALSE
|
|
for valid_separator in valid_separators:
|
|
result = LineParser._parse_line(b, valid_separator, flip=False)
|
|
if result is not None:
|
|
return result
|
|
# FLIP=TRUE
|
|
for valid_separator in valid_separators:
|
|
result = LineParser._parse_line(b, valid_separator, flip=True)
|
|
if result is not None:
|
|
return result
|
|
# print(f'DISCARD: {s.strip()}')
|
|
return None
|
|
|
|
|
|
class Processor:
|
|
|
|
def __init__(self, w=None, f=None, cython_acceleration=False): # for big files -> writes to disk directly.
|
|
if cython_acceleration:
|
|
print('Cython acceleration enabled.')
|
|
self.bind = c_parseline
|
|
else:
|
|
print('Cython acceleration disabled.')
|
|
self.bind = LineParser.parseline
|
|
self.w = w # cannot be serialized for multi processing.
|
|
self.f = f # cannot be serialized for multi processing.
|
|
|
|
def process(self, filename: Path, is_big_file: bool = False):
|
|
success_list = []
|
|
failure_list = []
|
|
with open(str(filename), 'r', encoding='utf8', errors='ignore') as r:
|
|
buf_size = 10 ** 6 # for very large files.
|
|
while True:
|
|
lines = r.readlines(buf_size)
|
|
if len(lines) == 0:
|
|
break
|
|
for line in lines:
|
|
result = self.bind(line)
|
|
if result is not None:
|
|
mail, password, password_type = result
|
|
if password_type == PasswordType.DECRYPTED:
|
|
# mail is case insensitive.
|
|
success_str = ':'.join([str(mail).lower().strip(), password.strip()]) + '\n'
|
|
if is_big_file:
|
|
self.w.write(success_str)
|
|
else:
|
|
success_list.append(success_str)
|
|
else:
|
|
# stop writing binary data to file...
|
|
fail_line = ''.join(list(filter(lambda x: x in string.printable, line)))
|
|
if is_big_file:
|
|
self.f.write(fail_line)
|
|
else:
|
|
failure_list.append(fail_line)
|
|
return success_list, failure_list
|
|
|
|
|
|
def parse_to_files(input_dir, success_filename, failure_filename, cython_acceleration):
|
|
files = [p for p in Path(input_dir).glob('**/*') if p.is_file()]
|
|
shuffle(files)
|
|
# restart from where we stopped.
|
|
progress_filename = 'progress.txt'
|
|
if os.path.exists(progress_filename):
|
|
lines = read_lines(progress_filename)
|
|
else:
|
|
lines = []
|
|
lines = {n: 1 for n in lines}
|
|
files = [f for f in files if str(f) not in lines]
|
|
shuffle(files)
|
|
# files.sort(key=lambda f: os.stat(f).st_size) # smallest to largest
|
|
|
|
success_total = 0
|
|
failure_total = 0
|
|
num_threads = os.cpu_count()
|
|
start_time = time()
|
|
with open(progress_filename, 'a+') as p:
|
|
with open(failure_filename, 'a+', encoding='utf8') as f:
|
|
with open(success_filename, 'a+', encoding='utf8') as w:
|
|
large_files_processor = Processor(w=w, f=f, cython_acceleration=cython_acceleration)
|
|
small_files_processor = Processor(cython_acceleration=cython_acceleration)
|
|
with tqdm(files) as bar:
|
|
for chunk in chunks(files, num_threads):
|
|
# more than 300MB?
|
|
any_large_files = any([c.stat().st_size / 1e6 > 300 for c in chunk])
|
|
if any_large_files:
|
|
# single thread and write directly to the files.
|
|
for chunky in chunk:
|
|
large_files_processor.process(chunky)
|
|
else:
|
|
# as many threads as we want.
|
|
results = parallel_function(small_files_processor.process, chunk, len(chunk))
|
|
|
|
for single_result in results:
|
|
success_list, failure_list = single_result
|
|
|
|
for single_success in success_list:
|
|
w.write(single_success)
|
|
for single_success in failure_list:
|
|
f.write(single_success)
|
|
|
|
success_total += len(success_list)
|
|
failure_total += len(failure_list)
|
|
|
|
bar.update(len(chunk))
|
|
bar.set_description(f'success = {success_total:,} failure = {failure_total:,}')
|
|
for single_chunk in chunk:
|
|
p.write(str(single_chunk) + '\n')
|
|
print(f'Time elapsed {time() - start_time:.3f} seconds.')
|
|
|
|
|
|
def statistics(input_dir):
|
|
files = glob(os.path.expanduser(input_dir) + '/**/*.*', recursive=True)
|
|
files = [f for f in files if not f.endswith('.tar.gz')]
|
|
shuffle(files)
|
|
parsed_lines = 0
|
|
with open('stats.txt', 'w') as w:
|
|
w.write(','.join(['success', 'failure', 'failure_rate', 'filename']) + '\n')
|
|
for filename in files:
|
|
success = 0
|
|
failure = 0
|
|
lines = read_lines(filename)
|
|
parsed_lines += len(lines)
|
|
for line in lines:
|
|
result = LineParser.parseline(line)
|
|
if result is not None:
|
|
success += 1
|
|
else:
|
|
failure += 1
|
|
try:
|
|
failure_rate = failure / (success + failure) * 100
|
|
except ZeroDivisionError:
|
|
failure_rate = 0.0
|
|
w.write(','.join([str(success), str(failure), f'{failure_rate:.3f}', filename]) + '\n')
|
|
print(f'Parsed lines = {parsed_lines:,}.')
|
|
print(f'filename = {filename}, success = {success:,}, '
|
|
f'failure = {failure:,}, failure rate = {failure_rate:.3f}%')
|