diff --git a/breach/api.py b/breach/api.py index 304c43a..0a7a605 100644 --- a/breach/api.py +++ b/breach/api.py @@ -1,6 +1,7 @@ import os import random import shutil +import string import subprocess from glob import glob from pathlib import Path @@ -10,7 +11,7 @@ from tqdm import tqdm from breach.parser import parse_to_files from breach.query import QueryAPI, get_query_manager from breach.splitter import get_output_file, write_line_to_file -from breach.utils import alpha_num_lookup, shuffle, read_lines +from breach.utils import alpha_num_lookup, shuffle, read_lines, create_dir_for_file class API: @@ -119,6 +120,16 @@ class API: subprocess.check_output(cmd) f.unlink() + @staticmethod + def filter_lines(file, out, less_than, more_than): + create_dir_for_file(out) + with open(out, 'w', encoding='utf8') as w: + with open(file, 'r', encoding='utf8', errors='ignore') as r: + for line in r: + line2 = ''.join(list(filter(lambda x: x in string.printable, line))) + if less_than <= len(line2.strip()) <= more_than: + w.write(line2) + def find_files(path: str): path2 = Path(path).expanduser() diff --git a/breach/cli.py b/breach/cli.py index 6b6a8d8..a927fe7 100644 --- a/breach/cli.py +++ b/breach/cli.py @@ -4,7 +4,6 @@ from pathlib import Path import click from breach.api import API -from breach.query import DATASET_CHOICES from breach.splitter import split_file from breach.utils import Ct, num_open_files_limit @@ -99,5 +98,14 @@ def evaluate(old: str, new: str): API.evaluate(old, new) +@cli.command(short_help='Filters lines less than N and more than M.') +@click.option('--file', required=True, type=Ct.input_file()) +@click.option('--out', required=True, type=Ct.output_file()) +@click.option('--less_than', required=True, type=int) +@click.option('--more_than', required=True, type=int) +def filter_lines(file: str, out: str, less_than: int, more_than: int): + API.filter_lines(file, out, less_than, more_than) + + if __name__ == '__main__': cli()