add filter lines

breach/api.py

@@ -1,6 +1,7 @@
 import os
 import random
 import shutil
+import string
 import subprocess
 from glob import glob
 from pathlib import Path
@@ -10,7 +11,7 @@ from tqdm import tqdm
 from breach.parser import parse_to_files
 from breach.query import QueryAPI, get_query_manager
 from breach.splitter import get_output_file, write_line_to_file
-from breach.utils import alpha_num_lookup, shuffle, read_lines
+from breach.utils import alpha_num_lookup, shuffle, read_lines, create_dir_for_file
 
 
 class API:
@@ -119,6 +120,16 @@ class API:
                 subprocess.check_output(cmd)
                 f.unlink()
 
+    @staticmethod
+    def filter_lines(file, out, less_than, more_than):
+        create_dir_for_file(out)
+        with open(out, 'w', encoding='utf8') as w:
+            with open(file, 'r', encoding='utf8', errors='ignore') as r:
+                for line in r:
+                    line2 = ''.join(list(filter(lambda x: x in string.printable, line)))
+                    if less_than <= len(line2.strip()) <= more_than:
+                        w.write(line2)
+
 
 def find_files(path: str):
     path2 = Path(path).expanduser()

breach/cli.py

@@ -4,7 +4,6 @@ from pathlib import Path
 import click
 
 from breach.api import API
-from breach.query import DATASET_CHOICES
 from breach.splitter import split_file
 from breach.utils import Ct, num_open_files_limit
 
@@ -99,5 +98,14 @@ def evaluate(old: str, new: str):
     API.evaluate(old, new)
 
 
+@cli.command(short_help='Filters lines less than N and more than M.')
+@click.option('--file', required=True, type=Ct.input_file())
+@click.option('--out', required=True, type=Ct.output_file())
+@click.option('--less_than', required=True, type=int)
+@click.option('--more_than', required=True, type=int)
+def filter_lines(file: str, out: str, less_than: int, more_than: int):
+    API.filter_lines(file, out, less_than, more_than)
+
+
 if __name__ == '__main__':
     cli()