awoo

Signed-off-by: Balazs Horvath <[email protected]>

scripts/check_captions_for_single_line.py

@@ -0,0 +1,45 @@
+"""
+This script is designed to search for .txt files in a given directory and its subdirectories.
+It skips files named 'sample-prompt.txt' and any files ending with '-sample-prompts.txt'.
+For each .txt file, it checks if the file contains more than one line.
+If a file contains more than one line, it prints the file name.
+
+The script uses the glob and os modules to perform the file search and manipulation.
+It starts the search from a specified directory path.
+
+The script is executed by calling the check_files function with the directory path as an argument.
+"""
+
+import glob
+import os
+
+
+def check_files(path):
+    """
+    This function recursively searches for .txt files in a given directory and its subdirectories.
+    It skips files named 'sample-prompts.txt' and any files ending with '-sample-prompts.txt'.
+    For each .txt file, it checks if the file contains more than one line.
+    If a file contains more than one line, it prints the file name.
+
+    Args:
+        path (str): The directory path where the search begins.
+
+    Returns:
+        None. The function prints the file names directly.
+    """
+    # Use glob to recursively find all .txt files
+    for filename in glob.iglob(path + "**/*.txt", recursive=True):
+        # Skip 'sample-prompts.txt' and '*-sample-prompts.txt' files
+        if os.path.basename(filename) == "sample-prompts.txt" or os.path.basename(
+            filename
+        ).endswith("-sample-prompts.txt"):
+            continue
+        # Open each text file and check if it contains more than one line
+        with open(filename, "r", encoding="utf-8") as file:
+            lines = file.readlines()
+            if len(lines) > 1:
+                print(filename)
+
+
+# Call the function with the directory path
+check_files("E:\\training_dir\\")

scripts/search_for_tag.py

@@ -0,0 +1,59 @@
+"""
+This script is used to search for the word "anthrofied" in all .txt files
+within a specified directory and its subdirectories. It uses multiprocessing
+to speed up the search by checking multiple files simultaneously.
+"""
+
+import glob
+import os
+import multiprocessing
+
+
+def check_file(filename):
+    """
+    Checks if a .txt file contains the word "anthrofied".
+
+    Args:
+        filename (str): The path of the file to check.
+
+    Returns:
+        str: The filename if it contains "anthrofied", otherwise None.
+    """
+    # Skip 'sample-prompts.txt' and '*-sample-prompts.txt' files
+    if os.path.basename(filename) == "sample-prompts.txt" or os.path.basename(
+        filename
+    ).endswith("-sample-prompts.txt"):
+        return None
+    # Open each text file and check if it contains the word "anthrofied"
+    with open(filename, "r", encoding="utf-8") as file:
+        content = file.read()
+        if "anthrofied" in content:
+            return filename
+    return None
+
+
+def check_files(path):
+    """
+    Searches for the word "anthrofied" in all .txt files within a specified
+    directory and its subdirectories.
+
+    Args:
+        path (str): The path of the directory to search.
+    """
+    # Use glob to recursively find all .txt files
+    filenames = glob.glob(path + "**/*.txt", recursive=True)
+
+    # Create a pool of processes
+    with multiprocessing.Pool() as pool:
+        # Use the pool to check each file
+        results = pool.map(check_file, filenames)
+
+    # Print the filenames that contain "anthrofied"
+    for result in results:
+        if result is not None:
+            print(result)
+
+
+if __name__ == "__main__":
+    # Call the function with the directory path
+    check_files("E:\\training_dir\\")

scripts/search_for_tags.py

@@ -0,0 +1,97 @@
+# Define the search terms
+SEARCH_TERMS = [
+    r"anthrofied",
+    r"canid",
+    r"canis",
+    r"pokemorph",
+    r"pokephilia",
+    r"alpha channel",
+    r"mythological scalie",
+    r"mythological creature",
+    r"collaboration",
+    r"mythology",
+    r"mammal,",
+    r"grandfathered content",
+    r"mephitid",
+    r"generation 1 pokemon",
+    r"generation 2 pokemon",
+    r"generation 3 pokemon",
+    r"generation 4 pokemon",
+    r"generation 5 pokemon",
+    r"generation 6 pokemon",
+    r"generation 7 pokemon",
+    r"generation 8 pokemon",
+    r"generation 9 pokemon",
+    r"generation 10 pokemon",
+]
+
+import glob
+import os
+import multiprocessing
+import re
+from rich.console import Console
+from rich.table import Table
+
+# Initialize the console
+console = Console()
+
+
+def check_file(filename):
+    """
+    Checks if a .txt file contains any of the specified search terms.
+
+    Args:
+        filename (str): The path of the file to check.
+
+    Returns:
+        tuple: A tuple containing the filename and a list of found search terms.
+    """
+    # Skip 'sample-prompts.txt' and '*-sample-prompts.txt' files
+    if "sample-prompts.txt" in os.path.basename(filename):
+        return None
+    found_terms = []
+    # Open each text file and check if it contains any of the search terms
+    with open(filename, "r", encoding="utf-8") as file:
+        content = file.read()
+        for term in SEARCH_TERMS:
+            # Use a regular expression to search for the match 'term' followed by a comma
+            pattern = r"\b" + re.escape(term) + r"\b"
+            if re.search(pattern, content):
+                found_terms.append(term)
+    return (filename, found_terms) if found_terms else None
+
+
+def check_files(path):
+    """
+    Searches for the specified search terms in all .txt files within a specified
+    directory and its subdirectories.
+
+    Args:
+        path (str): The path of the directory to search.
+    """
+    # Use glob to recursively find all .txt files
+    filenames = glob.glob(os.path.join(path, "**/*.txt"), recursive=True)
+
+    # Create a pool of processes
+    with multiprocessing.Pool() as pool:
+        # Use the pool to check each file
+        results = pool.map(check_file, filenames)
+
+    # Create a table
+    table = Table(show_header=True, header_style="bold magenta")
+    table.add_column("Filename", style="dim")
+    table.add_column("Found Search Terms")
+
+    # Print the filenames and found search terms in a table
+    for result in results:
+        if result is not None:
+            filename, terms = result
+            table.add_row(filename, ", ".join(terms))
+
+    # Print the table to the console
+    console.print(table)
+
+
+if __name__ == "__main__":
+    # Call the function with the directory path
+    check_files("E:\\training_dir\\")

tests/blotter-v1e400.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6837cd9106d4ed13f780e956f9559bc0cdfa613e8aa84789825202107f3d4ffe
+size 240590368