aawoo

Signed-off-by: Balazs Horvath <[email protected]>

README.md

@@ -29,7 +29,6 @@ The Yiff Toolkit is a comprehensive set of tools designed to enhance your creati
     - [Create the `training_dir` Directory](#create-the-training_dir-directory)
     - [Grabber](#grabber)
     - [Manual Method](#manual-method)
-  - [LoRA Training Guide](#lora-training-guide)
     - [Installation Tips](#installation-tips)
     - [Pony Training](#pony-training)
       - [Download Pony in Diffusers Format](#download-pony-in-diffusers-format)
@@ -257,7 +256,7 @@ This will put a link to the JSON next to the download button on e621.net and e6a
 
 ---
 
-## LoRA Training Guide
+****## LoRA Training Guide
 
 ### Installation Tips
 

dataset_tools/Convert WebP to PNG.ipynb

@@ -19,7 +19,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Converted C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\iee8nodoji731.webp to C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\iee8nodoji731.png\n"
+      "Converted C:\\Users\\kade\\Desktop\\ayaya\\497f176b92d00fa048723b2df8caa0148ffdf8a7_high.webp to C:\\Users\\kade\\Desktop\\ayaya\\497f176b92d00fa048723b2df8caa0148ffdf8a7_high.png\n",
+      "Converted E:\\training_dir\\_\\2Va3uwI-cQCarU2e7Bz30MSWVDYtejU0ByigESJ0h2A.webp to E:\\training_dir\\_\\2Va3uwI-cQCarU2e7Bz30MSWVDYtejU0ByigESJ0h2A.png\n",
+      "Converted E:\\training_dir\\_\\iee8nodoji731.webp to E:\\training_dir\\_\\iee8nodoji731.png\n"
      ]
     }
    ],
@@ -42,10 +44,12 @@
     "                except Exception as e:\n",
     "                    print(f\"Error converting {webp_path}: {e}\")\n",
     "\n",
-    "#directory = r'E:\\training_dir'\n",
     "directory = r'C:\\Users\\kade\\Desktop\\training_dir_staging'\n",
-    "#directory = r'C:\\Users\\kade\\Desktop\\ayaya'\n",
-    "convert_webp_to_png(directory)"
+    "convert_webp_to_png(directory)\n",
+    "directory = r'C:\\Users\\kade\\Desktop\\ayaya'\n",
+    "convert_webp_to_png(directory)\n",
+    "#directory = r'E:\\training_dir'\n",
+    "#convert_webp_to_png(directory)"
    ]
   }
  ],

dataset_tools/Escape Parentheses.ipynb

@@ -7,7 +7,11 @@
     "## Escape parentheses\n",
     "----\n",
     "\n",
-    "Recursively escape unescaped parentheses in all `.txt` files within the specified directory and its subdirectories. "
+    "This script is designed to process a directory and its subdirectories for text files. For each text file, it reads the content and checks for any unescaped parentheses. If it finds any, it escapes them and writes the modified content back to the file. If any modifications are made, it prints a warning message along with the original and modified content. The directory to be processed is specified by the `directory_path` variable. In this case, it is set to `C:\\Users\\kade\\Desktop\\training_dir_staging`. The script starts processing from this directory. \n",
+    "\n",
+    "The **`escape_parentheses(file_path)`** function takes a file path as an argument, reads the file content, and checks for unescaped parentheses. It uses regular expressions to find unescaped parentheses and escapes them. If the content is modified, it prints a warning message along with the original and modified content. Then, it writes the modified content back to the file.\n",
+    "\n",
+    "The **`process_directory(directory)`** function takes a directory path as an argument and processes it recursively. It uses the `os.walk()` function to iterate over the directory, its subdirectories, and files. For each text file (files ending with `.txt`), it calls the `escape_parentheses(file_path)` function to escape unescaped parentheses. It also processes each subdirectory in the same way. "
    ]
   },
   {
@@ -20,13 +24,21 @@
     "import re\n",
     "\n",
     "def escape_parentheses(file_path):\n",
-    "    with open(file_path, 'r') as file:\n",
+    "    with open(file_path, \"r\") as file:\n",
     "        content = file.read()\n",
+    "        original_content = content\n",
+    "\n",
     "        # Escape unescaped opening parentheses\n",
-    "        content = re.sub(r'(?<!\\\\)(\\()', r'\\\\\\1', content)\n",
+    "        content = re.sub(r\"(?<!\\\\)(\\\\*)(\\\\()\", r\"\\1\\\\\\2\", content)\n",
     "        # Escape unescaped closing parentheses\n",
-    "        content = re.sub(r'(?<!\\\\)(\\))', r'\\\\\\1', content)\n",
-    "    with open(file_path, 'w') as file:\n",
+    "        content = re.sub(r\"(?<!\\\\)(\\\\*)(\\\\))\", r\"\\1\\\\\\2\", content)\n",
+    "\n",
+    "    if content != original_content:\n",
+    "        print(f\"Warning: File '{file_path}' was modified.\")\n",
+    "        print(f\"Original: {original_content}\")\n",
+    "        print(f\"Modified: {content}\")\n",
+    "\n",
+    "    with open(file_path, \"w\") as file:\n",
     "        file.write(content)\n",
     "\n",
     "def process_directory(directory):\n",
@@ -35,11 +47,113 @@
     "            if file.endswith(\".txt\"):\n",
     "                file_path = os.path.join(root, file)\n",
     "                escape_parentheses(file_path)\n",
-    "        # Recurse into subdirectories\n",
     "        for dir in dirs:\n",
     "            process_directory(os.path.join(root, dir))\n",
     "\n",
-    "directory_path = r'C:\\Users\\kade\\Desktop\\training_dir_staging'\n",
+    "directory_path = r\"E:\\training_dir\"\n",
+    "# directory_path = r\"C:\\Users\\kade\\Desktop\\training_dir_staging\"\n",
+    "process_directory(directory_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Needs Testing\n",
+    "\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\1vw3q7q5knu71.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, anthro, clothing, solo, outside, clothed, 3d \\(artwork\\), detailed background, fur, water, black fur, white fur, white body, black body, feet, underwater, in the image anthro male dharrel jex a species characterized by its white and black furred head is depicted in a solo adventure underwater., this canine-like mammal is clothed in dark bottomwear and topwear in the 3d artwork., the jex’s dominantly white fur contrasts with the dark surroundings making it a focal point in the scene., the detailed background reveals ancient city submerged in water its ruins hinting at a forgotten past., small fish and aquatic plants add life to the scene their presence illuminated by the light filtering through the water., the feet of the dharrel are visible suggesting movement and exploration., he has small black and white feathers on his ears., open eyes, purple eyes, there are small air bubbles around the character and a few big ones in the scene.\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, anthro, clothing, solo, outside, clothed, 3d \\\\(artwork\\\\), detailed background, fur, water, black fur, white fur, white body, black body, feet, underwater, in the image anthro male dharrel jex a species characterized by its white and black furred head is depicted in a solo adventure underwater., this canine-like mammal is clothed in dark bottomwear and topwear in the 3d artwork., the jex’s dominantly white fur contrasts with the dark surroundings making it a focal point in the scene., the detailed background reveals ancient city submerged in water its ruins hinting at a forgotten past., small fish and aquatic plants add life to the scene their presence illuminated by the light filtering through the water., the feet of the dharrel are visible suggesting movement and exploration., he has small black and white feathers on his ears., open eyes, purple eyes, there are small air bubbles around the character and a few big ones in the scene.\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\2h164c8us2b61.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, anthro, plant, mammal, outside, clothing, clothed, detailed background, fur, white body, 3d \\(artwork\\), water, male, white fur, black fur, black body, In this 3D artwork a solo anthro male dharrel jex stands outside his black body and white fur contrasting sharply against the detailed background., He is clothed in dark purple topwear that complements his mammal features with fur that glistens from the surrounding water., The scene is set amidst vibrant plants and the character’s white body stands out., open eyes, purple eyes, \n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, anthro, plant, mammal, outside, clothing, clothed, detailed background, fur, white body, 3d \\\\(artwork\\\\), water, male, white fur, black fur, black body, In this 3D artwork a solo anthro male dharrel jex stands outside his black body and white fur contrasting sharply against the detailed background., He is clothed in dark purple topwear that complements his mammal features with fur that glistens from the surrounding water., The scene is set amidst vibrant plants and the character’s white body stands out., open eyes, purple eyes, \n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\3yjns0br1xv31.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, mammal, outside, clothed, fur, 3d \\(artwork\\), beverage, white body, topwear, white fur, container, cup, detailed background, coffee, male, holding object\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, mammal, outside, clothed, fur, 3d \\\\(artwork\\\\), beverage, white body, topwear, white fur, container, cup, detailed background, coffee, male, holding object\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\8174w4ticq431.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, outside, male, detailed background, water, clothed, 3d \\(artwork\\), topwear, fur, pants, bottomwear, tail, white body\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, outside, male, detailed background, water, clothed, 3d \\\\(artwork\\\\), topwear, fur, pants, bottomwear, tail, white body\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\br9aqfq4fzn31.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, solo, rating_safe, 3d \\(artwork\\), mammal, anthro, clothing, hoodie, furniture, looking at viewer, male, clothed, purple eyes, bed, inside, topwear,\n",
+      "Modified: jex, dharrel, feathered ears, solo, rating_safe, 3d \\\\(artwork\\\\), mammal, anthro, clothing, hoodie, furniture, looking at viewer, male, clothed, purple eyes, bed, inside, topwear,\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\dhar_sleeps_2.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, lying on bed, solo, rating_safe, 3d \\(artwork\\), mammal, anthro, furniture, looking at viewer, male, clothed, closed eyes, bed, inside\n",
+      "Modified: jex, dharrel, feathered ears, lying on bed, solo, rating_safe, 3d \\\\(artwork\\\\), mammal, anthro, furniture, looking at viewer, male, clothed, closed eyes, bed, inside\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\fbhwa0y9w4s21.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, anthro, outside, clothing, hoodie, fur, clothed, detailed background, 3d \\(artwork\\), tree, topwear, night, male, water\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, anthro, outside, clothing, hoodie, fur, clothed, detailed background, 3d \\\\(artwork\\\\), tree, topwear, night, male, water\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\freg1n3a2ko21.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, anthro, duo, 3d \\(artwork\\), rating_safe, night, male, outside, kissing, clothing, hair, water, green body, clothed, fur, eyes closed, green hair, felid, female, moon, plant, green fur\n",
+      "Modified: jex, dharrel, feathered ears, anthro, duo, 3d \\\\(artwork\\\\), rating_safe, night, male, outside, kissing, clothing, hair, water, green body, clothed, fur, eyes closed, green hair, felid, female, moon, plant, green fur\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\hxrxro437u461.txt' was modified.\n",
+      "Original: jex, dharrel,feathered ears, rating_safe, solo, mammal, anthro, clothing, purple eyes, city, clothed, fur, 3d \\(artwork\\), detailed background, eyewear, horn, white body, male, looking at viewer, white fur, night, fingers, feathered ears\n",
+      "Modified: jex, dharrel,feathered ears, rating_safe, solo, mammal, anthro, clothing, purple eyes, city, clothed, fur, 3d \\\\(artwork\\\\), detailed background, eyewear, horn, white body, male, looking at viewer, white fur, night, fingers, feathered ears\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\iee8nodoji731.txt' was modified.\n",
+      "Original: jex, dharrel, rating_safe, solo, mammal, anthro, clothing, purple eyes, clothed, fur, detailed background, eyewear, white body, male, looking at viewer, white fur, fingers, holding object, feathered ears, 3d \\(artwork\\)\n",
+      "Modified: jex, dharrel, rating_safe, solo, mammal, anthro, clothing, purple eyes, clothed, fur, detailed background, eyewear, white body, male, looking at viewer, white fur, fingers, holding object, feathered ears, 3d \\\\(artwork\\\\)\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\iqs60ikmcgw21.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, anthro, 3d \\(artwork\\), outside, female, hair, detailed background, tree, water, night, clothing, forest, plant, fur, lamp, solo, lantern, male, clothed\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, anthro, 3d \\\\(artwork\\\\), outside, female, hair, detailed background, tree, water, night, clothing, forest, plant, fur, lamp, solo, lantern, male, clothed\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\jn9rvqwuya241.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, blep, solo, rating_safe, clothing, mammal, hoodie, anthro, fur, clothed, 3d \\(artwork\\), topwear, looking at viewer, outside, tongue, white body, white fur, male\n",
+      "Modified: jex, dharrel, feathered ears, blep, solo, rating_safe, clothing, mammal, hoodie, anthro, fur, clothed, 3d \\\\(artwork\\\\), topwear, looking at viewer, outside, tongue, white body, white fur, male\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\mpcr44i6q0z21.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, solo, rating_safe, anthro, underwater, clothing, water, male, topless, clothed, hair, fur, 3d \\(artwork\\)\n",
+      "Modified: jex, dharrel, feathered ears, solo, rating_safe, anthro, underwater, clothing, water, male, topless, clothed, hair, fur, 3d \\\\(artwork\\\\)\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\promising-weather-second-life-v0-wdbxcxkftsh81.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, clothing, anthro, water, outside, 3d \\(artwork\\), clothed, detailed background, male, fur, standing, topwear, plant, bottomwear, tree, pants, white fur, river, white body\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, clothing, anthro, water, outside, 3d \\\\(artwork\\\\), clothed, detailed background, male, fur, standing, topwear, plant, bottomwear, tree, pants, white fur, river, white body\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\rdjho7a12w781.txt' was modified.\n",
+      "Original: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, mammal, fur, clothed, blurred background, purple eyes, topwear, 3d \\(artwork\\), beverage, outside, holding object, jacket, hair, male, black hair\n",
+      "Modified: jex, dharrel, feathered ears, rating_safe, solo, anthro, clothing, mammal, fur, clothed, blurred background, purple eyes, topwear, 3d \\\\(artwork\\\\), beverage, outside, holding object, jacket, hair, male, black hair\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\s0v2nbu28m031.txt' was modified.\n",
+      "Original: jex, dharrel, rating_safe, anthro, clothing, solo, male, clothed, fur, topwear, 3d \\(artwork\\), bottomwear, shirt, pants, hair, group, white fur, white body, crowd\n",
+      "Modified: jex, dharrel, rating_safe, anthro, clothing, solo, male, clothed, fur, topwear, 3d \\\\(artwork\\\\), bottomwear, shirt, pants, hair, group, white fur, white body, crowd\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\viicura4r3u51.txt' was modified.\n",
+      "Original: jex, feathered ears, dharrel, rating_safe, solo, mammal, clothing, anthro, outside, fur, clothed, 3d \\(artwork\\), detailed background, white fur, topwear, red clothing, hoodie, white body, male, day, tree, red topwear\n",
+      "Modified: jex, feathered ears, dharrel, rating_safe, solo, mammal, clothing, anthro, outside, fur, clothed, 3d \\\\(artwork\\\\), detailed background, white fur, topwear, red clothing, hoodie, white body, male, day, tree, red topwear\n",
+      "Warning: File 'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_dharrel\\z9122r3siwm31.txt' was modified.\n",
+      "Original: jex, dharrel, rating_safe, solo, anthro, 3d \\(artwork\\), clothing, fur, clothed, eyewear, male, glasses, topwear, white fur, photo background, blurred background, feathered ears\n",
+      "Modified: jex, dharrel, rating_safe, solo, anthro, 3d \\\\(artwork\\\\), clothing, fur, clothed, eyewear, male, glasses, topwear, white fur, photo background, blurred background, feathered ears\n"
+     ]
+    }
+   ],
+   "source": [
+    "import re\n",
+    "from pathlib import Path\n",
+    "\n",
+    "def escape_parentheses(file_path):\n",
+    "    with open(file_path, \"r\") as file:\n",
+    "        content = file.read()\n",
+    "        original_content = content\n",
+    "        # Escape unescaped opening parentheses\n",
+    "        content = re.sub(r\"(?<!\\\\)(?<!\\\\\\\\)(\\\\\\\\\\*)\\\\(\", r\"\\\\1\\\\\\\\(\", content)\n",
+    "        # Escape unescaped closing parentheses\n",
+    "        content = re.sub(r\"(?<!\\\\)(?<!\\\\\\\\)(\\\\\\\\\\*)\\\\)\", r\"\\\\1\\\\\\\\)\", content)\n",
+    "        if content != original_content:\n",
+    "            print(f\"Warning: File '{file_path}' was modified.\")\n",
+    "            print(f\"Original: {original_content}\")\n",
+    "            print(f\"Modified: {content}\")\n",
+    "            with open(file_path, \"w\") as file:\n",
+    "                file.write(content)\n",
+    "\n",
+    "def process_directory(directory):\n",
+    "    for file_path in Path(directory).rglob(\"*.txt\"):\n",
+    "        escape_parentheses(str(file_path))\n",
+    "\n",
+    "directory_path = r\"C:\\Users\\kade\\Desktop\\training_dir_staging\"\n",
     "process_directory(directory_path)"
    ]
   }

ponyxl_loras/jex-v2e400.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe6547307a42a853f9cf14bd475fa9b556fe2be600524ef6d17d473b5e3f22c
+size 240576848