k4d3
/

yiff_toolkit

Diffusers

TensorBoard

English

Not-For-All-Audiences

Model card Files Files and versions Metrics Training metrics Community

k4d3 commited on Apr 22

Commit

85d0442

•

1 Parent(s): 7c5072b

awoo

Browse files

Signed-off-by: Balazs Horvath <[email protected]>

Files changed (1) hide show

dataset_tools/e621 JSON to txt.ipynb +12 -16

dataset_tools/e621 JSON to txt.ipynb CHANGED Viewed

@@ -22,16 +22,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 55,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Define tags to be ignored using regular expressions for exact matching\n",
     "ignored_tags = [r\"\\bblizzard entertainment\\b\", r\"\\bwarcraft\\b\",\n",
-    "    r\"\\b1998\\b\", r\"\\b1999\\b\", r\"\\b2000\\b\",  r\"\\b2001\\b\", r\"\\b2002\\b\", r\"\\b2003\\b\", r\"\\b2004\\b\",\n",
-    "    r\"\\b2005\\b\", r\"\\b2006\\b\", r\"\\b2007\\b\",  r\"\\b2008\\b\", r\"\\b2009\\b\", r\"\\b2010\\b\", r\"\\b2011\\b\",\n",
-    "    r\"\\b2012\\b\", r\"\\b2013\\b\", r\"\\b2014\\b\",  r\"\\b2015\\b\", r\"\\b2016\\b\", r\"\\b2017\\b\", r\"\\b2018\\b\",\n",
-    "    r\"\\b2019\\b\", r\"\\b2020\\b\", r\"\\b2021\\b\",  r\"\\b2022\\b\", r\"\\b2023\\b\", r\"\\b2024\\b\",\n",
     "    r\"\\bdetailed\\b\", r\"\\bwidescreen\\b\", r\"\\b4k\\b\",\n",
     "    r\"\\babsurd res\\b\", r\"\\bhi res\\b\", r\"\\bshaded\\b\",  r\"\\bdetailed\\b\",\n",
     "    r\"\\btagme\\b\",\n",
@@ -50,16 +47,17 @@
     "    r\"\\bmammal\\b\", r\"\\bcanis\\b\", r\"\\bcanine\\b\", r\"\\bcanid\\b\",\n",
     "    r\"\\bdigimon\\b\", r\"\\bbandai namco\\b\",\n",
     "    r\"\\bpokemon \\(species\\)\\b\", r\"\\bpokemon\\b\", r\"\\bnintendo\\b\",\n",
-    "    r\"\\b5:4\\b\", r\"\\b5:3\\b\", r\"\\b4:5\\b\", r\"\\b1:1\\b\", r\"\\b5:6\\b\", r\"\\b6:5\\b\", r\"\\b16:9\\b\", r\"\\b3:4\\b\",\n",
     "    r\"\\\\bby conditional dnp\\\\b\",\n",
     "    r\"\\\\bconditional dnp\\\\b\",\n",
     "    r\"\\\\bconditional_dnp\\\\b\",\n",
-    "    r\"\\\\bby\\\\s+conditional\\\\s+dnp\\\\b\" ]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
    "metadata": {},
    "outputs": [
     {
@@ -1458,8 +1456,7 @@
        "│     fur, cervine genitalia, cervine penis, duo, erection, eyes closed, fur, genitals, grey body, grey fur,      │\n",
        "│    interspecies, knotting, lying, male, male/male, male penetrated, male penetrating, male penetrating male,    │\n",
        "│     nipples, nude, on back, penetration, penile, penile penetration, penis, penis in ass, sex, table lotus      │\n",
-       "│ position, by peregrine pegs, beastars, legoshi \\(beastars\\), louis \\(beastars\\), cervine, deer, red deer, wolf, │\n",
-       "│                                                       3:2                                                       │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
@@ -1472,8 +1469,7 @@
        "│     fur, cervine genitalia, cervine penis, duo, erection, eyes closed, fur, genitals, grey body, grey fur,      │\n",
        "│    interspecies, knotting, lying, male, male/male, male penetrated, male penetrating, male penetrating male,    │\n",
        "│     nipples, nude, on back, penetration, penile, penile penetration, penis, penis in ass, sex, table lotus      │\n",
-       "│ position, by peregrine pegs, beastars, legoshi \\(beastars\\), louis \\(beastars\\), cervine, deer, red deer, wolf, │\n",
-       "│                                                       3:2                                                       │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },
@@ -3130,7 +3126,7 @@
        "│  penetration, penile, penile penetration, penis, penis in ass, plant, quadruped, ranged weapon, rifle, saliva,  │\n",
        "│  scut tail, sex, sheath, short tail, standing, standing sex, tail, teeth, thrusting, tongue, tongue out, tree,  │\n",
        "│    weapon, by backlash91, by conditional dnp, bambi \\(film\\), the great prince of the forest, cervine, deer,    │\n",
-       "│                                             human, 4:3, artist name                                             │\n",
        "└────────────��────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
@@ -3149,7 +3145,7 @@
        "│  penetration, penile, penile penetration, penis, penis in ass, plant, quadruped, ranged weapon, rifle, saliva,  │\n",
        "│  scut tail, sex, sheath, short tail, standing, standing sex, tail, teeth, thrusting, tongue, tongue out, tree,  │\n",
        "│    weapon, by backlash91, by conditional dnp, bambi \\(film\\), the great prince of the forest, cervine, deer,    │\n",
-       "│                                             human, 4:3, artist name                                             │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },
@@ -3371,7 +3367,7 @@
        "│    piercing, plant, predator/prey, restraints, rope, rope bondage, sex, sheath, side view, size difference,     │\n",
        "│ smaller male, smile, spread legs, spreading, standing, submissive, submissive male, tree, tree bondage, wrists  │\n",
        "│  together, by conditional dnp, by xenoforge, ruska, tanner james, cervine, cross fox, deer, elk, fox, red fox,  │\n",
-       "│                                                  true fox, 4:3                                                  │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
@@ -3387,7 +3383,7 @@
        "│    piercing, plant, predator/prey, restraints, rope, rope bondage, sex, sheath, side view, size difference,     │\n",
        "│ smaller male, smile, spread legs, spreading, standing, submissive, submissive male, tree, tree bondage, wrists  │\n",
        "│  together, by conditional dnp, by xenoforge, ruska, tanner james, cervine, cross fox, deer, elk, fox, red fox,  │\n",
-       "│                                                  true fox, 4:3                                                  │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },

   },
   {
    "cell_type": "code",
+   "execution_count": 57,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Define tags to be ignored using regular expressions for exact matching\n",
     "ignored_tags = [r\"\\bblizzard entertainment\\b\", r\"\\bwarcraft\\b\",\n",
+    "    r\"(?:\\d{4})|(?:\\d+:\\d+)\",\n",
     "    r\"\\bdetailed\\b\", r\"\\bwidescreen\\b\", r\"\\b4k\\b\",\n",
     "    r\"\\babsurd res\\b\", r\"\\bhi res\\b\", r\"\\bshaded\\b\",  r\"\\bdetailed\\b\",\n",
     "    r\"\\btagme\\b\",\n",
     "    r\"\\bmammal\\b\", r\"\\bcanis\\b\", r\"\\bcanine\\b\", r\"\\bcanid\\b\",\n",
     "    r\"\\bdigimon\\b\", r\"\\bbandai namco\\b\",\n",
     "    r\"\\bpokemon \\(species\\)\\b\", r\"\\bpokemon\\b\", r\"\\bnintendo\\b\",\n",
     "    r\"\\\\bby conditional dnp\\\\b\",\n",
     "    r\"\\\\bconditional dnp\\\\b\",\n",
     "    r\"\\\\bconditional_dnp\\\\b\",\n",
+    "    r\"\\\\bby\\\\s+conditional\\\\s+dnp\\\\b\",\n",
+    "    r\"\\\\bgeneration\\s+\\d+\\s+pokemon\\\\b\",\n",
+    "]"
    ]
   },
   {
    "cell_type": "code",
+   "execution_count": 58,
    "metadata": {},
    "outputs": [
     {
        "│     fur, cervine genitalia, cervine penis, duo, erection, eyes closed, fur, genitals, grey body, grey fur,      │\n",
        "│    interspecies, knotting, lying, male, male/male, male penetrated, male penetrating, male penetrating male,    │\n",
        "│     nipples, nude, on back, penetration, penile, penile penetration, penis, penis in ass, sex, table lotus      │\n",
+       "│ position, by peregrine pegs, beastars, legoshi \\(beastars\\), louis \\(beastars\\), cervine, deer, red deer, wolf  │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
        "│     fur, cervine genitalia, cervine penis, duo, erection, eyes closed, fur, genitals, grey body, grey fur,      │\n",
        "│    interspecies, knotting, lying, male, male/male, male penetrated, male penetrating, male penetrating male,    │\n",
        "│     nipples, nude, on back, penetration, penile, penile penetration, penis, penis in ass, sex, table lotus      │\n",
+       "│ position, by peregrine pegs, beastars, legoshi \\(beastars\\), louis \\(beastars\\), cervine, deer, red deer, wolf  │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },
        "│  penetration, penile, penile penetration, penis, penis in ass, plant, quadruped, ranged weapon, rifle, saliva,  │\n",
        "│  scut tail, sex, sheath, short tail, standing, standing sex, tail, teeth, thrusting, tongue, tongue out, tree,  │\n",
        "│    weapon, by backlash91, by conditional dnp, bambi \\(film\\), the great prince of the forest, cervine, deer,    │\n",
+       "│                                               human, artist name                                                │\n",
        "└────────────��────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
        "│  penetration, penile, penile penetration, penis, penis in ass, plant, quadruped, ranged weapon, rifle, saliva,  │\n",
        "│  scut tail, sex, sheath, short tail, standing, standing sex, tail, teeth, thrusting, tongue, tongue out, tree,  │\n",
        "│    weapon, by backlash91, by conditional dnp, bambi \\(film\\), the great prince of the forest, cervine, deer,    │\n",
+       "│                                               human, artist name                                                │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },
        "│    piercing, plant, predator/prey, restraints, rope, rope bondage, sex, sheath, side view, size difference,     │\n",
        "│ smaller male, smile, spread legs, spreading, standing, submissive, submissive male, tree, tree bondage, wrists  │\n",
        "│  together, by conditional dnp, by xenoforge, ruska, tanner james, cervine, cross fox, deer, elk, fox, red fox,  │\n",
+       "│                                                    true fox                                                     │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
        "</pre>\n"
       ],
        "│    piercing, plant, predator/prey, restraints, rope, rope bondage, sex, sheath, side view, size difference,     │\n",
        "│ smaller male, smile, spread legs, spreading, standing, submissive, submissive male, tree, tree bondage, wrists  │\n",
        "│  together, by conditional dnp, by xenoforge, ruska, tanner james, cervine, cross fox, deer, elk, fox, red fox,  │\n",
+       "│                                                    true fox                                                     │\n",
        "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n"
       ]
      },