{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## e621.net/e6ai.net JSON to `.txt` tags\n", "----\n", "\n", "This Python script is designed to process JSON files found within a specified directory and its subdirectories. Each JSON file is expected to contain data related to image posts, particularly sourced from online platforms such as image boards. The script parses these JSON files, extracts relevant information such as image URL, ratings, and tags, and generates caption files (`.txt`) based on this data.\n", "\n", "Here's a breakdown of what the script does:\n", "\n", "1. **Ignoring Tags**: The script defines a list of tags to be ignored during processing, such as \"hi res\", \"shaded\", etc.\n", "\n", "2. **Processing Files**: The `process_file` function is responsible for processing each JSON file. It reads the JSON data, extracts the URL of the image file, determines its rating, and extracts tags associated with the image.\n", "\n", "3. **Generating Caption File**: For each image, a caption file is generated with the same name as the image file but with a .txt extension. The rating of the image is written first, followed by processed tags.\n", "\n", "4. **Processing Tags**: Tags are processed to replace underscores with spaces and to handle special cases such as artist tags. Ignored tags are filtered out." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "Define tags to be ignored using regular expressions for exact matching\n", "\"\"\"\n", "ignored_tags = [r\"\\bblizzard entertainment\\b\", r\"\\bwarcraft\\b\",\n", " r\"(?:\\d{4})|(?:\\d+:\\d+)\",\n", " r\"\\bdetailed\\b\", r\"\\bwidescreen\\b\", r\"\\b4k\\b\",\n", " r\"\\babsurd res\\b\", r\"\\bhi res\\b\", r\"\\bshaded\\b\", r\"\\bdetailed\\b\",\n", " r\"\\btagme\\b\",\n", " r\"\\bdota\\b\",\n", " r\"\\bcreative commons\\b\", r\"\\bcc-by-nc-nd\\b\",\n", " r\"\\bsquare enix\\b\", r\"\\bfinal fantasy xiv\\b\", r\"\\bfinal fantasy\\b\",\n", " r\"\\bmythological canine\\b\", r\"\\basian mythology\\b\", r\"\\bmythological scalie\\b\",\n", " r\"\\bancient pokemon\\b\", r\"\\bmythological creature\\b\", r\"\\blegendary pokemon\\b\",\n", " r\"\\bfelis\\b\", r\"\\bfelid\\b\",\n", " r\"\\bsega\\b\",\n", " r\"\\bhasbro\\b\",\n", " r\"\\bzootopia\\b\",\n", " r\"\\bfive nights at freddy's\\b\",\n", " r\"\\beeveelution\\b\",\n", " r\"\\bdisney\\b\",\n", " r\"\\bmammal\\b\", r\"\\bcanis\\b\", r\"\\bcanine\\b\", r\"\\bcanid\\b\",\n", " r\"\\bdigimon\\b\", r\"\\bbandai namco\\b\",\n", " r\"\\bpokemon (species)\\b\",\n", " r\"\\bpal (species)\\b\",\n", " r\"\\bpokemon\\b\", r\"\\bnintendo\\b\",\n", " r\"\\\\bby conditional dnp\\\\b\",\n", " r\"\\\\bconditional dnp\\\\b\",\n", " r\"\\\\bconditional_dnp\\\\b\",\n", " r\"\\\\bby\\\\s+conditional\\\\s+dnp\\\\b\",\n", " r\"\\bgeneration\\s+\\d+\\s+pokemon\\b\",\n", "]" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\package-lock.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\package-lock.json\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\package.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\package.json\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\2_anthro_weasel\\3051834.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\2_anthro_weasel\\\u001b[0m\u001b[1;36m3051834.j\u001b[0m\u001b[1mson\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃                  E:\\training_dir_staging\\2_anthro_weasel\\8a5ed4f684745414e5e57b2e3c9276ff.txt                   ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│    4 toes, anthro, balls, bandanna, blush, brown body, brown fur, chastity cage, chastity device, clothing,     │\n",
       "│  dialogue, dildo, exclamation point, feet, fur, genitals, hair, hindpaw, kerchief, legs up, lying, male, male   │\n",
       "│  wearing strapon, masturbation, mostly nude, offscreen character, on back, pawpads, paws, penetrable sex toy,   │\n",
       "│    penis, pink background, pouting, question mark, sex toy, simple background, solo, speech bubble, strapon,    │\n",
       "│    strapon masturbation, strapon over chastity, strapon over penis, tan body, tan fur, teasing, text, toes,     │\n",
       "│        underwear, white hair, by bittenbun, merlin \\(lllmaddy\\), digital media \\(artwork\\), english text        │\n",
       "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1;35m \u001b[0m\u001b[1;35m E:\\training_dir_staging\\2_anthro_weasel\\8a5ed4f684745414e5e57b2e3c9276ff.txt \u001b[0m\u001b[1;35m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", "│ 4 toes, anthro, balls, bandanna, blush, brown body, brown fur, chastity cage, chastity device, clothing, │\n", "│ dialogue, dildo, exclamation point, feet, fur, genitals, hair, hindpaw, kerchief, legs up, lying, male, male │\n", "│ wearing strapon, masturbation, mostly nude, offscreen character, on back, pawpads, paws, penetrable sex toy, │\n", "│ penis, pink background, pouting, question mark, sex toy, simple background, solo, speech bubble, strapon, │\n", "│ strapon masturbation, strapon over chastity, strapon over penis, tan body, tan fur, teasing, text, toes, │\n", "│ underwear, white hair, by bittenbun, merlin \\(lllmaddy\\), digital media \\(artwork\\), english text │\n", "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\2_anthro_weasel\\3584444.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\2_anthro_weasel\\\u001b[0m\u001b[1;36m3584444.j\u001b[0m\u001b[1mson\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃                  E:\\training_dir_staging\\2_anthro_weasel\\ecff451612ee44a571db893d44c910ce.txt                   ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│  anthro, anus, athletic, athletic anthro, athletic male, backsack, balls, big penis, butt, chandelier, claws,   │\n",
       "│  crouching, erection, eyebrows, fur, genitals, hair, humanoid genitalia, humanoid penis, imminent facesitting,  │\n",
       "│  indoor nudity, inside, licking, licking lips, licking own lips, looking at viewer, looking down, looking down  │\n",
       "│ at viewer, low-angle view, male, nipples, nude, nude anthro, nude male, open mouth, open smile, pawpads, pecs,  │\n",
       "│   penis, perineum, presenting, presenting anus, presenting hindquarters, presenting penis, self lick, smile,    │\n",
       "│           solo, teeth, thick thighs, tongue, tongue out, tuft, worm's-eye view, by kihu, barnaby kane           │\n",
       "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1;35m \u001b[0m\u001b[1;35m E:\\training_dir_staging\\2_anthro_weasel\\ecff451612ee44a571db893d44c910ce.txt \u001b[0m\u001b[1;35m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", "│ anthro, anus, athletic, athletic anthro, athletic male, backsack, balls, big penis, butt, chandelier, claws, │\n", "│ crouching, erection, eyebrows, fur, genitals, hair, humanoid genitalia, humanoid penis, imminent facesitting, │\n", "│ indoor nudity, inside, licking, licking lips, licking own lips, looking at viewer, looking down, looking down │\n", "│ at viewer, low-angle view, male, nipples, nude, nude anthro, nude male, open mouth, open smile, pawpads, pecs, │\n", "│ penis, perineum, presenting, presenting anus, presenting hindquarters, presenting penis, self lick, smile, │\n", "│ solo, teeth, thick thighs, tongue, tongue out, tuft, worm's-eye view, by kihu, barnaby kane │\n", "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\2_anthro_weasel\\4264190.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\2_anthro_weasel\\\u001b[0m\u001b[1;36m4264190.j\u001b[0m\u001b[1mson\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃                  E:\\training_dir_staging\\2_anthro_weasel\\3394ff7098766b5d13ff9728f8178895.txt                   ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│    4 toes, after sex, anthro, balls, beach, beach towel, beach umbrella, bedroom eyes, being watched, bodily    │\n",
       "│ fluids, butt, claws, cloud, crowd, cum, cum covered, cum in ass, cum inside, cum on anus, cum on belly, cum on  │\n",
       "│ body, cum on butt, cum on chest, cum on face, cum on feet, cum on head, cum on leg, cum on paw, cum on pawpads, │\n",
       "│ cum on penis, cum on tail, erection, excessive cum, excessive genital fluids, exhibitionism, feet, fur, genital │\n",
       "│    fluids, genitals, group, half-closed eyes, humanoid genitalia, humanoid penis, inner ear fluff, legs up,     │\n",
       "│ looking at another, looking at viewer, male, male anthro, messy, narrowed eyes, nude, orange body, orange fur,  │\n",
       "│  outdoor nudity, outside, parasol, pawpads, paws, penis, pinup, pose, presenting, presenting balls, presenting  │\n",
       "│  penis, public, public nudity, realistic penis size, sand, seaside, seductive, sky, smile, smiling at viewer,   │\n",
       "│            solo, spread legs, spreading, tail, toe claws, toes, towel, tuft, whiskers, by oxfort2199            │\n",
       "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1;35m \u001b[0m\u001b[1;35m E:\\training_dir_staging\\2_anthro_weasel\\3394ff7098766b5d13ff9728f8178895.txt \u001b[0m\u001b[1;35m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", "│ 4 toes, after sex, anthro, balls, beach, beach towel, beach umbrella, bedroom eyes, being watched, bodily │\n", "│ fluids, butt, claws, cloud, crowd, cum, cum covered, cum in ass, cum inside, cum on anus, cum on belly, cum on │\n", "│ body, cum on butt, cum on chest, cum on face, cum on feet, cum on head, cum on leg, cum on paw, cum on pawpads, │\n", "│ cum on penis, cum on tail, erection, excessive cum, excessive genital fluids, exhibitionism, feet, fur, genital │\n", "│ fluids, genitals, group, half-closed eyes, humanoid genitalia, humanoid penis, inner ear fluff, legs up, │\n", "│ looking at another, looking at viewer, male, male anthro, messy, narrowed eyes, nude, orange body, orange fur, │\n", "│ outdoor nudity, outside, parasol, pawpads, paws, penis, pinup, pose, presenting, presenting balls, presenting │\n", "│ penis, public, public nudity, realistic penis size, sand, seaside, seductive, sky, smile, smiling at viewer, │\n", "│ solo, spread legs, spreading, tail, toe claws, toes, towel, tuft, whiskers, by oxfort2199 │\n", "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
Processing file: E:\\training_dir_staging\\2_anthro_weasel\\521244.json\n",
       "
\n" ], "text/plain": [ "Processing file: \u001b[1mE:\\training_dir_staging\\2_anthro_weasel\\\u001b[0m\u001b[1;36m521244.j\u001b[0m\u001b[1mson\u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃                  E:\\training_dir_staging\\2_anthro_weasel\\12e58e17f13c0e71cbe40059fb8cc169.txt                   ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│ 4 toes, anal, anal penetration, animal genitalia, animal penis, anthro, anthro on anthro, balls, bodily fluids, │\n",
       "│  canine genitalia, canine penis, claws, cum, cum in ass, cum inside, digitigrade, duo, erection, eyes closed,   │\n",
       "│  eyewear, feet, finger fetish, finger in mouth, finger play, fingers, fur, genital fluids, genitals, glasses,   │\n",
       "│     hindpaw, humanoid genitalia, humanoid penis, knot, licking, licking lips, lying, male, male/male, male      │\n",
       "│ penetrated, male penetrating, male penetrating male, nude, on back, pawpads, paws, penetration, penis, precum,  │\n",
       "│    self lick, sex, slightly chubby, soles, spread legs, spreading, toes, tongue, tongue out, white balls, by    │\n",
       "│                       conditional dnp, by garnetto, arcshep, tre, tre \\(milligram smile\\)                       │\n",
       "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1;35m \u001b[0m\u001b[1;35m E:\\training_dir_staging\\2_anthro_weasel\\12e58e17f13c0e71cbe40059fb8cc169.txt \u001b[0m\u001b[1;35m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", "│ 4 toes, anal, anal penetration, animal genitalia, animal penis, anthro, anthro on anthro, balls, bodily fluids, │\n", "│ canine genitalia, canine penis, claws, cum, cum in ass, cum inside, digitigrade, duo, erection, eyes closed, │\n", "│ eyewear, feet, finger fetish, finger in mouth, finger play, fingers, fur, genital fluids, genitals, glasses, │\n", "│ hindpaw, humanoid genitalia, humanoid penis, knot, licking, licking lips, lying, male, male/male, male │\n", "│ penetrated, male penetrating, male penetrating male, nude, on back, pawpads, paws, penetration, penis, precum, │\n", "│ self lick, sex, slightly chubby, soles, spread legs, spreading, toes, tongue, tongue out, white balls, by │\n", "│ conditional dnp, by garnetto, arcshep, tre, tre \\(milligram smile\\) │\n", "└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import os\n", "import glob\n", "import re\n", "import json\n", "from rich.console import Console\n", "from rich.table import Table\n", "\n", "console = Console()\n", "\n", "\n", "def should_ignore_tag(tag, all_tags):\n", " for ignored_tag_pattern in ignored_tags:\n", " pattern = re.compile(ignored_tag_pattern, re.VERBOSE | re.IGNORECASE)\n", " if any(re.search(pattern, t) for t in all_tags):\n", " return True\n", " return False\n", "\n", "\n", "def process_tags(tags_dict):\n", " processed_tags = []\n", " for category, tags_list in tags_dict.items():\n", " category_tags = []\n", " if category == \"artist\":\n", " category_tags = [\n", " f\"by {tag.replace('_', ' ').replace(' (artist)', '')}\"\n", " for tag in tags_list\n", " if tag\n", " ]\n", " else:\n", " for tag in tags_list:\n", " tag = tag.replace(\"_\", \" \")\n", " tag = re.sub(r\"(?