Update

Signed-off-by: Balazs Horvath <[email protected]>

.zshrc

@@ -769,6 +769,52 @@ chop_lora() {
     done
 }
 
+# Function cs1
+# This function chops blocks from an SDXL LoRA's safetensors file to preserve the style information only.
+# It uses a specific block configuration and saves the output with a modified filename.
+cs1() {
+    # Get the target safetensors file path from the first argument
+    local target_safetensors=$1
+    
+    # Extract the base name of the target safetensors file (without the .safetensors extension)
+    local base_name=$(basename "$target_safetensors" .safetensors)
+    
+    # Extract the version and step string from the base name (e.g., v1s400)
+    local version_step=$(echo "$base_name" | grep -o 'v[0-9]*s[0-9]*')
+    
+    # Remove the version and step string from the base name to avoid duplication
+    local base_name_no_version=$(echo "$base_name" | sed "s/${version_step}//")
+    
+    # Construct the output safetensors filename by appending c1 to the version and step string
+    local output_safetensors="${base_name_no_version}${version_step}c1.safetensors"
+    
+    # Run the chop_blocks command with the specified block configuration and output filename
+    ~/toolkit/chop_blocks "$target_safetensors" 1,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,1,1,0,0,0 -o "$output_safetensors"
+}
+
+# Function cs2
+# This function chops blocks from an SDXL LoRA's safetensors file to preserve the style information only.
+# It uses a different block configuration and saves the output with a modified filename.
+cs2() {
+    # Get the target safetensors file path from the first argument
+    local target_safetensors=$1
+    
+    # Extract the base name of the target safetensors file (without the .safetensors extension)
+    local base_name=$(basename "$target_safetensors" .safetensors)
+    
+    # Extract the version and step string from the base name (e.g., v1s400)
+    local version_step=$(echo "$base_name" | grep -o 'v[0-9]*s[0-9]*')
+    
+    # Remove the version and step string from the base name to avoid duplication
+    local base_name_no_version=$(echo "$base_name" | sed "s/${version_step}//")
+    
+    # Construct the output safetensors filename by appending c2 to the version and step string
+    local output_safetensors="${base_name_no_version}${version_step}c2.safetensors"
+    
+    # Run the chop_blocks command with the specified block configuration and output filename
+    ~/toolkit/chop_blocks "$target_safetensors" 1,0,0,0,1,1,0,1,1,0,0,0,1,1,1,1,1,1,0,0,0 -o "$output_safetensors"
+}
+
 # Function: swch (Switch Git Branch)
 # Description:
 #   This function facilitates switching between Git branches while ensuring a clean working directory.

crawl/crawl

@@ -125,7 +125,7 @@ def save_result(target_url):
     # Choose the appropriate base path based on the operating system
     if platform.system() == "Windows":
         base_path = "E:\\datasets\\knowledgebase\\Saved Websites\\"
-    elsek:
+    else:
         base_path = "/home/kade/datasets/knowledgebase/Saved Websites"
 
     save_dir = os.path.join(base_path, sanitized_title)

paper-qa.code-workspace

@@ -1,11 +0,0 @@
-{
-	"folders": [
-		{
-			"path": "."
-		},
-		{
-			"path": "../miniconda3/lib/python3.12/site-packages/paperqa"
-		}
-	],
-	"settings": {}
-}

stats

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+from collections import Counter
+
+# ANSI color codes
+
+RED = "\033[91m"
+GREEN = "\033[92m"
+ORANGE = "\033[93m"
+BLUE = "\033[94m"
+MAGENTA = "\033[95m"
+CYAN = "\033[96m"
+RESET = "\033[0m"
+
+EXT2COLOR = {
+    "jxl": CYAN,
+    "png": MAGENTA,
+    "jpg": RED,
+    "jpeg": RED,
+    "webp": MAGENTA,
+    "caption": BLUE,
+    "txt": BLUE,
+}
+EXT2ORDER = {ext: i for i, ext in enumerate(EXT2COLOR.keys())}
+SORT_KEYS = ["name", "count", "image", "text", *EXT2COLOR.keys()]
+
+TEXT_FORMATS = {"txt", "caption"}
+IMAGE_FORMATS = EXT2COLOR.keys() - TEXT_FORMATS
+
+
+def count_files(directory):
+    file_counts = Counter()
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            base_name, ext = os.path.splitext(file)
+            if len(ext) > 1:
+                ext = ext[1:]
+                file_counts[ext] += 1
+                if ext in IMAGE_FORMATS:
+                    file_counts["image"] += 1
+                elif ext in TEXT_FORMATS:
+                    file_counts["text"] += 1
+
+    return file_counts
+
+
+def main():
+    sort_key_name = "name"
+    sort_reverse = False
+    if len(sys.argv) > 1:
+        sort_key_name = sys.argv[1]
+    if sort_key_name.endswith("_r"):
+        sort_reverse = True
+        sort_key_name = sort_key_name[:-2]
+
+    if sort_key_name == "name":
+        sort_key = lambda x: x[0]
+    elif sort_key_name == "count":
+        sort_key = lambda x: x[1]
+    elif sort_key_name in SORT_KEYS:
+        sort_key = lambda x: x[2].get(sort_key_name, 0)
+    else:
+        print(f'Valid short key are {", ".join(f'"{k}"' for k in SORT_KEYS)}')
+        print('Prepending "_r" to reverse the sort order')
+        sys.exit(1)
+
+    current_directory = os.getcwd()
+    directories = (
+        d
+        for d in os.listdir(current_directory)
+        if os.path.isdir(os.path.join(current_directory, d))
+    )
+
+    stats = []
+    grand_total = Counter()
+    for directory in directories:
+        dir_path = os.path.join(current_directory, directory)
+        counts = count_files(dir_path)
+        total_files = sum(v for k,v in counts.items() if k in EXT2ORDER)
+        stats.append((directory, total_files, counts))
+        grand_total.update(counts)
+
+    stats.sort(key=sort_key, reverse=sort_reverse)
+    stats.append((None, sum(v for k,v in grand_total.items() if k in EXT2ORDER), grand_total))
+
+    for directory, total_files, counts in stats:
+        if total_files == 0:
+            continue
+        if directory is None:
+            print(f'Grand Total: ')
+        print(f"Directory: {directory}")
+        for ext, count in sorted(
+            counts.items(), key=lambda x: EXT2ORDER.get(x[0], -1)
+        ):
+            if counts[ext] == 0 or ext not in EXT2COLOR:
+                continue
+            print(f"{EXT2COLOR[ext]}{ext} files: {counts[ext]}{RESET}")
+        tally_color = GREEN if total_files >= 200 else ORANGE
+        print(
+            f"{tally_color}Total files: {total_files}{RESET} ({counts['image']} images, {counts['text']} texts)"
+        )
+        print()
+
+
+if __name__ == "__main__":
+    main()