els_journal / quran.py
neuralworm's picture
rollback
3b58069
import logging
logger = logging.getLogger(__name__)
import json
import os
import re
from deep_translator import GoogleTranslator
from gematria import calculate_gematria
import math
import logging
import json
import re
import math
import glob
from deep_translator import GoogleTranslator
from gematria import calculate_gematria # Ensure this function is correctly defined
import csv
# Configure the logger
# You can uncomment the next line to enable debugging logs
# logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s')
logger = logging.getLogger(__name__)
def process_json_files(start=1, end=114, step=1, rounds="1", length=0, tlang="en", strip_spaces=True,
strip_in_braces=True, strip_diacritics=True, translate=False):
"""
Processes Quran JSON files and performs various text manipulations.
Parameters:
- start (int): Start number of the Quran sura (1-114).
- end (int): End number of the Quran sura (1-114).
- step (int): Step size for character selection.
- rounds (str): Comma-separated list of round numbers (can include negative values).
- length (int): Maximum length of the result text.
- tlang (str): Target language for translation.
- strip_spaces (bool): Whether to remove spaces from the text.
- strip_in_braces (bool): Whether to remove text within braces.
- strip_diacritics (bool): Whether to remove diacritics from the text.
- translate (bool): Whether to translate the result text.
Returns:
- list: A list of dictionaries containing processed data or error messages.
"""
base_path = "texts/quran"
translator = GoogleTranslator(source='ar', target=tlang) if translate else None
results = []
for i in range(start, end + 1):
file_name = f"{base_path}/{i:03}.json" # Updated file name formatting
try:
with open(file_name, 'r', encoding='utf-8') as file:
data = json.load(file)
# Extract text from verses
full_text = ""
for verse_key, verse_text in data.get("verse", {}).items():
full_text += verse_text + " "
# Remove BOM if present
full_text = full_text.replace("\ufeff", "")
clean_text = full_text
if strip_in_braces:
# Remove content within [], {}, <> brackets
clean_text = re.sub(r"\[.*?\]|\{.*?\}|\<.*?\>", "", clean_text, flags=re.DOTALL)
if strip_diacritics:
# Remove Arabic diacritics and punctuation
clean_text = re.sub(
r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED]+", "",
clean_text)
# Optionally, remove additional unwanted characters
clean_text = re.sub(r'[:?!\'\-]', '', clean_text)
# Normalize spaces
clean_text = clean_text.replace("\n\n ", " ")
clean_text = clean_text.replace("\n", " ")
clean_text = re.sub(r'\s+', ' ', clean_text).strip()
if strip_spaces:
clean_text = clean_text.replace(" ", "")
text_length = len(clean_text)
logger.debug(f"Sura {i}: Clean text length = {text_length}")
if text_length == 0:
logger.warning(f"Sura {i}: No text available after cleaning.")
continue # Skip processing if no text is available
# Parse 'rounds', allow floats
try:
rounds_list = list(map(float, rounds.split(',')))
except ValueError as e:
logger.error(f"Sura {i}: Invalid 'rounds' format: {rounds}")
results.append({"error": f"Sura {i}: Invalid 'rounds' format: {rounds}"})
continue # Skip this Sura due to invalid 'rounds' input
result_text = ""
for r in rounds_list:
abs_r = abs(r)
# Determine the number of full passes and the remainder.
full_passes = math.floor(abs_r)
remainder = abs_r - full_passes
# Base number of characters per pass
base_chars = text_length // step
if base_chars == 0:
if abs_r > 1: # Changed from >=1 to >1
# When step > text_length and rounds >1, pick 1 character per full pass
chars_per_full_pass = 1
logger.debug(f"Sura {i}: step > text_length ({step} > {text_length}), selecting 1 character per full pass.")
else:
# No characters to pick
chars_per_full_pass = 0
logger.debug(f"Sura {i}: step > text_length ({step} > {text_length}) and rounds <=1, no characters selected.")
# For remainder, since base_chars=0, no remainder characters
chars_for_remainder = 0
else:
# Normal case
chars_per_full_pass = base_chars
chars_for_remainder = math.floor(base_chars * remainder) # Partial pass
logger.debug(f"Sura {i}: Normal case, chars_per_full_pass = {chars_per_full_pass}, chars_for_remainder = {chars_for_remainder}")
if r > 0:
current_index = (step - 1) % text_length
direction = 1
else:
current_index = (text_length - step) % text_length
direction = -1
pass_result = ""
# Full passes, keep only the last pass
for pass_num in range(1, full_passes + 1):
current_pass_chars = ""
for _ in range(chars_per_full_pass):
if chars_per_full_pass == 0:
break
current_pass_chars += clean_text[current_index]
current_index = (current_index + direction * step) % text_length
# Keep only the last full pass
if pass_num == full_passes:
pass_result = current_pass_chars
logger.debug(f"Sura {i}: Pass {pass_num}, pass_result = {pass_result}")
# Remainder pass for fractional rounds
if remainder > 0 and chars_for_remainder > 0:
current_pass_chars = ""
for _ in range(chars_for_remainder):
current_pass_chars += clean_text[current_index]
current_index = (current_index + direction * step) % text_length
pass_result += current_pass_chars
logger.debug(f"Sura {i}: Remainder pass_result = {pass_result}")
# Handle cases where step > text_length and chars_per_full_pass=1
if base_chars == 0 and chars_per_full_pass == 1 and full_passes > 0:
# pass_result already contains the last character picked
pass
elif base_chars == 0 and chars_per_full_pass == 0 and full_passes > 0:
# When no characters are picked, skip appending
pass
result_text += pass_result
# Translate the result text if required
try:
translated_text = translator.translate(result_text) if translator and result_text else ""
except Exception as e:
logger.error(f"Translation error: {e}")
translated_text = ""
if length != 0:
result_text = result_text[:length]
logger.debug(f"Sura {i}: Result text truncated to length {length}.")
# Calculate the Gematria sum
try:
result_sum = calculate_gematria(result_text)
except Exception as e:
logger.error(f"Sura {i}: Gematria calculation error: {e}")
result_sum = None
if result_text:
results.append({
"book": f"Quran {i}.",
"title": data.get("name", "Unknown Title"), # Use "name" instead of "title"
"result_text": result_text,
"result_sum": result_sum,
"translated_text": translated_text,
"source_language": "ar"
})
except FileNotFoundError:
logger.error(f"File {file_name} not found.")
results.append({"error": f"File {file_name} not found."})
except json.JSONDecodeError as e:
logger.error(f"File {file_name} could not be read as JSON: {e}")
results.append({"error": f"File {file_name} could not be read as JSON: {e}"})
except KeyError as e:
logger.error(f"Expected key 'verse' is missing in {file_name}: {e}")
results.append({"error": f"Expected key 'verse' is missing in {file_name}: {e}"})
except Exception as e:
logger.error(f"An unexpected error occurred while processing {file_name}: {e}")
results.append({"error": f"An unexpected error occurred while processing {file_name}: {e}"})
return results if results else None
# Tests
test_results = [
#(process_json_files(0, 0, 21, rounds="3", length=0), "שרק"),
]
all_tests_passed = True
for result, expected in test_results:
if expected is None: # Check if no result is expected
if not result:
logger.info(f"Test passed: Expected no results, got no results.")
else:
logger.error(f"Test failed: Expected no results, but got: {result}")
all_tests_passed = False
else:
# Check if result is not empty before accessing elements
if result:
#result_text = result[0]['result_text']
result_text = None
if result_text == expected:
logger.info(f"Test passed: Expected '{expected}', got '{result_text}'")
else:
logger.error(f"Test failed: Expected '{expected}', but got '{result_text}'")
all_tests_passed = False
else:
logger.error(f"Test failed: Expected '{expected}', but got no results")
all_tests_passed = False
if all_tests_passed:
logger.info("All round tests passed.")