Spaces:
Running
Running
import logging | |
logger = logging.getLogger(__name__) | |
import json | |
import os | |
import re | |
from deep_translator import GoogleTranslator | |
from gematria import calculate_gematria | |
import math | |
import logging | |
import json | |
import re | |
import math | |
import glob | |
from deep_translator import GoogleTranslator | |
from gematria import calculate_gematria # Ensure this function is correctly defined | |
import csv | |
# Configure the logger | |
# You can uncomment the next line to enable debugging logs | |
# logging.basicConfig(level=logging.DEBUG, format='%(levelname)s:%(message)s') | |
logger = logging.getLogger(__name__) | |
def process_json_files(start=1, end=114, step=1, rounds="1", length=0, tlang="en", strip_spaces=True, | |
strip_in_braces=True, strip_diacritics=True, translate=False): | |
""" | |
Processes Quran JSON files and performs various text manipulations. | |
Parameters: | |
- start (int): Start number of the Quran sura (1-114). | |
- end (int): End number of the Quran sura (1-114). | |
- step (int): Step size for character selection. | |
- rounds (str): Comma-separated list of round numbers (can include negative values). | |
- length (int): Maximum length of the result text. | |
- tlang (str): Target language for translation. | |
- strip_spaces (bool): Whether to remove spaces from the text. | |
- strip_in_braces (bool): Whether to remove text within braces. | |
- strip_diacritics (bool): Whether to remove diacritics from the text. | |
- translate (bool): Whether to translate the result text. | |
Returns: | |
- list: A list of dictionaries containing processed data or error messages. | |
""" | |
base_path = "texts/quran" | |
translator = GoogleTranslator(source='ar', target=tlang) if translate else None | |
results = [] | |
for i in range(start, end + 1): | |
file_name = f"{base_path}/{i:03}.json" # Updated file name formatting | |
try: | |
with open(file_name, 'r', encoding='utf-8') as file: | |
data = json.load(file) | |
# Extract text from verses | |
full_text = "" | |
for verse_key, verse_text in data.get("verse", {}).items(): | |
full_text += verse_text + " " | |
# Remove BOM if present | |
full_text = full_text.replace("\ufeff", "") | |
clean_text = full_text | |
if strip_in_braces: | |
# Remove content within [], {}, <> brackets | |
clean_text = re.sub(r"\[.*?\]|\{.*?\}|\<.*?\>", "", clean_text, flags=re.DOTALL) | |
if strip_diacritics: | |
# Remove Arabic diacritics and punctuation | |
clean_text = re.sub( | |
r"[\u0610-\u061A\u064B-\u065F\u0670\u06D6-\u06DC\u06DF-\u06E4\u06E7\u06E8\u06EA-\u06ED]+", "", | |
clean_text) | |
# Optionally, remove additional unwanted characters | |
clean_text = re.sub(r'[:?!\'\-]', '', clean_text) | |
# Normalize spaces | |
clean_text = clean_text.replace("\n\n ", " ") | |
clean_text = clean_text.replace("\n", " ") | |
clean_text = re.sub(r'\s+', ' ', clean_text).strip() | |
if strip_spaces: | |
clean_text = clean_text.replace(" ", "") | |
text_length = len(clean_text) | |
logger.debug(f"Sura {i}: Clean text length = {text_length}") | |
if text_length == 0: | |
logger.warning(f"Sura {i}: No text available after cleaning.") | |
continue # Skip processing if no text is available | |
# Parse 'rounds', allow floats | |
try: | |
rounds_list = list(map(float, rounds.split(','))) | |
except ValueError as e: | |
logger.error(f"Sura {i}: Invalid 'rounds' format: {rounds}") | |
results.append({"error": f"Sura {i}: Invalid 'rounds' format: {rounds}"}) | |
continue # Skip this Sura due to invalid 'rounds' input | |
result_text = "" | |
for r in rounds_list: | |
abs_r = abs(r) | |
# Determine the number of full passes and the remainder. | |
full_passes = math.floor(abs_r) | |
remainder = abs_r - full_passes | |
# Base number of characters per pass | |
base_chars = text_length // step | |
if base_chars == 0: | |
if abs_r > 1: # Changed from >=1 to >1 | |
# When step > text_length and rounds >1, pick 1 character per full pass | |
chars_per_full_pass = 1 | |
logger.debug(f"Sura {i}: step > text_length ({step} > {text_length}), selecting 1 character per full pass.") | |
else: | |
# No characters to pick | |
chars_per_full_pass = 0 | |
logger.debug(f"Sura {i}: step > text_length ({step} > {text_length}) and rounds <=1, no characters selected.") | |
# For remainder, since base_chars=0, no remainder characters | |
chars_for_remainder = 0 | |
else: | |
# Normal case | |
chars_per_full_pass = base_chars | |
chars_for_remainder = math.floor(base_chars * remainder) # Partial pass | |
logger.debug(f"Sura {i}: Normal case, chars_per_full_pass = {chars_per_full_pass}, chars_for_remainder = {chars_for_remainder}") | |
if r > 0: | |
current_index = (step - 1) % text_length | |
direction = 1 | |
else: | |
current_index = (text_length - step) % text_length | |
direction = -1 | |
pass_result = "" | |
# Full passes, keep only the last pass | |
for pass_num in range(1, full_passes + 1): | |
current_pass_chars = "" | |
for _ in range(chars_per_full_pass): | |
if chars_per_full_pass == 0: | |
break | |
current_pass_chars += clean_text[current_index] | |
current_index = (current_index + direction * step) % text_length | |
# Keep only the last full pass | |
if pass_num == full_passes: | |
pass_result = current_pass_chars | |
logger.debug(f"Sura {i}: Pass {pass_num}, pass_result = {pass_result}") | |
# Remainder pass for fractional rounds | |
if remainder > 0 and chars_for_remainder > 0: | |
current_pass_chars = "" | |
for _ in range(chars_for_remainder): | |
current_pass_chars += clean_text[current_index] | |
current_index = (current_index + direction * step) % text_length | |
pass_result += current_pass_chars | |
logger.debug(f"Sura {i}: Remainder pass_result = {pass_result}") | |
# Handle cases where step > text_length and chars_per_full_pass=1 | |
if base_chars == 0 and chars_per_full_pass == 1 and full_passes > 0: | |
# pass_result already contains the last character picked | |
pass | |
elif base_chars == 0 and chars_per_full_pass == 0 and full_passes > 0: | |
# When no characters are picked, skip appending | |
pass | |
result_text += pass_result | |
# Translate the result text if required | |
try: | |
translated_text = translator.translate(result_text) if translator and result_text else "" | |
except Exception as e: | |
logger.error(f"Translation error: {e}") | |
translated_text = "" | |
if length != 0: | |
result_text = result_text[:length] | |
logger.debug(f"Sura {i}: Result text truncated to length {length}.") | |
# Calculate the Gematria sum | |
try: | |
result_sum = calculate_gematria(result_text) | |
except Exception as e: | |
logger.error(f"Sura {i}: Gematria calculation error: {e}") | |
result_sum = None | |
if result_text: | |
results.append({ | |
"book": f"Quran {i}.", | |
"title": data.get("name", "Unknown Title"), # Use "name" instead of "title" | |
"result_text": result_text, | |
"result_sum": result_sum, | |
"translated_text": translated_text, | |
"source_language": "ar" | |
}) | |
except FileNotFoundError: | |
logger.error(f"File {file_name} not found.") | |
results.append({"error": f"File {file_name} not found."}) | |
except json.JSONDecodeError as e: | |
logger.error(f"File {file_name} could not be read as JSON: {e}") | |
results.append({"error": f"File {file_name} could not be read as JSON: {e}"}) | |
except KeyError as e: | |
logger.error(f"Expected key 'verse' is missing in {file_name}: {e}") | |
results.append({"error": f"Expected key 'verse' is missing in {file_name}: {e}"}) | |
except Exception as e: | |
logger.error(f"An unexpected error occurred while processing {file_name}: {e}") | |
results.append({"error": f"An unexpected error occurred while processing {file_name}: {e}"}) | |
return results if results else None | |
# Tests | |
test_results = [ | |
#(process_json_files(0, 0, 21, rounds="3", length=0), "שרק"), | |
] | |
all_tests_passed = True | |
for result, expected in test_results: | |
if expected is None: # Check if no result is expected | |
if not result: | |
logger.info(f"Test passed: Expected no results, got no results.") | |
else: | |
logger.error(f"Test failed: Expected no results, but got: {result}") | |
all_tests_passed = False | |
else: | |
# Check if result is not empty before accessing elements | |
if result: | |
#result_text = result[0]['result_text'] | |
result_text = None | |
if result_text == expected: | |
logger.info(f"Test passed: Expected '{expected}', got '{result_text}'") | |
else: | |
logger.error(f"Test failed: Expected '{expected}', but got '{result_text}'") | |
all_tests_passed = False | |
else: | |
logger.error(f"Test failed: Expected '{expected}', but got no results") | |
all_tests_passed = False | |
if all_tests_passed: | |
logger.info("All round tests passed.") | |