mirror of
https://github.com/langgenius/dify-docs.git
synced 2026-03-26 13:18:34 +07:00
feat: remove backup
git has backup already
This commit is contained in:
Alter-xyz
@@ -1,38 +1,25 @@
|
||||
import os
|
||||
import yaml # pip install pyyaml
|
||||
import yaml # pip install pyyaml
|
||||
import re
|
||||
import datetime
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
|
||||
class Config:
|
||||
# --- Path Setup ---
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
LANGUAGES = ["zh", "en", "ja"] # Languages to process
|
||||
# Still useful for potential internal archiving if needed
|
||||
TIMESTAMP = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
# --- Directory Naming Templates ---
|
||||
# Original directory for a language, e.g., "plugin_dev_zh"
|
||||
# This will be renamed to become the source for processing.
|
||||
ORIGINAL_LANG_DIR_TEMPLATE = "plugin_dev_{lang}"
|
||||
# This is the directory we look for, operate within, and is the final name.
|
||||
LANG_DIR_TEMPLATE = "plugin_dev_{lang}"
|
||||
|
||||
# Name template for the source directory after renaming the original.
|
||||
# e.g., "plugin_dev_zh_20231027_103000"
|
||||
SOURCE_DIR_FROM_ORIGINAL_TEMPLATE = "plugin_dev_{lang}_{timestamp}"
|
||||
|
||||
# Name template for an empty source if the original lang dir doesn't exist.
|
||||
# e.g., "plugin_dev_zh_empty_source_20231027_103000"
|
||||
EMPTY_SOURCE_DIR_TEMPLATE = "plugin_dev_{lang}_empty_source_{timestamp}"
|
||||
|
||||
# Name template for the final output directory (will be newly created).
|
||||
# e.g., "plugin_dev_zh"
|
||||
TARGET_DIR_TEMPLATE = "plugin_dev_{lang}"
|
||||
|
||||
# Prefix for archiving the TARGET_DIR if it unexpectedly exists before creation.
|
||||
# e.g., "plugin_dev_zh_processed_archive_"
|
||||
# Note: TARGET_DIR should ideally be empty or non-existent after source dir renaming.
|
||||
# This archive is a safety net.
|
||||
ARCHIVE_TARGET_PREFIX_TEMPLATE = "plugin_dev_{lang}_archive_"
|
||||
# Prefix for archiving a LANG_DIR_TEMPLATE if (for some external reason)
|
||||
# we wanted to back it up before processing. Not used in the main flow currently
|
||||
# but kept as a utility.
|
||||
ARCHIVE_LANG_DIR_PREFIX_TEMPLATE = "plugin_dev_{lang}_archive_pre_processing_"
|
||||
|
||||
# --- PWXY Mappings ---
|
||||
PRIMARY_TYPE_MAP = {
|
||||
@@ -63,23 +50,26 @@ class Config:
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
|
||||
def extract_front_matter(content: str):
|
||||
match = re.match(r"^\s*---\s*$(.*?)^---\s*$(.*)", content, re.DOTALL | re.MULTILINE)
|
||||
match = re.match(r"^\s*---\s*$(.*?)^---\s*$(.*)",
|
||||
content, re.DOTALL | re.MULTILINE)
|
||||
if match:
|
||||
yaml_str = match.group(1).strip()
|
||||
markdown_content = match.group(2).strip()
|
||||
try:
|
||||
front_matter = yaml.safe_load(yaml_str)
|
||||
if front_matter is None: # Handles empty YAML (--- \n ---)
|
||||
if front_matter is None: # Handles empty YAML (--- \n ---)
|
||||
return {}, markdown_content
|
||||
return (
|
||||
front_matter if isinstance(front_matter, dict) else {}
|
||||
), markdown_content
|
||||
except yaml.YAMLError as e:
|
||||
print(f" [Error] YAML Parsing Failed: {e}")
|
||||
return None, content # Indicate error
|
||||
return None, content # Indicate error
|
||||
else:
|
||||
return {}, content # No front matter found
|
||||
return {}, content # No front matter found
|
||||
|
||||
|
||||
def sanitize_filename_part(part: str) -> str:
|
||||
if not isinstance(part, str):
|
||||
@@ -91,6 +81,7 @@ def sanitize_filename_part(part: str) -> str:
|
||||
part = part.strip(".-_")
|
||||
return part or "untitled"
|
||||
|
||||
|
||||
def _calculate_pwxy_and_warnings(front_matter: dict, config: Config) -> tuple[int, int, int, int, list[str]]:
|
||||
"""Calculates P, W, X, Y values and generates warnings for missing/unmapped data."""
|
||||
warnings_messages = []
|
||||
@@ -117,20 +108,25 @@ def _calculate_pwxy_and_warnings(front_matter: dict, config: Config) -> tuple[in
|
||||
if primary is None:
|
||||
warnings_messages.append("Missing dimensions.type.primary")
|
||||
elif W == config.DEFAULT_W:
|
||||
warnings_messages.append(f"Unmapped primary type: '{primary}'. Using W={config.DEFAULT_W}")
|
||||
warnings_messages.append(
|
||||
f"Unmapped primary type: '{primary}'. Using W={config.DEFAULT_W}")
|
||||
if detail is None:
|
||||
warnings_messages.append("Missing dimensions.type.detail")
|
||||
elif X == config.DEFAULT_X and primary in config.DETAIL_TYPE_MAPS:
|
||||
warnings_messages.append(f"Unmapped detail type: '{detail}' for primary '{primary}'. Using X={config.DEFAULT_X}")
|
||||
warnings_messages.append(
|
||||
f"Unmapped detail type: '{detail}' for primary '{primary}'. Using X={config.DEFAULT_X}")
|
||||
elif primary not in config.DETAIL_TYPE_MAPS and primary is not None:
|
||||
warnings_messages.append(f"No detail map defined for primary type: '{primary}'. Using X={config.DEFAULT_X}")
|
||||
warnings_messages.append(
|
||||
f"No detail map defined for primary type: '{primary}'. Using X={config.DEFAULT_X}")
|
||||
if level is None:
|
||||
warnings_messages.append("Missing dimensions.level")
|
||||
elif Y == config.DEFAULT_Y:
|
||||
warnings_messages.append(f"Unmapped level: '{level}'. Using Y={config.DEFAULT_Y}")
|
||||
warnings_messages.append(
|
||||
f"Unmapped level: '{level}'. Using Y={config.DEFAULT_Y}")
|
||||
|
||||
return P, W, X, Y, warnings_messages
|
||||
|
||||
|
||||
def _generate_filename_parts(
|
||||
P: int, W: int, X: int, Y: int,
|
||||
front_matter: dict,
|
||||
@@ -139,195 +135,200 @@ def _generate_filename_parts(
|
||||
"""Generates padded prefix, sanitized title, language suffix, and any warnings."""
|
||||
warnings_messages = []
|
||||
|
||||
# Padded Prefix
|
||||
prefix_str = f"{P}{W}{X}{Y}"
|
||||
try:
|
||||
numeric_prefix = int(prefix_str)
|
||||
padded_prefix = f"{numeric_prefix:04d}"
|
||||
except ValueError:
|
||||
# This case should ideally not happen if P,W,X,Y are always numeric
|
||||
warnings_messages.append(f"Could not form numeric prefix from P={P},W={W},X={X},Y={Y}. Using '0000'.")
|
||||
padded_prefix = "0000" # Fallback, but indicates an issue
|
||||
warnings_messages.append(
|
||||
f"Could not form numeric prefix from P={P},W={W},X={X},Y={Y}. Using '0000'.")
|
||||
padded_prefix = "0000"
|
||||
|
||||
# Sanitized Title
|
||||
standard_title = front_matter.get("standard_title")
|
||||
title_part_to_use = standard_title
|
||||
if not title_part_to_use:
|
||||
warnings_messages.append("Missing 'standard_title'. Using original filename stem as fallback.")
|
||||
warnings_messages.append(
|
||||
"Missing 'standard_title'. Using original filename stem as fallback.")
|
||||
title_part_to_use = original_filename_stem
|
||||
sanitized_title = sanitize_filename_part(title_part_to_use)
|
||||
|
||||
# Language Suffix
|
||||
lang_suffix = ""
|
||||
language_fm = front_matter.get("language") # Language from frontmatter
|
||||
language_fm = front_matter.get("language")
|
||||
if language_fm:
|
||||
lang_code = str(language_fm).strip().lower()
|
||||
if lang_code:
|
||||
lang_suffix = f".{lang_code}"
|
||||
else:
|
||||
warnings_messages.append("Empty 'language' field in frontmatter. Omitting language suffix.")
|
||||
warnings_messages.append(
|
||||
"Empty 'language' field in frontmatter. Omitting language suffix.")
|
||||
else:
|
||||
warnings_messages.append("Missing 'language' field in frontmatter. Omitting language suffix.")
|
||||
warnings_messages.append(
|
||||
"Missing 'language' field in frontmatter. Omitting language suffix.")
|
||||
|
||||
return padded_prefix, sanitized_title, lang_suffix, warnings_messages
|
||||
|
||||
# --- Core Processing Functions ---
|
||||
|
||||
def setup_paths_for_lang(lang: str, config: Config) -> tuple[Path | None, Path | None]:
|
||||
|
||||
def get_or_create_lang_dir(lang: str, config: Config) -> tuple[Path | None, bool]:
|
||||
"""
|
||||
Sets up source and target paths for a given language.
|
||||
Renames original lang_dir to be the source_dir for processing.
|
||||
Returns (source_dir_path, target_dir_path) or (None, None) on critical error.
|
||||
Identifies the language-specific directory. Creates it if it doesn't exist.
|
||||
This directory will be processed in-place.
|
||||
|
||||
Returns:
|
||||
- Path | None: The path to the language directory, or None on critical error.
|
||||
- bool: True if the directory was newly created (was_absent), False otherwise.
|
||||
"""
|
||||
original_lang_dir_name = config.ORIGINAL_LANG_DIR_TEMPLATE.format(lang=lang)
|
||||
original_lang_dir_path = config.BASE_DIR / original_lang_dir_name
|
||||
lang_dir_name = config.LANG_DIR_TEMPLATE.format(lang=lang)
|
||||
lang_dir_path = config.BASE_DIR / lang_dir_name
|
||||
was_newly_created = False
|
||||
|
||||
target_dir_name = config.TARGET_DIR_TEMPLATE.format(lang=lang)
|
||||
target_dir_path = config.BASE_DIR / target_dir_name
|
||||
|
||||
source_dir_path: Path
|
||||
source_dir_created_empty = False
|
||||
|
||||
if original_lang_dir_path.exists():
|
||||
if not original_lang_dir_path.is_dir():
|
||||
print(f"[ERROR] Path '{original_lang_dir_path}' exists but is not a directory. Skipping language '{lang}'.")
|
||||
return None, None
|
||||
|
||||
source_dir_name = config.SOURCE_DIR_FROM_ORIGINAL_TEMPLATE.format(lang=lang, timestamp=config.TIMESTAMP)
|
||||
source_dir_path = config.BASE_DIR / source_dir_name
|
||||
try:
|
||||
# Ensure no conflict if a previous timestamped dir exists (unlikely but possible)
|
||||
if source_dir_path.exists():
|
||||
print(f"[WARNING] Timestamped source dir '{source_dir_path}' already exists. This might be from a rapid re-run or manual creation. Trying to use it.")
|
||||
else:
|
||||
original_lang_dir_path.rename(source_dir_path)
|
||||
print(f"Using '{source_dir_path}' (renamed from '{original_lang_dir_path}') as source for '{lang}'.")
|
||||
except OSError as e:
|
||||
print(f"[ERROR] Failed to rename '{original_lang_dir_path}' to '{source_dir_path}': {e}. Skipping language '{lang}'.")
|
||||
return None, None
|
||||
if lang_dir_path.exists():
|
||||
if not lang_dir_path.is_dir():
|
||||
print(
|
||||
f"[ERROR] Path '{lang_dir_path}' exists but is not a directory. Skipping language '{lang}'.")
|
||||
return None, False
|
||||
print(
|
||||
f"Using existing directory '{lang_dir_path.name}' for in-place processing of '{lang}'.")
|
||||
else:
|
||||
print(f"Warning: Original directory '{original_lang_dir_path}' not found for language '{lang}'.")
|
||||
source_dir_name = config.EMPTY_SOURCE_DIR_TEMPLATE.format(lang=lang, timestamp=config.TIMESTAMP)
|
||||
source_dir_path = config.BASE_DIR / source_dir_name
|
||||
source_dir_path.mkdir(parents=True, exist_ok=True)
|
||||
source_dir_created_empty = True
|
||||
print(f"Created empty source directory: '{source_dir_path}' for '{lang}'.")
|
||||
print(
|
||||
f"Directory '{lang_dir_path.name}' not found for language '{lang}'. Creating it.")
|
||||
try:
|
||||
# exist_ok=False to ensure it's new
|
||||
lang_dir_path.mkdir(parents=True, exist_ok=False)
|
||||
was_newly_created = True
|
||||
print(f"Created directory: '{lang_dir_path.name}' for '{lang}'.")
|
||||
except FileExistsError: # Should not happen due to prior .exists() check, but for safety
|
||||
print(
|
||||
f"[ERROR] Directory '{lang_dir_path.name}' unexpectedly created by another process. Attempting to use it.")
|
||||
if not lang_dir_path.is_dir(): # Verify it's a dir
|
||||
print(
|
||||
f"[ERROR] Path '{lang_dir_path}' is not a directory after attempted creation. Skipping language '{lang}'.")
|
||||
return None, False
|
||||
was_newly_created = False # It existed.
|
||||
except OSError as e:
|
||||
print(
|
||||
f"[ERROR] Failed to create directory '{lang_dir_path}': {e}. Skipping language '{lang}'.")
|
||||
return None, False
|
||||
|
||||
return source_dir_path, target_dir_path, source_dir_created_empty, source_dir_name
|
||||
return lang_dir_path, was_newly_created
|
||||
|
||||
# This function is now a general utility, not directly tied to the old finalization flow.
|
||||
# It could be used if a pre-processing archive step is desired.
|
||||
|
||||
|
||||
def archive_and_create_target_dir(target_dir_path: Path, archive_prefix: str, timestamp: str) -> bool:
|
||||
def archive_existing_directory(path_to_archive: Path, archive_prefix_template: str, lang: str, timestamp: str) -> bool:
|
||||
"""
|
||||
Archives the target directory if it exists, then creates a new empty target directory.
|
||||
Returns True on success, False on failure.
|
||||
Archives the given directory if it exists.
|
||||
The archive_prefix_template should be like "plugin_dev_{lang}_archive_".
|
||||
Returns True if path is clear for use (was archived or didn't exist), False on error or if path is not a dir.
|
||||
"""
|
||||
if target_dir_path.exists():
|
||||
if target_dir_path.is_dir():
|
||||
archive_dir_name = f"{archive_prefix}{timestamp}"
|
||||
archive_dir_path = target_dir_path.parent / archive_dir_name
|
||||
if path_to_archive.exists():
|
||||
if path_to_archive.is_dir():
|
||||
archive_base_name = archive_prefix_template.format(lang=lang)
|
||||
archive_dir_name = f"{archive_base_name}{timestamp}"
|
||||
archive_dir_path = path_to_archive.parent / archive_dir_name
|
||||
try:
|
||||
# shutil.move is more robust for renaming across different filesystems (though unlikely here)
|
||||
# and can overwrite if archive_dir_path somehow exists (very unlikely)
|
||||
if archive_dir_path.exists():
|
||||
print(f" [Warning] Archive destination '{archive_dir_path}' already exists. Removing it first.")
|
||||
shutil.rmtree(archive_dir_path) # Or handle differently
|
||||
shutil.move(str(target_dir_path), str(archive_dir_path))
|
||||
print(f" Archived existing target directory to: {archive_dir_path}")
|
||||
if archive_dir_path.exists(): # Safety: if target archive name exists, remove it.
|
||||
print(
|
||||
f" [Warning] Archive destination '{archive_dir_path}' already exists. Removing it first to avoid error during move.")
|
||||
shutil.rmtree(archive_dir_path)
|
||||
shutil.move(str(path_to_archive), str(archive_dir_path))
|
||||
print(
|
||||
f" Archived existing directory '{path_to_archive.name}' to '{archive_dir_path.name}'.")
|
||||
return True # Path is now clear because original was moved
|
||||
except OSError as e:
|
||||
print(f" [Error] Failed to archive existing target dir '{target_dir_path}': {e}")
|
||||
print(" Aborting for this language to prevent data loss.")
|
||||
print(
|
||||
f" [Error] Failed to archive existing directory '{path_to_archive.name}' to '{archive_dir_path.name}': {e}")
|
||||
return False
|
||||
else:
|
||||
print(f" [Error] Target path '{target_dir_path}' exists but is not a file/directory. Please remove/rename manually.")
|
||||
print(" Aborting for this language.")
|
||||
print(
|
||||
f" [Error] Path '{path_to_archive}' exists but is not a directory. Cannot archive.")
|
||||
return False
|
||||
try:
|
||||
target_dir_path.mkdir(parents=True, exist_ok=False) # Should not exist now
|
||||
print(f" Created new target directory: {target_dir_path}")
|
||||
return True
|
||||
except OSError as e:
|
||||
print(f" [Error] Failed to create target directory '{target_dir_path}': {e}")
|
||||
print(" Aborting for this language.")
|
||||
return False
|
||||
return True # Path didn't exist, so it's clear
|
||||
|
||||
|
||||
def process_single_mdx_file(
|
||||
mdx_filepath: Path,
|
||||
target_dir: Path,
|
||||
config: Config
|
||||
) -> dict:
|
||||
"""
|
||||
Processes a single MDX file: extracts metadata for new filename,
|
||||
and copies the original file content to the new location.
|
||||
Processes a single MDX file: extracts metadata, generates new filename,
|
||||
and renames the file in place.
|
||||
Returns stats.
|
||||
"""
|
||||
stats = {"status": "processed", "warnings": [], "error_message": None}
|
||||
relative_path = mdx_filepath.relative_to(config.BASE_DIR).as_posix()
|
||||
display_path = mdx_filepath.name
|
||||
if mdx_filepath.parent != config.BASE_DIR:
|
||||
try:
|
||||
# Show relative path from the language directory's parent (BASE_DIR)
|
||||
display_path = mdx_filepath.relative_to(
|
||||
mdx_filepath.parent.parent).as_posix()
|
||||
except ValueError:
|
||||
display_path = mdx_filepath.relative_to(config.BASE_DIR).as_posix()
|
||||
|
||||
file_warnings = []
|
||||
|
||||
try:
|
||||
content = mdx_filepath.read_text(encoding="utf-8")
|
||||
front_matter, _ = extract_front_matter(content) # markdown_content not needed for re-writing
|
||||
front_matter, _ = extract_front_matter(content)
|
||||
|
||||
if front_matter is None: # YAML parsing error from extract_front_matter
|
||||
if front_matter is None:
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = "YAML Error in file."
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f"\nProcessing: {display_path}")
|
||||
print(f" [Skipping] {stats['error_message']}")
|
||||
return stats
|
||||
|
||||
# Calculate PWXY and related warnings
|
||||
P, W, X, Y, pwxy_warnings = _calculate_pwxy_and_warnings(front_matter, config)
|
||||
P, W, X, Y, pwxy_warnings = _calculate_pwxy_and_warnings(
|
||||
front_matter, config)
|
||||
file_warnings.extend(pwxy_warnings)
|
||||
|
||||
# Generate filename parts and related warnings
|
||||
padded_prefix, sanitized_title, lang_suffix, fname_warnings = _generate_filename_parts(
|
||||
P, W, X, Y, front_matter, mdx_filepath.stem
|
||||
)
|
||||
file_warnings.extend(fname_warnings)
|
||||
|
||||
if padded_prefix is None: # Critical error in prefix generation
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = "Could not form numeric prefix."
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f" [Error] {stats['error_message']} P={P},W={W},X={X},Y={Y}. Skipping.")
|
||||
return stats
|
||||
|
||||
# padded_prefix has a fallback to "0000", so it should not be None
|
||||
new_filename = f"{padded_prefix}-{sanitized_title}{lang_suffix}.mdx"
|
||||
target_filepath = target_dir / new_filename
|
||||
new_filepath = mdx_filepath.with_name(new_filename)
|
||||
|
||||
if target_filepath.exists():
|
||||
stats["status"] = "skipped_exists"
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f" [Skipping] Target file already exists: {new_filename}")
|
||||
return stats
|
||||
|
||||
# Copy original file instead of rewriting content
|
||||
try:
|
||||
shutil.copy2(mdx_filepath, target_filepath) # copy2 preserves metadata
|
||||
except Exception as copy_error:
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = f"Failed to copy file: {copy_error}"
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f" [Error] {stats['error_message']}")
|
||||
return stats
|
||||
if new_filepath == mdx_filepath:
|
||||
stats["status"] = "skipped_no_change"
|
||||
elif new_filepath.exists():
|
||||
stats["status"] = "skipped_target_exists"
|
||||
# Do not print here, summary will handle it. Let main loop print progress.
|
||||
else:
|
||||
try:
|
||||
mdx_filepath.rename(new_filepath)
|
||||
except Exception as rename_error:
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = f"Failed to rename file to '{new_filename}': {rename_error}"
|
||||
# Defer printing to main loop for consistency
|
||||
return stats
|
||||
|
||||
stats["warnings"] = file_warnings
|
||||
if file_warnings:
|
||||
print(f"\nProcessing: {relative_path} -> {new_filename}")
|
||||
# Only print details if there are warnings or an actual change/error for this file
|
||||
if file_warnings or (stats["status"] == "processed" and new_filepath != mdx_filepath) or stats["status"].startswith("error") or stats["status"] == "skipped_target_exists":
|
||||
print(
|
||||
f"\nProcessing: {display_path} -> {new_filename if new_filepath != mdx_filepath else '(no change)'}")
|
||||
for warning_msg in file_warnings:
|
||||
print(f" [Warning] {warning_msg}")
|
||||
# No print needed for successful processing without warnings unless verbose mode
|
||||
if stats["status"] == "skipped_target_exists":
|
||||
print(
|
||||
f" [Skipping] Target filename '{new_filename}' already exists in this directory.")
|
||||
if stats["error_message"]:
|
||||
print(f" [Error] {stats['error_message']}")
|
||||
|
||||
except FileNotFoundError:
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = f"File not found during processing: {mdx_filepath}"
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f"\nProcessing: {display_path}")
|
||||
print(f" [Error] {stats['error_message']}")
|
||||
except Exception as e:
|
||||
stats["status"] = "error"
|
||||
stats["error_message"] = f"Unexpected error: {e}"
|
||||
print(f"\nProcessing: {relative_path}")
|
||||
print(f"\nProcessing: {display_path}")
|
||||
print(f" [Error] Unexpected error processing file: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
@@ -335,114 +336,178 @@ def process_single_mdx_file(
|
||||
|
||||
|
||||
def run_processing_for_language(
|
||||
source_dir: Path,
|
||||
target_dir: Path,
|
||||
archive_prefix: str,
|
||||
lang_dir_path: Path,
|
||||
config: Config
|
||||
) -> dict:
|
||||
"""Processes all MDX files in the source_dir and outputs them to target_dir."""
|
||||
print(f"Starting processing for source: {source_dir}")
|
||||
print(f"Target directory: {target_dir}")
|
||||
"""Processes all MDX files in the lang_dir_path by renaming them in place."""
|
||||
print(f"Starting in-place processing for: {lang_dir_path.name}")
|
||||
|
||||
lang_stats = {
|
||||
"processed_count": 0,
|
||||
"skipped_count": 0,
|
||||
"skipped_no_change_count": 0,
|
||||
"skipped_target_exists_count": 0,
|
||||
"error_count": 0,
|
||||
"warning_files_count": 0, # Files with at least one warning
|
||||
"status": "OK"
|
||||
"warning_files_count": 0,
|
||||
"status": "OK",
|
||||
# For summary
|
||||
"dir_path_str": str(lang_dir_path.relative_to(config.BASE_DIR))
|
||||
}
|
||||
|
||||
if not source_dir.exists() or not source_dir.is_dir():
|
||||
print(f"[Error] Source directory '{source_dir}' does not exist or is not a directory.")
|
||||
lang_stats["status"] = "SOURCE_DIR_ERROR"
|
||||
if not lang_dir_path.exists() or not lang_dir_path.is_dir():
|
||||
print(
|
||||
f"[Error] Language directory '{lang_dir_path.name}' does not exist or is not a directory.")
|
||||
lang_stats["status"] = "LANG_DIR_ERROR"
|
||||
return lang_stats
|
||||
|
||||
# Archive existing target directory (if any) and create a new one
|
||||
if not archive_and_create_target_dir(target_dir, archive_prefix, config.TIMESTAMP):
|
||||
lang_stats["status"] = "TARGET_DIR_SETUP_ERROR"
|
||||
return lang_stats # Abort if target dir setup fails
|
||||
|
||||
mdx_files = list(source_dir.rglob("*.mdx"))
|
||||
# Sort for consistent processing order
|
||||
mdx_files = sorted(list(lang_dir_path.rglob("*.mdx")))
|
||||
total_files = len(mdx_files)
|
||||
print(f"Found {total_files} MDX files to process in '{source_dir}'.")
|
||||
|
||||
if not mdx_files and source_dir.name.startswith(config.ORIGINAL_LANG_DIR_TEMPLATE.format(lang="")[:-1]): # Heuristic check if it was an original dir
|
||||
pass # It's fine for an original directory to be empty.
|
||||
print(f"Found {total_files} MDX files to process in '{lang_dir_path.name}'.")
|
||||
|
||||
for i, mdx_filepath in enumerate(mdx_files):
|
||||
result = process_single_mdx_file(mdx_filepath, target_dir, config)
|
||||
result = process_single_mdx_file(mdx_filepath, config)
|
||||
|
||||
if result["status"] == "processed":
|
||||
lang_stats["processed_count"] += 1
|
||||
if result["warnings"]:
|
||||
lang_stats["warning_files_count"] +=1
|
||||
elif result["status"] == "skipped_exists":
|
||||
lang_stats["skipped_count"] += 1
|
||||
elif result["status"] == "skipped_no_change":
|
||||
lang_stats["skipped_no_change_count"] += 1
|
||||
elif result["status"] == "skipped_target_exists":
|
||||
lang_stats["skipped_target_exists_count"] += 1
|
||||
elif result["status"] == "error":
|
||||
lang_stats["error_count"] += 1
|
||||
|
||||
if (i + 1) % 10 == 0 or (i + 1) == total_files:
|
||||
if total_files > 0 : # Avoid division by zero for empty source
|
||||
print(f"Progress: {i+1}/{total_files} files evaluated.", end="\r")
|
||||
|
||||
print("\n" + "-" * 20) # Newline after progress
|
||||
print(f"Language Processing Summary ({source_dir.name}):")
|
||||
print(f" Successfully processed: {lang_stats['processed_count']}")
|
||||
print(f" Skipped (target exists): {lang_stats['skipped_count']}")
|
||||
print(f" Files with warnings: {lang_stats['warning_files_count']}")
|
||||
print(f" Errors encountered: {lang_stats['error_count']}")
|
||||
if result["warnings"]: # Count files with warnings regardless of status
|
||||
lang_stats["warning_files_count"] += 1
|
||||
|
||||
if total_files > 0:
|
||||
progress = (i + 1) / total_files * 100
|
||||
print(
|
||||
f"Progress for {lang_dir_path.name}: {i+1}/{total_files} files ({progress:.1f}%) evaluated.", end="\r")
|
||||
|
||||
if total_files > 0:
|
||||
print() # Newline after progress bar
|
||||
|
||||
print("-" * 20)
|
||||
print(f"Language Processing Summary ({lang_dir_path.name}):")
|
||||
print(
|
||||
f" Successfully processed (renamed): {lang_stats['processed_count']}")
|
||||
# Clarified term
|
||||
print(
|
||||
f" Checked (filename no change): {lang_stats['skipped_no_change_count']}")
|
||||
print(
|
||||
f" Skipped (target filename exists): {lang_stats['skipped_target_exists_count']}")
|
||||
print(f" Files with warnings: {lang_stats['warning_files_count']}")
|
||||
print(
|
||||
f" Errors encountered during file processing: {lang_stats['error_count']}")
|
||||
print("-" * 20)
|
||||
|
||||
if lang_stats["error_count"] > 0:
|
||||
lang_stats["status"] = "ERRORS_IN_PROCESSING"
|
||||
return lang_stats
|
||||
|
||||
# --- Main Orchestration ---
|
||||
|
||||
|
||||
def main():
|
||||
config = Config()
|
||||
print(f"Base directory: {config.BASE_DIR}")
|
||||
print(f"Timestamp for this run: {config.TIMESTAMP}")
|
||||
|
||||
overall_summary = {}
|
||||
# Store if the lang dir was newly created for cleanup decisions
|
||||
lang_dir_newly_created_flags = {}
|
||||
# Store the Path object of the language directory for each lang
|
||||
lang_dirs_map = {}
|
||||
|
||||
for lang in config.LANGUAGES:
|
||||
print(f"\n{'='*10} Processing Language: {lang.upper()} {'='*10}")
|
||||
|
||||
source_dir_path, target_dir_path, source_created_empty, source_actual_name = setup_paths_for_lang(lang, config)
|
||||
current_lang_dir, was_newly_created = get_or_create_lang_dir(
|
||||
lang, config)
|
||||
lang_dir_newly_created_flags[lang] = was_newly_created
|
||||
lang_dirs_map[lang] = current_lang_dir
|
||||
|
||||
if not source_dir_path or not target_dir_path:
|
||||
overall_summary[lang] = {"status": "SETUP_ERROR", "message": f"Failed to setup paths for {lang}."}
|
||||
continue # Skip to next language
|
||||
if not current_lang_dir:
|
||||
overall_summary[lang] = {
|
||||
"status": "SETUP_ERROR", "message": f"Failed to get or create language directory for {lang}."}
|
||||
continue
|
||||
|
||||
archive_prefix_for_lang = config.ARCHIVE_TARGET_PREFIX_TEMPLATE.format(lang=lang)
|
||||
lang_results = run_processing_for_language(source_dir_path, target_dir_path, archive_prefix_for_lang, config)
|
||||
# Optional: Add a pre-processing archive step if desired for non-Git backups
|
||||
# For example:
|
||||
# if current_lang_dir.exists() and any(current_lang_dir.iterdir()): # if dir exists and is not empty
|
||||
# print(f"Attempting to archive '{current_lang_dir.name}' before processing...")
|
||||
# if not archive_existing_directory(current_lang_dir, config.ARCHIVE_LANG_DIR_PREFIX_TEMPLATE, lang, config.TIMESTAMP):
|
||||
# print(f" [CRITICAL ERROR] Archiving failed. Skipping processing for {lang}.")
|
||||
# overall_summary[lang] = {"status": "PRE_ARCHIVE_ERROR", "message": f"Failed to archive existing directory {current_lang_dir.name}."}
|
||||
# continue
|
||||
# # After archiving, the original path is gone, so we need to re-create it to process into
|
||||
# current_lang_dir, was_newly_created = get_or_create_lang_dir(lang, config) # This will re-create it empty
|
||||
# lang_dir_newly_created_flags[lang] = True # Mark as newly created for potential cleanup
|
||||
# lang_dirs_map[lang] = current_lang_dir
|
||||
# if not current_lang_dir:
|
||||
# overall_summary[lang] = {"status": "SETUP_ERROR_POST_ARCHIVE", "message": f"Failed to re-create lang directory for {lang} after archiving."}
|
||||
# continue
|
||||
|
||||
lang_results = run_processing_for_language(current_lang_dir, config)
|
||||
overall_summary[lang] = lang_results
|
||||
|
||||
# Clean up empty source directory if it was created for this run
|
||||
if source_created_empty:
|
||||
try:
|
||||
# Check if it's truly empty (no files processed into it by mistake)
|
||||
if not any(source_dir_path.iterdir()):
|
||||
source_dir_path.rmdir()
|
||||
print(f"Removed temporary empty source directory: {source_dir_path}")
|
||||
else:
|
||||
print(f"Note: Temporary empty source '{source_dir_path}' was not empty. Not removed.")
|
||||
except OSError as e:
|
||||
print(f"Note: Could not remove temporary empty source directory '{source_dir_path}': {e}")
|
||||
|
||||
print("\n\n" + "=" * 20 + " Overall Summary " + "=" * 20)
|
||||
for lang, summary in overall_summary.items():
|
||||
print(f"\nLanguage: {lang.upper()}")
|
||||
if summary.get("status") == "OK":
|
||||
print(f" Status: OK")
|
||||
print(f" Processed: {summary.get('processed_count', 0)}")
|
||||
print(f" Skipped: {summary.get('skipped_count', 0)}")
|
||||
print(f" Files with Warnings: {summary.get('warning_files_count', 0)}")
|
||||
print(f" Errors: {summary.get('error_count', 0)}")
|
||||
else:
|
||||
print(f" Status: {summary.get('status', 'UNKNOWN_ERROR')}")
|
||||
if "message" in summary:
|
||||
print(f" Message: {summary['message']}")
|
||||
print("=" * (40 + len(" Overall Summary ")))
|
||||
# --- Finalization for this language (mainly cleanup) ---
|
||||
if current_lang_dir: # Should always be true if we reached here
|
||||
# Processed, even if with errors
|
||||
if lang_results["status"] in ["OK", "ERRORS_IN_PROCESSING"]:
|
||||
# If the directory was newly created by this script AND it's still empty after processing
|
||||
# (e.g., no MDX files were found or created in it), then remove it.
|
||||
if was_newly_created and current_lang_dir.exists() and not any(current_lang_dir.iterdir()):
|
||||
try:
|
||||
current_lang_dir.rmdir()
|
||||
print(
|
||||
f" Removed empty newly created language directory: {current_lang_dir.name}")
|
||||
lang_dirs_map[lang] = None # It's gone
|
||||
lang_results["message"] = lang_results.get(
|
||||
"message", "") + " Empty newly created directory removed."
|
||||
except OSError as e:
|
||||
print(
|
||||
f" Note: Could not remove empty newly created directory '{current_lang_dir.name}': {e}")
|
||||
# No renaming logic needed as we operate in-place.
|
||||
# The status messages in lang_results already indicate success/failure of content processing.
|
||||
|
||||
print("\n\n" + "=" * 20 + " Overall Script Summary " + "=" * 20)
|
||||
for lang_code in config.LANGUAGES:
|
||||
summary = overall_summary.get(lang_code, {})
|
||||
lang_dir_path_obj = lang_dirs_map.get(lang_code)
|
||||
|
||||
print(f"\nLanguage: {lang_code.upper()}")
|
||||
status = summary.get("status", "UNKNOWN")
|
||||
print(f" Status: {status}")
|
||||
|
||||
if "message" in summary: # Print setup/archive messages
|
||||
print(f" Message: {summary['message']}")
|
||||
|
||||
if status not in ["SETUP_ERROR", "SETUP_ERROR_POST_ARCHIVE", "PRE_ARCHIVE_ERROR", "LANG_DIR_ERROR"]:
|
||||
print(f" Directory: {summary.get('dir_path_str', 'N/A')}")
|
||||
print(
|
||||
f" Processed (renamed): {summary.get('processed_count', 0)}")
|
||||
print(
|
||||
f" Checked (no name change): {summary.get('skipped_no_change_count', 0)}")
|
||||
print(
|
||||
f" Skipped (target exists): {summary.get('skipped_target_exists_count', 0)}")
|
||||
print(
|
||||
f" Files with Warnings: {summary.get('warning_files_count', 0)}")
|
||||
print(
|
||||
f" Errors during file processing: {summary.get('error_count', 0)}")
|
||||
|
||||
if lang_dir_path_obj and lang_dir_path_obj.exists():
|
||||
print(f" Final directory location: {lang_dir_path_obj.name}")
|
||||
# If it was new and now gone
|
||||
elif lang_dir_newly_created_flags.get(lang_code) and not lang_dir_path_obj:
|
||||
print(" Note: Empty newly created directory was removed as expected.")
|
||||
elif not lang_dir_path_obj and status != "SETUP_ERROR": # If it wasn't a setup error but dir is None
|
||||
print(
|
||||
f" Note: Language directory '{config.LANG_DIR_TEMPLATE.format(lang=lang_code)}' may have been archived or removed.")
|
||||
|
||||
print("=" * (40 + len(" Overall Script Summary ")))
|
||||
print("\nScript finished. Please review changes and commit to Git if satisfied.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user