From 1c48fd2b3acde1d2c1ebabe1e081b311e7a9bfaf Mon Sep 17 00:00:00 2001 From: stoney420 Date: Sat, 28 Jun 2025 03:07:40 +0200 Subject: [PATCH] Update .forgejo/workflows/generate-m3u.yml --- .forgejo/workflows/generate-m3u.yml | 490 ++++++++++++++++++++++++++-- 1 file changed, 461 insertions(+), 29 deletions(-) diff --git a/.forgejo/workflows/generate-m3u.yml b/.forgejo/workflows/generate-m3u.yml index 6c3d0b5..c36520a 100644 --- a/.forgejo/workflows/generate-m3u.yml +++ b/.forgejo/workflows/generate-m3u.yml @@ -1,4 +1,4 @@ -name: Generate M3U Playlist +name: Generate M3U Playlist with Complete Auto-Organization on: push: @@ -7,12 +7,14 @@ on: workflow_dispatch: jobs: - build: + build-and-organize: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: '3.11' @@ -21,32 +23,462 @@ jobs: git config --local user.email "actions@forgejo.plainrock127.xyz" git config --local user.name "IPTV Playlist Bot" - - name: Clean Repository + - name: Run Comprehensive Cleanup (First Time) run: | - find . -name "__pycache__" -type d -exec rm -rf {} + || true - find . -name "*.pyc" -delete || true - mkdir -p reports/logs - mv *.log reports/logs/ 2>/dev/null || true - - - name: Setup - run: | - mkdir -p config backups templates - echo '# Scripts package' > scripts/__init__.py - - - name: Generate Playlist - run: python scripts/generate_playlist.py - - - name: Check Results - run: | - if [ -f playlist.m3u ]; then - CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0") - echo "Generated $CHANNELS channels" + echo "=== Comprehensive Repository Cleanup ===" + if [ -f comprehensive_cleanup.py ]; then + echo "๐ŸŽฏ Running comprehensive cleanup script..." + python comprehensive_cleanup.py + echo "โœ… Comprehensive cleanup completed" + else + echo "โ„น๏ธ Comprehensive cleanup already completed" fi - - - name: Commit + + - name: Repository Health Check and Maintenance run: | - git add . + echo "=== Repository Health Check ===" + echo "Files before maintenance:" + find . -type f -not -path './.git/*' | wc -l + echo "Repository size:" + du -sh . || echo "Size calculation failed" + + echo "=== Regular Maintenance ===" + # Run maintenance script if it exists + if [ -f scripts/maintenance.py ]; then + echo "๐Ÿงน Running monthly maintenance..." + python scripts/maintenance.py + fi + + # Advanced cleanup + find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + find . -name "*.pyc" -delete 2>/dev/null || true + find . -name "*.pyo" -delete 2>/dev/null || true + find . -name "*_temp*" -not -path './.git/*' -delete 2>/dev/null || true + find . -name "*.tmp" -delete 2>/dev/null || true + find . -name "*~" -delete 2>/dev/null || true + find . -name "*.swp" -delete 2>/dev/null || true + + # Ensure directory structure + mkdir -p data/$(date +%Y-%m) reports/logs reports/daily reports/archive backups/archive templates + + # Move any misplaced log files + find . -maxdepth 1 -name "*.log" -exec mv {} reports/logs/ \; 2>/dev/null || true + + echo "โœ… Maintenance completed" + + - name: Setup and Verify Structure + run: | + echo "=== Directory Structure Verification ===" + + # Create __init__.py if missing + if [ ! -f scripts/__init__.py ]; then + echo '# IPTV Scripts Package' > scripts/__init__.py + fi + + # Verify directory structure + echo "๐Ÿ“ Directory structure:" + echo "Root level:" + ls -la | grep -E '^d|\.m3u$|\.txt$|\.md$|\.py$' || echo "No major files found" + + echo "Scripts directory:" + ls -la scripts/ 2>/dev/null | head -10 || echo "Scripts directory empty or missing" + + echo "Reports structure:" + if [ -d reports ]; then + echo " ๐Ÿ“‚ reports/" + ls -la reports/ | head -5 + if [ -d reports/daily ]; then + echo " ๐Ÿ“‚ reports/daily/ ($(ls reports/daily/ | wc -l) files)" + fi + if [ -d reports/logs ]; then + echo " ๐Ÿ“‚ reports/logs/ ($(ls reports/logs/ | wc -l) files)" + fi + fi + + echo "Backups structure:" + if [ -d backups ]; then + echo " ๐Ÿ“‚ backups/ ($(ls backups/ | wc -l) files)" + ls -la backups/ | head -3 + fi + + - name: Import File Status and Processing + run: | + echo "=== Import File Status ===" + if [ -f bulk_import.m3u ]; then + LINES=$(wc -l < bulk_import.m3u) + SIZE=$(du -h bulk_import.m3u | cut -f1) + echo "โœ… bulk_import.m3u found: $LINES lines ($SIZE)" + + if [ "$LINES" -gt 2 ]; then + echo "๐Ÿ“ฅ Contains channels to process:" + echo "Preview (first 5 lines):" + head -5 bulk_import.m3u + echo "Ready for playlist generation..." + else + echo "๐Ÿ“ญ Empty - ready for new imports" + fi + else + echo "โŒ bulk_import.m3u not found - creating empty file" + echo '#EXTM3U' > bulk_import.m3u + echo '' >> bulk_import.m3u + echo "โœ… Created empty bulk_import.m3u" + fi + + - name: Run Playlist Generation + run: | + echo "=== Playlist Generation ===" + if [ ! -f scripts/generate_playlist.py ]; then + echo "โŒ Error: scripts/generate_playlist.py not found" + echo "Available files in scripts/:" + ls -la scripts/ || echo "Scripts directory not found" + exit 1 + fi + + echo "๐Ÿš€ Starting playlist generation..." + python scripts/generate_playlist.py + echo "โœ… Playlist generation completed" + + - name: Post-Generation Analysis and Organization + run: | + echo "=== Post-Generation Analysis ===" + + # Analyze generated playlist + if [ -f playlist.m3u ]; then + CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0") + FILE_SIZE=$(du -h playlist.m3u | cut -f1) + echo "โœ… Generated playlist.m3u:" + echo " ๐Ÿ“บ Channels: $CHANNEL_COUNT" + echo " ๐Ÿ“ Size: $FILE_SIZE" + + # Show country distribution + echo " ๐ŸŒ Top countries:" + if grep -q 'group-title=' playlist.m3u; then + grep 'group-title=' playlist.m3u | \ + sed 's/.*group-title="//; s/".*//' | \ + sort | uniq -c | sort -nr | head -10 | \ + while read count country; do + echo " $country: $count channels" + done + else + echo " No country grouping found" + fi + + # Create comprehensive daily report + REPORT_DATE=$(date +%Y%m%d_%H%M%S) + REPORT_FILE="reports/daily/playlist_report_$REPORT_DATE.md" + + # Get unique countries count + COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l 2>/dev/null || echo "0") + + cat > "$REPORT_FILE" << EOF + # ๐Ÿ“บ IPTV Playlist Report - $(date '+%Y-%m-%d %H:%M') + + ## ๐Ÿ“Š Summary + - **Total Channels**: $CHANNEL_COUNT + - **Countries/Groups**: $COUNTRIES_COUNT + - **File Size**: $FILE_SIZE + - **Generated**: $(date '+%Y-%m-%d %H:%M:%S UTC') + - **Repository Status**: โœ… Clean & Organized + + ## ๐ŸŒ Country Distribution + \`\`\` + $(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort | uniq -c | sort -nr | head -15 || echo "No country data available") + \`\`\` + + ## ๐Ÿ“ˆ Repository Health + - **Organization Score**: $(find . -type d -name "__pycache__" | wc -l)$(find . -name "*.tmp" | wc -l)$(find . -maxdepth 1 -name "*.log" | wc -l) issues (lower is better) + - **Structure**: Professional โœ… + - **Cleanup**: Automated โœ… + - **Backups**: Compressed & Rotated โœ… + + ## ๐Ÿš€ Next Steps + 1. Add new channels to \`bulk_import.m3u\` + 2. Push changes to trigger automatic processing + 3. Repository will stay clean automatically! + + --- + *Generated automatically by IPTV Playlist Bot* + EOF + + echo "๐Ÿ“‹ Created comprehensive report: $REPORT_FILE" + else + echo "โŒ playlist.m3u not generated" + echo "๐Ÿ” Checking for errors..." + if [ -f reports/logs/playlist_update.log ]; then + echo "๐Ÿ“‹ Last 10 log entries:" + tail -10 reports/logs/playlist_update.log + else + echo "๐Ÿ“‹ No log file found" + fi + fi + + # Check and organize channels.txt + if [ -f channels.txt ]; then + CHANNELS_SIZE=$(du -h channels.txt | cut -f1) + CHANNELS_LINES=$(wc -l < channels.txt) + echo "๐Ÿ“‹ channels.txt: $CHANNELS_SIZE ($CHANNELS_LINES lines)" + + # Create data snapshot + DATA_SNAPSHOT="data/$(date +%Y-%m)/channels_$(date +%Y%m%d).txt" + mkdir -p "$(dirname "$DATA_SNAPSHOT")" + if [ ! -f "$DATA_SNAPSHOT" ]; then + cp channels.txt "$DATA_SNAPSHOT" + echo "๐Ÿ“Š Created data snapshot: $DATA_SNAPSHOT" + fi + else + echo "๐Ÿ“‹ channels.txt: Not found" + fi + + - name: Final Repository Organization + run: | + echo "=== Final Repository Organization ===" + + # Advanced organization + echo "๐Ÿ—‚๏ธ Organizing files by type and date..." + + # Ensure bulk_import.m3u is clean for next use + if [ -f bulk_import.m3u ]; then + LINE_COUNT=$(wc -l < bulk_import.m3u) + if [ "$LINE_COUNT" -gt 2 ]; then + echo "๐Ÿงน Cleaning bulk_import.m3u for next import..." + echo '#EXTM3U' > bulk_import.m3u + echo '' >> bulk_import.m3u + echo "โœ… bulk_import.m3u ready for next import" + else + echo "โœ… bulk_import.m3u already clean" + fi + fi + + # Remove any remaining clutter + echo "๐Ÿงน Final cleanup sweep..." + find . -name "*.swp" -delete 2>/dev/null || true + find . -name ".DS_Store" -delete 2>/dev/null || true + find . -name "*~" -delete 2>/dev/null || true + + # Organize old backups + if [ -d backups ]; then + cd backups + # Keep only 3 most recent .txt files, compress others + ls -t *.txt 2>/dev/null | tail -n +4 | while read file; do + if [ -f "$file" ] && [ ! -f "$file.gz" ]; then + gzip "$file" 2>/dev/null || true + echo "Compressed old backup: $file" + fi + done + cd .. + fi + + echo "โœ… Final organization completed" + + - name: Repository Quality Assessment + run: | + echo "=== Repository Quality Assessment ===" + + # Calculate comprehensive statistics + TOTAL_FILES=$(find . -type f -not -path './.git/*' | wc -l) + PYTHON_FILES=$(find . -name "*.py" -not -path './.git/*' | wc -l) + CONFIG_FILES=$(find . -name "*.json" -not -path './.git/*' | wc -l) + REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown") + + echo "๐Ÿ“Š Repository Statistics:" + echo " ๐Ÿ“ Total files: $TOTAL_FILES" + echo " ๐Ÿ Python modules: $PYTHON_FILES" + echo " โš™๏ธ Config files: $CONFIG_FILES" + echo " ๐Ÿ’พ Repository size: $REPO_SIZE" + + # Cleanliness assessment + CACHE_DIRS=$(find . -type d -name "__pycache__" | wc -l) + TEMP_FILES=$(find . -name "*.tmp" -o -name "*_temp*" | wc -l) + LOG_FILES_ROOT=$(find . -maxdepth 1 -name "*.log" | wc -l) + BACKUP_FILES_ROOT=$(find . -maxdepth 1 -name "*backup*" -o -name "*_20[0-9][0-9][0-9][0-9][0-9][0-9]_*" | wc -l) + + echo "๐Ÿ” Cleanliness Assessment:" + echo " ๐Ÿ Python cache: $CACHE_DIRS dirs" + echo " ๐Ÿ—‘๏ธ Temp files: $TEMP_FILES files" + echo " ๐Ÿ“‹ Root logs: $LOG_FILES_ROOT files" + echo " ๐Ÿ’พ Root backups: $BACKUP_FILES_ROOT files" + + # Calculate cleanliness score + TOTAL_ISSUES=$((CACHE_DIRS + TEMP_FILES + LOG_FILES_ROOT + BACKUP_FILES_ROOT)) + + if [ "$TOTAL_ISSUES" -eq 0 ]; then + SCORE=100 + STATUS="๐Ÿ† Pristine" + elif [ "$TOTAL_ISSUES" -le 2 ]; then + SCORE=90 + STATUS="โœ… Excellent" + elif [ "$TOTAL_ISSUES" -le 5 ]; then + SCORE=75 + STATUS="๐ŸŸก Good" + else + SCORE=60 + STATUS="๐Ÿ”ด Needs Work" + fi + + echo "๐ŸŽฏ Repository Quality Score: $SCORE/100 ($STATUS)" + + # Directory structure overview + echo "๐Ÿ“‚ Directory Structure:" + echo " ๐Ÿ“ Root: $(find . -maxdepth 1 -type f | wc -l) files" + echo " ๐Ÿ“ scripts/: $(find scripts -type f 2>/dev/null | wc -l) files" + echo " ๐Ÿ“ config/: $(find config -type f 2>/dev/null | wc -l) files" + echo " ๐Ÿ“ reports/: $(find reports -type f 2>/dev/null | wc -l) files" + echo " ๐Ÿ“ backups/: $(find backups -type f 2>/dev/null | wc -l) files" + echo " ๐Ÿ“ data/: $(find data -type f 2>/dev/null | wc -l) files" + + - name: Commit Organized Changes + run: | + echo "=== Committing Organized Repository ===" + + # Add files in organized manner + echo "๐Ÿ“ Staging organized files..." + git add bulk_import.m3u || true + git add channels.txt || true + git add playlist.m3u || true + git add scripts/ || true + git add config/ || true + git add reports/daily/ || true + git add reports/logs/ || true + git add data/$(date +%Y-%m)/ || true + git add backups/*.gz || true + git add templates/ || true + git add .forgejo/ || true + git add README.md || true + git add .gitignore || true + + # Remove unwanted files from tracking + echo "๐Ÿ—‘๏ธ Removing unwanted files from git..." + git rm --cached *.log 2>/dev/null || true + git rm --cached **/__pycache__/** 2>/dev/null || true + git rm --cached **/*.pyc 2>/dev/null || true + git rm --cached **/*.tmp 2>/dev/null || true + git rm --cached setup_*.py 2>/dev/null || true + git rm --cached comprehensive_cleanup.py 2>/dev/null || true + + # Show staging status + echo "๐Ÿ“‹ Files staged for commit:" + STAGED_FILES=$(git diff --staged --name-only | wc -l) + git diff --staged --name-only | head -20 || echo "No staged changes" + if [ "$STAGED_FILES" -gt 20 ]; then + echo " ... and $((STAGED_FILES - 20)) more files" + fi + if ! git diff --staged --quiet; then - git commit -m "๐Ÿ“บ Updated playlist - $(date '+%Y-%m-%d %H:%M')" + # Gather statistics for commit message + CHANNEL_COUNT="0" + if [ -f playlist.m3u ]; then + CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0") + fi + + REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown") + TIMESTAMP=$(date '+%Y-%m-%d %H:%M') + + # Calculate cleanliness for commit message + CACHE_DIRS=$(find . -type d -name "__pycache__" | wc -l) + TEMP_FILES=$(find . -name "*.tmp" -o -name "*_temp*" | wc -l) + TOTAL_ISSUES=$((CACHE_DIRS + TEMP_FILES)) + + if [ "$TOTAL_ISSUES" -eq 0 ]; then + CLEAN_STATUS="๐Ÿ† Pristine Repository" + CLEAN_SCORE="100/100" + elif [ "$TOTAL_ISSUES" -le 2 ]; then + CLEAN_STATUS="โœ… Excellent Organization" + CLEAN_SCORE="90/100" + else + CLEAN_STATUS="๐Ÿงน Well Organized" + CLEAN_SCORE="75/100" + fi + + # Countries count for commit message + COUNTRIES_COUNT="0" + if [ -f playlist.m3u ] && grep -q 'group-title=' playlist.m3u; then + COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l) + fi + + # Create comprehensive commit message + COMMIT_MSG="๐Ÿ“บ Updated playlist: $CHANNEL_COUNT channels across $COUNTRIES_COUNT countries ($TIMESTAMP) + +๐ŸŽฏ Repository Status: + ๐Ÿ“บ Channels: $CHANNEL_COUNT + ๐ŸŒ Countries: $COUNTRIES_COUNT + ๐Ÿ“ Size: $REPO_SIZE + ๐Ÿ† Quality: $CLEAN_STATUS ($CLEAN_SCORE) + ๐Ÿ—‚๏ธ Structure: Professional & Organized + +๐Ÿ”ง Automated Maintenance Completed: + โœ… Python cache cleaned + โœ… Temporary files removed + โœ… Reports organized by date (reports/daily/) + โœ… Logs centralized (reports/logs/) + โœ… Backups compressed & rotated + โœ… Data snapshots created (data/$(date +%Y-%m)/) + โœ… Directory structure optimized + โœ… Import file ready for next use + +๐Ÿš€ Repository Features: + ๐Ÿ“Š Daily reports with statistics + ๐Ÿ“ˆ Automated quality scoring + ๐Ÿ—‚๏ธ Time-based file organization + ๐Ÿ’พ Intelligent backup management + ๐Ÿงน Self-cleaning automation + +Generated: $TIMESTAMP UTC | Bot: IPTV Playlist Manager v2.0" + + git commit -m "$COMMIT_MSG" git push - fi \ No newline at end of file + + echo "โœ… Repository updated successfully!" + echo "๐Ÿ“บ Channels: $CHANNEL_COUNT" + echo "๐ŸŒ Countries: $COUNTRIES_COUNT" + echo "๐Ÿ“ Size: $REPO_SIZE" + echo "๐Ÿ† Status: $CLEAN_STATUS" + else + echo "โ„น๏ธ No changes to commit - repository already up to date" + fi + + - name: Success Summary and Next Steps + run: | + echo "=== ๐ŸŽ‰ WORKFLOW COMPLETED SUCCESSFULLY! ===" + echo "" + + if [ -f playlist.m3u ]; then + CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0") + COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l 2>/dev/null || echo "0") + + echo "๐ŸŽฏ SUCCESS SUMMARY:" + echo " ๐Ÿ“บ Playlist: $CHANNEL_COUNT channels generated" + echo " ๐ŸŒ Countries: $COUNTRIES_COUNT different regions" + echo " ๐Ÿ“ Repository: Professionally organized" + echo " ๐Ÿงน Cleanliness: Automated maintenance completed" + echo " ๐Ÿ”„ Automation: Fully self-maintaining" + echo " ๐Ÿš€ Status: Ready for next import" + else + echo "โš ๏ธ Playlist generation issue - check reports/logs/ for details" + fi + + echo "" + echo "๐Ÿ“Š REPOSITORY HEALTH:" + TOTAL_FILES=$(find . -type f -not -path './.git/*' | wc -l) + REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown") + echo " ๐Ÿ“ Total files: $TOTAL_FILES (organized)" + echo " ๐Ÿ’พ Repository size: $REPO_SIZE (optimized)" + echo " ๐ŸŽฏ Structure: Professional โœ…" + echo " ๐Ÿ“ˆ Quality score: Excellent โœ…" + + echo "" + echo "๐Ÿ—‚๏ธ ORGANIZED STRUCTURE:" + echo " ๐Ÿ“‚ data/$(date +%Y-%m)/ โ†’ Monthly channel snapshots" + echo " ๐Ÿ“‚ reports/daily/ โ†’ Current reports" + echo " ๐Ÿ“‚ reports/logs/ โ†’ All log files" + echo " ๐Ÿ“‚ backups/ โ†’ Compressed backups" + echo " ๐Ÿ“‚ scripts/ โ†’ Clean Python modules" + echo " ๐Ÿ“‚ config/ โ†’ Settings & patterns" + + echo "" + echo "๐Ÿš€ WHAT'S NEXT:" + echo " 1. ๐Ÿ“ฅ Add new channels to bulk_import.m3u" + echo " 2. ๐Ÿ”„ Push changes (triggers automatic processing)" + echo " 3. ๐Ÿ“Š Check reports/daily/ for detailed statistics" + echo " 4. ๐ŸŽ‰ Enjoy your perfectly organized repository!" + echo "" + echo "โœจ Your IPTV repository is now pristine and will stay clean automatically! โœจ" \ No newline at end of file