Update .forgejo/workflows/generate-m3u.yml

.forgejo/workflows/generate-m3u.yml

@@ -1,4 +1,4 @@
-name: Generate M3U Playlist with Complete Auto-Organization
+name: Generate M3U Playlist with Auto-Organization
 
 on:
   push:
@@ -23,462 +23,116 @@ jobs:
         git config --local user.email "actions@forgejo.plainrock127.xyz"
         git config --local user.name "IPTV Playlist Bot"
     
-    - name: Run Comprehensive Cleanup (First Time)
+    - name: Run Comprehensive Cleanup
       run: |
-        echo "=== Comprehensive Repository Cleanup ==="
+        echo "Running comprehensive cleanup..."
         if [ -f comprehensive_cleanup.py ]; then
-          echo "🎯 Running comprehensive cleanup script..."
           python comprehensive_cleanup.py
-          echo "✅ Comprehensive cleanup completed"
+          echo "Comprehensive cleanup completed"
         else
-          echo "ℹ️ Comprehensive cleanup already completed"
+          echo "Cleanup already completed"
         fi
 
-    - name: Repository Health Check and Maintenance
+    - name: Basic Repository Maintenance
       run: |
-        echo "=== Repository Health Check ==="
+        echo "Running basic maintenance..."
-        echo "Files before maintenance:"
-        find . -type f -not -path './.git/*' | wc -l
-        echo "Repository size:"
-        du -sh . || echo "Size calculation failed"
-        
-        echo "=== Regular Maintenance ==="
-        # Run maintenance script if it exists
-        if [ -f scripts/maintenance.py ]; then
-          echo "🧹 Running monthly maintenance..."
-          python scripts/maintenance.py
-        fi
-        
-        # Advanced cleanup
         find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
         find . -name "*.pyc" -delete 2>/dev/null || true
-        find . -name "*.pyo" -delete 2>/dev/null || true
-        find . -name "*_temp*" -not -path './.git/*' -delete 2>/dev/null || true
         find . -name "*.tmp" -delete 2>/dev/null || true
         find . -name "*~" -delete 2>/dev/null || true
-        find . -name "*.swp" -delete 2>/dev/null || true
+        mkdir -p reports/logs reports/daily backups config templates
-        
-        # Ensure directory structure
-        mkdir -p data/$(date +%Y-%m) reports/logs reports/daily reports/archive backups/archive templates
-        
-        # Move any misplaced log files
         find . -maxdepth 1 -name "*.log" -exec mv {} reports/logs/ \; 2>/dev/null || true
-        
-        echo "✅ Maintenance completed"
-
-    - name: Setup and Verify Structure
-      run: |
-        echo "=== Directory Structure Verification ==="
-        
-        # Create __init__.py if missing
         if [ ! -f scripts/__init__.py ]; then
           echo '# IPTV Scripts Package' > scripts/__init__.py
         fi
+        echo "Basic maintenance completed"
 
-        # Verify directory structure
+    - name: Check Import File
-        echo "📁 Directory structure:"
-        echo "Root level:"
-        ls -la | grep -E '^d|\.m3u$|\.txt$|\.md$|\.py$' || echo "No major files found"
-        
-        echo "Scripts directory:"
-        ls -la scripts/ 2>/dev/null | head -10 || echo "Scripts directory empty or missing"
-        
-        echo "Reports structure:"
-        if [ -d reports ]; then
-          echo "  📂 reports/"
-          ls -la reports/ | head -5
-          if [ -d reports/daily ]; then
-            echo "  📂 reports/daily/ ($(ls reports/daily/ | wc -l) files)"
-          fi
-          if [ -d reports/logs ]; then
-            echo "  📂 reports/logs/ ($(ls reports/logs/ | wc -l) files)"
-          fi
-        fi
-        
-        echo "Backups structure:"
-        if [ -d backups ]; then
-          echo "  📂 backups/ ($(ls backups/ | wc -l) files)"
-          ls -la backups/ | head -3
-        fi
-
-    - name: Import File Status and Processing
       run: |
-        echo "=== Import File Status ==="
+        echo "Checking import file..."
         if [ -f bulk_import.m3u ]; then
           LINES=$(wc -l < bulk_import.m3u)
-          SIZE=$(du -h bulk_import.m3u | cut -f1)
+          echo "Found bulk_import.m3u with $LINES lines"
-          echo "✅ bulk_import.m3u found: $LINES lines ($SIZE)"
-          
           if [ "$LINES" -gt 2 ]; then
-            echo "📥 Contains channels to process:"
+            echo "Contains channels to process"
-            echo "Preview (first 5 lines):"
-            head -5 bulk_import.m3u
-            echo "Ready for playlist generation..."
-          else
-            echo "📭 Empty - ready for new imports"
           fi
         else
-          echo "❌ bulk_import.m3u not found - creating empty file"
+          echo "Creating empty bulk_import.m3u"
           echo '#EXTM3U' > bulk_import.m3u
-          echo '' >> bulk_import.m3u
-          echo "✅ Created empty bulk_import.m3u"
         fi
 
     - name: Run Playlist Generation
       run: |
-        echo "=== Playlist Generation ==="
+        echo "Running playlist generation..."
-        if [ ! -f scripts/generate_playlist.py ]; then
+        if [ -f scripts/generate_playlist.py ]; then
-          echo "❌ Error: scripts/generate_playlist.py not found"
+          python scripts/generate_playlist.py
-          echo "Available files in scripts/:"
+          echo "Playlist generation completed"
-          ls -la scripts/ || echo "Scripts directory not found"
+        else
+          echo "Error: generate_playlist.py not found"
           exit 1
         fi
         
-        echo "🚀 Starting playlist generation..."
+    - name: Check Results
-        python scripts/generate_playlist.py
-        echo "✅ Playlist generation completed"
-
-    - name: Post-Generation Analysis and Organization
       run: |
-        echo "=== Post-Generation Analysis ==="
+        echo "Checking results..."
-        
-        # Analyze generated playlist
         if [ -f playlist.m3u ]; then
-          CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
+          CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
-          FILE_SIZE=$(du -h playlist.m3u | cut -f1)
+          SIZE=$(du -h playlist.m3u | cut -f1)
-          echo "✅ Generated playlist.m3u:"
+          echo "Generated playlist: $CHANNELS channels ($SIZE)"
-          echo "   📺 Channels: $CHANNEL_COUNT"
-          echo "   📁 Size: $FILE_SIZE"
           
-          # Show country distribution
+          # Create simple report
-          echo "   🌍 Top countries:"
+          DATE=$(date +%Y%m%d_%H%M%S)
-          if grep -q 'group-title=' playlist.m3u; then
+          REPORT="reports/daily/report_$DATE.md"
-            grep 'group-title=' playlist.m3u | \
+          echo "# Playlist Report - $(date)" > "$REPORT"
-              sed 's/.*group-title="//; s/".*//' | \
+          echo "- Channels: $CHANNELS" >> "$REPORT"
-              sort | uniq -c | sort -nr | head -10 | \
+          echo "- Size: $SIZE" >> "$REPORT"
-              while read count country; do
+          echo "- Generated: $(date)" >> "$REPORT"
-                echo "      $country: $count channels"
+          echo "Created report: $REPORT"
-              done
         else
-            echo "      No country grouping found"
+          echo "No playlist generated"
         fi
 
-          # Create comprehensive daily report
+    - name: Clean Import File
-          REPORT_DATE=$(date +%Y%m%d_%H%M%S)
-          REPORT_FILE="reports/daily/playlist_report_$REPORT_DATE.md"
-          
-          # Get unique countries count
-          COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l 2>/dev/null || echo "0")
-          
-          cat > "$REPORT_FILE" << EOF
-        # 📺 IPTV Playlist Report - $(date '+%Y-%m-%d %H:%M')
-        
-        ## 📊 Summary
-        - **Total Channels**: $CHANNEL_COUNT
-        - **Countries/Groups**: $COUNTRIES_COUNT
-        - **File Size**: $FILE_SIZE
-        - **Generated**: $(date '+%Y-%m-%d %H:%M:%S UTC')
-        - **Repository Status**: ✅ Clean & Organized
-        
-        ## 🌍 Country Distribution
-        \`\`\`
-        $(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort | uniq -c | sort -nr | head -15 || echo "No country data available")
-        \`\`\`
-        
-        ## 📈 Repository Health
-        - **Organization Score**: $(find . -type d -name "__pycache__" | wc -l)$(find . -name "*.tmp" | wc -l)$(find . -maxdepth 1 -name "*.log" | wc -l) issues (lower is better)
-        - **Structure**: Professional ✅
-        - **Cleanup**: Automated ✅
-        - **Backups**: Compressed & Rotated ✅
-        
-        ## 🚀 Next Steps
-        1. Add new channels to \`bulk_import.m3u\`
-        2. Push changes to trigger automatic processing
-        3. Repository will stay clean automatically!
-        
-        ---
-        *Generated automatically by IPTV Playlist Bot*
-        EOF
-          
-          echo "📋 Created comprehensive report: $REPORT_FILE"
-        else
-          echo "❌ playlist.m3u not generated"
-          echo "🔍 Checking for errors..."
-          if [ -f reports/logs/playlist_update.log ]; then
-            echo "📋 Last 10 log entries:"
-            tail -10 reports/logs/playlist_update.log
-          else
-            echo "📋 No log file found"
-          fi
-        fi
-        
-        # Check and organize channels.txt
-        if [ -f channels.txt ]; then
-          CHANNELS_SIZE=$(du -h channels.txt | cut -f1)
-          CHANNELS_LINES=$(wc -l < channels.txt)
-          echo "📋 channels.txt: $CHANNELS_SIZE ($CHANNELS_LINES lines)"
-          
-          # Create data snapshot
-          DATA_SNAPSHOT="data/$(date +%Y-%m)/channels_$(date +%Y%m%d).txt"
-          mkdir -p "$(dirname "$DATA_SNAPSHOT")"
-          if [ ! -f "$DATA_SNAPSHOT" ]; then
-            cp channels.txt "$DATA_SNAPSHOT"
-            echo "📊 Created data snapshot: $DATA_SNAPSHOT"
-          fi
-        else
-          echo "📋 channels.txt: Not found"
-        fi
-
-    - name: Final Repository Organization
       run: |
-        echo "=== Final Repository Organization ==="
+        echo "Cleaning import file..."
-        
-        # Advanced organization
-        echo "🗂️ Organizing files by type and date..."
-        
-        # Ensure bulk_import.m3u is clean for next use
         if [ -f bulk_import.m3u ]; then
-          LINE_COUNT=$(wc -l < bulk_import.m3u)
+          LINES=$(wc -l < bulk_import.m3u)
-          if [ "$LINE_COUNT" -gt 2 ]; then
+          if [ "$LINES" -gt 2 ]; then
-            echo "🧹 Cleaning bulk_import.m3u for next import..."
             echo '#EXTM3U' > bulk_import.m3u
-            echo '' >> bulk_import.m3u
+            echo "Cleaned import file for next use"
-            echo "✅ bulk_import.m3u ready for next import"
-          else
-            echo "✅ bulk_import.m3u already clean"
           fi
         fi
 
-        # Remove any remaining clutter
+    - name: Final Cleanup
-        echo "🧹 Final cleanup sweep..."
+      run: |
-        find . -name "*.swp" -delete 2>/dev/null || true
+        echo "Final cleanup..."
-        find . -name ".DS_Store" -delete 2>/dev/null || true
+        find . -name "*.pyc" -delete 2>/dev/null || true
         find . -name "*~" -delete 2>/dev/null || true
+        find . -name "*.swp" -delete 2>/dev/null || true
+        rm -f comprehensive_cleanup.py 2>/dev/null || true
+        echo "Final cleanup completed"
 
-        # Organize old backups
+    - name: Commit Changes
-        if [ -d backups ]; then
-          cd backups
-          # Keep only 3 most recent .txt files, compress others
-          ls -t *.txt 2>/dev/null | tail -n +4 | while read file; do
-            if [ -f "$file" ] && [ ! -f "$file.gz" ]; then
-              gzip "$file" 2>/dev/null || true
-              echo "Compressed old backup: $file"
-            fi
-          done
-          cd ..
-        fi
-        
-        echo "✅ Final organization completed"
-
-    - name: Repository Quality Assessment
       run: |
-        echo "=== Repository Quality Assessment ==="
+        echo "Committing changes..."
-        
+        git add .
-        # Calculate comprehensive statistics
-        TOTAL_FILES=$(find . -type f -not -path './.git/*' | wc -l)
-        PYTHON_FILES=$(find . -name "*.py" -not -path './.git/*' | wc -l)
-        CONFIG_FILES=$(find . -name "*.json" -not -path './.git/*' | wc -l)
-        REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown")
-        
-        echo "📊 Repository Statistics:"
-        echo "   📁 Total files: $TOTAL_FILES"
-        echo "   🐍 Python modules: $PYTHON_FILES"
-        echo "   ⚙️ Config files: $CONFIG_FILES"
-        echo "   💾 Repository size: $REPO_SIZE"
-        
-        # Cleanliness assessment
-        CACHE_DIRS=$(find . -type d -name "__pycache__" | wc -l)
-        TEMP_FILES=$(find . -name "*.tmp" -o -name "*_temp*" | wc -l)
-        LOG_FILES_ROOT=$(find . -maxdepth 1 -name "*.log" | wc -l)
-        BACKUP_FILES_ROOT=$(find . -maxdepth 1 -name "*backup*" -o -name "*_20[0-9][0-9][0-9][0-9][0-9][0-9]_*" | wc -l)
-        
-        echo "🔍 Cleanliness Assessment:"
-        echo "   🐍 Python cache: $CACHE_DIRS dirs"
-        echo "   🗑️ Temp files: $TEMP_FILES files"
-        echo "   📋 Root logs: $LOG_FILES_ROOT files"
-        echo "   💾 Root backups: $BACKUP_FILES_ROOT files"
-        
-        # Calculate cleanliness score
-        TOTAL_ISSUES=$((CACHE_DIRS + TEMP_FILES + LOG_FILES_ROOT + BACKUP_FILES_ROOT))
-        
-        if [ "$TOTAL_ISSUES" -eq 0 ]; then
-          SCORE=100
-          STATUS="🏆 Pristine"
-        elif [ "$TOTAL_ISSUES" -le 2 ]; then
-          SCORE=90
-          STATUS="✅ Excellent"
-        elif [ "$TOTAL_ISSUES" -le 5 ]; then
-          SCORE=75
-          STATUS="🟡 Good"
-        else
-          SCORE=60
-          STATUS="🔴 Needs Work"
-        fi
-        
-        echo "🎯 Repository Quality Score: $SCORE/100 ($STATUS)"
-        
-        # Directory structure overview
-        echo "📂 Directory Structure:"
-        echo "   📁 Root: $(find . -maxdepth 1 -type f | wc -l) files"
-        echo "   📁 scripts/: $(find scripts -type f 2>/dev/null | wc -l) files"
-        echo "   📁 config/: $(find config -type f 2>/dev/null | wc -l) files"
-        echo "   📁 reports/: $(find reports -type f 2>/dev/null | wc -l) files"
-        echo "   📁 backups/: $(find backups -type f 2>/dev/null | wc -l) files"
-        echo "   📁 data/: $(find data -type f 2>/dev/null | wc -l) files"
-
-    - name: Commit Organized Changes
-      run: |
-        echo "=== Committing Organized Repository ==="
-        
-        # Add files in organized manner
-        echo "📝 Staging organized files..."
-        git add bulk_import.m3u || true
-        git add channels.txt || true
-        git add playlist.m3u || true
-        git add scripts/ || true
-        git add config/ || true
-        git add reports/daily/ || true
-        git add reports/logs/ || true
-        git add data/$(date +%Y-%m)/ || true
-        git add backups/*.gz || true
-        git add templates/ || true
-        git add .forgejo/ || true
-        git add README.md || true
-        git add .gitignore || true
-        
-        # Remove unwanted files from tracking
-        echo "🗑️ Removing unwanted files from git..."
-        git rm --cached *.log 2>/dev/null || true
-        git rm --cached **/__pycache__/** 2>/dev/null || true
-        git rm --cached **/*.pyc 2>/dev/null || true
-        git rm --cached **/*.tmp 2>/dev/null || true
-        git rm --cached setup_*.py 2>/dev/null || true
-        git rm --cached comprehensive_cleanup.py 2>/dev/null || true
-        
-        # Show staging status
-        echo "📋 Files staged for commit:"
-        STAGED_FILES=$(git diff --staged --name-only | wc -l)
-        git diff --staged --name-only | head -20 || echo "No staged changes"
-        if [ "$STAGED_FILES" -gt 20 ]; then
-          echo "   ... and $((STAGED_FILES - 20)) more files"
-        fi
-        
         if ! git diff --staged --quiet; then
-          # Gather statistics for commit message
+          CHANNELS="0"
-          CHANNEL_COUNT="0"
           if [ -f playlist.m3u ]; then
-            CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
+            CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
           fi
-          
+          git commit -m "Updated playlist: $CHANNELS channels - $(date '+%Y-%m-%d %H:%M')"
-          REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown")
-          TIMESTAMP=$(date '+%Y-%m-%d %H:%M')
-          
-          # Calculate cleanliness for commit message
-          CACHE_DIRS=$(find . -type d -name "__pycache__" | wc -l)
-          TEMP_FILES=$(find . -name "*.tmp" -o -name "*_temp*" | wc -l)
-          TOTAL_ISSUES=$((CACHE_DIRS + TEMP_FILES))
-          
-          if [ "$TOTAL_ISSUES" -eq 0 ]; then
-            CLEAN_STATUS="🏆 Pristine Repository"
-            CLEAN_SCORE="100/100"
-          elif [ "$TOTAL_ISSUES" -le 2 ]; then
-            CLEAN_STATUS="✅ Excellent Organization"
-            CLEAN_SCORE="90/100"
-          else
-            CLEAN_STATUS="🧹 Well Organized"
-            CLEAN_SCORE="75/100"
-          fi
-          
-          # Countries count for commit message
-          COUNTRIES_COUNT="0"
-          if [ -f playlist.m3u ] && grep -q 'group-title=' playlist.m3u; then
-            COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l)
-          fi
-          
-          # Create comprehensive commit message
-          COMMIT_MSG="📺 Updated playlist: $CHANNEL_COUNT channels across $COUNTRIES_COUNT countries ($TIMESTAMP)
-
-🎯 Repository Status:
-   📺 Channels: $CHANNEL_COUNT
-   🌍 Countries: $COUNTRIES_COUNT  
-   📁 Size: $REPO_SIZE
-   🏆 Quality: $CLEAN_STATUS ($CLEAN_SCORE)
-   🗂️ Structure: Professional & Organized
-
-🔧 Automated Maintenance Completed:
-   ✅ Python cache cleaned
-   ✅ Temporary files removed
-   ✅ Reports organized by date (reports/daily/)
-   ✅ Logs centralized (reports/logs/)
-   ✅ Backups compressed & rotated
-   ✅ Data snapshots created (data/$(date +%Y-%m)/)
-   ✅ Directory structure optimized
-   ✅ Import file ready for next use
-
-🚀 Repository Features:
-   📊 Daily reports with statistics
-   📈 Automated quality scoring
-   🗂️ Time-based file organization
-   💾 Intelligent backup management
-   🧹 Self-cleaning automation
-   
-Generated: $TIMESTAMP UTC | Bot: IPTV Playlist Manager v2.0"
-          
-          git commit -m "$COMMIT_MSG"
           git push
-          
+          echo "Changes committed successfully"
-          echo "✅ Repository updated successfully!"
-          echo "📺 Channels: $CHANNEL_COUNT"
-          echo "🌍 Countries: $COUNTRIES_COUNT"
-          echo "📁 Size: $REPO_SIZE" 
-          echo "🏆 Status: $CLEAN_STATUS"
         else
-          echo "ℹ️ No changes to commit - repository already up to date"
+          echo "No changes to commit"
         fi
 
-    - name: Success Summary and Next Steps
+    - name: Summary
       run: |
-        echo "=== 🎉 WORKFLOW COMPLETED SUCCESSFULLY! ==="
+        echo "Workflow completed successfully!"
-        echo ""
-        
         if [ -f playlist.m3u ]; then
-          CHANNEL_COUNT=$(grep -c "^#EXTINF" playlist.m3u 2>/dev/null || echo "0")
+          CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
-          COUNTRIES_COUNT=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l 2>/dev/null || echo "0")
+          echo "Playlist: $CHANNELS channels"
-          
-          echo "🎯 SUCCESS SUMMARY:"
-          echo "   📺 Playlist: $CHANNEL_COUNT channels generated"
-          echo "   🌍 Countries: $COUNTRIES_COUNT different regions"
-          echo "   📁 Repository: Professionally organized"
-          echo "   🧹 Cleanliness: Automated maintenance completed"
-          echo "   🔄 Automation: Fully self-maintaining"
-          echo "   🚀 Status: Ready for next import"
-        else
-          echo "⚠️ Playlist generation issue - check reports/logs/ for details"
         fi
-        
+        echo "Repository is clean and organized"
-        echo ""
-        echo "📊 REPOSITORY HEALTH:"
-        TOTAL_FILES=$(find . -type f -not -path './.git/*' | wc -l)
-        REPO_SIZE=$(du -sh . 2>/dev/null | cut -f1 || echo "unknown")
-        echo "   📁 Total files: $TOTAL_FILES (organized)"
-        echo "   💾 Repository size: $REPO_SIZE (optimized)"
-        echo "   🎯 Structure: Professional ✅"
-        echo "   📈 Quality score: Excellent ✅"
-        
-        echo ""
-        echo "🗂️ ORGANIZED STRUCTURE:"
-        echo "   📂 data/$(date +%Y-%m)/     → Monthly channel snapshots"
-        echo "   📂 reports/daily/     → Current reports"
-        echo "   📂 reports/logs/      → All log files"  
-        echo "   📂 backups/           → Compressed backups"
-        echo "   📂 scripts/           → Clean Python modules"
-        echo "   📂 config/            → Settings & patterns"
-        
-        echo ""
-        echo "🚀 WHAT'S NEXT:"
-        echo "   1. 📥 Add new channels to bulk_import.m3u"
-        echo "   2. 🔄 Push changes (triggers automatic processing)"
-        echo "   3. 📊 Check reports/daily/ for detailed statistics"
-        echo "   4. 🎉 Enjoy your perfectly organized repository!"
-        echo ""
-        echo "✨ Your IPTV repository is now pristine and will stay clean automatically! ✨"