Update .forgejo/workflows/generate-m3u.yml

.forgejo/workflows/generate-m3u.yml

@@ -1,43 +1,92 @@
-name: Generate M3U Playlist with Auto-Organization
+name: Enhanced IPTV Playlist with Discovery
 
 on:
   push:
     branches:
       - main
   workflow_dispatch:
+  schedule:
+    # Run discovery daily at 6 AM UTC
+    - cron: '0 6 * * *'
 
 jobs:
-  build-and-organize:
+  enhance-and-discover:
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v4
     
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    
+    - name: Install Discovery Dependencies
+      run: |
+        echo "Installing discovery dependencies..."
+        pip install requests beautifulsoup4
+        echo "Dependencies installed"
+    
     - name: Configure Git
       run: |
         git config --local user.email "actions@forgejo.plainrock127.xyz"
-        git config --local user.name "IPTV Playlist Bot"
+        git config --local user.name "IPTV Discovery Bot"
     
-    - name: Basic Setup
+    - name: Setup Directories
       run: |
-        echo "Setting up directories..."
+        echo "Setting up directory structure..."
-        mkdir -p reports/daily
+        mkdir -p reports/daily config
         echo "Setup completed"
 
+    - name: Run Channel Discovery
+      run: |
+        echo "Running automated channel discovery..."
+        if [ -f scripts/source_scraper.py ]; then
+          python3 scripts/source_scraper.py
+          echo "Channel discovery completed"
+        else
+          echo "Discovery script not found, skipping discovery"
+        fi
+
+    - name: Enable Stream Health Checking (Optional)
+      run: |
+        echo "Checking health check configuration..."
+        if [ -f config/settings.json ]; then
+          # Check if health checking should be enabled
+          HEALTH_ENABLED=$(python3 -c "
+import json
+try:
+    with open('config/settings.json', 'r') as f:
+        settings = json.load(f)
+    print(settings.get('enable_health_check', False))
+except:
+    print(False)
+" 2>/dev/null || echo "False")
+          
+          if [ "$HEALTH_ENABLED" = "True" ]; then
+            echo "Health checking is enabled"
+          else
+            echo "Health checking is disabled (recommended for faster processing)"
+          fi
+        fi
+
     - name: Check Import File
       run: |
         echo "Checking import file..."
         if [ -f bulk_import.m3u ]; then
           LINES=$(wc -l < bulk_import.m3u)
           echo "Found bulk_import.m3u with $LINES lines"
+          if [ "$LINES" -gt 2 ]; then
+            echo "Contains channels to process"
+          fi
         else
           echo "Creating empty bulk_import.m3u"
           echo '#EXTM3U' > bulk_import.m3u
         fi
 
-    - name: Run Playlist Generation
+    - name: Run Enhanced Playlist Generation
       run: |
-        echo "Running playlist generation..."
+        echo "Running enhanced playlist generation..."
         if [ -f scripts/generate_playlist.py ]; then
           python3 scripts/generate_playlist.py
           echo "Playlist generation completed"
@@ -46,36 +95,91 @@ jobs:
           exit 1
         fi
 
-    - name: Create Simple Report
+    - name: Generate Enhanced Reports
       run: |
-        echo "Creating report..."
+        echo "Generating enhanced reports..."
+        
         if [ -f playlist.m3u ]; then
           CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
-          echo "Found $CHANNELS channels in playlist"
+          echo "Generated playlist: $CHANNELS channels"
           
+          # Count countries and categories from existing system
+          COUNTRIES=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l || echo "0")
+          
+          # Create enhanced summary report
           DATE=$(date +%Y%m%d_%H%M%S)
-          REPORT="reports/daily/report_$DATE.md"
+          ENHANCED_REPORT="reports/daily/enhanced_report_$DATE.md"
           
-          echo "# Playlist Report" > "$REPORT"
+          # Check for discovery report
-          echo "Generated: $(date)" >> "$REPORT"
+          DISCOVERY_REPORT=""
-          echo "Channels: $CHANNELS" >> "$REPORT"
+          if ls reports/daily/discovery_report_*.md 1> /dev/null 2>&1; then
+            DISCOVERY_REPORT=$(ls -t reports/daily/discovery_report_*.md | head -1)
+            DISCOVERED_COUNT=$(grep "Total Channels Discovered:" "$DISCOVERY_REPORT" | grep -o '[0-9]*' || echo "0")
+          else
+            DISCOVERED_COUNT="0"
+          fi
           
-          echo "Report created: $REPORT"
+          cat > "$ENHANCED_REPORT" << EOF
+# Enhanced IPTV System Report
+**Generated:** $(date)
+
+## System Overview
+- **Total Channels:** $CHANNELS
+- **Countries/Groups:** $COUNTRIES
+- **Discovery Session:** $DISCOVERED_COUNT new channels found
+- **System Status:** ✅ Fully Operational
+
+## Features Active
+- ✅ **Automated Discovery** - Finding channels from multiple sources
+- ✅ **Smart Categorization** - Country-based grouping with pattern matching
+- ✅ **Duplicate Detection** - Signature-based deduplication
+- ✅ **Quality Detection** - 4K/FHD/HD/SD identification
+- ✅ **Content Filtering** - Adult content filtering available
+- 🔄 **Health Checking** - Available but disabled for performance
+
+## Recent Activity
+EOF
+
+          if [ "$DISCOVERED_COUNT" -gt "0" ]; then
+            echo "- 🔍 **Discovery:** $DISCOVERED_COUNT new channels discovered and processed" >> "$ENHANCED_REPORT"
+          else
+            echo "- 🔍 **Discovery:** No new channels found this session" >> "$ENHANCED_REPORT"
+          fi
+          
+          echo "- 📺 **Processing:** Generated playlist with $CHANNELS total channels" >> "$ENHANCED_REPORT"
+          echo "- 🌍 **Organization:** Channels organized into $COUNTRIES country groups" >> "$ENHANCED_REPORT"
+          
+          cat >> "$ENHANCED_REPORT" << EOF
+
+## Next Enhancement Phases
+- [ ] **Phase 2:** Genre-based sub-categorization (News, Sports, Movies, etc.)
+- [ ] **Phase 3:** Stream health validation and backup URL rotation
+- [ ] **Phase 4:** AI-powered channel metadata enhancement
+
+## Quick Access
+- 📺 [Download Playlist](../playlist.m3u)
+- 📋 [View Channels](../channels.txt)
+- 🔍 [Discovery Config](../config/discovery_sources.json)
+
+---
+*Enhanced IPTV System with Automated Discovery*
+EOF
+          
+          echo "Enhanced report created: $ENHANCED_REPORT"
         else
-          echo "No playlist found"
+          echo "No playlist generated"
         fi
 
     - name: Clean Old Reports
       run: |
         echo "Cleaning old reports..."
         cd reports/daily
-        # Count markdown files
         if ls *.md >/dev/null 2>&1; then
           COUNT=$(ls *.md | wc -l)
           echo "Found $COUNT reports"
-          if [ "$COUNT" -gt 3 ]; then
+          if [ "$COUNT" -gt 5 ]; then
-            echo "Removing excess reports..."
+            echo "Keeping 5 most recent reports..."
-            ls -t *.md | tail -n +4 | xargs rm -f
+            ls -t *.md | tail -n +6 | xargs rm -f
             echo "Cleanup done"
           fi
         else
@@ -89,28 +193,97 @@ jobs:
         echo '#EXTM3U' > bulk_import.m3u
         echo "Import file reset"
 
+    - name: Update Discovery Statistics
+      run: |
+        echo "Updating discovery statistics..."
+        
+        # Create or update discovery stats
+        if [ ! -f config/discovery_stats.json ]; then
+          echo '{"total_sessions": 0, "total_discovered": 0, "last_discovery": ""}' > config/discovery_stats.json
+        fi
+        
+        # Update stats if discovery report exists
+        if ls reports/daily/discovery_report_*.md 1> /dev/null 2>&1; then
+          LATEST_DISCOVERY=$(ls -t reports/daily/discovery_report_*.md | head -1)
+          DISCOVERED_COUNT=$(grep "Total Channels Discovered:" "$LATEST_DISCOVERY" | grep -o '[0-9]*' || echo "0")
+          
+          python3 -c "
+import json
+from datetime import datetime
+
+try:
+    with open('config/discovery_stats.json', 'r') as f:
+        stats = json.load(f)
+except:
+    stats = {'total_sessions': 0, 'total_discovered': 0, 'last_discovery': ''}
+
+stats['total_sessions'] += 1
+stats['total_discovered'] += $DISCOVERED_COUNT
+stats['last_discovery'] = datetime.now().isoformat()
+
+with open('config/discovery_stats.json', 'w') as f:
+    json.dump(stats, f, indent=2)
+
+print(f'Updated stats: {stats[\"total_sessions\"]} sessions, {stats[\"total_discovered\"]} total discovered')
+"
+        fi
+
     - name: Commit Changes
       run: |
-        echo "Committing changes..."
+        echo "Committing enhanced changes..."
         git add .
         if git diff --staged --quiet; then
           echo "No changes to commit"
         else
           CHANNELS="0"
+          COUNTRIES="0"
           if [ -f playlist.m3u ]; then
             CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
+            COUNTRIES=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l || echo "0")
           fi
-          git commit -m "📺 Updated playlist with $CHANNELS channels - $(date '+%Y-%m-%d %H:%M')"
+          
+          # Check if discovery was active
+          DISCOVERY_NOTE=""
+          if ls reports/daily/discovery_report_*.md 1> /dev/null 2>&1; then
+            LATEST_DISCOVERY=$(ls -t reports/daily/discovery_report_*.md | head -1)
+            DISCOVERED_COUNT=$(grep "Total Channels Discovered:" "$LATEST_DISCOVERY" | grep -o '[0-9]*' || echo "0")
+            if [ "$DISCOVERED_COUNT" -gt "0" ]; then
+              DISCOVERY_NOTE=" +$DISCOVERED_COUNT discovered"
+            fi
+          fi
+          
+          git commit -m "🚀 Enhanced IPTV: $CHANNELS channels across $COUNTRIES groups$DISCOVERY_NOTE - $(date '+%Y-%m-%d %H:%M')"
           git push
-          echo "Changes committed"
+          echo "Changes committed successfully"
         fi
 
-    - name: Summary
+    - name: Success Summary
       run: |
-        echo "=== WORKFLOW COMPLETE ==="
+        echo "=== ENHANCED IPTV SYSTEM COMPLETE ==="
         if [ -f playlist.m3u ]; then
           CHANNELS=$(grep -c "^#EXTINF" playlist.m3u || echo "0")
-          echo "✅ Playlist has $CHANNELS channels"
+          COUNTRIES=$(grep 'group-title=' playlist.m3u | sed 's/.*group-title="//; s/".*//' | sort -u | wc -l || echo "0")
+          echo "✅ Playlist: $CHANNELS channels across $COUNTRIES groups"
         fi
-        echo "✅ Reports updated"
+        
-        echo "✅ Repository cleaned"
+        # Show discovery stats if available
+        if [ -f config/discovery_stats.json ]; then
+          python3 -c "
+import json
+try:
+    with open('config/discovery_stats.json', 'r') as f:
+        stats = json.load(f)
+    print(f'✅ Discovery: {stats[\"total_sessions\"]} sessions, {stats[\"total_discovered\"]} total found')
+except:
+    pass
+"
+        fi
+        
+        echo "✅ Enhanced features active:"
+        echo "   🔍 Automated discovery"
+        echo "   🏷️ Smart categorization" 
+        echo "   🔄 Duplicate detection"
+        echo "   📊 Enhanced reporting"
+        echo ""
+        echo "🎯 Ready for Phase 2: Genre categorization"
+        echo "🎯 Ready for Phase 3: Stream health validation"